diff --git a/apps/web/PRODUCT.md b/apps/web/PRODUCT.md index 4c14c1539..a434ffeb5 100644 --- a/apps/web/PRODUCT.md +++ b/apps/web/PRODUCT.md @@ -32,10 +32,10 @@ The console runs beside the administrator's own Core, with execution, files and - **Monitor**: Overview (service status, running Sessions, sandbox slots, the Needs attention count for the last 24 hours, 24-hour Session activity, a compact inventory of Core and up to four nodes, prioritizing offline and degraded nodes when the list is full, where Core opens Core metrics, a node opens its page on Nodes and E2B opens Sandbox metrics, the Needs attention table, whose rows name each Session, its project, its one-line reason and last activity and open it, with older ones left to the Session log, usage by project), Core metrics (whether Core can run Turns: execution slots, the Turn queue, its p95 wait and interrupted Turns, connected daemons, database latency, the Core process's CPU and memory, and background jobs; Go heap, goroutines and the database pool sit in a collapsed Diagnostics area), Agent metrics (requests, errors, duration, tokens, models, tools, Agents and API keys for 1 h / 6 h / 24 h / 7 d), Sandbox metrics (hosted Runtimes across projects over time, active against total; a sandbox opens in a dialog with its figures and CPU and memory charts; an E2B deployment adds what Core holds in E2B's cloud), Request log (recent public or administration HTTP requests, status, duration, safe error code and correlation IDs; bounded to the serving Core instance, with manual refresh and no body capture), Session log (every Session, read-only, with a failed Session's reason under its status, opening one Session's history, which jumps to its failed Turns; a self-hosted Session's page also has its environment's executor credentials). Session facts and Agent metrics link to an Agent's page only when that saved resource exists; inline snapshots and snapshots of deleted Agents remain readable without a broken resource link. Agent metrics' failed Turns open the Agent's Sessions in the Session log. - **Resources**: Agents, Environment templates, Skills, Files, Vaults. Each list shows one project or all projects, with a Project column when all are shown and a Creator column naming the creating key. Detail pages show the resource's facts and offer Delete. An empty list says, in one sentence, that applications create these resources through the Agents API, and links to that resource's section of the API guide; Agent metrics does the same for Sessions. - **Platform**: Projects and keys (projects, their assets and usage, named keys, write history), Nodes (the node list, each node's capacity and limits, host figures and charts, allocations, Add node, Edit node and individual removal operations), and Settings, the single entry for platform configuration. Settings has four groups: Sandbox and scheduling, Workspace storage, Models and credentials, and Service and deployment. Sandbox and scheduling owns setup, resource edits, rollout, reset and the idle policy, and links to Nodes for nodes and their capacity. Add node on Nodes asks for limits before issuing its one-time command; installers use Core's public URL and require supported node artifacts. Edit node changes a node's name and limits; removal stays on the node page and offers the host's uninstall command. Models and credentials owns each Harness's default model and shows whether each Harness is ready: Ready when it is enabled and has a default model provider, otherwise the missing step, Model provider not set (with Set as the primary action) or Disabled (with a link to the startup settings). Getting started uses the same readiness. Service and deployment owns installation facts (the public and API base URLs, with the installation ID and source commit behind Installation details), Domain and HTTPS actions and read-only startup settings, each named by its environment variable, with a link to the configuration guide's section on changing them. -- A node whose provider is not ready names the reason as one Provider-neutral readiness class (provider unavailable, host unsupported, provider files or Runtime image missing, Runtime download failed, a host too small) and its fix in the help tip beside its status, wherever that status shows. +- A node whose provider is not ready names the reason as one Provider-neutral readiness class (provider unavailable, host unsupported, provider files or Runtime image missing, a host too small) and its fix in the help tip beside its status, wherever that status shows. - A node enrolled with an earlier Core address gets no new sandboxes, so on the Nodes list and its page its status is Old address, with "Remove and add again", never Available. - **Sandbox reset** is an explicit administrator operation in Settings → Sandbox and scheduling. Auto clear is the default, with a one-hour deadline (5 minutes–24 hours); Force clear requires destructive confirmation. Reset stops new hosted Session admission, clears idle, suspended and pending hosted work, and waits for busy Turns and file writes until Core forces the remaining work. It does not affect self-hosted execution. Histories and persisted Files/Artifacts remain; archived Sessions cannot resume, and unpersisted workspace contents may be lost. Cancel stops further clearing without undoing archives. Core alone reports progress and completion, including resources blocked on named offline nodes; force does not bypass their cleanup. Completion clears the backend configuration and retires old nodes/enrollment credentials. A new configuration is then a separate deliberate save. -- **Online sandbox configuration** changes the same backend's resources, Runtime or E2B template without retiring existing nodes or changing existing Sessions' resource ownership. New placement follows Core's qualified capacity; saving a target does not promise immediate placement on it. Changes keep the saved Runtime. When it differs from the Runtime this console serves, Sandbox and scheduling adds a Runtime row with both source commits and Use this installation's Runtime, which opens Change resources on its review with that Runtime; a console without a Runtime manifest leaves the release to Advanced settings. Configuration rollout shows Core's target preparation and retained previous-generation sandbox count. A settled rollout can still have failed, update-required or unknown nodes and old resources. An offline node stays offline even when it has a recorded serving generation. Node and allocation detail distinguish the serving pin, target preparation and each resource's configuration generation. +- **Online sandbox configuration** changes the same backend's resources or E2B template without retiring existing nodes or changing existing Sessions' resource ownership. New placement follows Core's qualified capacity; saving a target does not promise immediate placement on it. Configuration rollout shows Core's target preparation and retained previous-generation sandbox count. A settled rollout can still have failed or unknown nodes and old resources. An offline node stays offline even when it has a recorded serving generation. Node and allocation detail distinguish the serving pin, target preparation and each resource's configuration generation. - **E2B credential replacement** uses the same configuration form. Setup requires a key; leaving it blank during an update keeps the saved key. An explicit key, even the same value, is verified as a replacement and advances the target generation after successful verification. Another backend or E2B team requires a deliberate reset. A rejected or uncertain replacement never clears the committed configuration or replays the write. - **E2B deployments** have no machines: Nodes offers a link to Settings → Sandbox and scheduling. Overview and Sandbox metrics show the sandboxes Core holds in E2B's cloud (running, starting, size, template build) instead of node capacity, with no Add node action; a sandbox's dialog adds its disk use. - **Automatic suspension** appears when the deployment declares a suspension policy. Nodes shows each node's suspended count and retained limit, the most sandboxes it keeps running and suspended together, with Edit node for its limits; Overview adds the suspended total of all registered nodes under its sandbox slots. Settings → Sandbox and scheduling owns **Automatic suspension**: deployment-wide idle seconds are editable there; snapshot retention remains read-only. Necessary policy explanations belong in help tips, not persistent helper paragraphs. These are Core records, not measured CPU or memory savings. Node allocations show the recorded compute phase and elapsed phase time, never an inferred snapshot-expiry countdown. Sandbox metrics' Hosted Runtimes has one Status column: Core's lifecycle record of each sandbox (Active, Suspended, Changing state, Waiting for allocation or Stopped), which Core derives from the allocation's recorded phase. The Session page's Runtime state reads the same way. When an active sandbox has no sample, its CPU and Memory cells name why. Independent workspace persistence is explained only when the deployment declares it. Nodes refreshes its inventory every 30 seconds while visible; reset and active preparation retain their faster polling. diff --git a/apps/web/e2e/console.ts b/apps/web/e2e/console.ts index 2735ebc8c..fbe39052b 100644 --- a/apps/web/e2e/console.ts +++ b/apps/web/e2e/console.ts @@ -55,7 +55,7 @@ export async function failNext(request: APIRequestContext, failure: { method: st } /** Registers a node, or changes one, as a host running the last enrollment command (or, with its `enrollment_id`, another one) would. */ -export async function setNode(request: APIRequestContext, node: { id: string; name?: string; online?: boolean; provider_ready?: boolean; diagnostic?: string; enrollment_id?: string; rollout?: SandboxNodeRollout }) { +export async function setNode(request: APIRequestContext, node: { id: string; name?: string; online?: boolean; provider_ready?: boolean; enrollment_id?: string; rollout?: SandboxNodeRollout }) { await request.post(`${fixture}/__fixture/node`, { data: node }); } diff --git a/apps/web/e2e/data/routes.mjs b/apps/web/e2e/data/routes.mjs index 124a7966c..48ac025cd 100644 --- a/apps/web/e2e/data/routes.mjs +++ b/apps/web/e2e/data/routes.mjs @@ -85,7 +85,7 @@ export function buildDemo(now = Math.floor(Date.now() / 1000), publicUrl = "http sessions.sort((a, b) => b.created_at - a.created_at); const nodes = [ { rollout: { state: "ready", ready_generation: 1 }, id: "node-local", name: "core-01", provider: "docker", online: true, provider_ready: true, cpu_count: 16, available_memory_bytes: 38 * 2 ** 30, available_disk_bytes: 410 * 2 ** 30, running: 5, snapshots: 2, last_seen_at: new Date((now - 8) * 1000).toISOString(), max_active: 8, max_retained: 16, active: 5, reserved: 1, retained: 2, cleanup_pending: 0, created_at: new Date((now - 86400 * 30) * 1000).toISOString() }, - { rollout: { state: "failed", ready_generation: null, diagnostic: "host_unsupported" }, id: "node-gpu", name: "gpu-worker-02", provider: "docker", online: true, provider_ready: false, diagnostic: "host_unsupported", cpu_count: 32, available_memory_bytes: 12 * 2 ** 30, available_disk_bytes: 96 * 2 ** 30, running: 7, snapshots: 5, last_seen_at: new Date((now - 12) * 1000).toISOString(), max_active: 8, max_retained: 16, active: 7, reserved: 0, retained: 5, cleanup_pending: 1, created_at: new Date((now - 86400 * 12) * 1000).toISOString() }, + { rollout: { state: "failed", ready_generation: null, diagnostic: "host_unsupported" }, id: "node-gpu", name: "gpu-worker-02", provider: "docker", online: true, provider_ready: false, cpu_count: 32, available_memory_bytes: 12 * 2 ** 30, available_disk_bytes: 96 * 2 ** 30, running: 7, snapshots: 5, last_seen_at: new Date((now - 12) * 1000).toISOString(), max_active: 8, max_retained: 16, active: 7, reserved: 0, retained: 5, cleanup_pending: 1, created_at: new Date((now - 86400 * 12) * 1000).toISOString() }, { rollout: { state: "unknown", ready_generation: 1 }, id: "node-edge", name: "edge-03", provider: "docker", online: false, provider_ready: false, cpu_count: 8, available_memory_bytes: null, available_disk_bytes: null, running: 0, snapshots: 0, last_seen_at: new Date((now - 5400) * 1000).toISOString(), max_active: 4, max_retained: 8, active: 0, reserved: 0, retained: 0, cleanup_pending: 0, created_at: new Date((now - 86400 * 3) * 1000).toISOString() }, ]; const hosted = sessions.filter((session) => session.environment.type === "openai_hosted"); diff --git a/apps/web/e2e/fixture-console.mjs b/apps/web/e2e/fixture-console.mjs index 811f0d9e1..0922789b8 100644 --- a/apps/web/e2e/fixture-console.mjs +++ b/apps/web/e2e/fixture-console.mjs @@ -22,10 +22,7 @@ const LOCKOUT_AFTER = 3; const LOCKOUT_SECONDS = 30; let state; -const hex = (c) => c.repeat(64); -/** The Runtime release this fixture's distribution manifest describes. */ -const release = { source_commit: "c0ffee".padEnd(40, "0"), image_id: `sha256:${hex("1")}`, image_manifest_digest: `sha256:${hex("2")}`, microsandbox_ref: `oac-runtime@sha256:${hex("3")}`, runtime_sha256: hex("4"), firmware_sha256: hex("5") }; -const manifest = { platform: "linux/amd64", source_commit: release.source_commit, images: { runtime: release.image_id }, image_manifest_digests: { runtime: release.image_manifest_digest }, runtime_ref: release.microsandbox_ref, microsandbox: { runtime_sha256: release.runtime_sha256, firmware_sha256: release.firmware_sha256 }, artifacts: {} }; +const SOURCE_COMMIT = "c0ffee".padEnd(40, "0"); /** * config.json's public_url. "public": an HTTPS address, so applications get an API base URL; @@ -50,7 +47,7 @@ function installation() { return { // As Core: api_base_url is always public_url followed by /v1; local_only marks a loopback public_url. object: "core.installation", installation_id: INSTALLATION_ID, public_url: publicUrl(), api_base_url: `${publicUrl()}/v1`, - local_only: local, source_commit: release.source_commit, + local_only: local, source_commit: SOURCE_COMMIT, configuration: { settings: [ // The settings Core reports (services/core/internal/processconfig), with the enabled harnesses of this fixture. @@ -77,7 +74,7 @@ function installation() { const noNodeRollout = () => ({ state: "settled", previous_generation_sandboxes: 0, nodes: null }); function nodeRollout(previous = 0) { - const nodes = { ready: 0, preparing: 0, failed: 0, update_required: 0, unknown: 0 }; + const nodes = { ready: 0, preparing: 0, failed: 0, unknown: 0 }; for (const node of state.nodes) nodes[node.rollout.state]++; return { state: nodes.preparing > 0 ? "preparing" : "settled", previous_generation_sandboxes: previous, nodes }; } @@ -87,7 +84,7 @@ function unconfiguredDeployment(generation = 0, ownerEpoch = 3) { } function configuredDeployment() { - return { installation_id: INSTALLATION_ID, provider: "docker", credential_configured: false, configuration: {}, metadata: {}, core_url: publicUrl(), reset: null, rollout: nodeRollout(), owner_epoch: 3, generation: 1, mode: "nodes", resources: { allocations: 0, pending: 0 }, specification: { resources: { cpus: 2, memory_mib: 4096 }, runtime: release }, specification_digest: "fixture", suspension: null }; + return { installation_id: INSTALLATION_ID, provider: "docker", credential_configured: false, configuration: {}, metadata: {}, core_url: publicUrl(), reset: null, rollout: nodeRollout(), owner_epoch: 3, generation: 1, mode: "nodes", resources: { allocations: 0, pending: 0 }, specification: { resources: { cpus: 2, memory_mib: 4096 } }, specification_digest: "fixture", suspension: null }; } /** The E2B template build as Core read it when the selection was saved. */ const templateBuild = { status: "ready", resources: { cpus: 2, memory_mib: 2048, root_disk_mib: 10240 } }; @@ -397,7 +394,7 @@ async function sandboxRoute(request, response, path, url) { if (initialize && state.deployment.provider) return error(response, 409, "The sandbox deployment is already configured.", "sandbox_deployment_conflict"); if (!initialize && !state.deployment.provider) return error(response, 409, "The sandbox deployment is not configured.", "sandbox_deployment_conflict"); const e2b = input.provider === "e2b"; - if (!e2b && (!input.resources || !input.runtime)) return error(response, 400, "resources and runtime are required.", "invalid_sandbox_configuration"); + if (!e2b && !input.resources) return error(response, 400, "resources are required.", "invalid_sandbox_configuration"); // As Core (ErrSandboxPublicURLUnreachable): E2B sandboxes reach Core over the internet, which a loopback public_url cannot serve. if (e2b && state.installation === "local") return error(response, 409, "E2B sandboxes reach Core over the internet. Set an HTTPS public URL that is not loopback (public_url in config.json, OAC_PUBLIC_URL for Core).", "sandbox_configuration_error"); // Synthetic classifier outcomes only; never persist or echo submitted keys. @@ -409,7 +406,7 @@ async function sandboxRoute(request, response, path, url) { // As Core: E2B may omit resources and adopt its template build's CPU and memory; only microsandbox suspends. const resources = input.resources ?? { cpus: templateBuild.resources.cpus, memory_mib: templateBuild.resources.memory_mib }; const previous = state.deployment; - const specification = { resources, ...(input.runtime ? { runtime: input.runtime } : {}) }; + const specification = { resources }; const explicitKey = e2b && Object.hasOwn(input, "credential"); const sameSelection = !initialize && JSON.stringify(specification) === JSON.stringify(previous.specification) && (!e2b || input.configuration.template === previous.configuration?.template); // Omission can be a no-op; every explicit key, including identical bytes, @@ -516,8 +513,7 @@ async function executorCredentialRoute(request, response, projectId, environment /** * A node as it first registers with the last enrollment command: not yet connected, - * provider not ready, with the token's limits and enrollment ID. Like Core, it has no - * diagnostic field until one is reported. + * provider not ready, with the token's limits and enrollment ID. */ function registeredNode(nodeId) { const { enrollment_id = null, ...limits } = state.enrollment ?? { max_active: 1, max_retained: 1 }; @@ -623,21 +619,20 @@ async function fixtureRoute(request, response, url) { return send(response, 200, { ok: true }); } if (url.pathname === "/__fixture/node" && request.method === "POST") { - // { id, ...fields }: registers the node on first use, then applies the fields (online, provider_ready, diagnostic, - // or another command's enrollment_id); an empty diagnostic removes the field, as Core omits it. + // { id, ...fields }: registers the node on first use, then applies the fields (online, provider_ready, + // or another command's enrollment_id). const { id: nodeId, ...fields } = await body(request); let node = state.nodes.find((entry) => entry.id === nodeId); if (!node) state.nodes.push(node = registeredNode(nodeId)); Object.assign(node, fields); // A control may supply an explicit target observation. Existing enrollment // cases only supply connection facts, so give that newly joined target a pin. - if (!fields.rollout && ("online" in fields || "provider_ready" in fields || "diagnostic" in fields)) { + if (!fields.rollout && ("online" in fields || "provider_ready" in fields)) { node.rollout = !node.online ? { state: "unknown", ready_generation: node.rollout.ready_generation } : node.provider_ready ? { state: "ready", ready_generation: state.deployment.generation } - : { state: "failed", ready_generation: node.rollout.ready_generation, diagnostic: node.diagnostic || "provider_unavailable" }; + : { state: "failed", ready_generation: node.rollout.ready_generation, diagnostic: "provider_unavailable" }; } if (state.deployment.mode === "nodes") state.deployment.rollout = nodeRollout(state.deployment.rollout.previous_generation_sandboxes); - if (!node.diagnostic) delete node.diagnostic; return send(response, 200, node); } if (url.pathname === "/__fixture/requests") return send(response, 200, { violations: state.violations, writes: state.writes }); @@ -648,8 +643,6 @@ http.createServer(async (request, response) => { const url = new URL(request.url, `http://127.0.0.1:${port}`); try { if (url.pathname.startsWith("/__fixture/")) return await fixtureRoute(request, response, url); - // The console service serves its distribution manifest to anyone, as nodes download it. - if (url.pathname === "/node-install/manifest.json") return send(response, 200, manifest); if (request.headers.authorization) state.violations.push(`Authorization header on ${request.method} ${url.pathname}`); if (url.pathname.startsWith("/console/")) return await consoleRoute(request, response, url); const signedIn = state.auth.mode === "authenticated" && request.headers.cookie?.includes(SESSION_COOKIE); diff --git a/apps/web/e2e/nodes.spec.ts b/apps/web/e2e/nodes.spec.ts index 51c22b49b..09e9f8c0c 100644 --- a/apps/web/e2e/nodes.spec.ts +++ b/apps/web/e2e/nodes.spec.ts @@ -91,11 +91,11 @@ test("adds a node: host requirements, a root/sudo command, a countdown, the same await expect(problem).toContainText("Not connected yet"); await expect(problem).toContainText("sudo journalctl -u oac-node-7f3c2a90-5b1e-4c2d-9e3f-0a1b2c3d4e5f.service"); await expect(add.getByText("No sudo on this host?")).toHaveCount(0); - // Connected, it reports why its provider isn't ready; once ready, the node is connected. - await setNode(request, { id: "node-new", online: true, diagnostic: "host_unsupported" }); - await expect(problem).toContainText("Host unsupported"); + // Connected but not ready, it keeps waiting for its provider; once ready, the node is connected. + await setNode(request, { id: "node-new", online: true }); + await expect(problem).toContainText("Sandbox provider unavailable"); await expect(progress).toContainText("Waiting for Docker"); - await setNode(request, { id: "node-new", provider_ready: true, diagnostic: "" }); + await setNode(request, { id: "node-new", provider_ready: true }); await expect(progress).toHaveText("edge-04 · Connected"); await add.getByRole("button", { name: "Done" }).click(); await expect(add).toBeHidden(); @@ -192,7 +192,7 @@ test("gives the host's uninstall command even when the installation must be read await expect(cleanup.getByLabel("Uninstall command", { exact: true })).toHaveValue(/'https:\/\/core\.example\.com\/node-install\/node-install\.pyz'/); }); -test("sets up own-machine sandboxes page by page, with the Runtime from the distribution", async ({ page, request }) => { +test("sets up own-machine sandboxes page by page", async ({ page, request }) => { await openConsole(page, request, "settings?id=sandbox", { sandbox: "none" }); await expect(page.getByRole("heading", { name: "Where should sandboxes run?" })).toBeVisible(); await page.getByRole("button", { name: "Own machines" }).click(); @@ -211,9 +211,6 @@ test("sets up own-machine sandboxes page by page, with the Runtime from the dist expect(await writes(request)).toEqual(["POST /core/v1/sandbox/deployment"]); await add.getByRole("button", { name: "Close dialog" }).click(); await page.getByRole("button", { name: "Settings", exact: true }).click(); - // The saved specification carries the Runtime read from the console's manifest. - await page.getByRole("button", { name: "Configuration details", exact: true }).click(); - await expect(page.locator(".help-tip-popover").getByText("c0ffee000000", { exact: true })).toBeVisible(); }); test("preselects microsandbox and asks once before switching to Docker", async ({ page, request }) => { @@ -305,11 +302,9 @@ test("shows the retained E2B build while a replacement key is checked", async ({ await expect(edit.getByRole("button", { name: "Next" })).toBeEnabled(); }); -test("edits only the saved backend, preserving a custom size and Runtime", async ({ page, request }) => { - const runtime = { source_commit: "0".repeat(40), image_id: `sha256:${"a".repeat(64)}`, image_manifest_digest: `sha256:${"b".repeat(64)}`, - microsandbox_ref: `oac-runtime@sha256:${"b".repeat(64)}`, runtime_sha256: "c".repeat(64), firmware_sha256: "d".repeat(64) }; - const current = { resources: { cpus: 7, memory_mib: 8192 }, runtime }; - let deployment = { configuration: {}, metadata: {}, credential_configured: false, installation_id: "94be54a1-138c-4f30-bc87-b13686272dbe", provider: "docker", core_url: "https://core.example", reset: null, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 0, preparing: 0, failed: 0, update_required: 0, unknown: 0 } }, +test("edits only the saved backend, preserving a custom size", async ({ page, request }) => { + const current = { resources: { cpus: 7, memory_mib: 8192 } }; + let deployment = { configuration: {}, metadata: {}, credential_configured: false, installation_id: "94be54a1-138c-4f30-bc87-b13686272dbe", provider: "docker", core_url: "https://core.example", reset: null, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 0, preparing: 0, failed: 0, unknown: 0 } }, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 0, pending: 0 }, specification: current, specification_digest: "e".repeat(64), suspension: null }; let submitted: Record | null = null; @@ -327,7 +322,7 @@ test("edits only the saved backend, preserving a custom size and Runtime", async await expect(page.getByRole("button", { name: "E2B cloud" })).toHaveCount(0); await page.getByRole("button", { name: /^Current/ }).click(); await page.getByRole("button", { name: "Save configuration" }).click(); - await expect.poll(() => submitted).toMatchObject({ provider: "docker", expected_generation: 1, resources: current.resources, runtime }); + await expect.poll(() => submitted).toMatchObject({ provider: "docker", expected_generation: 1, resources: current.resources }); expect(submitted).not.toHaveProperty("core_url"); }); @@ -351,9 +346,6 @@ test("keeps the page usable when Core refuses a sandbox change, and shows a safe const added = page.getByRole("dialog", { name: "Add node" }); await expect(added).toBeVisible(); await added.getByRole("button", { name: "Close dialog" }).click(); - await page.getByRole("button", { name: "Settings", exact: true }).click(); - await page.getByRole("button", { name: "Configuration details", exact: true }).click(); - await expect(page.locator(".help-tip-popover").getByText("c0ffee000000", { exact: true })).toBeVisible(); }); test("renames a node and sets how many sandboxes run on it at once", async ({ page, request }) => { diff --git a/apps/web/e2e/sandbox-generation.spec.ts b/apps/web/e2e/sandbox-generation.spec.ts index 5f6fd2aef..7f19b121a 100644 --- a/apps/web/e2e/sandbox-generation.spec.ts +++ b/apps/web/e2e/sandbox-generation.spec.ts @@ -105,22 +105,6 @@ test("online configuration retains existing resources and stops rapid polling wh expect(await writes(request)).toEqual([`PUT ${deploymentPath}`]); }); -test("a saved Runtime moves to the one this installation serves through the reviewed update", async ({ page, request }) => { - await openConsole(page, request, "settings?id=sandbox"); - const served = await (await page.request.get(new URL("/node-install/manifest.json", page.url()).href)).json() as { source_commit: string; images: { runtime: string } }; - const hex = (c: string) => c.repeat(64); - const saved = { source_commit: "a".repeat(40), image_id: `sha256:${hex("a")}`, image_manifest_digest: `sha256:${hex("b")}`, microsandbox_ref: `oac-runtime@sha256:${hex("c")}`, runtime_sha256: hex("d"), firmware_sha256: hex("e") }; - await setDeployment(request, { specification: { resources: { cpus: 2, memory_mib: 4096 }, runtime: saved } }); - await page.getByRole("button", { name: "Refresh sandbox state", exact: true }).click(); - await expect(page.getByText(`Saved ${saved.source_commit.slice(0, 12)} · this installation serves ${served.source_commit.slice(0, 12)}`)).toBeVisible(); - const use = page.getByRole("button", { name: "Use this installation's Runtime", exact: true }); - await use.click(); - const sent = await saveConfiguration(page); - expect(sent.input.runtime).toMatchObject({ source_commit: served.source_commit, image_id: served.images.runtime }); - await expect(use).toHaveCount(0); - await inspectRollout(page, { "Core preparation": "Preparing configuration", "Target generation": "2" }); -}); - test("unknown target preparation preserves live old-generation service while an offline pin stays offline", async ({ page, request }, info) => { await page.setViewportSize({ width: 1280, height: 1000 }); await page.emulateMedia({ colorScheme: "light", reducedMotion: "reduce" }); @@ -128,9 +112,9 @@ test("unknown target preparation preserves live old-generation service while an await setDeployment(request, { generation: 2 }); await setNode(request, { id: "node-local", online: true, provider_ready: true, rollout: { state: "unknown", ready_generation: 1 } }); await setNode(request, { id: "node-edge", online: false, provider_ready: true, rollout: { state: "unknown", ready_generation: 1 } }); - await setNode(request, { id: "node-gpu", rollout: { state: "update_required", ready_generation: 1 } }); + await setNode(request, { id: "node-gpu", rollout: { state: "failed", ready_generation: 1 } }); await page.getByRole("button", { name: "Refresh sandbox state", exact: true }).click(); - await inspectRollout(page, { "Node software incompatible": "1" }); + await inspectRollout(page, { "Preparation failed": "1" }); await page.getByRole("button", { name: "Nodes", exact: true }).click(); const serving = page.getByRole("row", { name: /core-01/ }); await expect(serving).toContainText("Available"); @@ -149,7 +133,7 @@ test("unknown target preparation preserves live old-generation service while an await expect(setup).toContainText("Done"); await expect(rollout(page)).toHaveCount(0); await page.getByRole("button", { name: "Settings", exact: true }).click(); - await inspectRollout(page, { "Target generation": "2", "Target readiness unknown": "2", "Node software incompatible": "1" }); + await inspectRollout(page, { "Target generation": "2", "Target readiness unknown": "2", "Preparation failed": "1" }); await capture(page, info, "generation-configuration-summary-en", rollout(page)); expect(await writes(request)).toEqual([]); }); diff --git a/apps/web/e2e/sandbox-rollout-details.spec.ts b/apps/web/e2e/sandbox-rollout-details.spec.ts index bf9618dd2..4695ac5ee 100644 --- a/apps/web/e2e/sandbox-rollout-details.spec.ts +++ b/apps/web/e2e/sandbox-rollout-details.spec.ts @@ -9,7 +9,7 @@ test("keeps rollout summary brief and opens all Core observations in details", a await page.route("**/core/v1/sandbox/deployment", async (route) => { const response = await route.fetch(); const deployment = await response.json(); - await route.fulfill({ response, json: { ...deployment, generation: 4, resources: { allocations: 8, pending: 0 }, rollout: { state: "settled", previous_generation_sandboxes: 8, nodes: { ready: 1, preparing: 0, failed: 2, update_required: 3, unknown: 4 } } } }); + await route.fulfill({ response, json: { ...deployment, generation: 4, resources: { allocations: 8, pending: 0 }, rollout: { state: "settled", previous_generation_sandboxes: 8, nodes: { ready: 1, preparing: 0, failed: 2, unknown: 4 } } } }); }); await openConsole(page, request, "settings?id=sandbox"); const summary = page.getByRole("region", { name: "Configuration rollout", exact: true }); @@ -23,7 +23,6 @@ test("keeps rollout summary brief and opens all Core observations in details", a await expect(fact(dialog, "Target generation")).toHaveText("4"); await expect(fact(dialog, "Previous-generation sandboxes")).toHaveText("8"); await expect(fact(dialog, "Preparation failed")).toHaveText("2"); - await expect(fact(dialog, "Node software incompatible")).toHaveText("3"); await expect(fact(dialog, "Target readiness unknown")).toHaveText("4"); await page.keyboard.press("Escape"); await expect(dialog).not.toBeVisible(); @@ -35,7 +34,7 @@ test("labels unknown target readiness without declaring settled nodes ready", as await page.route("**/core/v1/sandbox/deployment", async (route) => { const response = await route.fetch(); const deployment = await response.json(); - await route.fulfill({ response, json: { ...deployment, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, update_required: 0, unknown: 1 } } } }); + await route.fulfill({ response, json: { ...deployment, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 1 } } } }); }); await openConsole(page, request, "settings?id=sandbox"); const summary = page.getByRole("region", { name: "Configuration rollout", exact: true }); diff --git a/apps/web/src/features/fleet/SandboxResetNotice.test.tsx b/apps/web/src/features/fleet/SandboxResetNotice.test.tsx index 427c1c816..0cb7598a1 100644 --- a/apps/web/src/features/fleet/SandboxResetNotice.test.tsx +++ b/apps/web/src/features/fleet/SandboxResetNotice.test.tsx @@ -6,7 +6,7 @@ import { SandboxResetNotice } from "./SandboxResetNotice"; const reset: SandboxReset = { clear: "auto", requested_at: "2026-09-27T10:00:00Z", deadline_at: "2026-09-27T11:00:00Z", forced_at: null, remaining: { busy: 1, idle: 0, cleanup: 2, on_offline_nodes: 1, offline_nodes: [{ node_id: "n1", name: "Node 1", resources: 1 }] } }; -const deployment = (reset: SandboxReset | null): SandboxDeployment => ({ credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, update_required: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "http://core", reset, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 3, pending: 0 }, suspension: null }); +const deployment = (reset: SandboxReset | null): SandboxDeployment => ({ credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "http://core", reset, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 3, pending: 0 }, suspension: null }); const render = (value: SandboxDeployment | undefined, failed = false) => renderToStaticMarkup( {}} />); describe("reset notices on read-only surfaces", () => { diff --git a/apps/web/src/features/fleet/fleet-model.test.ts b/apps/web/src/features/fleet/fleet-model.test.ts index c8de45dbc..700ea2250 100644 --- a/apps/web/src/features/fleet/fleet-model.test.ts +++ b/apps/web/src/features/fleet/fleet-model.test.ts @@ -15,7 +15,7 @@ describe("fleet model", () => { it("classifies node health", () => { expect(nodeHealth(node("a"))).toBe("available"); - expect(nodeHealth(node("a", { diagnostic: "provider_unavailable" }))).toBe("degraded"); + expect(nodeHealth(node("a", { provider_ready: false }))).toBe("degraded"); expect(nodeHealth(node("a", { online: false }))).toBe("offline"); }); diff --git a/apps/web/src/features/fleet/fleet-model.ts b/apps/web/src/features/fleet/fleet-model.ts index 48cd3a1c6..cbbdbc1bf 100644 --- a/apps/web/src/features/fleet/fleet-model.ts +++ b/apps/web/src/features/fleet/fleet-model.ts @@ -10,7 +10,7 @@ export type NodeHealth = "available" | "degraded" | "offline"; export function nodeHealth(node: SandboxNode): NodeHealth { if (!node.online) return "offline"; - return node.provider_ready && !node.diagnostic ? "available" : "degraded"; + return node.provider_ready ? "available" : "degraded"; } /** Live provider evidence is independent of target preparation and durable serving pins. */ diff --git a/apps/web/src/features/fleet/fleet-queries.test.ts b/apps/web/src/features/fleet/fleet-queries.test.ts index 963af5991..e0eb11cc1 100644 --- a/apps/web/src/features/fleet/fleet-queries.test.ts +++ b/apps/web/src/features/fleet/fleet-queries.test.ts @@ -5,7 +5,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { sandboxAdmin, sandboxDeploymentQuery, sandboxScope } from "../sandbox/sandbox-queries"; import { fleetQuery } from "./fleet-queries"; -const deployment: SandboxDeployment = { credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, update_required: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "http://core", owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 1, pending: 0 }, suspension: null, +const deployment: SandboxDeployment = { credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "http://core", owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 1, pending: 0 }, suspension: null, reset: { clear: "auto", requested_at: "2026-09-27T10:00:00Z", deadline_at: "2026-09-27T11:00:00Z", forced_at: null, remaining: { busy: 1, idle: 0, cleanup: 0, on_offline_nodes: 0, offline_nodes: [] } } }; afterEach(() => { vi.restoreAllMocks(); vi.unstubAllGlobals(); }); diff --git a/apps/web/src/features/fleet/use-sandbox-fleet.test.tsx b/apps/web/src/features/fleet/use-sandbox-fleet.test.tsx index c97c61b5e..756ae7660 100644 --- a/apps/web/src/features/fleet/use-sandbox-fleet.test.tsx +++ b/apps/web/src/features/fleet/use-sandbox-fleet.test.tsx @@ -10,7 +10,7 @@ import { fleetQuery, type FleetSnapshot } from "./fleet-queries"; import { FleetReadNotice } from "./FleetReadNotice"; import { fleetSnapshot, useSandboxFleet } from "./use-sandbox-fleet"; -const configured: SandboxDeployment = { credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, update_required: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "http://core", reset: null, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 1, pending: 0 }, suspension: null }; +const configured: SandboxDeployment = { credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "http://core", reset: null, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 1, pending: 0 }, suspension: null }; function Probe() { const { state, deployment } = useSandboxFleet(); @@ -51,7 +51,7 @@ describe("fleet evidence after deployment changes", () => { }); it("retains older resource evidence across an online generation change with an explicit qualification", () => { - const html = render({ ...configured, generation: 2, rollout: { state: "preparing", previous_generation_sandboxes: 1, nodes: { ready: 0, preparing: 1, failed: 0, update_required: 0, unknown: 0 } } }); + const html = render({ ...configured, generation: 2, rollout: { state: "preparing", previous_generation_sandboxes: 1, nodes: { ready: 0, preparing: 1, failed: 0, unknown: 0 } } }); expect(html).toContain('data-fleet="ready"'); expect(html).toContain("1 nodes / 1 allocations"); expect(html).toContain("configuration generation 1; the current target is 2"); diff --git a/apps/web/src/features/overview/getting-started.test.ts b/apps/web/src/features/overview/getting-started.test.ts index 927960707..3952215bf 100644 --- a/apps/web/src/features/overview/getting-started.test.ts +++ b/apps/web/src/features/overview/getting-started.test.ts @@ -6,7 +6,7 @@ import { checklistStorageKey, checklistView, gettingStartedSteps, nextStepAfterN import { node, project } from "./test-fixtures"; const deployment = (overrides: Partial = {}): SandboxDeployment => ({ credential_configured: false, configuration: {}, metadata: {}, - rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, update_required: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "http://core", reset: null, owner_epoch: 1, generation: 1, mode: "nodes", + rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "http://core", reset: null, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 0, pending: 0 }, suspension: null, ...overrides, }); const fleet = (value: SandboxDeployment, nodes = [node("n1")]): FleetState => ({ status: "ready", snapshot: { deployment: value, nodes, allocations: [], loadedAt: 0 }, targetGeneration: value.generation, refreshing: false, error: null }); @@ -16,7 +16,7 @@ const harness = (id: CoreHarness["id"], fields: Partial = {}): Core describe("Getting started steps", () => { it("uses live provider readiness independently of target state or a durable pin", () => { - for (const state of ["preparing", "failed", "update_required", "unknown"] as const) { + for (const state of ["preparing", "failed", "unknown"] as const) { const serving = node("n1", { rollout: { state, ready_generation: 1 } }); expect(sandboxes(fleet(deployment({ generation: 2 }), [serving])).state).toBe("done"); expect(sandboxes(fleet(deployment({ generation: 2 }), [{ ...serving, provider_ready: false }])).state).toBe("todo"); diff --git a/apps/web/src/features/sandbox/NodeList.test.ts b/apps/web/src/features/sandbox/NodeList.test.ts index d90258500..5aa689af4 100644 --- a/apps/web/src/features/sandbox/NodeList.test.ts +++ b/apps/web/src/features/sandbox/NodeList.test.ts @@ -10,7 +10,7 @@ const core = "https://core.example"; describe("node state", () => { it("keeps live serving health independent of target preparation and durable pins", () => { - for (const state of ["unknown", "preparing", "failed", "update_required"] as const) { + for (const state of ["unknown", "preparing", "failed"] as const) { const serving = node("a", { online: true, provider_ready: true, rollout: { state, ready_generation: 1 } }); expect(nodeState(serving, [], false, core)).toBe("available"); expect(nodeState({ ...serving, online: false }, [], false, core)).toBe("offline"); diff --git a/apps/web/src/features/sandbox/NodeRolloutStatus.tsx b/apps/web/src/features/sandbox/NodeRolloutStatus.tsx index 4e41978a8..02205b850 100644 --- a/apps/web/src/features/sandbox/NodeRolloutStatus.tsx +++ b/apps/web/src/features/sandbox/NodeRolloutStatus.tsx @@ -1,11 +1,11 @@ import type { SandboxNode, SandboxNodeRollout } from "@oac/agents-client"; import { useTranslation } from "react-i18next"; -import { HelpTip, StatusDot, type Tone } from "../../components/console-ui"; +import { StatusDot, type Tone } from "../../components/console-ui"; import type { ParseKeys } from "i18next"; import { DiagnosticTip } from "../fleet/DiagnosticTip"; -const labels: Record> = { ready: "Ready for target", preparing: "Preparing target", failed: "Preparation failed", update_required: "Node software incompatible", unknown: "Target readiness unknown" }; -const tones: Record = { ready: "ok", preparing: "neutral", failed: "warning", update_required: "warning", unknown: "neutral" }; +const labels: Record> = { ready: "Ready for target", preparing: "Preparing target", failed: "Preparation failed", unknown: "Target readiness unknown" }; +const tones: Record = { ready: "ok", preparing: "neutral", failed: "warning", unknown: "neutral" }; /** A serving pin is historical ownership, not proof of a live connection or capacity. */ export function NodeRolloutStatus({ node, stale = false }: { node: SandboxNode; stale?: boolean }) { @@ -14,6 +14,5 @@ export function NodeRolloutStatus({ node, stale = false }: { node: SandboxNode; return {state === "failed" && node.rollout.diagnostic ? : null} - {state === "update_required" ? {t("This node can't prepare new sandbox configurations and keeps serving the one it registered with. To move it to the current configuration, choose Remove node, then add the host again with Add node.")} : null} ; } diff --git a/apps/web/src/features/sandbox/SandboxDeploymentSettings.tsx b/apps/web/src/features/sandbox/SandboxDeploymentSettings.tsx index 3b1affcaf..1f2170049 100644 --- a/apps/web/src/features/sandbox/SandboxDeploymentSettings.tsx +++ b/apps/web/src/features/sandbox/SandboxDeploymentSettings.tsx @@ -1,15 +1,13 @@ import { useState, type ReactNode } from "react"; -import type { UpdateSandboxDeployment, SandboxDeployment, SandboxRuntimeRelease } from "@oac/agents-client"; -import { useQuery } from "@tanstack/react-query"; -import { Trans, useTranslation } from "react-i18next"; +import type { UpdateSandboxDeployment, SandboxDeployment } from "@oac/agents-client"; +import { useTranslation } from "react-i18next"; import { Modal } from "../../components/Modal"; import { HelpTip, RefreshButton } from "../../components/console-ui"; import { formatBytes, formatPeriod, MISSING } from "../../lib/format"; import type { ParseKeys } from "i18next"; import { sandboxProviderLabel } from "../../lib/sandbox-labels"; import { Fact } from "../system/Fact"; -import { RUNTIME_RELEASE_FIELDS, sandboxSize, templateBuildSize, templateBuildStatus } from "./deployment-specification"; -import { distributionRuntimeQuery } from "./sandbox-queries"; +import { sandboxSize, templateBuildSize, templateBuildStatus } from "./deployment-specification"; import { SandboxRolloutSummary } from "./SandboxRolloutSummary"; import { SandboxSetupWizard } from "./SandboxSetupWizard"; @@ -31,21 +29,14 @@ export function SandboxDeploymentSettings({ deployment, disabled, fresh, onUpdat const locale = i18n.resolvedLanguage?.startsWith("zh") ? "zh-CN" : "en"; const [changing, setChanging] = useState(false); const [editKey, setEditKey] = useState(0); - const [proposedRuntime, setProposedRuntime] = useState(); const spec = deployment.specification; - // Only Docker and microsandbox save a Runtime. Changes keep it; moving to the - // one this console serves is an explicit change of its own. - const savedRuntime = spec?.runtime; - const served = useQuery({ ...distributionRuntimeQuery, enabled: Boolean(savedRuntime) }).data; - const offered = savedRuntime && served && !deployment.reset && RUNTIME_RELEASE_FIELDS.some((field) => savedRuntime[field] !== served[field]) ? served : null; // An E2B selection may adopt its template build's size instead of saving one. const size = sandboxSize(deployment); const build = deployment.metadata?.template_build; const buildSize = templateBuildSize(deployment); const sizeLabel = (value: { cpus: number; memory_mib: number }) => t("{{cpus}} CPU · {{memory}}", { cpus: value.cpus, memory: formatBytes(value.memory_mib * MIB) }); const canEdit = fresh && !deployment.reset; - const change = (runtime?: SandboxRuntimeRelease) => { setEditKey((key) => key + 1); setProposedRuntime(runtime); setChanging(true); }; - const short = (release: SandboxRuntimeRelease) => {release.source_commit.slice(0, 12)}; + const change = () => { setEditKey((key) => key + 1); setChanging(true); }; return
@@ -54,7 +45,6 @@ export function SandboxDeploymentSettings({ deployment, disabled, fresh, onUpdat {t("One provider serves this deployment. Reset it before choosing a different backend.")} {deployment.mode === "direct" ? ` ${t("Core creates E2B sandboxes directly. No node enrollment is needed.")} ${t("Saved configuration does not confirm execution readiness. Session and Environment state report actual execution.")}` : ""}
- {spec?.runtime ?
{t("Runtime")}
{spec.runtime.source_commit.slice(0, 12)}
: null} {deployment.suspension ?
{t("Idle suspension")}
{t("After {{idle}} · kept {{retention}}", { idle: formatPeriod(deployment.suspension.idle_seconds, i18n.resolvedLanguage), retention: formatPeriod(deployment.suspension.retention_seconds, i18n.resolvedLanguage) })}
: null}
{t("Allocated resources")}
{deployment.resources?.allocations ?? t("Unknown state")}
{t("Pending environments")}
{deployment.resources?.pending ?? t("Unknown state")}
@@ -63,15 +53,12 @@ export function SandboxDeploymentSettings({ deployment, disabled, fresh, onUpdat
{!deployment.reset ?
{t("Changes keep this backend and existing Sessions and nodes. Core prepares the new target for future work; placement may continue on qualified earlier generations.")} - +
: null}
{sandboxProviderLabel(deployment.provider, locale)} {size ? `${sizeLabel(size)}${size.root_disk_mib ? ` · ${spec?.workspace ? t("Root disk {{root}} · external workspace", { root: formatBytes(size.root_disk_mib * MIB) }) : t("Root disk {{root}} · data disk {{data}}", { root: formatBytes(size.root_disk_mib * MIB), data: formatBytes((size.environment_disk_mib ?? 0) * MIB) })}` : ""}` : MISSING} - {savedRuntime && offered ? change(offered)}>{t("Use this installation's Runtime")}}> - - : null} {deployment.provider === "e2b" ? <> {deployment.configuration?.api_url} {deployment.configuration?.domain} @@ -92,7 +79,6 @@ export function SandboxDeploymentSettings({ deployment, disabled, fresh, onUpdat current={deployment.provider ? { provider: deployment.provider, specification: deployment.specification, e2bTemplate: deployment.configuration?.template, e2bAPIURL: deployment.configuration?.api_url, e2bDomain: deployment.configuration?.domain } : undefined} disabled={disabled || !canEdit} editing - proposedRuntime={proposedRuntime} onSubmit={async (input) => { if (await onUpdate(input)) setChanging(false); }} /> {writeFailure} diff --git a/apps/web/src/features/sandbox/SandboxResetControls.test.tsx b/apps/web/src/features/sandbox/SandboxResetControls.test.tsx index 7d95ba13c..c5184de13 100644 --- a/apps/web/src/features/sandbox/SandboxResetControls.test.tsx +++ b/apps/web/src/features/sandbox/SandboxResetControls.test.tsx @@ -5,7 +5,7 @@ import { SandboxResetControls } from "./SandboxResetControls"; const reset: SandboxReset = { clear: "auto", requested_at: "2020-01-01T10:00:00Z", deadline_at: "2020-01-01T11:00:00Z", forced_at: null, remaining: { busy: 2, idle: 1, cleanup: 3, on_offline_nodes: 2, offline_nodes: [{ node_id: "offline-a", name: "Offline A", resources: 2 }] } }; -const deployment = (value: SandboxReset | null): SandboxDeployment => ({ credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, update_required: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "https://core.example", reset: value, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 5, pending: 1 }, suspension: null }); +const deployment = (value: SandboxReset | null): SandboxDeployment => ({ credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "https://core.example", reset: value, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 5, pending: 1 }, suspension: null }); const render = (value: SandboxReset | null, stale = false) => renderToStaticMarkup( true} onCancel={async () => true} />); describe("authoritative reset progress", () => { diff --git a/apps/web/src/features/sandbox/SandboxRolloutSummary.test.tsx b/apps/web/src/features/sandbox/SandboxRolloutSummary.test.tsx index a6d724701..0364ada21 100644 --- a/apps/web/src/features/sandbox/SandboxRolloutSummary.test.tsx +++ b/apps/web/src/features/sandbox/SandboxRolloutSummary.test.tsx @@ -8,7 +8,7 @@ import { NodeRolloutStatus } from "./NodeRolloutStatus"; import { SandboxDeploymentSettings } from "./SandboxDeploymentSettings"; const deployment: SandboxDeployment = { credential_configured: false, configuration: {}, metadata: {}, installation_id: "i", owner_epoch: 1, generation: 4, provider: "docker", mode: "nodes", core_url: "https://core.example", resources: { allocations: 8, pending: 2 }, reset: null, suspension: null, - rollout: { state: "settled", previous_generation_sandboxes: 8, nodes: { ready: 1, preparing: 0, failed: 2, update_required: 3, unknown: 4 } } }; + rollout: { state: "settled", previous_generation_sandboxes: 8, nodes: { ready: 1, preparing: 0, failed: 2, unknown: 4 } } }; describe("authoritative configuration rollout", () => { it("keeps attention visible while moving detailed counts off the main page", () => { @@ -30,14 +30,14 @@ describe("authoritative configuration rollout", () => { }); it("does not call settled nodes ready when Core reports unknown target readiness", () => { - const html = renderToStaticMarkup(); + const html = renderToStaticMarkup(); expect(html).toContain("Target readiness unknown"); expect(html).not.toContain("No active preparation"); expect(html).not.toContain("Ready for target"); }); it("reports preparing and settled without turning settled into readiness", () => { - const quiet = { ...deployment, rollout: { state: "settled" as const, previous_generation_sandboxes: 0, nodes: { ready: 2, preparing: 0, failed: 0, update_required: 0, unknown: 0 } } }; + const quiet = { ...deployment, rollout: { state: "settled" as const, previous_generation_sandboxes: 0, nodes: { ready: 2, preparing: 0, failed: 0, unknown: 0 } } }; expect(renderToStaticMarkup()).toContain("No active preparation"); expect(renderToStaticMarkup()).toContain("Preparing configuration"); expect(renderToStaticMarkup()).toBe(""); diff --git a/apps/web/src/features/sandbox/SandboxRolloutSummary.tsx b/apps/web/src/features/sandbox/SandboxRolloutSummary.tsx index 67f3c521f..3749f300f 100644 --- a/apps/web/src/features/sandbox/SandboxRolloutSummary.tsx +++ b/apps/web/src/features/sandbox/SandboxRolloutSummary.tsx @@ -13,7 +13,7 @@ export function SandboxRolloutSummary({ deployment, stale = false, onOpen }: { d const [open, setOpen] = useState(false); if (!deployment.provider || deployment.reset) return null; const { rollout } = deployment; - const needsAttention = !!rollout.nodes && (rollout.nodes.failed > 0 || rollout.nodes.update_required > 0); + const needsAttention = !!rollout.nodes && (rollout.nodes.failed > 0); const unknown = !!rollout.nodes && rollout.nodes.unknown > 0; const label = stale ? "Rollout state unconfirmed" : needsAttention ? "Needs attention" : unknown ? "Target readiness unknown" : rollout.state === "preparing" ? "Preparing configuration" : "No active preparation"; const tone: Tone = stale || needsAttention || unknown ? "warning" : rollout.state === "preparing" ? "pending" : "neutral"; @@ -32,7 +32,7 @@ export function SandboxRolloutSummary({ deployment, stale = false, onOpen }: { d {t(stale ? "Rollout state is unconfirmed. These are the last confirmed observations." : rollout.state === "settled" ? "Core reports no active preparation. This does not mean every node is ready." : "Core is preparing the target configuration.")} - {needsAttention || unknown ? ` ${t("Review affected nodes for preparation errors, incompatible node software or an unconfirmed connection. Qualified earlier generations may still serve work.")}` : null} + {needsAttention || unknown ? ` ${t("Review affected nodes for preparation errors or an unconfirmed connection. Qualified earlier generations may still serve work.")}` : null}
@@ -43,7 +43,6 @@ export function SandboxRolloutSummary({ deployment, stale = false, onOpen }: { d
{t("Ready for target")}
{count(rollout.nodes.ready)}
{t("Preparing target")}
{count(rollout.nodes.preparing)}
{t("Preparation failed")}
{count(rollout.nodes.failed)}
-
{t("Node software incompatible")}
{count(rollout.nodes.update_required)}
{t("Target readiness unknown")}
{count(rollout.nodes.unknown)}
: null}
diff --git a/apps/web/src/features/sandbox/SandboxSetupWizard.tsx b/apps/web/src/features/sandbox/SandboxSetupWizard.tsx index b3a0ff0da..3f185fa88 100644 --- a/apps/web/src/features/sandbox/SandboxSetupWizard.tsx +++ b/apps/web/src/features/sandbox/SandboxSetupWizard.tsx @@ -1,4 +1,4 @@ -import { AgentCoreError, deploymentContract, type InitializeSandboxDeployment, type UpdateSandboxDeployment, type SandboxE2BReadyBuild, type SandboxE2BTemplate, type SandboxProvider, type SandboxResources, type SandboxRuntimeRelease, type SandboxSpecification } from "@oac/agents-client"; +import { AgentCoreError, deploymentContract, type InitializeSandboxDeployment, type UpdateSandboxDeployment, type SandboxE2BReadyBuild, type SandboxE2BTemplate, type SandboxProvider, type SandboxResources, type SandboxSpecification } from "@oac/agents-client"; import { useQuery } from "@tanstack/react-query"; import { AnimatePresence } from "motion/react"; import * as m from "motion/react-m"; @@ -15,10 +15,10 @@ import { formatBytes } from "../../lib/format"; import { installationQuery } from "../../lib/installation"; import type { ParseKeys } from "i18next"; import { sandboxConfigurationRejection, sandboxProviderLabel } from "../../lib/sandbox-labels"; -import { defaultSandboxResources, isRuntimeRelease, isRuntimeReleaseField, RUNTIME_RELEASE_FIELDS, savedSpecification, selectedWorkspace, validSandboxResources } from "./deployment-specification"; +import { defaultSandboxResources, savedSpecification, selectedWorkspace, validSandboxResources } from "./deployment-specification"; import { workspaceStorageQuery } from "../settings/workspace-storage"; import { e2bKeyReady, e2bUpdateSelection } from "./sandbox-update"; -import { distributionRuntimeQuery, sandboxAdmin } from "./sandbox-queries"; +import { sandboxAdmin } from "./sandbox-queries"; import "./sandbox-wizard.css"; type Where = "nodes" | "direct"; @@ -82,15 +82,6 @@ function presets(provider: SandboxProvider, workspace?: SandboxSpecification["wo return { small: scale(0.5), standard, large: scale(2) }; } -const releaseLabels: Record> = { - source_commit: "Source commit", - image_id: "Image ID", - image_manifest_digest: "Image manifest digest", - microsandbox_ref: "microsandbox reference", - runtime_sha256: "Runtime SHA-256", - firmware_sha256: "Firmware SHA-256", -}; - function presetOf(provider: SandboxProvider, resources: SandboxResources, workspace?: SandboxSpecification["workspace"]): Preset | null { const same = (a: SandboxResources, b: SandboxResources) => a.cpus === b.cpus && a.memory_mib === b.memory_mib && (a.root_disk_mib ?? 0) === (b.root_disk_mib ?? 0) && (a.environment_disk_mib ?? 0) === (b.environment_disk_mib ?? 0); @@ -103,12 +94,11 @@ function presetOf(provider: SandboxProvider, resources: SandboxResources, worksp * which backend (own machines, microsandbox preselected) or the E2B account, * how big each sandbox is (only for a Provider that declares a default size: * E2B sandboxes take the template build's size), then a review. The Provider's - * declarations decide the size, disk and Runtime inputs. Advanced settings - * hold the complete form. The Runtime release comes from this console's - * distribution manifest when it serves one. + * declarations decide the size and disk inputs. Advanced settings hold the + * complete form. * `current` pre-selects the saved choices when a deployment changes. Keeping - * the backend keeps its saved size and Runtime; another backend starts from its - * defaults and this console's Runtime. E2B updates can retain the saved key. + * the backend keeps its saved size; another backend starts from its defaults. + * E2B updates can retain the saved key. * Docker isolates less than microsandbox, so choosing it takes a confirmation, * once per wizard session; a saved Docker deployment has already made it. * Core's address is config.json's `public_url`: the review only shows it, and @@ -119,18 +109,16 @@ type WizardProps = { expectedGeneration: number; current?: { provider: SandboxProvider; specification?: SandboxSpecification; e2bTemplate?: string; e2bAPIURL?: string; e2bDomain?: string }; disabled: boolean; - /** A Runtime release an update proposes; the wizard opens on its review. */ - proposedRuntime?: SandboxRuntimeRelease; } & ({ editing: true; onSubmit: (input: UpdateSandboxDeployment) => Promise } | { editing?: false; onSubmit: (input: InitializeSandboxDeployment) => Promise }); -export function SandboxSetupWizard({ coreUrl, expectedGeneration, current, disabled, editing, proposedRuntime, onSubmit }: WizardProps) { +export function SandboxSetupWizard({ coreUrl, expectedGeneration, current, disabled, editing, onSubmit }: WizardProps) { const { t, i18n } = useTranslation("sandbox"); const { t: tCommon } = useTranslation("common"); const id = useId(); const locale = i18n.resolvedLanguage?.startsWith("zh") ? "zh-CN" : "en"; const currentMode: Where | null = current ? deploymentContract.providers[current.provider].mode : null; - const [step, setStep] = useState(proposedRuntime ? "review" : editing ? currentMode === "direct" ? "e2b" : "size" : "where"); + const [step, setStep] = useState(editing ? currentMode === "direct" ? "e2b" : "size" : "where"); const [where, setWhere] = useState(currentMode); const [provider, setProvider] = useState(current?.provider ?? null); const policy = provider ? deploymentContract.providers[provider] : null; @@ -155,7 +143,6 @@ export function SandboxSetupWizard({ coreUrl, expectedGeneration, current, disab const [discovery, setDiscovery] = useState<"idle" | "loading" | "ready" | "error">("idle"); const [buildDiscovery, setBuildDiscovery] = useState<"idle" | "loading" | "ready" | "error">("idle"); const [discoveryRetry, setDiscoveryRetry] = useState(0); - const [runtime, setRuntime] = useState>(proposedRuntime ?? {}); const [busy, setBusy] = useState(false); const [dockerConfirmed, setDockerConfirmed] = useState(current?.provider === "docker"); const [confirmingDocker, setConfirmingDocker] = useState(false); @@ -199,13 +186,6 @@ export function SandboxSetupWizard({ coreUrl, expectedGeneration, current, disab setTemplate(""); setSelectedTemplate(""); setTemplates([]); setBuilds([]); } - // A release the administrator entered or an update proposes comes first, then the saved one of the same backend, - // then the one this console distributes (the release its node installer verifies). - const matched = useQuery(distributionRuntimeQuery); - const release: Partial = Object.keys(runtime).length ? runtime : saved?.runtime ?? matched.data ?? {}; - - const needsRuntime = policy?.runtime ?? false; - const runtimeReady = !needsRuntime || isRuntimeRelease(release); // Initial setup requires a key; an update may retain the committed key. const keyReady = e2bKeyReady(Boolean(editing), replacementRequested, apiKey); const connectionChanged = Boolean(editing && (apiURL.trim() !== (current?.e2bAPIURL || E2B_PRESETS.official.apiURL) || domain.trim() !== (current?.e2bDomain || E2B_PRESETS.official.domain))); @@ -213,13 +193,13 @@ export function SandboxSetupWizard({ coreUrl, expectedGeneration, current, disab // A Provider without a declared default size takes it from its configuration, so the selection sends no resources. const sized = Boolean(policy?.default_resources); const sizeReady = provider !== null && (!sized || validSandboxResources(provider, resources, workspace)); - const ready = provider !== null && runtimeReady && e2bReady && sizeReady && !disabled && !busy; + const ready = provider !== null && e2bReady && sizeReady && !disabled && !busy; const order: Step[] = editing ? where === "direct" ? ["e2b", "review"] : ["size", "review"] : where === "direct" ? ["where", "e2b", "review"] : ["where", "backend", "size", "review"]; const index = Math.max(0, order.indexOf(step === "advanced" ? "review" : step)); const back = () => setStep(step === "advanced" ? "review" : order[Math.max(0, index - 1)]!); - // The saved backend keeps its size and Runtime; another starts from its declared default size and this console's Runtime. + // The saved backend keeps its size; another starts from its declared default size. function choose(next: SandboxProvider) { if (next !== provider) { setRejection(null); @@ -227,7 +207,6 @@ export function SandboxSetupWizard({ coreUrl, expectedGeneration, current, disab const proposed = kept?.resources ?? defaultSandboxResources(next, selectedWorkspace(next, storage)); if (proposed) setResources(proposed); setSize(kept ? presetOf(next, kept.resources, kept.workspace) ?? "current" : "standard"); - setRuntime({}); } setProvider(next); } @@ -247,7 +226,6 @@ export function SandboxSetupWizard({ coreUrl, expectedGeneration, current, disab const selection = { provider, expected_generation: expectedGeneration, ...(sized ? { resources } : {}), - ...(needsRuntime ? { runtime: release as SandboxRuntimeRelease } : {}), }; if (editing) await onSubmit({ ...selection, ...(provider === "e2b" ? { ...e2bUpdateSelection(template, apiKey), configuration: { template: template.trim(), api_url: apiURL.trim(), domain: domain.trim() } } : {}) }); else await onSubmit({ ...selection, ...(provider === "e2b" ? { credential: { api_key: apiKey.trim() }, configuration: { template: template.trim(), api_url: apiURL.trim(), domain: domain.trim() } } : {}) }); @@ -259,7 +237,7 @@ export function SandboxSetupWizard({ coreUrl, expectedGeneration, current, disab setFieldRejection(error); if (error instanceof AgentCoreError && error.param) { if (["credential", "configuration", "e2b.api_url", "e2b.domain"].includes(error.param) && error.code !== "sandbox_credential_ownership") setStep("e2b"); - else if (error.param === "runtime" || error.param.startsWith("resources.")) setStep("advanced"); + else if (error.param.startsWith("resources.")) setStep("advanced"); } setResetRequired(error instanceof AgentCoreError && ["sandbox_credential_ownership", "sandbox_reset_required"].includes(error.code ?? "")); setAddressRejected(error instanceof AgentCoreError && error.code === "sandbox_configuration_error"); @@ -383,12 +361,6 @@ export function SandboxSetupWizard({ coreUrl, expectedGeneration, current, disab {provider === "e2b" ?
{t("Sandbox API URL")}
{apiURL || "https://api.e2b.app"}
: null} {provider === "e2b" ?
{t("Sandbox data-plane domain")}
{domain || "e2b.app"}
: null} {provider === "e2b" && editing ?
{t("E2B credential")}
{t(replacementRequested ? "Replace saved key" : "Keep saved key")}
: null} - {needsRuntime ? ( -
-
{t("Runtime")}{t("The target Runtime release. Existing sandboxes keep their owned release while nodes prepare the target.")}
-
{runtimeReady ? {release.source_commit!.slice(0, 12)} : {t("Runtime release needed")}{t("This console serves no Runtime manifest. Enter the release under advanced settings.")}}
-
- ) : null}
{t("Core address")}{t("The address nodes and sandboxes use to reach Core.")}
@@ -439,20 +411,6 @@ export function SandboxSetupWizard({ coreUrl, expectedGeneration, current, disab : null}
: null} - {needsRuntime ? ( -
- {t("Runtime release")}{t("Filled in from this console's distribution when it serves one. Otherwise copy these from the distribution manifest that matches your nodes; image configuration IDs and manifest digests are different values.")} - {fieldError("runtime") ? : null} - {RUNTIME_RELEASE_FIELDS.map((field) => { - const value = release[field] ?? ""; - return ( - - { setRuntime({ ...release, [field]: event.target.value.trim() }); setFieldRejection(null); }} /> - - ); - })} -
- ) : null} {provider === "e2b" ? ( { setTemplate(event.target.value); setFieldRejection(null); }} autoComplete="off" spellCheck={false} /> diff --git a/apps/web/src/features/sandbox/deployment-specification.test.ts b/apps/web/src/features/sandbox/deployment-specification.test.ts index fcfb05cf7..9b6649e1f 100644 --- a/apps/web/src/features/sandbox/deployment-specification.test.ts +++ b/apps/web/src/features/sandbox/deployment-specification.test.ts @@ -1,25 +1,10 @@ -import { afterEach, describe, expect, it, vi } from "vitest"; -import { defaultSandboxResources, distributionRuntime, isRuntimeReleaseField, sandboxesThatFit, savedSpecification, selectedWorkspace, validSandboxResources } from "./deployment-specification"; +import { describe, expect, it } from "vitest"; +import { defaultSandboxResources, sandboxesThatFit, savedSpecification, selectedWorkspace, validSandboxResources } from "./deployment-specification"; import { deploymentContract, type SandboxProvider, type SandboxSpecification } from "@oac/agents-client"; -const manifest = { platform: "linux/amd64", source_commit: "0".repeat(40), images: { runtime: `sha256:${"a".repeat(64)}` }, - image_manifest_digests: { runtime: `sha256:${"b".repeat(64)}` }, runtime_ref: `oac-runtime@sha256:${"b".repeat(64)}`, - microsandbox: { runtime_sha256: "c".repeat(64), firmware_sha256: "d".repeat(64) } }; -afterEach(() => vi.unstubAllGlobals()); - -describe("deployment resources and Runtime", () => { - it("maps the exact six identities from one matched distribution", async () => { - const fetcher = vi.fn().mockResolvedValue(new Response(JSON.stringify(manifest))); - vi.stubGlobal("fetch", fetcher); - expect(await distributionRuntime(new AbortController().signal)).toEqual({ source_commit: manifest.source_commit, - image_id: manifest.images.runtime, image_manifest_digest: manifest.image_manifest_digests.runtime, - microsandbox_ref: manifest.runtime_ref, runtime_sha256: manifest.microsandbox.runtime_sha256, firmware_sha256: manifest.microsandbox.firmware_sha256 }); - expect(fetcher.mock.calls.map((call) => call[0])).toEqual(["/node-install/manifest.json"]); - }); - it("preserves saved resources and Runtime only for the same provider", () => { - const current: SandboxSpecification = { resources: { cpus: 7, memory_mib: 8192 }, runtime: { source_commit: manifest.source_commit, - image_id: manifest.images.runtime, image_manifest_digest: manifest.image_manifest_digests.runtime, - microsandbox_ref: manifest.runtime_ref, runtime_sha256: manifest.microsandbox.runtime_sha256, firmware_sha256: manifest.microsandbox.firmware_sha256 } }; +describe("deployment resources", () => { + it("preserves saved resources only for the same provider", () => { + const current: SandboxSpecification = { resources: { cpus: 7, memory_mib: 8192 } }; expect(savedSpecification("docker", "docker", current)).toEqual(current); expect(savedSpecification("docker", "docker", current)).not.toBe(current); expect(savedSpecification("microsandbox", "docker", current)).toBeNull(); @@ -68,26 +53,6 @@ describe("deployment resources and Runtime", () => { expect(sized(null).workspace).toBeUndefined(); expect(selectedWorkspace("docker", { tenant_capacity_mib: 0 })).toBeUndefined(); }); - it("accepts only Core's Runtime image in the Runtime reference", async () => { - const digest = "b".repeat(64); - const runtime_ref = `oac-runtime@sha256:${digest}`; - vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response(JSON.stringify({ ...manifest, runtime_ref })))); - expect((await distributionRuntime(new AbortController().signal)).microsandbox_ref).toBe(runtime_ref); - expect(isRuntimeReleaseField("microsandbox_ref", runtime_ref)).toBe(true); - for (const runtime_ref of [`custom-runtime@sha256:${digest}`, `oac-runtime:${digest}`, `oac-runtime@sha256:${"b".repeat(63)}`, `oac-runtime@sha256:${"B".repeat(64)}`, `@sha256:${digest}`]) { - vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response(JSON.stringify({ ...manifest, runtime_ref })))); - await expect(distributionRuntime(new AbortController().signal)).rejects.toThrow(); - expect(isRuntimeReleaseField("microsandbox_ref", runtime_ref)).toBe(false); - } - }); - it("rejects unavailable, mutable or incomplete release identities", async () => { - vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response("{}", { status: 404 }))); - await expect(distributionRuntime(new AbortController().signal)).rejects.toThrow(); - for (const invalid of [{ ...manifest, platform: "linux/arm64" }, { ...manifest, source_commit: "main" }, { ...manifest, runtime_ref: "oac-runtime:latest" }, { ...manifest, microsandbox: {} }]) { - vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response(JSON.stringify(invalid)))); - await expect(distributionRuntime(new AbortController().signal)).rejects.toThrow(); - } - }); }); describe("sandboxes a host holds", () => { diff --git a/apps/web/src/features/sandbox/deployment-specification.ts b/apps/web/src/features/sandbox/deployment-specification.ts index 93ef7ae13..3a62c15cf 100644 --- a/apps/web/src/features/sandbox/deployment-specification.ts +++ b/apps/web/src/features/sandbox/deployment-specification.ts @@ -1,13 +1,4 @@ -import { deploymentContract, type SandboxDeployment, type SandboxE2BTemplateBuild, type SandboxProvider, type SandboxResources, type SandboxRuntimeRelease, type SandboxSpecification } from "@oac/agents-client"; - -interface Manifest { - platform?: string; - source_commit?: string; - images?: { runtime?: string }; - image_manifest_digests?: { runtime?: string }; - runtime_ref?: string; - microsandbox?: { runtime_sha256?: string; firmware_sha256?: string }; -} +import { deploymentContract, type SandboxDeployment, type SandboxE2BTemplateBuild, type SandboxProvider, type SandboxResources, type SandboxSpecification } from "@oac/agents-client"; /** The size the Provider declares for setup to propose; null when its configuration selects the size. */ export function defaultSandboxResources(provider: SandboxProvider, workspace?: SandboxSpecification["workspace"]): SandboxResources | null { @@ -34,18 +25,6 @@ export function validSandboxResources(provider: SandboxProvider, resources: Sand }); } -const releasePatterns = Object.fromEntries(deploymentContract.runtime.map(({ name, pattern }) => [name, new RegExp(`^(?:${pattern})$`)])) as Record; - -export const RUNTIME_RELEASE_FIELDS = deploymentContract.runtime.map(({ name }) => name); - -export function isRuntimeReleaseField(field: keyof SandboxRuntimeRelease, value: string): boolean { - return releasePatterns[field].test(value); -} - -export function isRuntimeRelease(value: Partial): value is SandboxRuntimeRelease { - return RUNTIME_RELEASE_FIELDS.every((field) => typeof value[field] === "string" && releasePatterns[field].test(value[field])); -} - export function savedSpecification(provider: SandboxProvider, savedProvider?: SandboxProvider | "", specification?: SandboxSpecification): SandboxSpecification | null { return provider === savedProvider && specification ? structuredClone(specification) : null; } @@ -76,14 +55,3 @@ export function sandboxesThatFit(host: { cpus: number | null; memoryBytes: numbe if (!size || host.cpus === null || host.memoryBytes === null || size.cpus <= 0 || size.memory_mib <= 0) return null; return Math.min(Math.floor(host.cpus / size.cpus), Math.floor(host.memoryBytes / (size.memory_mib * 2 ** 20))); } - -/** The paired console serves one matched distribution; Core persists approval. */ -export async function distributionRuntime(signal: AbortSignal): Promise { - const response = await fetch("/node-install/manifest.json", { signal, credentials: "include", redirect: "error" }); - if (!response.ok) throw new Error("distribution unavailable"); - const manifest = await response.json() as Manifest; - const release = { source_commit: manifest.source_commit, image_id: manifest.images?.runtime, image_manifest_digest: manifest.image_manifest_digests?.runtime, - microsandbox_ref: manifest.runtime_ref, runtime_sha256: manifest.microsandbox?.runtime_sha256, firmware_sha256: manifest.microsandbox?.firmware_sha256 }; - if (manifest.platform !== "linux/amd64" || !isRuntimeRelease(release)) throw new Error("invalid distribution"); - return release; -} diff --git a/apps/web/src/features/sandbox/node-enrollment.test.ts b/apps/web/src/features/sandbox/node-enrollment.test.ts index 74025f841..91d6273a9 100644 --- a/apps/web/src/features/sandbox/node-enrollment.test.ts +++ b/apps/web/src/features/sandbox/node-enrollment.test.ts @@ -31,7 +31,6 @@ describe("node enrollment", () => { const connected = { ...fresh, online: true }; expect(enrollmentProgress(connected, 0, 1_000)).toEqual({ stage: "connected", problem: "" }); expect(enrollmentProgress(connected, 0, NODE_READY_WAIT_MS)).toEqual({ stage: "connected", problem: "provider_unavailable" }); - expect(enrollmentProgress({ ...connected, diagnostic: "host_unsupported" }, 0, 1_000)).toEqual({ stage: "connected", problem: "host_unsupported" }); expect(enrollmentProgress({ ...connected, provider_ready: true }, 0, NODE_READY_WAIT_MS * 2)).toEqual({ stage: "ready", problem: "" }); }); diff --git a/apps/web/src/features/sandbox/node-enrollment.ts b/apps/web/src/features/sandbox/node-enrollment.ts index 93328f795..067a41b9f 100644 --- a/apps/web/src/features/sandbox/node-enrollment.ts +++ b/apps/web/src/features/sandbox/node-enrollment.ts @@ -95,5 +95,5 @@ export function enrollmentProgress(node: SandboxNode | null, appearedAt: number if (node.online && node.provider_ready) return { stage: "ready", problem: "" }; const late = appearedAt !== undefined && now - appearedAt >= NODE_READY_WAIT_MS; if (!node.online) return { stage: "registered", problem: late ? "not_connected" : "" }; - return { stage: "connected", problem: node.diagnostic || late ? nodeProviderDiagnostic(node) : "" }; + return { stage: "connected", problem: late ? nodeProviderDiagnostic(node) : "" }; } diff --git a/apps/web/src/features/sandbox/sandbox-deployment-write.test.ts b/apps/web/src/features/sandbox/sandbox-deployment-write.test.ts index 552b4f541..be04e2036 100644 --- a/apps/web/src/features/sandbox/sandbox-deployment-write.test.ts +++ b/apps/web/src/features/sandbox/sandbox-deployment-write.test.ts @@ -5,7 +5,7 @@ import { sandboxAdmin, sandboxDeploymentQuery } from "./sandbox-queries"; import { writeSandboxDeployment } from "./sandbox-deployment-write"; import { sandboxWriteOwnershipQuery } from "./sandbox-write-ownership"; -const deployment: SandboxDeployment = { credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, update_required: 0, unknown: 0 } }, installation_id: "i", owner_epoch: 1, generation: 1, provider: "docker", core_url: "http://core", mode: "nodes", reset: null, resources: { allocations: 0, pending: 0 }, suspension: null }; +const deployment: SandboxDeployment = { credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "i", owner_epoch: 1, generation: 1, provider: "docker", core_url: "http://core", mode: "nodes", reset: null, resources: { allocations: 0, pending: 0 }, suspension: null }; function deferred() { let resolve!: (value: T) => void; let reject!: (error: Error) => void; diff --git a/apps/web/src/features/sandbox/sandbox-queries.test.ts b/apps/web/src/features/sandbox/sandbox-queries.test.ts index 643c2fab4..62bef4afb 100644 --- a/apps/web/src/features/sandbox/sandbox-queries.test.ts +++ b/apps/web/src/features/sandbox/sandbox-queries.test.ts @@ -9,7 +9,7 @@ const reset: SandboxReset = { remaining: { busy: 1, idle: 0, cleanup: 1, on_offline_nodes: 1, offline_nodes: [{ node_id: "node-a", name: "Offline host", resources: 1 }] }, }; const deployment = (overrides: Partial = {}): SandboxDeployment => ({ credential_configured: false, configuration: {}, metadata: {}, - rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, update_required: 0, unknown: 0 } }, installation_id: "installation-a", provider: "docker", core_url: "https://core.example", reset, + rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "installation-a", provider: "docker", core_url: "https://core.example", reset, owner_epoch: 1, generation: 2, mode: "nodes", resources: { allocations: 2, pending: 0 }, suspension: null, ...overrides, }); const clients: QueryClient[] = []; @@ -18,7 +18,7 @@ afterEach(() => { for (const client of clients.splice(0)) client.clear(); vi.res describe("sandbox reset reads", () => { it("polls only Core preparation or reset, never settled old resources or failed/unknown nodes", () => { - const settled = deployment({ reset: null, rollout: { state: "settled", previous_generation_sandboxes: 9, nodes: { ready: 0, preparing: 0, failed: 2, update_required: 1, unknown: 3 } } }); + const settled = deployment({ reset: null, rollout: { state: "settled", previous_generation_sandboxes: 9, nodes: { ready: 0, preparing: 0, failed: 2, unknown: 3 } } }); expect(sandboxResetPollInterval(settled)).toBe(false); expect(sandboxResetPollInterval({ ...settled, rollout: { ...settled.rollout, state: "preparing" } })).toBe(5000); expect(sandboxResetPollInterval({ ...settled, reset })).toBe(5000); diff --git a/apps/web/src/features/sandbox/sandbox-queries.ts b/apps/web/src/features/sandbox/sandbox-queries.ts index 5bf2fe68d..81029f64e 100644 --- a/apps/web/src/features/sandbox/sandbox-queries.ts +++ b/apps/web/src/features/sandbox/sandbox-queries.ts @@ -3,7 +3,6 @@ import { queryOptions, useQuery } from "@tanstack/react-query"; import { confirmSandboxRead, startSandboxRead } from "./sandbox-write-ownership"; import { sandboxConsoleConfig } from "./console-config"; -import { distributionRuntime } from "./deployment-specification"; /** * Cache entries for the sandbox deployment (`/core/v1/sandbox`). Every read @@ -25,14 +24,6 @@ export const sandboxConsoleConfigQuery = queryOptions({ }, }); -/** The Runtime release of this console's distribution manifest, the release its node installer verifies; null when it serves none. */ -export const distributionRuntimeQuery = queryOptions({ - queryKey: ["sandbox-runtime-release"], - queryFn: ({ signal }) => distributionRuntime(signal).catch(() => null), - staleTime: Infinity, - retry: false, -}); - /** The deployment alone, for pages that only describe it. */ export const sandboxDeploymentQuery = queryOptions({ queryKey: [...sandboxScope, "deployment"], diff --git a/apps/web/src/features/sandbox/use-sandbox-page-state.test.tsx b/apps/web/src/features/sandbox/use-sandbox-page-state.test.tsx index 8ce16999d..63c8667c2 100644 --- a/apps/web/src/features/sandbox/use-sandbox-page-state.test.tsx +++ b/apps/web/src/features/sandbox/use-sandbox-page-state.test.tsx @@ -6,7 +6,7 @@ import { node } from "../overview/test-fixtures"; import { sandboxAdmin, sandboxDeploymentQuery, sandboxSnapshotQuery } from "./sandbox-queries"; import { useSandboxPageState } from "./use-sandbox-page-state"; -const configured: SandboxDeployment = { credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, update_required: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "http://core", reset: null, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 1, pending: 0 }, suspension: null }; +const configured: SandboxDeployment = { credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "http://core", reset: null, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 1, pending: 0 }, suspension: null }; const resetting: SandboxDeployment = { ...configured, reset: { clear: "auto", requested_at: "2026-09-27T10:00:00Z", deadline_at: "2026-09-27T11:00:00Z", forced_at: null, remaining: { busy: 1, idle: 0, cleanup: 0, on_offline_nodes: 0, offline_nodes: [] } } }; const clients: QueryClient[] = []; function cache() { diff --git a/apps/web/src/i18n/locales/en/core-errors.ts b/apps/web/src/i18n/locales/en/core-errors.ts index 0b5e733d6..aa4ed5dc0 100644 --- a/apps/web/src/i18n/locales/en/core-errors.ts +++ b/apps/web/src/i18n/locales/en/core-errors.ts @@ -19,7 +19,7 @@ export const coreErrors = { "model_provider_protocol_unsupported": "This protocol is not supported by the Harness.", "model_provider_api_key_invalid": "Enter a valid API key without control characters.", "model_provider_token_limits_invalid": "Use valid token limits; output cannot exceed context.", - "invalid_sandbox_configuration": "Check the sandbox resources and Runtime release.", + "invalid_sandbox_configuration": "Check the sandbox resources.", "project_archived": "This Project is archived.", "project_exists": "A Project with this name already exists.", "project_api_key_exists": "An active API key with this name already exists.", @@ -57,6 +57,5 @@ export const coreErrorDetails = { "protocols": "Allowed protocols: {{protocols}}.", "keyLimit": "Enter a valid API key of at most {{max}} characters.", "resourceRange": "Enter a whole number from {{min}} to {{max}}.", - "resourceMin": "Enter a whole number of at least {{min}}.", - "runtime": "Use a valid immutable Runtime release for this backend." + "resourceMin": "Enter a whole number of at least {{min}}." } as const; diff --git a/apps/web/src/i18n/locales/zh-CN/core-errors.ts b/apps/web/src/i18n/locales/zh-CN/core-errors.ts index 53ac68599..9b117c70b 100644 --- a/apps/web/src/i18n/locales/zh-CN/core-errors.ts +++ b/apps/web/src/i18n/locales/zh-CN/core-errors.ts @@ -18,7 +18,7 @@ export const coreErrors = { "model_provider_protocol_unsupported": "此 Harness 不支持该协议。", "model_provider_api_key_invalid": "请输入有效的 API 密钥，不要包含控制字符。", "model_provider_token_limits_invalid": "请填写有效的 token 限制；输出不能超过上下文。", - "invalid_sandbox_configuration": "请检查沙箱资源和 Runtime 版本。", + "invalid_sandbox_configuration": "请检查沙箱资源。", "project_archived": "此项目已归档。", "project_exists": "此项目名称已被使用。", "project_api_key_exists": "此名称已被使用中的 API 密钥占用。", @@ -55,6 +55,5 @@ export const coreErrorDetails = { "protocols": "支持的协议：{{protocols}}。", "keyLimit": "请输入有效的 API 密钥，长度最多 {{max}} 个字符。", "resourceRange": "请输入 {{min}} 到 {{max}} 的整数。", - "resourceMin": "请输入不小于 {{min}} 的整数。", - "runtime": "请使用适用于此沙箱提供商的有效不可变 Runtime 版本。" + "resourceMin": "请输入不小于 {{min}} 的整数。" } as const; diff --git a/apps/web/src/i18n/locales/zh-CN/sandbox.ts b/apps/web/src/i18n/locales/zh-CN/sandbox.ts index 850fd74c3..f9cae0b04 100644 --- a/apps/web/src/i18n/locales/zh-CN/sandbox.ts +++ b/apps/web/src/i18n/locales/zh-CN/sandbox.ts @@ -45,7 +45,6 @@ export const sandbox = { "Ready for target": "目标配置已就绪", "Preparing target": "正在准备目标配置", "Preparation failed": "目标配置准备失败", - "Node software incompatible": "节点程序不兼容", "Target readiness unknown": "目标配置就绪状态未知", "Target preparation": "目标配置准备状态", "Serving generation": "服务代次", @@ -56,15 +55,13 @@ export const sandbox = { "Core is preparing the target configuration.": "Core 正在准备目标配置。", "Core reports no active preparation. This does not mean every node is ready.": "Core 报告当前没有正在进行的准备任务。这不代表所有节点都已就绪。", "Rollout state is unconfirmed. These are the last confirmed observations.": "配置更新状态尚未确认。当前显示的是上次确认的状态。", - "Review affected nodes for preparation errors, incompatible node software or an unconfirmed connection. Qualified earlier generations may still serve work.": "请检查相关节点的准备错误、不兼容的节点程序或尚未确认的连接。符合条件的旧代次仍可提供服务。", + "Review affected nodes for preparation errors or an unconfirmed connection. Qualified earlier generations may still serve work.": "请检查相关节点的准备错误或尚未确认的连接。符合条件的旧代次仍可提供服务。", "Existing sandboxes keep their configuration generation. Saving does not move them or prove the target is ready.": "现有沙箱保留所属配置代次。保存不会迁移沙箱，也不代表目标配置已就绪。", "Changes keep this backend and existing Sessions and nodes. Core prepares the new target for future work; placement may continue on qualified earlier generations.": "修改会保留当前沙箱提供商、现有 Session 和节点。Core 为后续任务准备新目标配置，期间仍可能将任务分配到符合条件的旧代次。", - "This node can't prepare new sandbox configurations and keeps serving the one it registered with. To move it to the current configuration, choose Remove node, then add the host again with Add node.": "该节点无法准备新的沙箱配置，会继续服务注册时的配置。要让它使用当前配置，请选择“移除节点”，再通过“添加节点”重新添加该主机。", "The saved serving generation is not proof that this node is online, ready or has free capacity.": "已保存的服务代次不代表节点当前在线、已就绪或有空闲容量。", "Node observations are from an earlier configuration generation. Refreshing does not change their owned sandboxes.": "节点观测来自较早的配置代次。刷新不会改变节点所属的沙箱。", "Leave blank to keep the saved key. Any key you enter is verified as a replacement, even if unchanged.": "留空可保留已保存的密钥。填写的密钥都会作为替换密钥进行验证，即使内容未变。", "These limits apply to the selected configuration generation. Existing sandboxes keep their limits. Concurrency is set per node.": "这些限制适用于所选配置代次。现有沙箱保留原有限制。并发数量按节点设置。", - "The target Runtime release. Existing sandboxes keep their owned release while nodes prepare the target.": "目标 Runtime 版本。节点准备目标版本时，现有沙箱仍使用其所属版本。", "Cancel editing to use Reset deployment. This change requires an explicit reset; the saved configuration is unchanged.": "取消编辑后可使用“重置部署”。此更改需要明确重置；已保存的配置未改变。", "Sandbox state is unconfirmed. Refresh before issuing a node command.": "沙箱状态尚未确认。请刷新后再生成节点命令。", @@ -263,17 +260,15 @@ export const sandbox = { "Sandbox ownership mismatch": "沙箱归属不一致", "Reconcile the assigned resource and its ownership record before resuming execution.": "请核对已分配资源及其归属记录，再恢复执行。", "Sandbox provider unavailable": "沙箱提供商不可用", - "Restore the provider on the assigned node, then refresh. Running the install command again on the host checks its requirements and names the fix; a manually registered node logs the local error.": "请恢复已分配节点上的沙箱提供商，然后刷新。在该主机上重新运行安装命令，会检查主机要求并指出修复方法；手动注册的节点会在日志中记录本地错误。", + "Restore the provider on the assigned node, then refresh. Running the install command again on the host checks its requirements and names the fix.": "请恢复已分配节点上的沙箱提供商，然后刷新。在该主机上重新运行安装命令，会检查主机要求并指出修复方法。", "Sandbox state needs attention": "沙箱状态需要检查", "Inspect the assigned node and resource, then refresh.": "请检查已分配节点和资源，然后刷新。", "Host unsupported": "主机不满足要求", - "The host lacks a capability its sandbox provider requires. Running the install command again on the host checks its requirements and names the fix; a manually registered node logs the local error.": "主机缺少沙箱提供商所需的能力。在该主机上重新运行安装命令，会检查主机要求并指出修复方法；手动注册的节点会在日志中记录本地错误。", + "The host lacks a capability its sandbox provider requires. Running the install command again on the host checks its requirements and names the fix.": "主机缺少沙箱提供商所需的能力。在该主机上重新运行安装命令，会检查主机要求并指出修复方法。", "Provider files missing": "沙箱提供商文件缺失", "Pinned provider files are missing or fail their checksum. Run the install command again.": "指定版本的沙箱提供商文件缺失或校验不通过。请重新运行安装命令。", - "Runtime download failed": "Runtime 下载失败", - "Runtime files could not be downloaded or verified. Check the node's network access and the configured Runtime release.": "Runtime 文件下载或验证失败。请检查节点网络连接及配置的 Runtime 发布版本。", "Runtime image missing": "缺少 Runtime 镜像", - "The pinned Runtime image isn't on the host. Run the install command again.": "主机上没有指定版本的 Runtime 镜像。请重新运行安装命令。", + "The Runtime image isn't on the host. Run the install command again.": "主机上没有 Runtime 镜像。请重新运行安装命令。", "Host too small": "主机资源不足", "The host has less CPU or memory than one sandbox needs. Use a bigger host or a smaller sandbox size.": "主机的 CPU 或内存不够运行一个沙箱。请换一台更大的主机，或调小沙箱规格。", "Connecting to this console's Core…": "正在连接此控制台的 Core…", @@ -315,7 +310,6 @@ export const sandbox = { "{{cpus}} CPUs, {{memory}} MiB of memory. microsandbox disks are at least {{disk}} MiB.": "CPU {{cpus}} 核，内存 {{memory}} MiB；microsandbox 的磁盘至少 {{disk}} MiB。", "Advanced settings": "高级设置", "CPUs": "CPU（核）", - "Check this value": "请检查这个值", "Connect E2B": "连接 E2B", "Core address": "Core 地址", "Custom size in advanced settings": "在高级设置里自定义规格", @@ -323,28 +317,16 @@ export const sandbox = { "microsandbox, the recommended default, runs each sandbox as a lightweight virtual machine: stronger isolation and its own root and data disks with size limits, but the host needs KVM. Docker runs each sandbox as a container on the host's kernel: CPU and memory limits but no disk quota, for trusted workloads or hosts without KVM.": "microsandbox（推荐的默认沙箱提供商）：每个沙箱是一个轻量虚拟机，隔离更强，有独立的根盘和数据盘并能限制大小，但主机需要支持 KVM。Docker：每个沙箱是一个共用主机内核的容器，能限制 CPU 和内存，但没有磁盘配额，只适合可信的工作负载或没有 KVM 的主机。", "E2B runs sandboxes in its cloud: no machines to manage, billed by E2B. Own machines run them on hosts you add, with microsandbox (recommended) or Docker.": "E2B 云端：沙箱跑在 E2B 的云上，不用管理机器，由 E2B 计费。自有机器：沙箱跑在你添加的主机上，用 microsandbox（推荐）或 Docker。", "Each sandbox": "每个沙箱", - "Filled in from this console's distribution when it serves one. Otherwise copy these from the distribution manifest that matches your nodes; image configuration IDs and manifest digests are different values.": "控制台提供发行清单时会自动填好；否则从与节点配套的发行清单里复制。镜像配置 ID 和清单摘要是两个不同的值。", - "Firmware SHA-256": "固件 SHA-256", "How big is each sandbox?": "每个沙箱多大？", - "Image ID": "镜像 ID", - "Image manifest digest": "镜像清单摘要", "Large": "大", "Memory (MiB)": "内存（MiB）", "Next": "下一步", "Review and save": "确认并保存", "Root disk (MiB)": "根盘（MiB）", "Root disk {{root}} · data disk {{data}}": "根盘 {{root}} · 数据盘 {{data}}", - "Runtime": "Runtime", - "Saved · this installation serves ": "已保存 · 本安装提供 ", - "Other changes keep the saved Runtime. Existing sandboxes keep their Runtime while nodes prepare the new one.": "其他修改会保留已保存的 Runtime。节点准备新 Runtime 期间，现有沙箱继续使用原来的 Runtime。", - "Use this installation's Runtime": "使用本安装的 Runtime", - "Runtime SHA-256": "Runtime SHA-256", - "Runtime release": "Runtime 版本", - "Runtime release needed": "需要 Runtime 版本", "Sandboxes run on": "运行在", "Saving…": "保存中…", "Small": "小", - "Source commit": "源码提交", "Standard": "标准", "Template build": "模板构建", "Ready": "就绪", @@ -355,7 +337,6 @@ export const sandbox = { "After {{idle}} · kept {{retention}}": "{{idle}}后 · 保留 {{retention}}", "Current": "当前规格", "The key is write-only: Core encrypts it and never shows it again.": "密钥只写入：Core 加密保存，之后不再显示。", - "This console serves no Runtime manifest. Enter the release under advanced settings.": "这个控制台没有提供 Runtime 清单，请在高级设置里填写。", "Where should sandboxes run?": "沙箱在哪里运行？", "Which sandbox backend?": "用哪个沙箱提供商？", "Recommended": "推荐", @@ -368,7 +349,6 @@ export const sandbox = { "Docker suits only trusted workloads, or hosts without KVM.": "Docker 只适合可信的工作负载，或没有 KVM 的主机。", "Use Docker": "使用 Docker", "Keep microsandbox": "保留 microsandbox", - "microsandbox reference": "microsandbox 引用", "{{cpus}} CPU · {{memory}}": "{{cpus}} 核 · {{memory}}", "Save configuration": "保存配置", "Change the sandbox configuration": "修改沙箱配置", diff --git a/apps/web/src/lib/core-error.ts b/apps/web/src/lib/core-error.ts index d8608e2b2..fb94ba984 100644 --- a/apps/web/src/lib/core-error.ts +++ b/apps/web/src/lib/core-error.ts @@ -18,7 +18,6 @@ export function knownCoreError(error: unknown, t: TFunction<"common">, nameUnit: const min = number("min"), max = number("max"); if (error.code === "invalid_node_capacity" && min !== undefined && max !== undefined && min <= max) return t("coreErrorDetails.capacityRange", { min, max }); if (error.code === "invalid_sandbox_configuration") { - if (error.param === "runtime") return t("coreErrorDetails.runtime"); if (resourceParams.has(error.param) && min !== undefined) { if (max !== undefined && min <= max) return t("coreErrorDetails.resourceRange", { min, max }); if (max === undefined) return t("coreErrorDetails.resourceMin", { min }); diff --git a/apps/web/src/lib/locale.test.ts b/apps/web/src/lib/locale.test.ts index 38278a9ad..15a4dcbaa 100644 --- a/apps/web/src/lib/locale.test.ts +++ b/apps/web/src/lib/locale.test.ts @@ -13,7 +13,7 @@ describe("sandbox localization", () => { }); it("localizes all diagnostic labels and advice", () => { for (const code of ["node_unavailable", "resource_missing", "compute_unconfirmed", "ownership_mismatch", "provider_unavailable", "host_unsupported", - "artifacts_unavailable", "runtime_download_failed", "runtime_image_unavailable", "capacity_insufficient", "unknown"]) { + "artifacts_unavailable", "runtime_image_unavailable", "capacity_insufficient", "unknown"]) { const message = sandboxDiagnosticMessage(code, "zh-CN"); expect(message?.label).toMatch(/[\u4e00-\u9fff]/); expect(message?.advice).toMatch(/[\u4e00-\u9fff]/); diff --git a/apps/web/src/lib/sandbox-diagnostic.test.ts b/apps/web/src/lib/sandbox-diagnostic.test.ts index 65a830b86..e1b5d934d 100644 --- a/apps/web/src/lib/sandbox-diagnostic.test.ts +++ b/apps/web/src/lib/sandbox-diagnostic.test.ts @@ -16,47 +16,17 @@ describe("sandbox diagnostics", () => { expect(sandboxDiagnosticMessage("ownership_mismatch")?.label).toBe("Sandbox ownership mismatch"); expect(sandboxDiagnosticMessage("provider_unavailable")?.label).toBe("Sandbox provider unavailable"); }); - it("names why a node's provider is not ready, reading an unknown code as provider_unavailable", () => { - expect(sandboxDiagnosticMessage(nodeProviderDiagnostic({ online: true, provider_ready: false, diagnostic: "host_unsupported" }))?.label).toBe("Host unsupported"); - expect(nodeProviderDiagnostic({ online: true, provider_ready: false, diagnostic: "future_code" as SandboxNodeDiagnostic })).toBe("provider_unavailable"); - expect(nodeProviderDiagnostic({ online: true, provider_ready: true })).toBe(""); - // An offline node's last code may no longer apply. - expect(nodeProviderDiagnostic({ online: false, provider_ready: false, diagnostic: "host_unsupported" })).toBe(""); + it("names an online node's readiness class from its rollout and hides an offline one", () => { + const failed = { state: "failed", ready_generation: null, diagnostic: "host_unsupported" } as const; + const unknown = { state: "unknown", ready_generation: null } as const; + expect(sandboxDiagnosticMessage(nodeProviderDiagnostic({ online: true, provider_ready: false, rollout: failed }))?.label).toBe("Host unsupported"); + expect(sandboxDiagnosticMessage(nodeProviderDiagnostic({ online: true, provider_ready: false, rollout: unknown }))?.label).toBe("Sandbox provider unavailable"); + expect(nodeProviderDiagnostic({ online: true, provider_ready: true, rollout: failed })).toBe(""); + expect(nodeProviderDiagnostic({ online: false, provider_ready: false, rollout: unknown })).toBe(""); }); it("does not expose an unknown raw error or turn it into a healthy state", () => { const message = sandboxDiagnosticMessage("private-provider-error-with-secret"); expect(message?.label).toBe("Sandbox state needs attention"); expect(JSON.stringify(message)).not.toContain("private-provider-error"); }); - it("preserves the typed download diagnostic on an online node and suppresses stale offline health", () => { - const diagnostic: SandboxNodeDiagnostic = "runtime_download_failed"; - expect(nodeProviderDiagnostic({ online: true, provider_ready: false, diagnostic })).toBe(diagnostic); - expect(nodeProviderDiagnostic({ online: false, provider_ready: false, diagnostic })).toBe(""); - }); - it("explains download and verification failures separately from provider image failures in both languages", () => { - const diagnostic: SandboxNodeDiagnostic = "runtime_download_failed"; - expect(sandboxDiagnosticMessage(diagnostic, "en")).toEqual({ - label: "Runtime download failed", - advice: "Runtime files could not be downloaded or verified. Check the node's network access and the configured Runtime release.", - }); - expect(sandboxDiagnosticMessage(diagnostic, "zh-CN")).toEqual({ - label: "Runtime 下载失败", - advice: "Runtime 文件下载或验证失败。请检查节点网络连接及配置的 Runtime 发布版本。", - }); - for (const locale of ["en", "zh-CN"] as const) { - expect(sandboxDiagnosticMessage(diagnostic, locale)).not.toEqual(sandboxDiagnosticMessage("runtime_image_unavailable", locale)); - } - }); -}); - -describe("shared node readiness diagnostics", () => { - it.each(sandboxNodeDiagnostics)("preserves %s with localized messages", (diagnostic) => { - expect(nodeProviderDiagnostic({ online: true, provider_ready: false, diagnostic })).toBe(diagnostic); - const english = sandboxDiagnosticMessage(diagnostic, "en"); - const chinese = sandboxDiagnosticMessage(diagnostic, "zh-CN"); - expect(english?.label).not.toBe("Sandbox state needs attention"); - expect(english?.advice).not.toBe("Inspect the assigned node and resource, then refresh."); - expect(chinese?.label).not.toBe(english?.label); - expect(chinese?.advice).not.toBe(english?.advice); - }); }); diff --git a/apps/web/src/lib/sandbox-diagnostic.ts b/apps/web/src/lib/sandbox-diagnostic.ts index aaf2ecbd0..17ae8d24d 100644 --- a/apps/web/src/lib/sandbox-diagnostic.ts +++ b/apps/web/src/lib/sandbox-diagnostic.ts @@ -1,4 +1,4 @@ -import { normalizeSandboxNodeDiagnostic, type SandboxNode } from "@oac/agents-client"; +import type { SandboxNode } from "@oac/agents-client"; import i18n, { type SupportedLanguage } from "../i18n"; import type { ParseKeys } from "i18next"; export interface SandboxDiagnosticMessage { label: string; advice: string } @@ -22,24 +22,20 @@ const diagnostics: Record; advice: ParseKe }, provider_unavailable: { label: "Sandbox provider unavailable", - advice: "Restore the provider on the assigned node, then refresh. Running the install command again on the host checks its requirements and names the fix; a manually registered node logs the local error.", + advice: "Restore the provider on the assigned node, then refresh. Running the install command again on the host checks its requirements and names the fix.", }, // Provider-neutral readiness classes; Core sends only the code, and the node keeps the local detail. host_unsupported: { label: "Host unsupported", - advice: "The host lacks a capability its sandbox provider requires. Running the install command again on the host checks its requirements and names the fix; a manually registered node logs the local error.", + advice: "The host lacks a capability its sandbox provider requires. Running the install command again on the host checks its requirements and names the fix.", }, artifacts_unavailable: { label: "Provider files missing", advice: "Pinned provider files are missing or fail their checksum. Run the install command again.", }, - runtime_download_failed: { - label: "Runtime download failed", - advice: "Runtime files could not be downloaded or verified. Check the node's network access and the configured Runtime release.", - }, runtime_image_unavailable: { label: "Runtime image missing", - advice: "The pinned Runtime image isn't on the host. Run the install command again.", + advice: "The Runtime image isn't on the host. Run the install command again.", }, capacity_insufficient: { label: "Host too small", @@ -47,14 +43,10 @@ const diagnostics: Record; advice: ParseKe }, }; -/** - * Why an online node's provider is not ready, as one fixed code; an unknown - * value reads as provider_unavailable. Empty while the provider is ready, and - * for an offline node, whose last code may no longer apply. - */ -export function nodeProviderDiagnostic(node: Pick): string { - if (!node.online || (node.provider_ready && !node.diagnostic)) return ""; - return normalizeSandboxNodeDiagnostic(node.diagnostic ?? ""); +/** Why an online node is not ready: its target generation's fixed code, else provider_unavailable; empty otherwise. */ +export function nodeProviderDiagnostic(node: Pick): string { + if (!node.online || node.provider_ready) return ""; + return node.rollout.diagnostic ?? "provider_unavailable"; } export function sandboxDiagnosticMessage(value?: string, locale: SupportedLanguage = "en"): SandboxDiagnosticMessage | null { diff --git a/contracts/agents-api/core-errors.md b/contracts/agents-api/core-errors.md index 306d6d18a..dd4dd2f67 100644 --- a/contracts/agents-api/core-errors.md +++ b/contracts/agents-api/core-errors.md @@ -67,9 +67,8 @@ Each code returns HTTP 400 with `type: "invalid_request_error"`. A missing, malf | `invalid_sandbox_configuration` | `resources.cpus` | `min`: 1, `max`: 255 | The CPU count is outside the supported bounds | | `invalid_sandbox_configuration` | `resources.memory_mib` | `min`: 512, `max`: 1048576 | Memory is outside the supported bounds | | `invalid_sandbox_configuration` | `resources.root_disk_mib` or `resources.environment_disk_mib` | `min`: 1024 for microsandbox; `min`: 0, `max`: 0 for Docker and E2B | Disk capacity is missing or unsupported by the provider | -| `invalid_sandbox_configuration` | `runtime` | omitted | The Runtime release is missing, mutable, invalid or not allowed for E2B | -Bounds are validation constants, never submitted values. Node names are limited in bytes; Project and key names in trimmed Unicode characters without control characters. Only the first failure is reported, in this order: model provider URL, protocol, key, limits, then the Harness's protocol; sandbox resources CPU, memory, disk, then Runtime. Model-provider field errors inside a `model_provider` object keep that object's field as `param`. An unknown sandbox provider returns an error without these fields. +Bounds are validation constants, never submitted values. Node names are limited in bytes; Project and key names in trimmed Unicode characters without control characters. Only the first failure is reported, in this order: model provider URL, protocol, key, limits, then the Harness's protocol; sandbox resources CPU, memory, then disk. Model-provider field errors inside a `model_provider` object keep that object's field as `param`. An unknown sandbox provider returns an error without these fields. ## Other administration errors diff --git a/contracts/agents-api/core.openapi.yaml b/contracts/agents-api/core.openapi.yaml index a38349cfd..32dcd2874 100644 --- a/contracts/agents-api/core.openapi.yaml +++ b/contracts/agents-api/core.openapi.yaml @@ -733,8 +733,6 @@ definitions: description: |- Per-sandbox limits, required for Docker and microsandbox. E2B may omit them; Core then uses the validated template build's cpus and memory_mib. - runtime: - $ref: '#/definitions/sandbox.RuntimeRelease' required: - expected_generation type: object @@ -754,8 +752,6 @@ definitions: description: |- Per-sandbox limits, required for Docker and microsandbox. E2B may omit them; Core then uses the validated template build's cpus and memory_mib. - runtime: - $ref: '#/definitions/sandbox.RuntimeRelease' required: - expected_generation type: object @@ -1276,10 +1272,6 @@ definitions: x-nullable: true created_at: type: string - diagnostic: - allOf: - - $ref: '#/definitions/sandbox.NodeDiagnosticCode' - description: Fixed reason for the last reported unreadiness; absent while the provider is ready. Clients treat an unknown value as provider_unavailable. enrollment_id: description: The enrollment_id of the command that registered this node (POST /core/v1/sandbox/enrollment-tokens); null for nodes enrolled before Core recorded it. type: string @@ -1300,6 +1292,7 @@ definitions: provider: type: string provider_ready: + description: True while the node is online and its serving generation is ready. type: boolean reserved: type: integer @@ -1405,10 +1398,6 @@ definitions: x-nullable: true created_at: type: string - diagnostic: - allOf: - - $ref: '#/definitions/sandbox.NodeDiagnosticCode' - description: Fixed reason for the last reported unreadiness; absent while the provider is ready. Clients treat an unknown value as provider_unavailable. enrollment_id: description: The enrollment_id of the command that registered this node (POST /core/v1/sandbox/enrollment-tokens); null for nodes enrolled before Core recorded it. type: string @@ -1433,6 +1422,7 @@ definitions: provider: type: string provider_ready: + description: True while the node is online and its serving generation is ready. type: boolean reserved: type: integer @@ -1518,14 +1508,12 @@ definitions: - ready - preparing - failed - - update_required - unknown type: string x-enum-varnames: - NodeRolloutReady - NodeRolloutPreparing - NodeRolloutFailed - - NodeRolloutUpdateRequired - NodeRolloutUnknown deployment.NodeUpdate: properties: @@ -1653,14 +1641,11 @@ definitions: type: integer unknown: type: integer - update_required: - type: integer required: - failed - preparing - ready - unknown - - update_required type: object deployment.RolloutState: enum: @@ -1879,8 +1864,6 @@ definitions: properties: resources: $ref: '#/definitions/sandbox.Resources' - runtime: - $ref: '#/definitions/sandbox.RuntimeRelease' workspace: allOf: - $ref: '#/definitions/workspacefs.Declaration' @@ -1894,7 +1877,6 @@ definitions: - provider_unavailable - host_unsupported - artifacts_unavailable - - runtime_download_failed - runtime_image_unavailable - capacity_insufficient type: string @@ -1902,7 +1884,6 @@ definitions: - NodeProviderUnavailable - NodeHostUnsupported - NodeArtifactsUnavailable - - NodeRuntimeDownloadFailed - NodeRuntimeImageUnavailable - NodeCapacityInsufficient sandbox.Resources: @@ -1919,28 +1900,6 @@ definitions: - cpus - memory_mib type: object - sandbox.RuntimeRelease: - properties: - firmware_sha256: - type: string - image_id: - type: string - image_manifest_digest: - type: string - microsandbox_ref: - type: string - runtime_sha256: - type: string - source_commit: - type: string - required: - - firmware_sha256 - - image_id - - image_manifest_digest - - microsandbox_ref - - runtime_sha256 - - source_commit - type: object sessions.AdminAssetCounts: properties: agents: @@ -7016,7 +6975,7 @@ paths: post: consumes: - application/json - description: Selects a provider, enforced resource limits and pinned Runtime release. Core derives the deployment's core_url from the installation public URL and rejects a core_url member with 400. E2B returns 409 sandbox_configuration_error while the public URL is loopback. E2B credentials are write-only. E2B may omit resources to adopt the validated template build's CPU and memory, returned in specification.resources. Requires explicit expected_generation, including zero at first setup. Stale retries reject before provider validation. An identical selection at the current generation is a no-op; a differing selection rejects. This does not create compute or execute work. + description: Selects a provider and its enforced resource limits. Core derives the deployment's core_url from the installation public URL and rejects a core_url member with 400. E2B returns 409 sandbox_configuration_error while the public URL is loopback. E2B credentials are write-only. E2B may omit resources to adopt the validated template build's CPU and memory, returned in specification.resources. Requires explicit expected_generation, including zero at first setup. Stale retries reject before provider validation. An identical selection at the current generation is a no-op; a differing selection rejects. This does not create compute or execute work. parameters: - description: Deployment selection in: body diff --git a/contracts/agents-api/node-generation-protocol.md b/contracts/agents-api/node-generation-protocol.md index 3f1fd4dc9..2d6976860 100644 --- a/contracts/agents-api/node-generation-protocol.md +++ b/contracts/agents-api/node-generation-protocol.md @@ -44,13 +44,13 @@ sequenceDiagram ``` 1. The node dials `/api/v1/sandbox-node/connect?node_id=` on its stored Core origin (`wss` for `https`) with its node credential as a Bearer header. Core answers 401 to a rejected credential, which the node treats as permanent; any other failure, including a 403 from a proxy, is retried with bounded backoff. Core refuses a second connection for a node identity while one is opening, live or closing, with 409. -2. Within 15 seconds the node sends `hello` with its identity (`node_id`, `installation_id`, `provider`, `backend_fingerprint`, the enrolled `deployment_generation` and `specification_digest`, `max_active`, `max_retained`), its first health report and, when it can prepare and retain several deployment generations, `generation_management: true`. Core closes the connection unless the identity matches the authenticated node. -3. Core records the node's presence, then replies `welcome` with a new `connection_id`, the current `owner_epoch` and, for a generation-managing node, `deployment`: the target `generation`, its `specification_digest` and a nullable `serving_generation`. The node stores a higher owner epoch and refuses a lower one. -4. Every 10 seconds the node sends `heartbeat` with `connection_id`, `owner_epoch` and health. On each heartbeat Core authenticates the node credential again and checks the owner epoch, records the health and replies `heartbeat_ack`, with `deployment` for a generation-managing node. Either peer closes the connection after 35 seconds without a frame. +2. Within 15 seconds the node sends `hello` with its identity (`node_id`, `installation_id`, `provider`, `backend_fingerprint`, the enrolled `deployment_generation` and `specification_digest`, `max_active`, `max_retained`), and its first health report. Core closes the connection unless the identity matches the authenticated node. +3. Core records the node's presence, then replies `welcome` with a new `connection_id`, the current `owner_epoch` and `deployment`: the target `generation`, its `specification_digest` and a nullable `serving_generation`. The node stores a higher owner epoch and refuses a lower one. +4. Every 10 seconds the node sends `heartbeat` with `connection_id`, `owner_epoch` and health. On each heartbeat Core authenticates the node credential again and checks the owner epoch, records the health and replies `heartbeat_ack` with `deployment`. Either peer closes the connection after 35 seconds without a frame. Core counts a node as online while it is connected under the current owner epoch and its last heartbeat is less than 45 seconds old. A heartbeat establishes provider readiness and the last host measurements, never Session activity. [Timeouts and limits](#timeouts-and-limits) lists every timer. -Health carries `provider_ready`, an optional fixed `diagnostic`, `observed_at`, `active_operations` (at most 32) and the host measurements that the [Runtime telemetry API](./runtime-observability-api.md#node-host-observations-and-history) reports. A node without generation management probes its provider for every report; an unready provider reports the Provider-neutral readiness class of its first failed check, from the class error the probe wraps, and the probe text, vendor detail and host paths stay on the node. A `diagnostic` in node or generation health that is not a declared class makes the frame invalid. The [nodes guide](../../docs/getting-started/nodes.md#readiness-codes) lists the classes and their causes. A generation-managing node reports readiness per generation instead, as described below. +Health carries `observed_at`, `active_operations` (at most 32) and the host measurements that the [Runtime telemetry API](./runtime-observability-api.md#node-host-observations-and-history) reports. A node reports readiness per generation, as described below. A `diagnostic` in a generation observation that is not a declared class makes the frame invalid. The [nodes guide](../../docs/getting-started/nodes.md#readiness-codes) lists the classes and their causes. ## Provider requests @@ -86,7 +86,7 @@ Each operation carries its own arguments and returns the following result on suc | `resume` | `Resume` | `resume` | `state` | | `delete_snapshot` | `DeleteSnapshot` | `snapshot` | None | -A request whose `connection_id`, `owner_epoch` or `sequence` does not match closes the connection. A malformed request gets an `invalid` response. A node without generation management accepts only its enrolled `deployment_generation`; a generation-managing node runs the request on that generation's provider and answers `unconfirmed` when it cannot. Core sends `create` and a `resume` that is not observe-only only to a generation that is ready on that node, and keeps at most 32 requests pending per connection. +A request whose `connection_id`, `owner_epoch` or `sequence` does not match closes the connection. A malformed request gets an `invalid` response. The node runs the request on the provider of the named `deployment_generation` and answers `unconfirmed` when it cannot. Core sends `create` and a `resume` that is not observe-only only to a generation that is ready on that node, and keeps at most 32 requests pending per connection. The budget is relative: the node anchors `timeout_ms` to its own clock on receipt and consumes it while the request waits in its queue, so the hosts' clocks need not agree. Core still bounds its own wait. A full node queue closes the connection. @@ -108,9 +108,9 @@ Create bootstrap and Resume requests may carry an optional workspace filesystem ## Generation control -A node without generation management serves only its enrolled generation, with fixed configuration, and receives no preparation or retention frames. It can report that exact generation in `Health.Generations`; a checkpoint-capable provider must do so, including its verified checkpoint qualification. `provider_ready` and the reported state must agree. A generation-managing node prepares the target generation Core announces in `welcome` and `heartbeat_ack` and keeps serving its durable serving generation while it does; target preparation is independent of the serving provider's readiness. +The node prepares the target generation Core announces in `welcome` and `heartbeat_ack` and keeps serving its durable serving generation while it does; target preparation is independent of the serving provider's readiness. -Its `hello` and heartbeats carry at most eight generation observations. Each names a positive signed-64-bit generation, its lowercase SHA-256 specification digest, a `ready`, `preparing` or `failed` state and an optional fixed diagnostic. The target and serving generations come first; other records rotate fairly. Eight bounds one message, not the number of generations a node may keep. An omitted observation never authorizes deletion or implies absence. A ready checkpoint-capable generation includes `checkpoint` with nonempty opaque `artifact_domain` and `execution_class` tokens, each at most 256 bytes. Unsupported providers omit it; preparing or failed generations never advertise it. Core persists the declaration with the authenticated connection and generation, and uses it only while that generation is ready on the current online connection. The [Provider contract](../../docs/sandbox-provider.md#checkpoint-transfer) defines archive and execution compatibility. +The node's `hello` and heartbeats carry at most eight generation observations. Each names a positive signed-64-bit generation, its lowercase SHA-256 specification digest, a `ready`, `preparing` or `failed` state and an optional fixed diagnostic. The target and serving generations come first; other records rotate fairly. Eight bounds one message, not the number of generations a node may keep. An omitted observation never authorizes deletion or implies absence. A ready checkpoint-capable generation includes `checkpoint` with nonempty opaque `artifact_domain` and `execution_class` tokens, each at most 256 bytes. Unsupported providers omit it; preparing or failed generations never advertise it. Core persists the declaration with the authenticated connection and generation, and uses it only while that generation is ready on the current online connection. The [Provider contract](../../docs/sandbox-provider.md#checkpoint-transfer) defines archive and execution compatibility. ```mermaid stateDiagram-v2 @@ -139,7 +139,7 @@ sequenceDiagram The readiness classes, one exported error and one code each, are authored in `services/core/internal/sandbox/node_diagnostic.go`. The shared `services/core/internal/sandbox/testdata/node-diagnostics.json` fixture checks the Go mapping, OpenAPI source annotations and generated enums, and the TypeScript client declaration. Web uses the client normalizer and checks localized messages for every declared code. Update these projections with a code change; clients read an unknown code as `provider_unavailable`. -Preparation diagnostics keep fixed typed causes. Only artifact transfer, checksum or release-provenance failures report `runtime_download_failed`; the private preparer signals that class through its exit category, without Core or the node parsing stderr. Provider, ownership, cancellation and unclassified failures keep their typed code or `provider_unavailable`. No raw provider text crosses the protocol. +Preparation diagnostics keep fixed typed causes. Provider, ownership, cancellation and unclassified failures keep their typed code or `provider_unavailable`. No raw provider text crosses the protocol. ## Timeouts and limits diff --git a/contracts/agents-api/runtime.openapi.yaml b/contracts/agents-api/runtime.openapi.yaml index 01fe6edeb..1ab2fdab4 100644 --- a/contracts/agents-api/runtime.openapi.yaml +++ b/contracts/agents-api/runtime.openapi.yaml @@ -117,8 +117,6 @@ definitions: properties: resources: $ref: '#/definitions/sandbox.Resources' - runtime: - $ref: '#/definitions/sandbox.RuntimeRelease' workspace: allOf: - $ref: '#/definitions/workspacefs.Declaration' @@ -141,28 +139,6 @@ definitions: - cpus - memory_mib type: object - sandbox.RuntimeRelease: - properties: - firmware_sha256: - type: string - image_id: - type: string - image_manifest_digest: - type: string - microsandbox_ref: - type: string - runtime_sha256: - type: string - source_commit: - type: string - required: - - firmware_sha256 - - image_id - - image_manifest_digest - - microsandbox_ref - - runtime_sha256 - - source_commit - type: object v1.APIError: properties: code: diff --git a/contracts/agents-api/sandbox-deployment.md b/contracts/agents-api/sandbox-deployment.md index 148253aad..fee77fb2d 100644 --- a/contracts/agents-api/sandbox-deployment.md +++ b/contracts/agents-api/sandbox-deployment.md @@ -2,7 +2,7 @@ title: "Sandbox deployment" --- -The sandbox deployment selects the Sandbox Provider, the per-sandbox resources and the immutable Runtime release for Core-managed `openai_hosted` execution. PostgreSQL holds one active selection per installation; Web and the Core API write the same configuration. A node's files hold an installed copy of it plus host-specific paths and cannot override its resources or Runtime. The selection is independent of the Harness. A deployment can stay unconfigured, with no nodes; it then refuses hosted admission. +The sandbox deployment selects the Sandbox Provider and the per-sandbox resources for Core-managed `openai_hosted` execution. PostgreSQL holds one active selection per installation; Web and the Core API write the same configuration. A node's files hold an installed copy of it plus host-specific paths and cannot override its resources. The Runtime is not part of the selection: each node runs the Runtime its own installation verified. The selection is independent of the Harness. A deployment can stay unconfigured, with no nodes; it then refuses hosted admission. This contract owns the Core API routes below and their semantics. The [nodes guide](../../docs/getting-started/nodes.md) owns the operator workflow, the [machine connection API](./machine-api.md#node-routes) the routes nodes call, and the [sandbox node protocol](./node-generation-protocol.md) the node connection. @@ -14,7 +14,7 @@ Every route requires the Core key. [Web's console server](../../docs/web/console | --- | --- | | `GET /core/v1/sandbox/deployment` | Read the safe active configuration, rollout, reset and resource counts | | `PATCH /core/v1/sandbox/deployment` | Update deployment-wide idle suspension without changing the configuration generation | -| `POST /core/v1/sandbox/deployment` | Select the initial provider, resources and Runtime | +| `POST /core/v1/sandbox/deployment` | Select the initial provider and resources | | `PUT /core/v1/sandbox/deployment` | Change the same provider's target online | | `POST /core/v1/sandbox/deployment/reset` | Start or escalate a durable clear of hosted resources | | `DELETE /core/v1/sandbox/deployment/reset?expected_generation=N` | Cancel the remaining clear | @@ -36,7 +36,6 @@ POST and PUT take the same complete selection and require `expected_generation` | `expected_generation` | Required nonnegative integer from GET; never refresh and replay it automatically | | `provider` | Exactly one of `docker`, `microsandbox`, `e2b` | | `resources` | Per-sandbox limits, below; required for Docker and microsandbox, optional for E2B | -| `runtime` | The immutable [Runtime release](#runtime-release); required for Docker and microsandbox, absent for E2B | | `configuration` | The provider's public selectors. E2B: the immutable `template` build and the optional paired `api_url` and `domain`. Docker and microsandbox accept only `{}` or omission | | `credential` | The provider's write-only credential. E2B: `{api_key}`, required at first setup and omitted on PUT to keep the current key; a null or empty key is invalid. Docker and microsandbox reject it | @@ -61,21 +60,6 @@ The optional response `specification.workspace` is an immutable capability recei To enable external storage for an existing owned deployment, select the filesystem configuration first, then update the sandbox deployment with compatible resources (`environment_disk_mib: 0` for a filesystem without quotas). Selecting the filesystem does not alter existing generations or allocations. The next sandbox update derives the external receipt; old owned allocations keep their owned disks. A filesystem selection must support the provider's declared requirements. An already external deployment requires the same `attachment` and `user_xattr`; the selection may change `capacity_quota`, after which new workspaces are refused until a sandbox update sets a matching `environment_disk_mib`. Existing workspaces keep the capacity they were admitted with. -### Runtime release - -Docker and microsandbox use every field of one verified distribution: - -| Field | Identity | -| --- | --- | -| `source_commit` | Lowercase 40-character commit SHA | -| `image_id` | Docker image configuration ID: `sha256:` and 64 lowercase hex characters | -| `image_manifest_digest` | OCI image manifest digest, in the same form | -| `microsandbox_ref` | `oac-runtime@sha256:` and 64 lowercase hex characters | -| `runtime_sha256` | SHA-256 of the native microsandbox runtime binary | -| `firmware_sha256` | SHA-256 of the matching firmware | - -Copy these identities from the matching distribution manifest. An image configuration ID and an OCI manifest digest identify different objects and never substitute for each other. The node installer verifies the saved release against its payload before registration and keeps the exact local image identity it imports. - ### E2B configuration E2B uses `configuration.template` in `template-id:build-uuid` form; the build UUID must be canonical and nonzero, and a mutable template alias alone is refused. The API key is encrypted in PostgreSQL and never appears in a response, bootstrap configuration, command argument or log. By default Core uses `https://api.e2b.app` and `e2b.app`. For a compatible service, set both `configuration.api_url` (an HTTPS origin without path, port, query, fragment or credentials) and `configuration.domain` (the sandbox data-plane DNS suffix); the API host must equal the domain or be a subdomain of it. Core rejects a sandbox whose data-plane domain lies outside the selected suffix before sending daemon credentials or using envd. @@ -95,9 +79,9 @@ GET and successful writes return `installation_id`, `provider`, `core_url` (read - Request `resources` and response `specification.resources` are per-sandbox limits. Response `resources.allocations` and `resources.pending` count unreleased allocations and pending hosted Environments without an allocation. - An unconfigured deployment has an empty provider and no specification. Docker and microsandbox use `mode: nodes`; E2B uses `mode: direct`, without a synthetic node. - `generation` identifies the saved selection. `owner_epoch` fences the execution owner and node connections; it does not replace `expected_generation`. -- `specification_digest` is the server's identity of the provider, limits, Runtime release and derived workspace capability receipt; enrollment echoes it unchanged. +- `specification_digest` is the server's identity of the provider, limits and derived workspace capability receipt; enrollment echoes it unchanged. -The typed `SandboxAdminClient` in `packages/agents-client` checks the deployment, node list, node detail and allocation responses against exactly these shapes. An unknown or missing member, or a wrong type, rejects the whole response with a 502 `invalid_admin_response` error. A node's `diagnostic` is absent or a code, never empty, and the client reads an unknown code as `provider_unavailable`. +The typed `SandboxAdminClient` in `packages/agents-client` checks the deployment, node list, node detail and allocation responses against exactly these shapes. An unknown or missing member, or a wrong type, rejects the whole response with a 502 `invalid_admin_response` error. A node's `rollout.diagnostic` is absent or a code, never empty, and the client reads an unknown code as `provider_unavailable`. ## Initial setup and same-provider changes @@ -139,27 +123,24 @@ Every deployment response includes `rollout`: "rollout": {"state": "settled", "previous_generation_sandboxes": 3, "nodes": null} ``` -`previous_generation_sandboxes` counts, from the same snapshot as the resource totals, unreleased allocations and allocation-less pending placements of older generations, never one resource twice. E2B and unconfigured deployments return null `nodes`. A node deployment returns `nodes` as counts `{ready, preparing, failed, update_required, unknown}`, and every node that is not removed falls in exactly one bucket: +`previous_generation_sandboxes` counts, from the same snapshot as the resource totals, unreleased allocations and allocation-less pending placements of older generations, never one resource twice. E2B and unconfigured deployments return null `nodes`. A node deployment returns `nodes` as counts `{ready, preparing, failed, unknown}`, and every node that is not removed falls in exactly one bucket: - a node that is offline on its current connection is `unknown`; -- an online node without generation management that enrolled at an older target is `update_required`; - otherwise the exact target observation on the current connection and owner epoch gives `ready`, `preparing` or `failed`, and a missing observation gives `unknown`. ```mermaid flowchart TD A{"Online on its current connection?"} -- no --> U["unknown"] - A -- yes --> B{"Generation management?"} - B -- "no, older target" --> UR["update_required"] - B -- otherwise --> C{"Target observation on this connection and epoch"} + A -- yes --> C{"Target observation on this connection and epoch"} C -- ready --> R["ready"] C -- preparing --> P["preparing"] C -- failed --> F["failed"] C -- missing --> U ``` -A node is [online](./node-generation-protocol.md#connection) when it is connected under the current owner epoch with a fresh heartbeat. Target rollout is independent of serving readiness: `unknown`, `preparing`, `failed` or `update_required` does not remove an independently confirmed older serving generation. `provider_ready` requires online presence and an exact observation of the serving generation on that connection and epoch; a node without generation management qualifies only its enrolled generation. Pins alone never imply readiness. +A node is [online](./node-generation-protocol.md#connection) when it is connected under the current owner epoch with a fresh heartbeat. Target rollout is independent of serving readiness: `unknown`, `preparing` or `failed` does not remove an independently confirmed older serving generation. `provider_ready` requires online presence and an exact observation of the serving generation on that connection and epoch. Pins alone never imply readiness. -Each node adds `rollout: {state, ready_generation, diagnostic?}`, where `ready_generation` is the nullable durable serving pin and `diagnostic` a fixed code for the target generation; allocation items add `deployment_generation`. Poll every five seconds only while `rollout.state` is `preparing` or `reset` is not null; old Sessions and failed, update-required or offline nodes alone do not keep polling active. +Each node adds `rollout: {state, ready_generation, diagnostic?}`, where `ready_generation` is the nullable durable serving pin and `diagnostic` a fixed code for the target generation; allocation items add `deployment_generation`. Poll every five seconds only while `rollout.state` is `preparing` or `reset` is not null; old Sessions and failed or offline nodes alone do not keep polling active. Node-backed creation validates the target deployment and commits the Session, pending Environment and any initial input without reserving compute. A full, offline or preparing fleet leaves that accepted work waiting; an unconfigured deployment, reset or unsupported combination still rejects admission. The common scheduler uses bounded rotating scans of unplaced demand, with pages ordered by recorded time and Environment ID and a fixed time boundary for each scan so continuous arrivals cannot prevent it from revisiting older work. It checks online presence, exact serving-generation readiness, address, shared capacity and the selected generation's Harness/filesystem compatibility, then prefers the newest qualifying pin. Placement is immutable once reserved until confirmed release or a settled checkpoint transfer. A bounded scan skips temporarily unavailable demand and resumes after a restart. If preparation fails before a new allocation is reserved, Core releases only that attempt’s node reservation, retaining any filesystem identity. Its next placement demand is ordered no earlier than release time, behind already-waiting work; successful retries use the newly selected generation’s specification. An allocation with an unknown compute outcome keeps its reservation. Existing suspended allocations reserve eligible checkpoint capacity through the same lock under the [checkpoint transfer contract](../../docs/sandbox-provider.md#checkpoint-transfer); this is not a global fairness guarantee across hot restores and unplaced work. Input retains its [original five-minute deadline](./environments.md#reservations), including time waiting for capacity. @@ -211,9 +192,9 @@ One mutation gate serializes setup, PUT, reset, cancellation and finalization. A Node capacity is approved by the administrator, separately from the deployment specification. `POST /core/v1/sandbox/enrollment-tokens` accepts optional `max_active` and `max_retained`, default 2 and 8, and returns `{token, expires_at, enrollment_id}`; `enrollment_id` is a public handle of that command, never a credential. microsandbox uses both limits. Docker never suspends, so Core replaces its `max_retained` with `max_active`, here and in PATCH. E2B has no nodes and answers 409 `sandbox_deployment_conflict`. Core stores the approval with the token and copies it to the node it registers; a node cannot submit capacity, and its local checks may refuse a deployment it cannot run but never raise the limits. [Node capacity](../../docs/configuration.md#node-capacity) explains the limits for operators. -`GET /core/v1/sandbox/nodes` returns `{data: [...]}` with, for each node: `id`, `name`, `provider`, `online`, `last_seen_at`, `created_at`, `max_active`, `max_retained`, the counts `active`, `reserved`, `running`, `retained`, `snapshots` and `cleanup_pending`, `provider_ready`, `diagnostic`, the host measurements `cpu_count`, `available_memory_bytes` and `available_disk_bytes`, `rollout`, `enrollment_id` and `core_url`. `enrollment_id` is the handle of the command that registered the node, or null when Core has none. `core_url` is the installation public URL at enrollment; a node whose `core_url` differs from the current public URL receives no new placements. Work already placed on it finishes there, including a placed Environment that has no allocation yet, and its retained sandboxes can still resume while the old address reaches Core. Remove it and add it again. +`GET /core/v1/sandbox/nodes` returns `{data: [...]}` with, for each node: `id`, `name`, `provider`, `online`, `last_seen_at`, `created_at`, `max_active`, `max_retained`, the counts `active`, `reserved`, `running`, `retained`, `snapshots` and `cleanup_pending`, `provider_ready`, the host measurements `cpu_count`, `available_memory_bytes` and `available_disk_bytes`, `rollout`, `enrollment_id` and `core_url`. `enrollment_id` is the handle of the command that registered the node, or null when Core has none. `core_url` is the installation public URL at enrollment; a node whose `core_url` differs from the current public URL receives no new placements. Work already placed on it finishes there, including a placed Environment that has no allocation yet, and its retained sandboxes can still resume while the old address reaches Core. Remove it and add it again. -A node without generation management reports its provider's readiness itself: `provider_ready`, and when it is unready one fixed `diagnostic` code. A node added with Web's command manages generations, so its readiness follows its serving generation and its fixed code for the target generation appears in `rollout.diagnostic`. The node classifies the first failed readiness check and sends only its code; a frame with any other value is invalid, and Core never stores or returns probe text or host paths. The codes are Provider-neutral classes: `provider_unavailable`, `host_unsupported`, `artifacts_unavailable`, `runtime_download_failed`, `runtime_image_unavailable` and `capacity_insufficient`. `runtime_download_failed` means the exact Runtime artifacts could not be transferred or verified; it never contains artifact URLs, credentials or transport output. [Readiness codes](../../docs/getting-started/nodes.md#readiness-codes) gives causes and operator actions. Core and nodes must come from the same distribution. +A node manages generations, so its `provider_ready` follows its serving generation and its fixed code for the target generation appears in `rollout.diagnostic`. The node classifies the first failed readiness check and sends only its code; a frame with any other value is invalid, and Core never stores or returns probe text or host paths. The codes are Provider-neutral classes: `provider_unavailable`, `host_unsupported`, `artifacts_unavailable`, `runtime_image_unavailable` and `capacity_insufficient`. [Readiness codes](../../docs/getting-started/nodes.md#readiness-codes) gives causes and operator actions. Core and nodes must come from the same distribution. `PATCH /core/v1/sandbox/nodes/{node_id}` takes `{name, max_active, max_retained}`; lowering a limit stops no running sandbox. `DELETE /core/v1/sandbox/nodes/{node_id}` refuses with 409 `runtime_node_in_use` while the node holds allocations, snapshots, reservations or pending cleanup, including while it is offline. Removal deletes no compute and retires the node's identity; the host can come back only as a new node. There is no node drain. @@ -227,7 +208,6 @@ Some fields keep one name across providers but differ in meaning, or do not appl | --- | --- | --- | --- | | Independent workspace filesystem | [Kernel NFS adapter](../../docs/workspace-provider.md#kernel-nfs-adapter); configured separately from compute | Unsupported | Unsupported | | Deployment `specification.resources` | `cpus`, `memory_mib`, `root_disk_mib` and `environment_disk_mib` | `cpus` and `memory_mib`, equal to the ready template build's and taken from it when omitted; no disk fields | `cpus` and `memory_mib`; no disk quota | -| Deployment `specification.runtime` | The full [release](#runtime-release); nodes match `microsandbox_ref`, `runtime_sha256` and `firmware_sha256` | Absent; `configuration.template` selects the build | The full [release](#runtime-release); nodes match `image_id` or `image_manifest_digest` | | Deployment `metadata.template_build` | Absent: `metadata` is empty | The build as Core read it when the selection was saved | Absent: `metadata` is empty | | Deployment `suspension` | `{idle_seconds, retention_seconds}` | `null`; Core does not suspend E2B sandboxes | `null` | | Deployment `resources.allocations`, `resources.pending` | Totals across all nodes | Core's unreleased E2B sandboxes, and hosted Environments waiting for one | Totals across all nodes | @@ -259,6 +239,6 @@ Storage and credential failures stay errors: an empty or failed read never prove ## Canonical node specification -`sandbox/deployment_contract.go` owns the resource bounds, release patterns and canonical field order, and each registered Provider's `sandbox.DeploymentPolicy` declares its requirements; `sandbox/deployment.go` applies them in Core. The installer consumes the generated declaration in `deploy/node/node_spec.py`, and the TypeScript client and Web the generated bounds, patterns and Provider declarations in `packages/agents-client/src/deployment-contract.ts`, so there is no second set of limits, patterns or providers. Regenerate both from the repository root with `go run ./services/core/cmd/specification-contract -write`; the sandbox Go tests, part of `make check`, reject a stale projection. +`sandbox/deployment_contract.go` owns the resource bounds and canonical field order, and each registered Provider's `sandbox.DeploymentPolicy` declares its requirements; `sandbox/deployment.go` applies them in Core. The installer consumes the generated declaration in `deploy/node/node_spec.py`, and the TypeScript client and Web the generated bounds and Provider declarations in `packages/agents-client/src/deployment-contract.ts`, so there is no second set of limits or providers. Regenerate both from the repository root with `go run ./services/core/cmd/specification-contract -write`; the sandbox Go tests, part of `make check`, reject a stale projection. -The specification digest is the SHA-256 of compact UTF-8 JSON with `provider` first, then `resources`, then `runtime` when the provider requires it, then `workspace` when external storage is selected. Workspace fields follow the filesystem declaration order (`attachment`, `user_xattr`, `capacity_quota`); its Boolean fields remain present. Resource and Runtime fields follow the contract's declaration order; zero optional disk fields are omitted and required fields stay present. Release identities are lowercase ASCII, and the digest never depends on the incoming field order or whitespace. `services/core/internal/sandbox/testdata/deployment-contract.json` holds shared acceptance cases, exact canonical bytes and digests that both the Go and Python tests consume. +The specification digest is the SHA-256 of compact UTF-8 JSON with `provider` first, then `resources`, then `workspace` when external storage is selected. Workspace fields follow the filesystem declaration order (`attachment`, `user_xattr`, `capacity_quota`); its Boolean fields remain present. Resource fields follow the contract's declaration order; zero optional disk fields are omitted and required fields stay present. The digest never depends on the incoming field order or whitespace. `services/core/internal/sandbox/testdata/deployment-contract.json` holds shared acceptance cases, exact canonical bytes and digests that both the Go and Python tests consume. diff --git a/contracts/agents-api/zh/core-errors.md b/contracts/agents-api/zh/core-errors.md index 99373de2f..c2af3a9c1 100644 --- a/contracts/agents-api/zh/core-errors.md +++ b/contracts/agents-api/zh/core-errors.md @@ -1,7 +1,7 @@ --- title: "Core 管理错误" source: contracts/agents-api/core-errors.md -source_hash: 814b5faa5534df0814f8037a947c24bc1f4d78960aff9f5c5bf42fe4d1ad11ec +source_hash: 35a964412b5cd64d552df2e10295dc23cb91e8ba43794a932957269f546b1125 --- `/core/v1` 上的错误使用此封装结构。`message` 是安全的英文文本；`code` 和 `param` 可以为 null。客户端依据稳定的 `code` 和可选的 `param` 进行处理，对未知代码显示 `message`，绝不解析消息，也绝不自动重试被拒绝的写操作。 @@ -69,9 +69,8 @@ Web 的控制台服务器在 `/core` 路径上发生自身故障时使用此封 | `invalid_sandbox_configuration` | `resources.cpus` | `min`：1，`max`：255 | CPU 数量超出支持范围 | | `invalid_sandbox_configuration` | `resources.memory_mib` | `min`：512，`max`：1048576 | 内存超出支持范围 | | `invalid_sandbox_configuration` | `resources.root_disk_mib` 或 `resources.environment_disk_mib` | `min`：microsandbox 为 1024；Docker 和 E2B 的 `min`：0，`max`：0 | 磁盘容量缺失或提供商不支持 | -| `invalid_sandbox_configuration` | `runtime` | 省略 | Runtime 版本缺失、可变、无效或 E2B 不允许 | -这些边界是验证常量，绝不是提交的值。节点名称按字节数限制；项目名称和键名称按去除首尾空白后的 Unicode 字符数限制，且不得包含控制字符。系统仅按以下顺序报告第一个失败项：模型提供商 URL、协议、密钥、限制，然后是 Harness 协议；沙箱资源依次为 CPU、内存、磁盘，然后是 Runtime。`model_provider` 对象内的模型提供商字段错误仍以该对象的相应字段作为 `param`。未知的沙箱提供商返回一个不含这些字段的错误。 +这些边界是验证常量，绝不是提交的值。节点名称按字节数限制；项目名称和键名称按去除首尾空白后的 Unicode 字符数限制，且不得包含控制字符。系统仅按以下顺序报告第一个失败项：模型提供商 URL、协议、密钥、限制，然后是 Harness 协议；沙箱资源依次为 CPU、内存，然后是磁盘。`model_provider` 对象内的模型提供商字段错误仍以该对象的相应字段作为 `param`。未知的沙箱提供商返回一个不含这些字段的错误。 ## 其他管理错误 {#other-administration-errors} diff --git a/contracts/agents-api/zh/node-generation-protocol.md b/contracts/agents-api/zh/node-generation-protocol.md index dfebf5a03..385a2e399 100644 --- a/contracts/agents-api/zh/node-generation-protocol.md +++ b/contracts/agents-api/zh/node-generation-protocol.md @@ -1,7 +1,7 @@ --- title: "沙箱节点协议" source: contracts/agents-api/node-generation-protocol.md -source_hash: 1ace9fb85fb4673fc4b671da08514c64716509ecc69461917dd2d4bed59b66e7 +source_hash: a4b0eae5408163c044b11016dda7a2d66bdcb6b6f90660d1388ffb5a5a2115e3 --- 沙箱节点协议连接 Core 与 Docker 或 microsandbox 主机：主机针对其本地提供商执行提供商操作，并报告就绪状态、主机测量值及其持有的部署代次。 @@ -46,13 +46,13 @@ sequenceDiagram ``` 1. 节点在其存储的 Core 源地址上发起对 `/api/v1/sandbox-node/connect?node_id=` 的连接（对于 `https` 使用 `wss`），并以 Bearer 请求头发送节点凭据。Core 对被拒绝的凭据返回 401，节点将其视为永久性拒绝；其他任何失败（包括代理返回的 403）都会使用有界退避进行重试。当某个节点身份已有一个连接正在建立、存活或关闭时，Core 会以 409 拒绝第二个连接。 -2. 节点须在 15 秒内发送 `hello`，其中包含节点身份（`node_id`、`installation_id`、`provider`、`backend_fingerprint`、已登记的 `deployment_generation` 和 `specification_digest`、`max_active`、`max_retained`）、首次健康报告；如果节点能够准备并保留多个部署代次，还包含 `generation_management: true`。除非身份与已认证节点匹配，否则 Core 会关闭连接。 -3. Core 记录节点的存在状态，然后回复 `welcome`，其中包含新的 `connection_id` 和当前 `owner_epoch`；对于支持代次管理的节点，还包含 `deployment`：目标 `generation`、其 `specification_digest` 和可空的 `serving_generation`。节点保存更高的所有者 epoch，并拒绝更低的值。 -4. 节点每 10 秒发送一次 `heartbeat`，其中包含 `connection_id`、`owner_epoch` 和健康状态。对于每次心跳，Core 都会再次认证节点凭据并检查所有者 epoch，记录健康状态并回复 `heartbeat_ack`；对于支持代次管理的节点，回复中还包含 `deployment`。任一端连续 35 秒未收到任何帧时都会关闭连接。 +2. 节点须在 15 秒内发送 `hello`，其中包含节点身份（`node_id`、`installation_id`、`provider`、`backend_fingerprint`、已登记的 `deployment_generation` 和 `specification_digest`、`max_active`、`max_retained`）和首次健康报告。除非身份与已认证节点匹配，否则 Core 会关闭连接。 +3. Core 记录节点的存在状态，然后回复 `welcome`，其中包含新的 `connection_id` 、当前 `owner_epoch` 和 `deployment`：目标 `generation`、其 `specification_digest` 和可空的 `serving_generation`。节点保存更高的所有者 epoch，并拒绝更低的值。 +4. 节点每 10 秒发送一次 `heartbeat`，其中包含 `connection_id`、`owner_epoch` 和健康状态。对于每次心跳，Core 都会再次认证节点凭据并检查所有者 epoch，记录健康状态并回复包含 `deployment` 的 `heartbeat_ack`。任一端连续 35 秒未收到任何帧时都会关闭连接。 只要节点在当前所有者 epoch 下保持连接，并且最近一次心跳距今不足 45 秒，Core 就会将该节点计为在线。心跳会确立提供商的就绪状态和最近的主机测量值，但绝不表示 Session 活动。[超时与限制](#timeouts-and-limits)列出所有计时器。 -健康报告包含 `provider_ready`、可选的固定 `diagnostic`、`observed_at`、最多为 32 的 `active_operations`，以及 [Runtime 遥测 API](runtime-observability-api.md#node-host-observations-and-history) 报告的主机测量值。未启用代次管理的节点每次报告时都会探测其提供商；未就绪的提供商会报告其首个失败检查所属的、与提供商无关的就绪类别，该类别取自探测所包装的类别错误；探测文本、厂商细节和主机路径保留在节点上。节点或代次健康状态中的 `diagnostic` 若不是已声明的类别，该帧即无效。[节点指南](../../../docs/zh/getting-started/nodes.md#readiness-codes) 列出了这些类别及其原因。支持代次管理的节点则按下文所述按代次报告就绪状态。 +健康报告包含 `observed_at`、最多为 32 的 `active_operations`，以及 [Runtime 遥测 API](runtime-observability-api.md#node-host-observations-and-history) 报告的主机测量值。节点按代次报告就绪状态，详见下文。代次观察记录中的 `diagnostic` 若不是已声明的类别，该帧即无效。[节点指南](../../../docs/zh/getting-started/nodes.md#readiness-codes) 列出了这些类别及其原因。 ## 提供商请求 {#provider-requests} @@ -88,7 +88,7 @@ Core 发送包含以下内容的 `request` 帧： | `resume` | `Resume` | `resume` | `state` | | `delete_snapshot` | `DeleteSnapshot` | `snapshot` | 无 | -只要 `connection_id`、`owner_epoch` 或 `sequence` 中任一值不匹配，请求就会关闭连接。格式错误的请求会得到 `invalid` 响应。未启用代次管理的节点仅接受其登记的 `deployment_generation`；支持代次管理的节点在对应代次的提供商上运行请求，无法运行时回复 `unconfirmed`。Core 仅向节点上已就绪的代次发送 `create` 和非 observe-only 的 `resume`，并且每条连接最多保留 32 个待处理请求。 +只要 `connection_id`、`owner_epoch` 或 `sequence` 中任一值不匹配，请求就会关闭连接。格式错误的请求会得到 `invalid` 响应。节点在所指 `deployment_generation` 的提供商上运行请求，无法运行时回复 `unconfirmed`。Core 仅向节点上已就绪的代次发送 `create` 和非 observe-only 的 `resume`，并且每条连接最多保留 32 个待处理请求。 预算采用相对计时：节点收到请求时以自己的时钟为基准锚定 `timeout_ms`，并在请求排队等待期间持续消耗该预算，因此各主机的时钟无需保持一致。Core 仍会限制自身等待时长。节点队列已满时会关闭连接。 @@ -110,7 +110,7 @@ Create 引导和 Resume 请求可携带可选的工作区文件系统绑定。 ## 代次控制 {#generation-control} -未启用代次管理的节点只服务其登记的代次，配置固定，并且不会收到准备或保留帧。它可以在 `Health.Generations` 报告该精确代次；支持检查点的提供商必须报告，并包含已验证的检查点资格。`provider_ready` 必须与报告的状态一致。支持代次管理的节点会准备 Core 在 `welcome` 和 `heartbeat_ack` 中通告的目标代次，并在此期间继续服务其持久化的服务代次；目标代次的准备独立于服务提供商的就绪状态。 +节点会准备 Core 在 `welcome` 和 `heartbeat_ack` 中通告的目标代次，并在此期间继续服务其持久化的服务代次；目标代次的准备独立于服务提供商的就绪状态。 节点的 `hello` 和心跳最多携带八条代次观察记录。每条记录指定一个正值的有符号 64 位代次编号、其小写 SHA-256 规范摘要、`ready`、`preparing` 或 `failed` 状态，以及可选的固定诊断信息。目标代次和服务代次的记录排在前面，其余记录公平轮换。八条记录限制的是单条消息，而不是节点可保留的代次数量。省略某条观察记录绝不会授权删除，也不会暗示不存在。 支持检查点的 ready 代次携带 `checkpoint`，包含非空且不透明的 `artifact_domain` 和 `execution_class` token，各最多 256 字节。不支持的提供商省略该字段；preparing 或 failed 代次不声明它。Core 将声明与已认证连接和代次一起持久化，仅在当前在线连接的该代次仍 ready 时使用。[提供商契约](../../../docs/zh/sandbox-provider.md#checkpoint-transfer) 定义归档与执行兼容性。 @@ -141,7 +141,7 @@ sequenceDiagram 就绪类别编写于 `services/core/internal/sandbox/node_diagnostic.go`，每个类别对应一个导出错误和一个代码。共享的 `services/core/internal/sandbox/testdata/node-diagnostics.json` 测试夹具检查 Go 映射、OpenAPI 源注释和生成的枚举，以及 TypeScript 客户端声明。Web 使用客户端规范化器，并检查每个已声明代码的本地化消息。代码变更时要同步更新这些投影；客户端将未知代码读作 `provider_unavailable`。 -准备诊断使用固定的类型化原因。只有制品传输、校验和或版本来源验证失败才会报告 `runtime_download_failed`；私有准备器通过退出类别指示这一类失败，Core 和节点都不解析 stderr。提供商故障、所有权故障、取消和未分类故障保留其类型化代码，或使用 `provider_unavailable`。协议中不会传输任何提供商原始文本。 +准备诊断使用固定的类型化原因。提供商故障、所有权故障、取消和未分类故障保留其类型化代码，或使用 `provider_unavailable`。协议中不会传输任何提供商原始文本。 ## 超时与限制 {#timeouts-and-limits} diff --git a/contracts/agents-api/zh/sandbox-deployment.md b/contracts/agents-api/zh/sandbox-deployment.md index 5b235b8e8..d6a1e9943 100644 --- a/contracts/agents-api/zh/sandbox-deployment.md +++ b/contracts/agents-api/zh/sandbox-deployment.md @@ -1,10 +1,10 @@ --- title: "沙箱部署" source: contracts/agents-api/sandbox-deployment.md -source_hash: cd4ce6e40836fbf598d368b1b448e13e2f853f9ad111ef122292d10d7fb1bce9 +source_hash: 7e94d713fd7de92996f17717201c8da58bf8e538dc78e18334f5c43a76802e86 --- -沙箱部署为 Core 管理的 `openai_hosted` 执行选择沙箱提供商、每个沙箱的资源以及不可变的 Runtime 发行版。PostgreSQL 为每个安装维护一个当前有效选择；Web 和 Core API 写入同一配置。节点文件保存其已安装副本和特定于主机的路径，且不能覆盖其资源或 Runtime。该选择独立于 Harness。部署可以保持未配置状态，没有节点；此时它拒绝托管准入。 +沙箱部署为 Core 管理的 `openai_hosted` 执行选择沙箱提供商和每个沙箱的资源。PostgreSQL 为每个安装维护一个当前有效选择；Web 和 Core API 写入同一配置。节点文件保存其已安装副本和特定于主机的路径，且不能覆盖其资源。Runtime 不属于该选择：每个节点运行其所属安装验证过的 Runtime。该选择独立于 Harness。部署可以保持未配置状态，没有节点；此时它拒绝托管准入。 本契约负责下列 Core API 路由及其语义。[节点指南](../../../docs/zh/getting-started/nodes.md)负责操作员工作流，[机器连接 API](machine-api.md#node-routes)负责节点调用的路由，[沙箱节点协议](node-generation-protocol.md)负责节点连接。 @@ -16,7 +16,7 @@ source_hash: cd4ce6e40836fbf598d368b1b448e13e2f853f9ad111ef122292d10d7fb1bce9 | --- | --- | | `GET /core/v1/sandbox/deployment` | 读取安全的当前配置、推出、重置和资源计数 | | `PATCH /core/v1/sandbox/deployment` | 更新整个部署的空闲挂起策略，不改变配置代次 | -| `POST /core/v1/sandbox/deployment` | 选择初始提供商、资源和 Runtime | +| `POST /core/v1/sandbox/deployment` | 选择初始提供商和资源 | | `PUT /core/v1/sandbox/deployment` | 在线更改同一提供商的目标 | | `POST /core/v1/sandbox/deployment/reset` | 启动托管资源的持久清除或将其升级为持久清除 | | `DELETE /core/v1/sandbox/deployment/reset?expected_generation=N` | 取消剩余清除 | @@ -38,7 +38,6 @@ POST 和 PUT 接受相同的完整选择，并要求提供先前 GET 返回的 ` | `expected_generation` | 必填的非负整数，来自 GET；绝不会自动刷新并重放 | | `provider` | 必须是 `docker`、`microsandbox`、`e2b` 中恰好一个 | | `resources` | 每个沙箱的限制，见下文；Docker 和 microsandbox 必填，E2B 可选 | -| `runtime` | 不可变的 [Runtime 发行版](#runtime-release)；Docker 和 microsandbox 必填，E2B 必须省略 | | `configuration` | 提供商的公开选择器。E2B：不可变的 `template` 构建以及可选且配套的 `api_url` 和 `domain`。Docker 和 microsandbox 仅接受 `{}` 或省略 | | `credential` | 提供商的只写凭据。E2B：`{api_key}`，首次设置时必填，在 PUT 中省略以保留当前密钥；null 或空密钥无效。Docker 和 microsandbox 拒绝该字段 | @@ -64,21 +63,6 @@ microsandbox 会配置 CPU、内存、托管根磁盘，以及位于 `/environme 为现有自有磁盘部署启用外部存储时，先选择文件系统配置，再使用兼容资源更新 Sandbox 部署（不支持配额的文件系统使用 `environment_disk_mib: 0`）。选择文件系统不会改变现有代次或 allocation。下一次 Sandbox 更新派生外部能力回执；旧的自有磁盘 allocation 保留原磁盘。文件系统选择必须支持提供商声明的要求。已采用外部存储的部署要求相同的 `attachment` 和 `user_xattr`；选择可以更改 `capacity_quota`，此后在 Sandbox 更新设置相匹配的 `environment_disk_mib` 之前拒绝新工作区。既有工作区保持其获准时的容量。 -### Runtime 发行版 {#runtime-release} - -Docker 和 microsandbox 使用一个经过验证发行包中的每个字段： - -| 字段 | 标识 | -| --- | --- | -| `source_commit` | 由 40 个字符组成的小写提交 SHA | -| `image_id` | Docker 镜像配置 ID：`sha256:` 加 64 个小写十六进制字符 | -| `image_manifest_digest` | OCI 镜像清单摘要，格式相同 | -| `microsandbox_ref` | `oac-runtime@sha256:` 加 64 个小写十六进制字符 | -| `runtime_sha256` | 原生 microsandbox Runtime 二进制文件的 SHA-256 | -| `firmware_sha256` | 匹配固件的 SHA-256 | - -请从匹配的发行清单中复制这些标识。镜像配置 ID 和 OCI 清单摘要标识不同的对象，绝不能相互替代。节点安装程序会在注册前根据载荷验证已保存的发行版，并保留其导入的精确本地镜像标识。 - ### E2B 配置 {#e2b-configuration} E2B 使用 `template-id:build-uuid` 形式的 `configuration.template`；构建 UUID 必须是规范形式且非零，仅提供可变模板别名将被拒绝。API 密钥在 PostgreSQL 中加密，绝不会出现在响应、引导配置、命令参数或日志中。默认情况下，Core 使用 `https://api.e2b.app` 和 `e2b.app`。对于兼容服务，请同时设置 `configuration.api_url`（一个不含路径、端口、查询、片段或凭据的 HTTPS 源地址）和 `configuration.domain`（沙箱数据平面 DNS 后缀）；API 主机必须等于该域名或其子域。Core 会在发送守护进程凭据或使用 envd 之前，拒绝数据平面域名位于所选后缀之外的沙箱。 @@ -98,9 +82,9 @@ GET 和成功的写入操作会返回 `installation_id`、`provider`、`core_url - 请求中的 `resources` 和响应中的 `specification.resources` 是每个沙箱的限制。响应中的 `resources.allocations` 和 `resources.pending` 分别计算尚未释放的分配，以及尚未分配沙箱的待处理托管 Environment。 - 未配置的部署具有空的提供商且没有 specification。Docker 和 microsandbox 使用 `mode: nodes`；E2B 使用 `mode: direct`，且没有合成节点。 - `generation` 标识已保存的选择。`owner_epoch` 用于对执行所有者和节点连接进行栅栏隔离；它不能替代 `expected_generation`。 -- `specification_digest` 是服务器对提供商、限制和 Runtime 发行版的标识；注册过程会原样回显该值。 +- `specification_digest` 是服务器对提供商、限制和派生的工作区能力回执的标识；注册过程会原样回显该值。 -`packages/agents-client` 中类型化的 `SandboxAdminClient` 会根据准确的这些结构检查部署、节点列表、节点详情和分配响应。未知成员、缺少成员或类型错误都会拒绝整个响应，并产生 502 `invalid_admin_response` 错误。节点的 `diagnostic` 可能不存在，也可能是一个代码，但绝不会为空；客户端会将未知代码读取为 `provider_unavailable`。 +`packages/agents-client` 中类型化的 `SandboxAdminClient` 会根据准确的这些结构检查部署、节点列表、节点详情和分配响应。未知成员、缺少成员或类型错误都会拒绝整个响应，并产生 502 `invalid_admin_response` 错误。节点的 `rollout.diagnostic` 可能不存在，也可能是一个代码，但绝不会为空；客户端会将未知代码读取为 `provider_unavailable`。 ## 初始设置与同提供商更改 {#initial-setup-and-same-provider-changes} @@ -142,27 +126,24 @@ POST 会在持久保存候选配置之前对其进行验证，并且不会创建 "rollout": {"state": "settled", "previous_generation_sandboxes": 3, "nodes": null} ``` -`previous_generation_sandboxes` 使用与资源总数相同的快照，统计旧代次中尚未释放的分配以及没有分配的待处理放置，绝不会重复计算同一资源。E2B 和未配置的部署返回 null `nodes`。节点部署将 `nodes` 返回为计数 `{ready, preparing, failed, update_required, unknown}`，每个尚未移除的节点恰好归入一个类别： +`previous_generation_sandboxes` 使用与资源总数相同的快照，统计旧代次中尚未释放的分配以及没有分配的待处理放置，绝不会重复计算同一资源。E2B 和未配置的部署返回 null `nodes`。节点部署将 `nodes` 返回为计数 `{ready, preparing, failed, unknown}`，每个尚未移除的节点恰好归入一个类别： - 在当前连接上离线的节点为 `unknown`； -- 不具备代次管理且在较旧目标上注册的在线节点为 `update_required`； - 否则，当前连接和所有者 epoch 上的精确目标观测会给出 `ready`、`preparing` 或 `failed`，缺少观测则给出 `unknown`。 ```mermaid flowchart TD A{"Online on its current connection?"} -- no --> U["unknown"] - A -- yes --> B{"Generation management?"} - B -- "no, older target" --> UR["update_required"] - B -- otherwise --> C{"Target observation on this connection and epoch"} + A -- yes --> C{"Target observation on this connection and epoch"} C -- ready --> R["ready"] C -- preparing --> P["preparing"] C -- failed --> F["failed"] C -- missing --> U ``` -节点在当前所有者 epoch 下保持连接且心跳未过期时，即为[在线](node-generation-protocol.md#connection)。目标推出与服务就绪状态相互独立：`unknown`、`preparing`、`failed` 或 `update_required` 不会移除已独立确认的较旧服务代次。`provider_ready` 要求节点在线，并且在该连接和 epoch 上观测到精确的服务代次；不具备代次管理的节点仅能使其注册时的代次满足此条件。固定状态本身绝不表示就绪。 +节点在当前所有者 epoch 下保持连接且心跳未过期时，即为[在线](node-generation-protocol.md#connection)。目标推出与服务就绪状态相互独立：`unknown`、`preparing` 或 `failed` 不会移除已独立确认的较旧服务代次。`provider_ready` 要求节点在线，并且在该连接和 epoch 上观测到精确的服务代次。固定状态本身绝不表示就绪。 -每个节点会添加 `rollout: {state, ready_generation, diagnostic?}`，其中 `ready_generation` 是可为 null 的持久服务固定状态，`diagnostic` 是目标代次的固定代码；分配项会添加 `deployment_generation`。仅当 `rollout.state` 为 `preparing` 或 `reset` 非 null 时，才每五秒轮询一次；旧 Session 以及失败、需要更新或离线的节点本身均不会使轮询保持活动状态。 +每个节点会添加 `rollout: {state, ready_generation, diagnostic?}`，其中 `ready_generation` 是可为 null 的持久服务固定状态，`diagnostic` 是目标代次的固定代码；分配项会添加 `deployment_generation`。仅当 `rollout.state` 为 `preparing` 或 `reset` 非 null 时，才每五秒轮询一次；旧 Session 以及失败或离线的节点本身均不会使轮询保持活动状态。 节点模式创建先验证目标部署，提交 Session、pending Environment 及初始输入，但不预留计算资源。节点全部已满、离线或准备中时，已接受的工作继续等待；未配置部署、reset 或不支持的组合仍拒绝准入。共同调度器对尚未放置的需求进行有界循环扫描，分页按需求记录时间和 Environment ID 排序，每轮使用固定时间上界，避免持续的新需求阻止重新访问较早的工作。它检查在线状态、精确服务代次就绪情况、地址、共享容量和所选代次的 Harness/文件系统兼容性，然后优先选择最新的合格固定代次。预留后的 placement 保持不可变，直到确认释放或已结清的检查点转移。有界扫描跳过暂时不可调度的需求，并在重启后继续。已有 suspended allocation 按[检查点转移契约](../../../docs/zh/sandbox-provider.md#checkpoint-transfer)，通过同一容量锁预留符合条件的检查点容量；这不构成热恢复和未放置工作之间的全局公平性保证。输入保留[原始五分钟期限](./environments.md#reservations)，等待容量的时间也计入其中。 @@ -216,9 +197,9 @@ stateDiagram-v2 节点容量由管理员批准，独立于部署 specification。`POST /core/v1/sandbox/enrollment-tokens` 接受可选的 `max_active` 和 `max_retained`，默认分别为 2 和 8，并返回 `{token, expires_at, enrollment_id}`；`enrollment_id` 是该命令的公共句柄，绝不是凭据。microsandbox 会使用两个限制。Docker 绝不暂停，因此 Core 会在此处和 PATCH 中用 `max_active` 替换其 `max_retained`。E2B 没有节点，并返回 409 `sandbox_deployment_conflict`。Core 将批准值与令牌一同存储，并复制到其注册的节点；节点不能提交容量，其本地检查可以拒绝其无法运行的部署，但绝不会提高限制。[节点容量](../../../docs/zh/configuration.md#node-capacity)为操作员说明了这些限制。 -`GET /core/v1/sandbox/nodes` 返回 `{data: [...]}`，其中每个节点包含 `id`、`name`、`provider`、`online`、`last_seen_at`、`created_at`、`max_active`、`max_retained`，计数项 `active`、`reserved`、`running`、`retained`、`snapshots` 和 `cleanup_pending`，以及 `provider_ready`、`diagnostic`、主机测量值 `cpu_count`、`available_memory_bytes` 和 `available_disk_bytes`、`rollout`、`enrollment_id` 和 `core_url`。`enrollment_id` 是注册该节点的命令的句柄；如果 Core 没有该句柄，则为 null。`core_url` 是注册时的安装公开 URL；`core_url` 与当前公开 URL 不同的节点不会收到新放置。已放置到该节点的工作会继续在那里完成，包括已放置但尚未获得分配的 Environment；只要旧地址仍能访问 Core，其保留沙箱仍可恢复。请移除该节点并重新添加。 +`GET /core/v1/sandbox/nodes` 返回 `{data: [...]}`，其中每个节点包含 `id`、`name`、`provider`、`online`、`last_seen_at`、`created_at`、`max_active`、`max_retained`，计数项 `active`、`reserved`、`running`、`retained`、`snapshots` 和 `cleanup_pending`，以及 `provider_ready`、主机测量值 `cpu_count`、`available_memory_bytes` 和 `available_disk_bytes`、`rollout`、`enrollment_id` 和 `core_url`。`enrollment_id` 是注册该节点的命令的句柄；如果 Core 没有该句柄，则为 null。`core_url` 是注册时的安装公开 URL；`core_url` 与当前公开 URL 不同的节点不会收到新放置。已放置到该节点的工作会继续在那里完成，包括已放置但尚未获得分配的 Environment；只要旧地址仍能访问 Core，其保留沙箱仍可恢复。请移除该节点并重新添加。 -不具备代次管理的节点会自行报告其提供商的就绪状态：`provider_ready`，未就绪时则报告一个固定的 `diagnostic` 代码。通过 Web 的命令添加的节点会管理代次，因此其就绪状态取决于服务代次，而目标代次的固定代码会出现在 `rollout.diagnostic` 中。节点会对首次失败的就绪检查进行分类，并且只发送代码；携带任何其他值的帧均无效，且 Core 绝不存储或返回探测文本或主机路径。这些代码是与提供商无关的类别：`provider_unavailable`、`host_unsupported`、`artifacts_unavailable`、`runtime_download_failed`、`runtime_image_unavailable` 和 `capacity_insufficient`。`runtime_download_failed` 表示无法传输或验证精确的 Runtime 制品；它绝不会包含制品 URL、凭据或传输输出。[就绪代码](../../../docs/zh/getting-started/nodes.md#readiness-codes)给出了原因和操作员应采取的措施。Core 和节点必须来自同一发行包。 +节点管理代次，因此其 `provider_ready` 取决于服务代次，目标代次的固定代码会出现在 `rollout.diagnostic` 中。节点会对首次失败的就绪检查进行分类，并且只发送代码；携带任何其他值的帧均无效，且 Core 绝不存储或返回探测文本或主机路径。这些代码是与提供商无关的类别：`provider_unavailable`、`host_unsupported`、`artifacts_unavailable`、`runtime_image_unavailable` 和 `capacity_insufficient`。[就绪代码](../../../docs/zh/getting-started/nodes.md#readiness-codes)给出了原因和操作员应采取的措施。Core 和节点必须来自同一发行包。 `PATCH /core/v1/sandbox/nodes/{node_id}` 接受 `{name, max_active, max_retained}`；降低限制不会停止任何正在运行的沙箱。当节点仍持有分配、快照、预留或待处理清理时，包括节点离线期间，`DELETE /core/v1/sandbox/nodes/{node_id}` 会拒绝操作并返回 409 `runtime_node_in_use`。移除操作不会删除任何计算资源，并且会停用该节点的身份；该主机只能作为新节点重新加入。不存在节点排空过程。 @@ -232,7 +213,6 @@ stateDiagram-v2 | --- | --- | --- | --- | | 独立工作区文件系统 | [Kernel NFS adapter](../../../docs/zh/workspace-provider.md#kernel-nfs-adapter)；与计算资源分开配置 | 不支持 | 不支持 | | 部署 `specification.resources` | `cpus`、`memory_mib`、`root_disk_mib` 和 `environment_disk_mib` | `cpus` 和 `memory_mib`，必须等于就绪模板构建中的值；省略时取自该构建；无磁盘字段 | `cpus` 和 `memory_mib`；无磁盘配额 | -| 部署 `specification.runtime` | 完整的[发行版](#runtime-release)；节点必须与 `microsandbox_ref`、`runtime_sha256` 和 `firmware_sha256` 匹配 | 不存在；`configuration.template` 用于选择构建 | 完整的[发行版](#runtime-release)；节点必须与 `image_id` 或 `image_manifest_digest` 匹配 | | 部署 `metadata.template_build` | 不存在：`metadata` 为空 | Core 保存选择时读取到的构建 | 不存在：`metadata` 为空 | | 部署 `suspension` | `{idle_seconds, retention_seconds}` | `null`；Core 不暂停 E2B 沙箱 | `null` | | 部署 `resources.allocations`、`resources.pending` | 所有节点的总数 | Core 中尚未释放的 E2B 沙箱，以及正在等待沙箱的托管 Environment | 所有节点的总数 | @@ -264,8 +244,8 @@ stateDiagram-v2 ## 规范的节点规格 {#canonical-node-specification} -`sandbox/deployment_contract.go`负责资源边界、发行版模式和规范字段顺序，每个已注册提供商的 `sandbox.DeploymentPolicy` 声明其要求；`sandbox/deployment.go`在 Core 中应用这些规则。安装程序会使用 `deploy/node/node_spec.py` 中生成的声明，TypeScript 客户端和 Web 使用 `packages/agents-client/src/deployment-contract.ts` 中生成的边界、模式和提供商声明，因此不存在第二套限制、模式或提供商列表。请在仓库根目录运行 `go run ./services/core/cmd/specification-contract -write` 重新生成两者；作为 `make check` 一部分的沙箱 Go 测试会拒绝过时的投影。 +`sandbox/deployment_contract.go`负责资源边界和规范字段顺序，每个已注册提供商的 `sandbox.DeploymentPolicy` 声明其要求；`sandbox/deployment.go`在 Core 中应用这些规则。安装程序会使用 `deploy/node/node_spec.py` 中生成的声明，TypeScript 客户端和 Web 使用 `packages/agents-client/src/deployment-contract.ts` 中生成的边界和提供商声明，因此不存在第二套限制或提供商列表。请在仓库根目录运行 `go run ./services/core/cmd/specification-contract -write` 重新生成两者；作为 `make check` 一部分的沙箱 Go 测试会拒绝过时的投影。 -规范摘要是紧凑 UTF-8 JSON 的 SHA-256，其中 `provider` 位于首位，其次是 `resources`，然后在提供商需要时放置 `runtime`。资源和 Runtime 字段遵循契约的声明顺序；值为零的可选磁盘字段会被省略，必填字段则保持存在。发行版标识采用小写 ASCII，摘要绝不会受传入字段顺序或空白字符影响。`services/core/internal/sandbox/testdata/deployment-contract.json`保存共享验收用例、精确的规范字节和摘要，Go 与 Python 测试都会使用这些内容。 +规范摘要是紧凑 UTF-8 JSON 的 SHA-256，其中 `provider` 位于首位，其次是 `resources`。资源字段遵循契约的声明顺序；值为零的可选磁盘字段会被省略，必填字段则保持存在。摘要绝不会受传入字段顺序或空白字符影响。`services/core/internal/sandbox/testdata/deployment-contract.json`保存共享验收用例、精确的规范字节和摘要，Go 与 Python 测试都会使用这些内容。 -规范摘要的 JSON 在 `runtime` 后包含可选的 `workspace`，字段按文件系统声明顺序排列：`attachment`、`user_xattr`、`capacity_quota`，布尔字段始终保留。该回执参与不可变 specification digest。 +规范摘要的 JSON 在 `resources` 后包含可选的 `workspace`，字段按文件系统声明顺序排列：`attachment`、`user_xattr`、`capacity_quota`，布尔字段始终保留。该回执参与不可变 specification digest。 diff --git a/deploy/install-source.py b/deploy/install-source.py index dd05469f7..461cf23b9 100644 --- a/deploy/install-source.py +++ b/deploy/install-source.py @@ -181,7 +181,6 @@ def install(bundle, destination, host, port, public_url): run(f'{step} failed; no service was changed. See the output above.', *compose, *check, cwd=destination) else: run('Compose configuration check failed; see the output above. Nothing was installed.', *compose, 'config', '--quiet', cwd=destination) - node_payload.activate(payload, revision) if not existing and native.is_dir(): os.replace(stage / 'native-installers', destination / 'native-installers') # compose.yaml goes last: it marks the directory as an installation. diff --git a/deploy/node/README.md b/deploy/node/README.md index 9a98e0645..5307ff177 100644 --- a/deploy/node/README.md +++ b/deploy/node/README.md @@ -8,7 +8,7 @@ These modules are packaged as `node-install.pyz`, which Web serves at `/node-ins | `distribution.py` | Verified artifacts, private temporary files, release metadata and Docker image identity | | `install_display.py`, `node_output.py` | Terminal output | | `provider_assets.py` | Node provider artifact names, generated by `go run ./services/core/cmd/provider-artifacts` | -| `node_payload.py` | Publish a release's node files without replacing bytes already installed, then activate a release | +| `node_payload.py` | Publish the installation's node files without replacing bytes already installed | ## Accounts and permissions @@ -40,7 +40,7 @@ The distribution manifest is the one download contract for nodes: flat versioned - A Compose installation keeps only the node metadata from its release archive; Core's image never acquires execution-only payloads. - A node obtains bootstrap metadata from the console that generated its command. Web serves artifacts it has locally and redirects missing declared execution artifacts to the versioned HTTPS release base in the verified manifest. Web never downloads or caches those bytes. - Only artifact requests may follow HTTPS redirects, and only without credentials or cookies. Metadata and enrollment requests stay on the configured console. The console publishes only fixed non-secret files and declared artifact names. -- Download into private temporary files, verify size and SHA-256 before an atomic rename, resume a leftover partial artifact download with an HTTP range (a server that cannot resume, or bytes that fail verification, restart the file) and clear leftover extracted files before retrying, and reuse only verified cache entries or exact image identities. Never select a release other than the pinned one. +- Download into private temporary files, verify size and SHA-256 before an atomic rename, resume a leftover partial artifact download with an HTTP range (a server that cannot resume, or bytes that fail verification, restart the file) and clear leftover extracted files before retrying, and reuse only verified cache entries or exact image identities. Never select a release other than the installation's own. - Release downloads are anonymous. Never add repository credentials to installed node or Runtime configuration. - Manual builds use the `build-` release tag and tag builds the `v*` tag. The manifest's download base must match the release tag; artifact file names and source provenance keep the full source SHA. @@ -52,41 +52,31 @@ A node stores the deployment generations it prepares under its installation stat A Core drop grant is necessary but not sufficient for collection. Queued and running provider calls, preparation, the local target and the serving pin all keep references; collection rechecks them and refuses on an already canceled connection. A canceled provider caller does not prove that its native helper stopped: the node counts a helper until its actual `Wait` returns. -Every generation owns a permanent private lease file, `state/node/generations/.lease`. Before starting a native helper, the node takes a shared flock on it, validates the durable lease identity under that lock, and requires the published final provider configuration and no preparing, collecting or dropped journal. The helper inherits the descriptor; the node closes its own copy only after `Wait` and never unlocks the shared open-file description, so a canceled caller or a node exit does not release a live helper's reference. Native helpers set the descriptor close-on-exec before calling the SDK, so VM and daemon descendants do not inherit it. +Every generation owns a permanent private lease file, `state/node/generations/.lease`. Before starting a native helper, the node takes a shared flock on it, validates the durable lease identity under that lock, and requires the published final provider configuration and no dropped journal. The helper inherits the descriptor; the node closes its own copy only after `Wait` and never unlocks the shared open-file description, so a canceled caller or a node exit does not release a live helper's reference. Native helpers set the descriptor close-on-exec before calling the SDK, so VM and daemon descendants do not inherit it. -Collection takes the exclusive nonblocking flock before it inspects references, removes shared images or release files, or publishes the dropped marker, and holds it through those changes. Lease files belong to stable node state and are never removed or replaced during collection; symlinks, multiply linked files, foreign ownership, unsafe permissions and replaced lock paths are refused. Before the first helper can start, the installer creates the lease exclusively and fsyncs it, then atomically persists and fsyncs a private `.lease-identity` record and its directory. The record binds installation, generation, specification digest, device and inode. The Python collector and the Go helper opener validate the same record on every open, including after a restart; neither adopts a missing identity, replaces its inode or erases it after collection. An installation interrupted before the identity is durable refuses re-adoption and is kept for inspection. A removed identity or a replaced lease refuses even when its current metadata agree. +Collection takes the exclusive nonblocking flock before it inspects references, or publishes the dropped marker, and holds it through that change. Lease files belong to stable node state and are never removed or replaced during collection; symlinks, multiply linked files, foreign ownership, unsafe permissions and replaced lock paths are refused. Before the first helper can start, the installer creates the lease exclusively and fsyncs it, then atomically persists and fsyncs a private `.lease-identity` record and its directory. The record binds installation, generation, specification digest, device and inode. The Python collector and the Go helper opener validate the same record on every open, including after a restart; neither adopts a missing identity, replaces its inode or erases it after collection. An installation interrupted before the identity is durable refuses re-adoption and is kept for inspection. A removed identity or a replaced lease refuses even when its current metadata agree. A dropped generation can never be prepared or used again. A helper's exit is evidence about local files only, not proof that a remote mutation or an uncertain provider receipt has been released; Core's durable allocation and placement retention stays independent. -### Matched fresh installation +### Installation payload -The host program release and Core's selected Runtime release are independent. A fresh node gets its executable and private preparer from the console's current release, and reads the exact Runtime source, image identities and native runtime and firmware digests from its authenticated configuration. When that Runtime is older, the console still serves its immutable `releases//` manifest, checksums and allowlisted artifacts: the Runtime helper, firmware, seccomp profile and image bytes come from the selected release, and artifact URLs stay pinned to their verified manifest even if the console's current release changes during a download. +One installation serves exactly one release. A node gets its executable, private preparer and Runtime from the console that generated its command, and the Runtime stays on the node until uninstall. The console serves a single flat payload: `node-install.pyz`, `manifest.json`, `SHA256SUMS`, `runtime/seccomp.json` and the declared `artifacts/`. Artifact URLs stay pinned to the verified manifest. A node program that belongs to another installation of Core is refused with the remove, uninstall and add steps before the installer writes the node identity, imports the Runtime, registers the node or starts its service. -A missing retained release refuses the installation rather than substituting the current Runtime, and so does a local bundle that holds only a different Runtime. These refusals happen before the installer writes the node identity, imports the Runtime, registers the node or starts its service. - -A published console release keeps its metadata and artifact bytes. Publishing it again first validates all metadata and every existing declared artifact, then may atomically add only missing, checksum-matched declared artifacts; any conflict prevents every addition, and nothing is overwritten. +A published console payload keeps its metadata and artifact bytes. Publishing it again first validates all metadata and every existing declared artifact, then may atomically add only missing, checksum-matched declared artifacts; any conflict prevents every addition, and nothing is overwritten. ### Restart recovery -Missing Runtime bytes never move a pinned placement to the current Runtime. The node keeps the original generation and specification digest as unready and asks Core's authenticated configuration route for that exact generation before recovery. Missing seccomp bytes may leave an unready provider placeholder; missing image or native artifacts found by a provider probe queue a repair without advertising readiness. +Missing Runtime bytes never move a placement to another Runtime. The node keeps the original generation and specification digest as unready and asks Core's authenticated configuration route for that exact generation before recovery. Missing seccomp bytes may leave an unready provider placeholder; a missing image or native artifact found by a provider probe keeps the generation unready, and rerunning the node's command restores the installation's missing bytes. -Preparation and repair are serialized. Target and serving generations come first, with bounded progress on the others. Each attempt has a 30-minute deadline, and failures back off for 1, 2, 5 and 10 minutes, then at most 30. Nothing is prepared before the connection has delivered deployment facts. Repair keeps existing configurations and paths, verifies the selected release and every existing sibling checksum, and downloads only missing immutable files. Conflicting bytes or a different retained specification refuse repair, and a missing provider configuration without an exact preparation plan is refused rather than reconstructed. +Preparation is serialized. Target and serving generations come first, with bounded progress on the others. Each attempt has a 30-minute deadline, and failures back off for 1, 2, 5 and 10 minutes, then at most 30. Nothing is prepared before the connection has delivered deployment facts. Conflicting bytes or a different retained specification refuse preparation. -Repair takes the same exclusive generation lease and installation lock as collection, so a live helper or a running collector keeps ownership and repair retries later. After the bytes are restored, the node still runs the provider's readiness probe; file presence and executable capability never establish readiness. +Preparation takes the same exclusive generation lease and installation lock as collection, so a live helper or a running collector keeps ownership and preparation retries later. The node still runs the provider's readiness probe; file presence and executable capability never establish readiness. ### Preparation and collection records -New preparation writes two distinct records. Before downloads, `.preparing` holds the installation, generation and specification identity, the private provider paths and `import_started: false`; it is a recovery and collection plan, not a provider. Before the importer runs, the plan records `import_started: true`. Python retention discovery and Go restart recovery both recognize pending-only plans but never build, probe or acquire a provider from them, and recovery or collection still needs authorization on the current connection. - -Only a successful preparation publishes the final `.json` provider configuration, which is write-once. Docker records the immutable local image ID its resolver returns; either the builder's config identity or the manifest identity can be valid for the same specification. Publication is durable before the preparation journal is cleared. An interruption between the two revalidates the same plan and final identity; plan, specification or path drift refuses. A canceled or failed import stays visible to Core's retention exchange without becoming a serving generation. - -Before any native or release deletion, the node persists a private collection journal bound to its installation, generation and specification digest. After a restart an unfinished journal is only a retention-exchange candidate: it cannot prepare, probe, acquire or advertise that generation, and a fresh correlated Core drop grant is needed to resume. Native completion is persisted before release files are removed, so a retry can finish a partly removed release without running an already removed helper. The digest-bound dropped marker follows durable file cleanup and prevents re-adoption; the small configuration and ownership journals stay as local identity records. - -For the installation-private microsandbox store, a successful, complete native image inventory distinguishes absence from a CLI failure; a failed query, malformed inventory, native in-use refusal or unknown ownership keeps the bytes. Shared microsandbox images and private releases stay until their last local reference. Docker images belong to the host's shared daemon: automatic collection never removes or prunes them, and only the host administrator can remove them after confirming that no installation on the host needs them. - -A fresh installation also records the verified checksums of its Runtime files separately from the host program. Collecting the original generation removes only the exact private Runtime helper, executable, firmware, seccomp and import-cache files that no retained configuration references. Every remaining file is checked before the first deletion; an unknown hash, changed bytes, links or missing ownership metadata refuse cleanup. The node executable, preparer, identity, base provider configuration and manifests stay, so a restarted node can still read its enrolled identity and construct a newer retained provider. Shared native paths are compared across all retained configurations before removal. +Preparation writes the generation's final `.json` provider configuration, which is write-once: a copy of the node's base provider configuration with the generation's specification and number, written after Core authorized the generation and the node confirmed that the installation's Runtime is still available. Every generation of a node shares that Runtime and, for microsandbox, its store. Publication is atomic, so an interrupted preparation leaves either nothing or the complete file, and a rerun revalidates it. -An interrupted download repairs only missing bytes at the original paths. When collection comes before any import attempt, the preparation journal proves that the generation has no imported native image. A generation whose native executable is missing and whose import may have started stays retained: missing files never prove native absence, and an empty native inventory never erases receipt or store history. +Collection writes a digest-bound dropped marker under the generation lease after a fresh correlated Core drop grant. The marker prevents re-adoption; the small configuration and ownership journals stay as local identity records. It never removes the installation's Runtime, images or stores. Docker images belong to the host's shared daemon, and uninstall keeps them and prints the command that removes them. ## Image identity diff --git a/deploy/node/node_generations.py b/deploy/node/node_generations.py index 584ceab92..911b653f8 100644 --- a/deploy/node/node_generations.py +++ b/deploy/node/node_generations.py @@ -4,13 +4,10 @@ import fcntl import io import stat -import hashlib import json import os from pathlib import Path import re -import shutil -import subprocess import sys import zipfile @@ -47,10 +44,6 @@ def helper_archive(args, installer): def install_helper(root, args, installer, archive=None): """Keep the executed, trusted installer available to the unprivileged node.""" - settings = root / "preparation.json" - value = {"source_url": args.source_url or args.core_url} - if settings.exists() and installer.private_json(settings) != value: - raise installer.InstallError("The retained node artifact origin differs; preserve its configuration") target = root / "generation-preparer.pyz" installer.existing_file(target) if archive is None: @@ -59,7 +52,6 @@ def install_helper(root, args, installer, archive=None): with os.fdopen(os.open(staged, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "wb") as output: output.write(archive) os.replace(staged, target) - atomic_json(settings, value) def generation_marker(root, generation, suffix, digest, installation, installer): @@ -71,7 +63,7 @@ def generation_marker(root, generation, suffix, digest, installation, installer) with os.fdopen(descriptor) as stream: info = os.fstat(stream.fileno()) if (not stat.S_ISREG(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o600 - or info.st_uid != os.getuid() or info.st_nlink != 1 or info.st_size > (16384 if suffix == ".preparing" else 4096)): + or info.st_uid != os.getuid() or info.st_nlink != 1 or info.st_size > 4096): raise installer.InstallError("Invalid generation ownership journal") try: value = json.load(stream) @@ -80,31 +72,11 @@ def generation_marker(root, generation, suffix, digest, installation, installer) named = path.lstat() if (named.st_dev, named.st_ino) != (info.st_dev, info.st_ino): raise installer.InstallError("Generation ownership journal was replaced") - if digest is None: - digest = value.get("specification_digest") if isinstance(value, dict) else None - if not isinstance(digest, str) or not re.fullmatch(r"[a-f0-9]{64}", digest): - raise installer.InstallError("Invalid preparation specification identity") expected = {"installation_id": installation, "generation": generation, "specification_digest": digest} if (not isinstance(value, dict) or type(value.get("generation")) is not int or any(value.get(key) != item for key, item in expected.items())): raise installer.InstallError("Invalid generation ownership journal") - keys = set(expected) - if suffix == ".collecting": - keys.add("native_complete") - if type(value.get("native_complete")) is not bool: - raise installer.InstallError("Invalid generation collection journal") - if suffix == ".preparing": - keys.add("import_started") - if "configuration" in value: - keys.add("configuration") - plan = value["configuration"] - if (not isinstance(plan, dict) or plan.get("installation_id") != installation or plan.get("generation") != generation - or plan.get("provider") not in ("docker", "microsandbox") - or installer.node_spec.digest(plan["provider"], plan.get("specification")) != digest): - raise installer.InstallError("Invalid generation preparation plan") - if type(value.get("import_started")) is not bool: - raise installer.InstallError("Invalid generation preparation journal") - if set(value) != keys: + if set(value) != set(expected): raise installer.InstallError("Invalid generation journal fields") return value @@ -114,68 +86,6 @@ def marker_identity(args): "specification_digest": args.specification_digest} -def record_root_runtime(root, args, manifest, sums, installer): - checksums = {"runtime/seccomp.json": sums["runtime/seccomp.json"]} - checksums.update({name: installer.distribution.artifact(manifest, name)["sha256"] - for name in installer.provider_assets.artifacts(args.provider, ("runtime",))}) - archive = installer.distribution.artifact(manifest, "images/runtime.tar.gz") - checksums["images/runtime.tar.gz"] = archive["sha256"] - checksums["images/runtime.tar"] = archive["unpacked_sha256"] - value = {"installation_id": args.installation_id, "source_commit": manifest["source_commit"], "sha256": checksums} - path = root / "runtime-artifacts.json" - if installer.existing_file(path): - if installer.private_json(path) != value: - raise installer.InstallError("Original Runtime artifact ownership differs") - return - atomic_json(path, value) - - -def root_runtime_files(root, value, others, installer): - """Return only verified original Runtime files that no retained config uses.""" - def paths(configuration): - if configuration["provider"] == "docker": - return [Path(configuration["native"]["seccomp_file"])] - return [Path(configuration["native"][key]) for key in ("helper_path", "runtime_path", "firmware_path")] - names = ["runtime/seccomp.json", "images/runtime.tar.gz", "images/runtime.tar"] - if value["provider"] == "microsandbox": - names.extend(installer.MICRO) - own_paths = paths(value) - if not any(path == root / name for path in own_paths for name in names): - return [] - referenced = {Path(os.path.realpath(path)) for item in others for path in paths(item)} - auxiliary = {root / name for name in ("runtime/seccomp.json", "images/runtime.tar.gz", "images/runtime.tar")} - # An original Runtime provider also retains its import cache and policy. - if any(path in referenced for path in own_paths): - referenced.update(auxiliary) - candidates = [root / name for name in names if root / name not in referenced] - if not candidates: - return [] - record = root / "runtime-artifacts.json" - installer.no_links(record) - if not installer.existing_file(record): - raise installer.InstallError("Original Runtime artifact ownership is missing; preserve its files") - saved = installer.private_json(record) - base = installer.private_json(root / "provider.json") - if not base or base.get("installation_id") != value["installation_id"] or base.get("provider") != value["provider"]: - raise installer.InstallError("Original Runtime configuration differs") - source = base["specification"]["runtime"]["source_commit"] - if (not saved or set(saved) != {"installation_id", "source_commit", "sha256"} - or saved["installation_id"] != value["installation_id"] or saved["source_commit"] != source - or not isinstance(saved["sha256"], dict) or set(saved["sha256"]) != set(names) - or any(not isinstance(digest, str) or not re.fullmatch(r"[a-f0-9]{64}", digest) for digest in saved["sha256"].values())): - raise installer.InstallError("Original Runtime artifact ownership differs") - if value["provider"] == "microsandbox" and any( - saved["sha256"][name] != base["specification"]["runtime"][field] - for name, field in zip(installer.MICRO[1:], ("runtime_sha256", "firmware_sha256"))): - raise installer.InstallError("Original Runtime artifact hashes differ from its specification") - for path in candidates: - installer.no_links(path) - if installer.existing_file(path): - if path.stat().st_nlink != 1 or installer.file_digest(path) != saved["sha256"][str(path.relative_to(root))]: - raise installer.InstallError("Original Runtime file differs; refusing deletion") - return candidates - - def retained_configs(root, installer): directory = root / "state/node/generations" result = {} @@ -193,73 +103,9 @@ def retained_configs(root, installer): if value["generation"] in result and result[value["generation"]] != value: raise installer.InstallError("Conflicting retained generation configuration") result[value["generation"]] = value - if directory.exists(): - for path in sorted(directory.glob("*.preparing")): - if not re.fullmatch(r"[1-9][0-9]*\.preparing", path.name): - raise installer.InstallError("Invalid preparation filename") - if not base: - raise installer.InstallError("Preparation installation identity is unavailable") - journal = generation_marker(root, int(path.stem), ".preparing", None, base["installation_id"], installer) - plan = journal.get("configuration") - if plan is None: - if int(path.stem) not in result: - raise installer.InstallError("Preparation plan is missing") - continue - validate_preparation_plan(root, plan, base, installer) - if generation_marker(root, plan["generation"], ".dropped", journal["specification_digest"], plan["installation_id"], installer): - continue - final = result.get(plan["generation"]) - if final is not None: - verify_plan_final(plan, final, installer) - else: - result[plan["generation"]] = plan return result -def validate_preparation_plan(root, plan, base, installer): - if (not base or any(plan.get(key) != base.get(key) for key in ("installation_id", "provider", "core_url")) - or type(plan.get("generation")) is not int): - raise installer.InstallError("Preparation plan installation differs") - runtime = plan["specification"]["runtime"] - source = runtime["source_commit"] - if not re.fullmatch(r"[a-f0-9]{40}", source): - raise installer.InstallError("Preparation release identity differs") - if plan["provider"] == "docker": - policy = Path(plan["native"]["seccomp_file"]) - if policy not in (root / "runtime/seccomp.json", root / "releases" / source / "runtime/seccomp.json"): - raise installer.InstallError("Preparation policy is outside its immutable release") - if plan["native"]["image"] not in (runtime["image_id"], runtime["image_manifest_digest"]): - raise installer.InstallError("Preparation image differs from the specification") - installer.no_links(policy) - else: - micro = plan["native"] - paths = [Path(micro[key]) for key in ("helper_path", "runtime_path", "firmware_path")] - if not any(paths == [release / name for name in installer.MICRO] for release in (root, root / "releases" / source)): - raise installer.InstallError("Preparation artifacts are outside their immutable release") - expected_home = generation_home(root, plan, base, installer) - if micro["checkpoint_root"] != base["native"]["checkpoint_root"]: - raise installer.InstallError("Preparation checkpoint store differs") - installer.prepare_checkpoint_root(Path(micro["checkpoint_root"]), initialize=False) - if Path(micro["runtime_home"]) != expected_home: - raise installer.InstallError("Preparation native store differs") - for path in paths + [expected_home]: - installer.no_links(path) - configuration = dict(plan, core_url=plan["core_url"].removesuffix("/api/v1")) - installer.node_spec.verify_provider(plan, configuration, plan.get("native", {}).get("image")) - - -def verify_plan_final(plan, final, installer): - expected = copy.deepcopy(plan) - if plan["provider"] == "docker": - image = final.get("native", {}).get("image") - runtime = plan["specification"]["runtime"] - if image not in (runtime["image_id"], runtime["image_manifest_digest"]): - raise installer.InstallError("Final Docker image differs from the preparation specification") - expected["native"]["image"] = image - if expected != final: - raise installer.InstallError("Final generation differs from its immutable preparation plan") - - def owned_root(args, installer): if os.getuid() == 0: raise installer.InstallError("Generation operations run only as the node service user") @@ -273,19 +119,8 @@ def owned_root(args, installer): return root, identity -def generation_home(root, configuration, base, installer): - runtime = configuration["specification"]["runtime"] - previous = base["specification"]["runtime"] - if (runtime["runtime_sha256"], runtime["firmware_sha256"]) == (previous["runtime_sha256"], previous["firmware_sha256"]): - return Path(base["native"]["runtime_home"]) - material = ":".join((configuration["installation_id"], runtime["runtime_sha256"], runtime["firmware_sha256"])) - home = Path.home() / ".oac/m" / hashlib.sha256(material.encode()).hexdigest()[:12] - if len(os.fsencode(home)) > 48: - raise installer.InstallError("HOME is too long for versioned microsandbox socket paths") - return home - - def image_available(value, installer): + """Whether the installation's Runtime that this configuration names is still on the node.""" try: if value["provider"] == "docker": seccomp = Path(value["native"]["seccomp_file"]) @@ -293,68 +128,22 @@ def image_available(value, installer): json.loads(seccomp.read_text()) raw = installer.checked(list(installer.DOCKER) + ["image", "inspect", value["native"]["image"], "--format", "{{.Id}} {{.Os}}/{{.Architecture}}"], "Cannot inspect pinned image") return raw.strip() == value["native"]["image"] + " linux/amd64" - micro, runtime = value["native"], value["specification"]["runtime"] + micro = value["native"] for key in ("helper_path", "runtime_path", "firmware_path"): if not installer.existing_file(Path(micro[key])): return False - if (installer.file_digest(Path(micro["runtime_path"])) != runtime["runtime_sha256"] - or installer.file_digest(Path(micro["firmware_path"])) != runtime["firmware_sha256"]): + if (installer.file_digest(Path(micro["runtime_path"])) != micro["runtime_sha256"] + or installer.file_digest(Path(micro["firmware_path"])) != micro["firmware_sha256"]): return False env = dict(os.environ, MSB_BACKEND="local", MSB_HOME=micro["runtime_home"], MSB_PATH=micro["runtime_path"], MSB_LIBKRUNFW_PATH=micro["firmware_path"]) - image = json.loads(installer.checked([micro["runtime_path"], "image", "inspect", runtime["microsandbox_ref"], "--format", "json"], "Cannot inspect pinned image", env=env)) - return image.get("digest") == runtime["microsandbox_ref"].split("@", 1)[1] and image.get("architecture") == "amd64" and image.get("os") == "linux" + image = json.loads(installer.checked([micro["runtime_path"], "image", "inspect", micro["image"], "--format", "json"], "Cannot inspect pinned image", env=env)) + return image.get("digest") == micro["image"].split("@", 1)[1] and image.get("architecture") == "amd64" and image.get("os") == "linux" except (installer.InstallError, OSError, ValueError): return False -def runtime_files(root, value, args, manifest, sums, installer): - """Restore only absent immutable bytes; existing conflicts are never replaced.""" - source = args.configuration["specification"]["runtime"]["source_commit"] - release = root / "releases" / source - if value is not None: - if args.provider == "microsandbox": - release = Path(value["native"]["helper_path"]).parents[2] - if any(Path(value["native"][key]) != release / name for key, name in zip( - ("helper_path", "runtime_path", "firmware_path"), installer.MICRO)): - raise installer.InstallError("Retained Runtime artifact paths differ") - else: - release = Path(value["native"]["seccomp_file"]).parents[1] - if Path(value["native"]["seccomp_file"]) != release / "runtime/seccomp.json": - raise installer.InstallError("Retained Runtime seccomp path differs") - if release not in (root, root / "releases" / source): - raise installer.InstallError("Retained Runtime artifacts are outside this installation") - installer.no_links(release) - installer.safe_directory(release) - names = installer.provider_assets.artifacts(args.provider, ("policy", "runtime")) - saved = installer.private_json(release / "manifest.json") - if (release / "manifest.json").exists(): - if saved is None: - raise installer.InstallError("Retained Runtime manifest is unreadable") - try: - installer.node_spec.verify_release(args.configuration, saved) - except installer.node_spec.SpecificationError as error: - raise installer.RuntimeDownloadError("Retained Runtime release provenance differs") from error - # Inspect every existing byte before starting any repair, so a missing file - # cannot hide a conflicting sibling or redirect a later download. - for name in names: - path = release / name - installer.no_links(path) - if installer.existing_file(path): - expected = sums[name] if name == "runtime/seccomp.json" else installer.distribution.artifact(manifest, name)["sha256"] - if installer.file_digest(path) != expected: - raise installer.RuntimeDownloadError("Retained Runtime artifact checksum differs") - for name in names: - installer.safe_directory((release / name).parent) - if name == "runtime/seccomp.json": - installer.download(args.source_url, name, release, sums[name], prefix="releases/" + source + "/") - else: - installer.distribution.obtain_artifact(manifest, name, release / name) - os.chmod(release / name, 0o700) - atomic_json(release / "manifest.json", manifest) - return release - - def prepare(args, installer): + """Write the generation's configuration: the installation's own Runtime under the specification Core authorized.""" root, identity = owned_root(args, installer) with installer.install_lock(root), collection_lease( root, args.generation, installer, marker_identity(args), @@ -362,82 +151,27 @@ def prepare(args, installer): and not (root / "state/node/generations" / (str(args.generation) + ".json")).exists()): directory = root / "state/node/generations" target = directory / (str(args.generation) + ".json") - if (generation_marker(root, args.generation, ".dropped", args.specification_digest, args.installation_id, installer) - or generation_marker(root, args.generation, ".collecting", args.specification_digest, args.installation_id, installer)): + if generation_marker(root, args.generation, ".dropped", args.specification_digest, args.installation_id, installer): raise installer.InstallError("A dropped generation cannot be adopted again") - preparation = generation_marker(root, args.generation, ".preparing", args.specification_digest, args.installation_id, installer) args.core_url = identity["core_url"] - args.configuration = installer.node_spec.fetch(args, "", identity, installer.open_request, - generation=args.generation, allow_selection_change=True) - if args.configuration["specification_digest"] != args.specification_digest: + configuration = installer.node_spec.fetch(args, "", identity, installer.open_request, + generation=args.generation, allow_selection_change=True) + if configuration["specification_digest"] != args.specification_digest: raise installer.InstallError("Core generation identity differs from its authorization") - args.provider = args.configuration["provider"] - configurations = retained_configs(root, installer) base = installer.private_json(root / "provider.json") - runtime = args.configuration["specification"]["runtime"] - value = configurations.get(args.generation) - finalized = target.exists() or base["generation"] == args.generation + value = retained_configs(root, installer).get(args.generation) if value is not None: - installer.node_spec.verify_provider(value, args.configuration, value.get("native", {}).get("image")) + installer.node_spec.verify_provider(value, configuration, value.get("native", {}).get("image")) else: - for candidate in configurations.values(): - # Unpublished plans must never be used as ready reuse candidates. - if any((directory / (str(candidate["generation"]) + suffix)).exists() for suffix in (".preparing", ".collecting", ".dropped")): - continue - if candidate["specification"]["runtime"] == runtime and image_available(candidate, installer): - value = copy.deepcopy(candidate) - value["generation"] = args.generation - value["specification"] = args.configuration["specification"] - break - if not finalized or value is None or not image_available(value, installer): - settings = installer.private_json(root / "preparation.json") - args.source_url = installer.origin(settings["source_url"]) - manifest, sums = installer.metadata(args.source_url, prefix="releases/" + runtime["source_commit"] + "/") - try: - installer.node_spec.verify_release(args.configuration, manifest) - except installer.node_spec.SpecificationError as error: - raise installer.RuntimeDownloadError("Runtime release provenance differs") from error - if args.provider == "microsandbox": - args.runtime_home = Path(value["native"]["runtime_home"]) if value else generation_home(root, args.configuration, base, installer) - args.checkpoint_root = Path(base["native"]["checkpoint_root"]) - if value is None: - value = installer.provider_config(root / "releases" / runtime["source_commit"], args, runtime["image_id"]) - if preparation is None: - preparation = dict(marker_identity(args), import_started=False, configuration=copy.deepcopy(value)) - atomic_json(directory / (str(args.generation) + ".preparing"), preparation) - release = runtime_files(root, value, args, manifest, sums, installer) - if args.provider == "microsandbox": - installer.safe_directory(args.runtime_home) - owner = args.runtime_home / "oac-installation.json" - if not owner.exists() and any(args.runtime_home.iterdir()): - raise installer.InstallError("Versioned microsandbox store contains unowned state") - installer.write_once(owner, installer.json_text({"installation_id": args.installation_id})) - preparation = dict(preparation, import_started=True) - atomic_json(directory / (str(args.generation) + ".preparing"), preparation) - runtime_image = installer.prepare_runtime(release, args, manifest) - if finalized: - installer.node_spec.verify_provider(value, args.configuration, runtime_image) - elif args.provider == "docker": - if runtime_image not in (runtime["image_id"], runtime["image_manifest_digest"]): - raise installer.InstallError("Resolved Docker image is outside the authorized specification") - value = copy.deepcopy(value) - value["native"]["image"] = runtime_image - if preparation and preparation.get("configuration"): - verify_plan_final(preparation["configuration"], value, installer) + value = copy.deepcopy(base) + value["generation"], value["specification"] = args.generation, configuration["specification"] + if not image_available(value, installer): + raise installer.InstallError("This installation's Runtime is missing on the node; remove the node, uninstall it and add it again") if installer.existing_file(target): if installer.private_json(target) != value: raise installer.InstallError("Immutable generation configuration differs") else: atomic_json(target, value) - # If interrupted after publication, restart rechecks the same plan and - # final identity. It never edits the published provider configuration. - pending = directory / (str(args.generation) + ".preparing") - pending.unlink(missing_ok=True) - descriptor = os.open(directory, os.O_RDONLY | os.O_DIRECTORY) - try: - os.fsync(descriptor) - finally: - os.close(descriptor) @contextlib.contextmanager @@ -507,94 +241,13 @@ def collection_lease(root, generation, installer, identity=None, initialize=Fals def collect(args, installer): root, _ = owned_root(args, installer) with installer.install_lock(root), collection_lease(root, args.generation, installer, marker_identity(args)): - configurations = retained_configs(root, installer) - value = configurations.get(args.generation) + value = retained_configs(root, installer).get(args.generation) if value is None: return if (value["installation_id"] != args.installation_id or installer.node_spec.digest(value["provider"], value["specification"]) != args.specification_digest): raise installer.InstallError("Collection grant does not match the local generation") - source = value["specification"]["runtime"]["source_commit"] - if not re.fullmatch(r"[a-f0-9]{40}", source): - raise installer.InstallError("Invalid retained release identity") - release = root / "releases" / source - installer.no_links(release) - directory = root / "state/node/generations" - journal = generation_marker(root, args.generation, ".collecting", args.specification_digest, args.installation_id, installer) - if journal is None: - journal = dict(marker_identity(args), native_complete=False) - atomic_json(directory / (str(args.generation) + ".collecting"), journal) - others = [item for generation, item in configurations.items() if generation != args.generation] - original_files = root_runtime_files(root, value, others, installer) - if not journal["native_complete"]: - preparation = generation_marker(root, args.generation, ".preparing", args.specification_digest, args.installation_id, installer) - if preparation is None or preparation["import_started"]: - collect_image(args, value, others, installer) - # Persist native completion before deleting its executable. A fresh - # Core grant is still required after restart to finish file cleanup. - journal["native_complete"] = True - atomic_json(directory / (str(args.generation) + ".collecting"), journal) - directory = root / "state/node/generations" - installer.safe_directory(directory) - if not any(item["specification"]["runtime"]["source_commit"] == source for item in others): - release = root / "releases" / source - installer.no_links(release) - if release.exists(): - shutil.rmtree(release) - if release.parent.exists(): - descriptor = os.open(release.parent, os.O_RDONLY | os.O_DIRECTORY) - try: - os.fsync(descriptor) - finally: - os.close(descriptor) - - for path in original_files: - path.unlink(missing_ok=True) - for parent in {path.parent for path in original_files}: - if parent.exists(): - descriptor = os.open(parent, os.O_RDONLY | os.O_DIRECTORY) - try: - os.fsync(descriptor) - finally: - os.close(descriptor) - atomic_json(directory / (str(args.generation) + ".dropped"), marker_identity(args)) - # Keep the immutable configuration and permanent journal as restart - # identity. They contain no provider credentials and are not readiness. - - -def collect_image(args, value, others, installer): - if value["provider"] == "microsandbox": - micro, image = value["native"], value["specification"]["runtime"]["microsandbox_ref"] - shared = [item for item in others if item["native"]["runtime_home"] == micro["runtime_home"]] - home = Path(micro["runtime_home"]) - installer.no_links(home) - if not home.is_dir() or installer.private_json(home / "oac-installation.json") != {"installation_id": args.installation_id}: - raise installer.InstallError("Microsandbox store ownership differs") - runtime_path = Path(micro["runtime_path"]) - installer.no_links(runtime_path) - if not installer.existing_file(runtime_path) or installer.file_digest(runtime_path) != value["specification"]["runtime"]["runtime_sha256"]: - raise installer.InstallError("Cannot verify retained microsandbox executable") - env = dict(os.environ, MSB_BACKEND="local", MSB_HOME=micro["runtime_home"], MSB_PATH=micro["runtime_path"], MSB_LIBKRUNFW_PATH=micro["firmware_path"]) - if not any(item["specification"]["runtime"]["microsandbox_ref"] == image for item in shared): - # A failed inspect/remove is not proof of absence. A successful full - # inventory must contain only understood immutable references. - raw = installer.checked([micro["runtime_path"], "image", "list", "--quiet"], "Cannot verify microsandbox image inventory", env=env) - references = raw.splitlines() - if any(not re.fullmatch(r"[^\s@]+@sha256:[a-f0-9]{64}", item) for item in references): - raise installer.InstallError("Cannot verify microsandbox image inventory") - if any(item.split("@", 1)[1] == image.split("@", 1)[1] for item in references): - installer.checked([micro["runtime_path"], "image", "remove", image, "--quiet"], "Runtime image is still in use", env=env) - if not shared: - raw = installer.checked([micro["runtime_path"], "sandbox", "list", "--format", "json"], "Cannot verify empty microsandbox store", env=env) - if json.loads(raw) != []: - raise installer.InstallError("Microsandbox store still contains native sandboxes") - home = Path(micro["runtime_home"]) - installer.safe_directory(home) - if installer.private_json(home / "oac-installation.json") != {"installation_id": args.installation_id}: - raise installer.InstallError("Microsandbox store ownership differs") - # Native emptiness never authorizes discarding receipt/lock history. - else: - # Docker imported content belongs to the host daemon, including idle - # serving pins in other installations. This installation has no authority - # to remove it; its private generation/release files can still be collected. - return + # The Runtime belongs to the installation; uninstall removes it. Keep the immutable + # configuration and this permanent journal as restart identity. They contain no + # provider credentials and are not readiness. + atomic_json(root / "state/node/generations" / (str(args.generation) + ".dropped"), marker_identity(args)) diff --git a/deploy/node/node_install.py b/deploy/node/node_install.py index 458543718..9ee3ee139 100644 --- a/deploy/node/node_install.py +++ b/deploy/node/node_install.py @@ -42,10 +42,6 @@ class InstallError(Exception): pass -class RuntimeDownloadError(InstallError): - """A fixed private helper exit category for transfer/provenance failures.""" - - MICRO = provider_assets.artifacts("microsandbox", ("runtime",)) DOCKER = ("docker", "--host", "unix:///var/run/docker.sock") DOCKER_SOCKET = Path("/var/run/docker.sock") @@ -136,7 +132,7 @@ def discard_unregistered(root): """Remove what an earlier attempt decided, so a new command can register this host. Only for a node Core never recorded: downloads, the image and the service file stay.""" - for name in ("installation.json", "provider.json", "runtime-artifacts.json", "preparation.json"): + for name in ("installation.json", "provider.json"): if existing_file(root / name): (root / name).unlink() if (root / "state/node").is_dir() and not (root / "state/node").is_symlink(): @@ -176,48 +172,48 @@ def fetch(source, name): # A body cut short by the connection reads as a normal end; the declared length tells. declared = getattr(response, "headers", {}).get("Content-Length", "") if len(raw) > 1024 * 1024: - raise RuntimeDownloadError("Node bootstrap metadata is too large: " + name) + raise InstallError("Node bootstrap metadata is too large: " + name) if declared.isdigit() and int(declared) != len(raw): raise http.client.IncompleteRead(raw, int(declared) - len(raw)) return io.BytesIO(raw) except (urllib.error.URLError, TimeoutError, ConnectionError, http.client.IncompleteRead) as error: if not transient(error) or attempt == 2: status = " (HTTP " + str(error.code) + ")" if isinstance(error, urllib.error.HTTPError) else "" - raise RuntimeDownloadError("Cannot download node metadata " + name + status + "; check the console URL, TLS and network, then rerun") from None + raise InstallError("Cannot download node metadata " + name + status + "; check the console URL, TLS and network, then rerun") from None time.sleep(attempt + 1) -def metadata(source, prefix=""): - read = lambda name: fetch(source, prefix + name) +def metadata(source): + read = lambda name: fetch(source, name) with read("SHA256SUMS") as response: raw = response.read(1024 * 1024 + 1) if len(raw) > 1024 * 1024: - raise RuntimeDownloadError("Invalid distribution checksum list") + raise InstallError("Invalid distribution checksum list") sums = {} try: for line in raw.decode().splitlines(): checksum, name = line.split(" ", 1) if name in sums or not re.fullmatch(r"[0-9a-f]{64}", checksum): - raise RuntimeDownloadError("Invalid distribution checksum entry") + raise InstallError("Invalid distribution checksum entry") sums[name] = checksum except ValueError: - raise RuntimeDownloadError("Invalid distribution checksum list") from None + raise InstallError("Invalid distribution checksum list") from None with read("manifest.json") as response: raw = response.read(1024 * 1024 + 1) if len(raw) > 1024 * 1024 or hashlib.sha256(raw).hexdigest() != sums.get("manifest.json"): - raise RuntimeDownloadError("Distribution manifest checksum mismatch") + raise InstallError("Distribution manifest checksum mismatch") try: manifest = json.loads(raw) except (ValueError, TypeError): - raise RuntimeDownloadError("Invalid distribution manifest") from None + raise InstallError("Invalid distribution manifest") from None if (not isinstance(manifest, dict) or manifest.get("platform") != "linux/amd64" or not re.fullmatch(r"[0-9a-f]{40}", manifest.get("source_commit", "")) or not re.fullmatch(r"sha256:[0-9a-f]{64}", manifest.get("images", {}).get("runtime", ""))): - raise RuntimeDownloadError("Unsupported node distribution") + raise InstallError("Unsupported node distribution") distribution.image_identities(manifest, "runtime") for name in dict.fromkeys(item["path"] for items in provider_assets.CATALOG.values() for item in items if item["role"] != "policy"): distribution.artifact(manifest, name) - # Metadata stays on the console; artifact requests may redirect to its pinned release. - manifest["artifact_base_url"] = source + "/node-install/releases/" + manifest["source_commit"] + "/artifacts" + # Metadata stays on the console; artifact requests may redirect to the release. + manifest["artifact_base_url"] = source + "/node-install/artifacts" return manifest, sums @@ -266,21 +262,21 @@ def file_digest(path): return result.hexdigest() -def download(source, name, root, expected, prefix="", replace=False): +def download(source, name, root, expected, replace=False): target = root / name safe_directory(target.parent) if existing_file(target): if file_digest(target) == expected: return if not replace: - raise RuntimeDownloadError("Installed node payload differs; refusing to overwrite it") + raise InstallError("Installed node payload differs; refusing to overwrite it") target.unlink() with distribution.temporary_file(target) as temporary: - with os.fdopen(os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "wb") as output, fetch(source, prefix + name) as response: + with os.fdopen(os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "wb") as output, fetch(source, name) as response: for block in iter(lambda: response.read(1024 * 1024), b""): output.write(block) if file_digest(Path(temporary)) != expected: - raise RuntimeDownloadError("Node payload checksum mismatch: " + name) + raise InstallError("Node payload checksum mismatch: " + name) os.replace(temporary, target) if name.startswith("native/"): os.chmod(target, 0o700) @@ -336,7 +332,7 @@ def prepare_checkpoint_root(directory, *, initialize=True): os.close(descriptor) -def provider_config(root, args, runtime_image): +def provider_config(root, args, manifest, runtime_image): result = {"installation_id": args.installation_id, "provider": args.provider, "core_url": args.core_url + "/api/v1", "specification": args.configuration["specification"], "generation": args.configuration["generation"]} if args.provider == "docker": @@ -350,7 +346,9 @@ def provider_config(root, args, runtime_image): core_rules = [{"action": "allow", "direction": "egress", "destination": address, "protocol": "tcp", "port": str(port)} for address in addresses] result["native"] = { "helper_path": str(root / MICRO[0]), "runtime_path": str(root / MICRO[1]), "firmware_path": str(root / MICRO[2]), - "runtime_home": str(getattr(args, "runtime_home", micro_home(args.installation_id))), + "image": manifest["runtime_ref"], "runtime_sha256": manifest["microsandbox"]["runtime_sha256"], + "firmware_sha256": manifest["microsandbox"]["firmware_sha256"], + "runtime_home": str(micro_home(args.installation_id)), "checkpoint_root": str(checkpoint_root(args)), "network": {"default_egress": "deny", "default_ingress": "deny", "rules": core_rules + [ {"action": "allow", "direction": "egress", "destination": "public"}, @@ -369,7 +367,7 @@ def drop_runtime_archives(root): def micro_env(root, args): - return dict(os.environ, MSB_BACKEND="local", MSB_HOME=str(getattr(args, "runtime_home", micro_home(args.installation_id))), + return dict(os.environ, MSB_BACKEND="local", MSB_HOME=str(micro_home(args.installation_id)), MSB_PATH=str(root / MICRO[1]), MSB_LIBKRUNFW_PATH=str(root / MICRO[2])) @@ -461,23 +459,13 @@ def install_lock(root): yield -NEWER_NODE_SOFTWARE = ("This host's node program differs from the one this console serves, usually because Core was upgraded " - "after the node was added. Nothing was changed. To move the node to this release, choose Remove node on " - "the Nodes page, uninstall it on this host, then add the host again with Add node.") - - -def refuse_newer_node_software(root, manifest, names): - """A registered node keeps its own program; a rerun cannot swap it for another release's.""" - for name in names: - target = root / name - if existing_file(target) and not distribution.matches(target, distribution.artifact(manifest, name)): - raise InstallError(NEWER_NODE_SOFTWARE) - +NOT_THIS_INSTALLATION = ("This host's node was added with another Core address, release or provider. Nothing was changed. To use it with this Core, " + "choose Remove node on the Nodes page, uninstall it on this host, then add the host again with Add node.") GIB = 1024 ** 3 -def require_disk(root, args, program_manifest, manifest, names): +def require_disk(root, args, manifest, names): """Refuse before downloading when the node account's disk cannot hold what is still missing. The Runtime archive and its unpacked copy exist only until the image is imported; a @@ -485,7 +473,7 @@ def require_disk(root, args, program_manifest, manifest, names): needed = 0 for name in names: if name != "runtime/seccomp.json": - entry = distribution.artifact(program_manifest if name in provider_assets.artifacts(args.provider, ("node",)) else manifest, name) + entry = distribution.artifact(manifest, name) target = root / name if not (target.is_file() and distribution.matches(target, entry)): needed += entry["size"] - distribution.resumable(distribution.partial_path(target), entry["size"]) @@ -511,33 +499,26 @@ def require_disk(root, args, program_manifest, manifest, names): def register_node(root, args, token, helper_archive=None, *, secret_path): """Download and verify the payload, prepare the Runtime and register; not the service.""" install_display.step("Downloading and verifying node files") - program_manifest, program_sums = metadata(args.source_url) - selected = args.configuration["specification"]["runtime"] - manifest, sums = program_manifest, program_sums - if selected["source_commit"] != program_manifest["source_commit"]: - manifest, sums = metadata(args.source_url, prefix="releases/" + selected["source_commit"] + "/") - node_spec.verify_release(args.configuration, manifest) - runtime_prefix = "releases/" + manifest["source_commit"] + "/" + manifest, sums = metadata(args.source_url) names = provider_assets.artifacts(args.provider, ("node", "runtime", "policy")) if "runtime/seccomp.json" not in sums: raise InstallError("The distribution is missing required node checksums") - node_names = provider_assets.artifacts(args.provider, ("node",)) - if (root / "registered.json").exists(): - refuse_newer_node_software(root, program_manifest, node_names) - require_disk(root, args, program_manifest, manifest, names) state = {"installation_id": args.installation_id, "provider": args.provider, "core_url": args.core_url, "source_commit": manifest["source_commit"], "generation": args.configuration["generation"], "specification_digest": args.configuration["specification_digest"]} + marker = root / "registered.json" + if marker.exists() and marker.read_text() != json_text(state): + raise InstallError(NOT_THIS_INSTALLATION) + require_disk(root, args, manifest, names) write_once(root / "installation.json", json_text(state)) - node_generations.record_root_runtime(root, args, manifest, sums, sys.modules[__name__]) for name in names: if name == "runtime/seccomp.json": - download(args.source_url, name, root, sums[name], prefix=runtime_prefix, replace=args.disposable) + download(args.source_url, name, root, sums[name], replace=args.disposable) else: target = root / name safe_directory(target.parent) existing_file(target) - distribution.obtain_artifact(program_manifest if name in node_names else manifest, name, target, args.disposable) + distribution.obtain_artifact(manifest, name, target, args.disposable) os.chmod(target, 0o700) node_generations.install_helper(root, args, sys.modules[__name__], helper_archive) safe_directory(root / "state/node") @@ -550,10 +531,9 @@ def register_node(root, args, token, helper_archive=None, *, secret_path): runtime_image = prepare_runtime(root, args, manifest) # Retain the original network policy when recovering a partial installation. if not existing_file(root / "provider.json"): - write_once(root / "provider.json", json_text(provider_config(root, args, runtime_image))) + write_once(root / "provider.json", json_text(provider_config(root, args, manifest, runtime_image))) else: node_spec.verify_provider(json.loads((root / "provider.json").read_text()), args.configuration, runtime_image) - marker = root / "registered.json" if not existing_file(marker): # The one-time credential is never passed through process arguments or service environments. with os.fdopen(os.open(secret_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "w") as secret: @@ -567,8 +547,6 @@ def register_node(root, args, token, helper_archive=None, *, secret_path): discard_unregistered(root) raise write_once(marker, json_text(state)) - elif marker.read_text() != json_text(state): - raise InstallError("Registered node identity differs; refusing to replace it") def prepare_service_node(args, token, helper_archive): @@ -1490,11 +1468,7 @@ def main(argv=None): if args.generation_action: if args.generation is None or not 1 <= args.generation <= 9223372036854775807 or not re.fullmatch(r"[0-9a-f]{64}", args.specification_digest or ""): parser.error("Invalid generation authorization") - try: - (node_generations.prepare if args.generation_action == "prepare" else node_generations.collect)(args, sys.modules[__name__]) - except (RuntimeDownloadError, distribution.ArtifactError): - print("Runtime artifact transfer or verification failed", file=sys.stderr) - raise SystemExit(65) from None + (node_generations.prepare if args.generation_action == "prepare" else node_generations.collect)(args, sys.modules[__name__]) return if os.geteuid() != 0: raise InstallError("Node installation and removal require root. Run this command with sudo.") diff --git a/deploy/node/node_payload.py b/deploy/node/node_payload.py index 6cbe8826b..405f147d0 100644 --- a/deploy/node/node_payload.py +++ b/deploy/node/node_payload.py @@ -1,8 +1,6 @@ -"""Publish release node payloads without replacing installed bytes, then activate one.""" +"""Publish the installation's node payload without replacing installed bytes.""" import json import os -from pathlib import Path -import re import shutil import tempfile @@ -14,19 +12,13 @@ class PayloadError(Exception): def publish(destination, source): - """Publish the release in source under destination/releases/ and return the commit. + """Copy the release in source into destination, which holds exactly one release. - Each release stays immutable and addressable while nodes retain it. The active pointer is not moved. - """ + Installed bytes are immutable: a file with other bytes is refused, and only absent files are added.""" if destination.is_symlink(): raise PayloadError("Invalid node payload directory") - destination.mkdir(mode=0o700, exist_ok=True) manifest = json.loads((source / "manifest.json").read_text()) - revision = manifest.get("source_commit", "") - if not re.fullmatch(r"[0-9a-f]{40}", revision): - raise PayloadError("Invalid node payload release identity") - metadata_names = ("node-install.pyz", "manifest.json", "SHA256SUMS", "runtime/seccomp.json") - names = list(metadata_names) + names = ["node-install.pyz", "manifest.json", "SHA256SUMS", "runtime/seccomp.json"] for logical in manifest.get("artifacts", {}): entry = artifact(manifest, logical) name = "artifacts/" + entry["filename"] @@ -37,91 +29,33 @@ def publish(destination, source): or path.stat().st_size != entry["size"] or digest(path) != entry["sha256"]): raise PayloadError("Offline artifact verification failed: " + logical) names.append(name) - target = destination / "releases" / revision - if target.is_symlink() or target.parent.is_symlink(): - raise PayloadError("Installed node payload differs; preserve it and inspect the distribution") - target.parent.mkdir(parents=True, exist_ok=True, mode=0o700) - if target.exists(): - # Validate the complete published metadata and every existing declared - # artifact before filling any absence. Existing bytes are immutable. - for name in metadata_names: - previous = target / name - if (previous.parent.is_symlink() or previous.is_symlink() or not previous.is_file() - or digest(previous) != digest(source / name)): + destination.mkdir(mode=0o700, exist_ok=True) + for name in names: + incoming, target = source / name, destination / name + if incoming.is_symlink() or not incoming.is_file(): + raise PayloadError("Invalid node payload source file") + target.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + if target.parent.is_symlink() or target.is_symlink(): + raise PayloadError("Installed node payload differs; preserve it and inspect the distribution") + if target.exists(): + if not target.is_file() or digest(target) != digest(incoming): raise PayloadError("Installed node payload differs; preserve it and inspect the distribution") - artifacts = target / "artifacts" - if artifacts.is_symlink() or artifacts.exists() and not artifacts.is_dir(): - raise PayloadError("Installed node artifact directory differs") - missing = [] - for logical in manifest.get("artifacts", {}): - entry = artifact(manifest, logical) - name = "artifacts/" + entry["filename"] - previous = target / name - if previous.is_symlink() or previous.exists() and (not previous.is_file() - or previous.stat().st_size != entry["size"] or digest(previous) != entry["sha256"]): - raise PayloadError("Installed node artifact differs; refusing repair") - if not previous.exists() and name in names: - missing.append((name, entry)) - if missing: - artifacts.mkdir(mode=0o700, exist_ok=True) - for name, entry in missing: - descriptor, temporary = tempfile.mkstemp(prefix=".payload-", dir=artifacts) - try: - with os.fdopen(descriptor, "wb") as outgoing, (source / name).open("rb") as incoming: - shutil.copyfileobj(incoming, outgoing) - outgoing.flush() - os.fsync(outgoing.fileno()) - if Path(temporary).stat().st_size != entry["size"] or digest(Path(temporary)) != entry["sha256"]: - raise PayloadError("Node artifact changed during repair") - # Publish without replacing bytes introduced concurrently. - os.link(temporary, target / name) - finally: - os.unlink(temporary) - for directory in (artifacts, target): - descriptor = os.open(directory, os.O_RDONLY | os.O_DIRECTORY) - try: - os.fsync(descriptor) - finally: - os.close(descriptor) - return revision - with tempfile.TemporaryDirectory(prefix=".payload-", dir=target.parent) as temporary: - stage = Path(temporary) / "release" - stage.mkdir(mode=0o700) - for name in names: - path = source / name - if path.is_symlink() or not path.is_file(): - raise PayloadError("Invalid node payload source file") - copied = stage / name - copied.parent.mkdir(parents=True, exist_ok=True, mode=0o700) - with path.open("rb") as incoming, copied.open("xb") as outgoing: - os.chmod(copied, 0o600) - shutil.copyfileobj(incoming, outgoing) + continue + descriptor, temporary = tempfile.mkstemp(prefix=".payload-", dir=target.parent) + try: + with os.fdopen(descriptor, "wb") as outgoing, incoming.open("rb") as stream: + shutil.copyfileobj(stream, outgoing) outgoing.flush() os.fsync(outgoing.fileno()) - os.rename(stage, target) - return revision - - -def activate(destination, revision): - """Atomically point destination/active.json at a published release.""" - pointer = destination / "active.json" - if pointer.is_symlink(): - raise PayloadError("Invalid active node payload pointer") - descriptor, temporary = tempfile.mkstemp(prefix=".active-", dir=destination) - try: - with os.fdopen(descriptor, "w") as stream: - json.dump({"source_commit": revision}, stream) - stream.write("\n") - stream.flush() - os.fsync(stream.fileno()) - # Web reads the pointer as another user while it serves the previous release. - os.chmod(temporary, 0o644) - os.replace(temporary, pointer) - descriptor = os.open(destination, os.O_RDONLY | os.O_DIRECTORY) + # Publish without replacing bytes introduced concurrently. + os.link(temporary, target) + except FileExistsError: + raise PayloadError("Installed node payload differs; preserve it and inspect the distribution") from None + finally: + os.unlink(temporary) + for directory in {(destination / name).parent for name in names}: + descriptor = os.open(directory, os.O_RDONLY | os.O_DIRECTORY) try: os.fsync(descriptor) finally: os.close(descriptor) - finally: - if os.path.exists(temporary): - os.unlink(temporary) diff --git a/deploy/node/node_spec.py b/deploy/node/node_spec.py index 6895e5b79..72b0e9534 100644 --- a/deploy/node/node_spec.py +++ b/deploy/node/node_spec.py @@ -1,4 +1,4 @@ -"""Read the Core-owned node specification and verify its matched release.""" +"""Read the Core-owned node specification.""" import hashlib import http.client import json @@ -22,24 +22,15 @@ class UnrecordedIdentity(SpecificationError): "and run it on this host.") -def release(manifest): - return {"source_commit": manifest["source_commit"], - "image_id": manifest["images"]["runtime"], - "image_manifest_digest": manifest["image_manifest_digests"]["runtime"], - "microsandbox_ref": manifest["runtime_ref"], - "runtime_sha256": manifest["microsandbox"]["runtime_sha256"], - "firmware_sha256": manifest["microsandbox"]["firmware_sha256"]} - - # BEGIN GENERATED DEPLOYMENT CONTRACT # Generated from sandbox/deployment_contract.go; do not edit. -_CONTRACT = json.loads("{\"workspace_fields\":[\"attachment\",\"user_xattr\",\"capacity_quota\"],\"resources\":[{\"name\":\"cpus\",\"min\":1,\"max\":255,\"omit_zero\":false},{\"name\":\"memory_mib\",\"min\":512,\"max\":1048576,\"omit_zero\":false},{\"name\":\"root_disk_mib\",\"min\":0,\"max\":4294967295,\"omit_zero\":true},{\"name\":\"environment_disk_mib\",\"min\":0,\"max\":4294967295,\"omit_zero\":true}],\"runtime\":[{\"name\":\"source_commit\",\"pattern\":\"[0-9a-f]{40}\"},{\"name\":\"image_id\",\"pattern\":\"sha256:[0-9a-f]{64}\"},{\"name\":\"image_manifest_digest\",\"pattern\":\"sha256:[0-9a-f]{64}\"},{\"name\":\"microsandbox_ref\",\"pattern\":\"oac-runtime@sha256:[0-9a-f]{64}\"},{\"name\":\"runtime_sha256\",\"pattern\":\"[0-9a-f]{64}\"},{\"name\":\"firmware_sha256\",\"pattern\":\"[0-9a-f]{64}\"}],\"providers\":{\"docker\":{\"mode\":\"nodes\",\"disk\":false,\"runtime\":true,\"default_resources\":{\"cpus\":2,\"memory_mib\":2048}},\"e2b\":{\"mode\":\"direct\",\"disk\":false,\"runtime\":false,\"default_resources\":null},\"microsandbox\":{\"mode\":\"nodes\",\"workspace\":{\"attachment\":\"host_directory\",\"user_xattr\":true},\"disk\":true,\"runtime\":true,\"default_resources\":{\"cpus\":2,\"memory_mib\":4096,\"root_disk_mib\":8192,\"environment_disk_mib\":8192}}},\"minimum_disk\":1024}") +_CONTRACT = json.loads("{\"workspace_fields\":[\"attachment\",\"user_xattr\",\"capacity_quota\"],\"resources\":[{\"name\":\"cpus\",\"min\":1,\"max\":255,\"omit_zero\":false},{\"name\":\"memory_mib\",\"min\":512,\"max\":1048576,\"omit_zero\":false},{\"name\":\"root_disk_mib\",\"min\":0,\"max\":4294967295,\"omit_zero\":true},{\"name\":\"environment_disk_mib\",\"min\":0,\"max\":4294967295,\"omit_zero\":true}],\"providers\":{\"docker\":{\"mode\":\"nodes\",\"disk\":false,\"default_resources\":{\"cpus\":2,\"memory_mib\":2048}},\"e2b\":{\"mode\":\"direct\",\"disk\":false,\"default_resources\":null},\"microsandbox\":{\"mode\":\"nodes\",\"workspace\":{\"attachment\":\"host_directory\",\"user_xattr\":true},\"disk\":true,\"default_resources\":{\"cpus\":2,\"memory_mib\":4096,\"root_disk_mib\":8192,\"environment_disk_mib\":8192}}},\"minimum_disk\":1024}") # END GENERATED DEPLOYMENT CONTRACT def canonical_spec(provider, specification, validate=True): rules = _CONTRACT["providers"][provider] - if validate and (not isinstance(specification, dict) or set(specification) - {"workspace"} != ({"resources", "runtime"} if rules["runtime"] else {"resources"})): + if validate and (not isinstance(specification, dict) or set(specification) - {"workspace"} != {"resources"}): raise ValueError("Invalid specification fields") workspace = specification.get("workspace") requirements = rules.get("workspace") @@ -69,17 +60,6 @@ def canonical_spec(provider, specification, validate=True): if value or not rule["omit_zero"]: ordered[name] = value result = {"provider": provider, "resources": ordered} - if rules["runtime"]: - runtime = specification["runtime"] - if validate and (not isinstance(runtime, dict) or set(runtime) != {rule["name"] for rule in _CONTRACT["runtime"]}): - raise ValueError("Invalid release fields") - ordered_runtime = {} - for rule in _CONTRACT["runtime"]: - value = runtime[rule["name"]] - if validate and (not isinstance(value, str) or not re.fullmatch(rule["pattern"], value)): - raise ValueError("Invalid release identity") - ordered_runtime[rule["name"]] = value - result["runtime"] = ordered_runtime if workspace is not None: result["workspace"] = {field: workspace[field] if field == "attachment" else bool(workspace.get(field, False)) for field in _CONTRACT["workspace_fields"]} return result @@ -98,7 +78,7 @@ def validate(data, args): if (provider not in ("docker", "microsandbox") or data["installation_id"] != args.installation_id or data["core_url"] != args.core_url or type(data["generation"]) is not int or data["generation"] < 1 or getattr(args, "provider", None) not in (None, provider) - or set(spec) - {"workspace"} != {"resources", "runtime"} + or set(spec) - {"workspace"} != {"resources"} or type(data["max_active"]) is not int or type(data["max_retained"]) is not int or not 1 <= data["max_active"] <= data["max_retained"] <= 1000000): raise ValueError() @@ -168,11 +148,6 @@ def fetch(args, token, retained, open_request, allow_enrollment=False, generatio return data -def verify_release(configuration, manifest): - if configuration["specification"]["runtime"] != release(manifest): - raise SpecificationError("Core Runtime release differs from this distribution; use the matched installation artifacts") - - def verify_provider(stored, configuration, runtime_image): spec, provider = configuration["specification"], configuration["provider"] if (stored.get("specification") != spec or stored.get("provider") != provider diff --git a/deploy/node/test_generation_review_regressions.py b/deploy/node/test_generation_review_regressions.py index 82eb11515..0fe09d62e 100644 --- a/deploy/node/test_generation_review_regressions.py +++ b/deploy/node/test_generation_review_regressions.py @@ -13,7 +13,6 @@ import node_generations import node_install as installer import node_payload -import test_node_install import test_node_generations @@ -24,64 +23,8 @@ def fixture(self, kind): self.addCleanup(case.doCleanups) return case - def test_containerd_resolution_recovers_interrupted_commit(self): - case = self.fixture(test_node_install.NodeInstallTests) - case.containerd = True - case.payloads["runtime/seccomp.json"] = b"{}" - case.refresh_manifest() - case.install() - case.manifest['source_commit'] = 'f' * 40 - case.refresh_manifest() - configuration = json.loads(case.configuration_response(None).read()) - configuration['generation'] = 2 - case.args.generation = 2 - case.args.specification_digest = configuration['specification_digest'] - path = case.root / 'state/node/generations/2.json' - original = node_generations.atomic_json - def interrupted(path, value): - original(path, value) - if path.name == '2.json': - raise OSError('interrupted after resolved configuration') - with mock.patch.object(installer.node_spec, 'fetch', return_value=configuration): - with mock.patch.object(node_generations, 'atomic_json', side_effect=interrupted): - with self.assertRaises(OSError): - node_generations.prepare(case.args, installer) - self.assertEqual(installer.private_json(path)['native']['image'], case.manifest['image_manifest_digests']['runtime']) - self.assertTrue(installer.private_json(path.with_suffix('.preparing'))['import_started']) - node_generations.prepare(case.args, installer) - saved = path.read_bytes() - node_generations.prepare(case.args, installer) - self.assertEqual(path.read_bytes(), saved) - self.assertFalse(path.with_suffix('.preparing').exists()) - self.assertTrue(node_generations.image_available(installer.private_json(path), installer)) - - def test_unresolved_import_remains_discoverable_and_collectible(self): - case = self.fixture(test_node_install.NodeInstallTests) - case.containerd = True - case.payloads["runtime/seccomp.json"] = b"{}" - case.refresh_manifest() - case.install() - case.manifest['source_commit'] = 'f' * 40 - case.refresh_manifest() - configuration = json.loads(case.configuration_response(None).read()) - configuration['generation'] = 2 - case.args.generation = 2 - case.args.specification_digest = configuration['specification_digest'] - with mock.patch.object(installer.node_spec, 'fetch', return_value=configuration), mock.patch.object(installer, 'prepare_runtime', side_effect=installer.InstallError('daemon unavailable')): - with self.assertRaises(installer.InstallError): - node_generations.prepare(case.args, installer) - self.assertIn(2, node_generations.retained_configs(case.root, installer)) - record = installer.private_json(case.root / 'state/node/generations/2.preparing') - self.assertTrue(record['import_started']) - self.assertEqual(record['configuration']['specification'], configuration['specification']) - self.assertFalse((case.root / 'state/node/generations/2.json').exists()) - node_generations.collect(case.args, installer) - node_generations.collect(case.args, installer) - self.assertNotIn(2, node_generations.retained_configs(case.root, installer)) - self.assertTrue((case.root / 'provider.json').exists()) - def test_live_helper_replaced_inode_is_refused_after_restart(self): - case = self.fixture(test_node_generations.CollectionTests) + case = self.fixture(test_node_generations.GenerationTests) lease = case.directory / '1.lease' helper = subprocess.Popen([sys.executable, '-c', 'import os,fcntl,sys; f=os.open(sys.argv[1],os.O_RDWR);fcntl.flock(f,fcntl.LOCK_SH);print("locked",flush=True);sys.stdin.read()', str(lease)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) try: @@ -96,7 +39,6 @@ def test_live_helper_replaced_inode_is_refused_after_restart(self): with self.assertRaises(installer.InstallError): node_generations.collect(case.args, installer) self.assertIsNone(helper.poll()) - self.assertTrue(case.release.exists()) self.assertFalse((case.directory / '1.dropped').exists()) with old.open('rb') as stream: with self.assertRaises(BlockingIOError): @@ -107,10 +49,10 @@ def test_live_helper_replaced_inode_is_refused_after_restart(self): helper.communicate(timeout=10) self.assertEqual(helper.returncode, 0) node_generations.collect(case.args, installer) - self.assertFalse(case.release.exists()) + self.assertTrue((case.directory / '1.dropped').exists()) def test_interrupted_lease_initialization_is_not_reclaimed(self): - case = self.fixture(test_node_generations.CollectionTests) + case = self.fixture(test_node_generations.GenerationTests) identity = dict(node_generations.marker_identity(case.args), generation=3) with mock.patch.object(node_generations, 'atomic_json', side_effect=OSError('power loss')): with self.assertRaises(OSError): @@ -122,7 +64,7 @@ def test_interrupted_lease_initialization_is_not_reclaimed(self): (case.directory / '1.lease-identity').unlink() with self.assertRaises(OSError): node_generations.collect(case.args, installer) - self.assertTrue(case.release.exists()) + self.assertFalse((case.directory / '1.dropped').exists()) def payload_fixture(self, directory): root = directory / 'installation'; root.mkdir() @@ -136,47 +78,45 @@ def payload_fixture(self, directory): for name, raw in files.items(): path = bundle / name; path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(raw) node_payload.publish(root / 'node-payload', bundle) - return root, bundle, root / 'node-payload/releases' / source, entries + return root, bundle, root / 'node-payload', entries - def test_publish_keeps_releases_and_activate_swaps_the_pointer(self): + def test_publish_is_flat_and_adds_and_repairs_verified_missing_artifacts(self): with tempfile.TemporaryDirectory() as temp: - root, bundle, old, _ = self.payload_fixture(Path(temp)) - payload = root / 'node-payload' - node_payload.activate(payload, old.name) - (bundle / 'manifest.json').write_text(json.dumps({'source_commit': 'b' * 40, 'artifacts': {}})) - self.assertEqual(node_payload.publish(payload, bundle), 'b' * 40) - self.assertEqual(json.loads((payload / 'active.json').read_text()), {'source_commit': old.name}) - node_payload.activate(payload, 'b' * 40) - self.assertEqual(json.loads((payload / 'active.json').read_text()), {'source_commit': 'b' * 40}) - self.assertEqual(sorted(path.name for path in (payload / 'releases').iterdir()), [old.name, 'b' * 40]) - - def test_thin_release_adds_and_repairs_verified_missing_artifact(self): - with tempfile.TemporaryDirectory() as temp: - root, bundle, release, entries = self.payload_fixture(Path(temp)) + root, bundle, payload, entries = self.payload_fixture(Path(temp)) + self.assertEqual(sorted(path.name for path in payload.iterdir()), ['SHA256SUMS', 'manifest.json', 'node-install.pyz', 'runtime']) (bundle / 'artifacts').mkdir() for name, entry in entries.items(): (bundle / 'artifacts' / entry['filename']).write_bytes((name + ' verified artifact').encode()) - original = (release / 'manifest.json').read_bytes() - node_payload.publish(root / 'node-payload', bundle) - artifact = release / 'artifacts' / entries['node']['filename'] + original = (payload / 'manifest.json').read_bytes() + node_payload.publish(payload, bundle) + artifact = payload / 'artifacts' / entries['node']['filename'] self.assertEqual(artifact.read_bytes(), b'node verified artifact') artifact.unlink() - node_payload.publish(root / 'node-payload', bundle) + node_payload.publish(payload, bundle) self.assertEqual(artifact.read_bytes(), b'node verified artifact') - self.assertEqual((release / 'manifest.json').read_bytes(), original) + self.assertEqual((payload / 'manifest.json').read_bytes(), original) self.assertEqual(artifact.stat().st_nlink, 1) def test_published_conflict_refuses_every_missing_repair(self): with tempfile.TemporaryDirectory() as temp: - root, bundle, release, entries = self.payload_fixture(Path(temp)) - (bundle / 'artifacts').mkdir(); (release / 'artifacts').mkdir() + root, bundle, payload, entries = self.payload_fixture(Path(temp)) + (bundle / 'artifacts').mkdir(); (payload / 'artifacts').mkdir() node = entries['node']['filename']; helper = entries['helper']['filename'] (bundle / 'artifacts' / node).write_bytes(b'node verified artifact') - (release / 'artifacts' / helper).write_bytes(b'changed') + (bundle / 'artifacts' / helper).write_bytes(b'helper verified artifact') + (payload / 'artifacts' / helper).write_bytes(b'changed') with self.assertRaises(node_payload.PayloadError): - node_payload.publish(root / 'node-payload', bundle) - self.assertFalse((release / 'artifacts' / node).exists()) - self.assertEqual((release / 'artifacts' / helper).read_bytes(), b'changed') + node_payload.publish(payload, bundle) + self.assertEqual((payload / 'artifacts' / helper).read_bytes(), b'changed') + + def test_another_release_cannot_replace_published_bytes(self): + with tempfile.TemporaryDirectory() as temp: + root, bundle, payload, _ = self.payload_fixture(Path(temp)) + before = (payload / 'manifest.json').read_bytes() + (bundle / 'manifest.json').write_text(json.dumps({'source_commit': 'b' * 40, 'artifacts': {}})) + with self.assertRaises(node_payload.PayloadError): + node_payload.publish(payload, bundle) + self.assertEqual((payload / 'manifest.json').read_bytes(), before) def test_transfer_classification_excludes_provider_and_path_errors(self): with self.assertRaises(distribution.ArtifactError): @@ -191,16 +131,8 @@ def test_transfer_classification_excludes_provider_and_path_errors(self): distribution.checked_path(root / 'link') self.assertNotIsInstance(got.exception, distribution.ArtifactError) - def test_private_helper_exit_classification_never_parses_text(self): - for expression, code in (("installer.RuntimeDownloadError('secret')", 65), ("installer.distribution.ArtifactError('secret')", 65), ("installer.InstallError('runtime_download_failed secret')", 1)): - script = "import sys,runpy,node_generations; sys.argv=['node_install.py','--installation-id','94be54a1-138c-4f30-bc87-b13686272dbe','--generation-action','prepare','--generation','2','--specification-digest','" + 'a' * 64 + "'];\ndef fail(args,installer): raise " + expression + "\nnode_generations.prepare=fail; runpy.run_module('node_install',run_name='__main__',alter_sys=True)" - result = subprocess.run([sys.executable, '-c', script], capture_output=True, text=True, timeout=10, cwd=Path(__file__).parent) - self.assertEqual(result.returncode, code, result.stderr) - if code == 65: - self.assertNotIn('secret', result.stderr) - def test_lease_record_is_fsynced_before_any_caller_can_enter(self): - case = self.fixture(test_node_generations.CollectionTests) + case = self.fixture(test_node_generations.GenerationTests) identity = dict(node_generations.marker_identity(case.args), generation=4) operations = [] original = node_generations.os.fsync @@ -213,63 +145,3 @@ def sync(fd): record = installer.private_json(case.directory / '4.lease-identity') lease = (case.directory / '4.lease').stat() self.assertEqual((record['device'], record['inode']), (lease.st_dev, lease.st_ino)) - - def test_unpublished_plan_refuses_path_or_installation_drift(self): - case = self.fixture(test_node_install.NodeInstallTests) - case.payloads["runtime/seccomp.json"] = b"{}" - case.refresh_manifest(); case.install() - case.manifest['source_commit'] = 'f' * 40; case.refresh_manifest() - configuration = json.loads(case.configuration_response(None).read()); configuration['generation'] = 2 - case.args.generation = 2; case.args.specification_digest = configuration['specification_digest'] - with mock.patch.object(installer.node_spec, 'fetch', return_value=configuration), mock.patch.object(installer, 'prepare_runtime', side_effect=installer.InstallError('stopped')): - with self.assertRaises(installer.InstallError): node_generations.prepare(case.args, installer) - path=case.root/'state/node/generations/2.preparing' - original=installer.private_json(path) - for field,value in (("installation_id","foreign"),("core_url","http://foreign/api/v1")): - changed=json.loads(json.dumps(original)); changed['configuration'][field]=value - node_generations.atomic_json(path,changed) - with self.assertRaises(installer.InstallError): node_generations.retained_configs(case.root,installer) - changed=json.loads(json.dumps(original)); changed['configuration']['native']['seccomp_file']=str(case.home/'foreign') - node_generations.atomic_json(path,changed) - with self.assertRaises(installer.InstallError): node_generations.collect(case.args,installer) - self.assertFalse((case.root/'state/node/generations/2.dropped').exists()) - node_generations.atomic_json(path,original) - node_generations.collect(case.args,installer) - - def test_pending_fifo_is_refused_without_opening_a_blocking_reader(self): - case = self.fixture(test_node_generations.CollectionTests) - import os - os.mkfifo(case.directory/'2.preparing',0o600) - with self.assertRaises(installer.InstallError): node_generations.retained_configs(case.root,installer) - - def test_final_identity_never_changes_on_resolver_drift(self): - case = self.fixture(test_node_install.NodeInstallTests) - case.payloads["runtime/seccomp.json"] = b"{}"; case.refresh_manifest(); case.install() - case.manifest['source_commit']='f'*40; case.refresh_manifest() - cfg=json.loads(case.configuration_response(None).read()); cfg['generation']=2 - case.args.generation=2;case.args.specification_digest=cfg['specification_digest'] - with mock.patch.object(installer.node_spec,'fetch',return_value=cfg): - node_generations.prepare(case.args,installer) - path=case.root/'state/node/generations/2.json';before=path.read_bytes() - # The same spec allows both IDs, but a finalized local provider is immutable. - case.containerd=True - with self.assertRaises(installer.node_spec.SpecificationError): node_generations.prepare(case.args,installer) - self.assertEqual(path.read_bytes(),before) - - def test_collecting_generation_is_retained_but_never_reused(self): - case=self.fixture(test_node_install.NodeInstallTests) - case.args.provider="microsandbox";case.install() - cfg=json.loads(case.configuration_response(None).read());cfg['generation']=2 - case.args.generation=2;case.args.specification_digest=cfg['specification_digest'] - original=installer.private_json(case.root/'provider.json') - marker=dict(installation_id=case.args.installation_id,generation=1,specification_digest=installer.node_spec.digest('microsandbox',original['specification']),native_complete=False) - node_generations.atomic_json(case.root/'state/node/generations/1.collecting',marker) - real=node_generations.image_available - def available(value,installer): - self.assertNotEqual(value['generation'],1,'collecting entry was considered for reuse') - return real(value,installer) - with mock.patch.object(installer.node_spec,'fetch',return_value=cfg),mock.patch.object(node_generations,'image_available',side_effect=available): - node_generations.prepare(case.args,installer) - final=installer.private_json(case.root/'state/node/generations/2.json') - self.assertNotEqual(final['native']['helper_path'],original['native']['helper_path']) - self.assertIn(1,node_generations.retained_configs(case.root,installer)) diff --git a/deploy/node/test_node_generations.py b/deploy/node/test_node_generations.py index 0b2c66311..2d210261f 100644 --- a/deploy/node/test_node_generations.py +++ b/deploy/node/test_node_generations.py @@ -1,8 +1,7 @@ -"""Collection retains bytes until local helper ownership settles.""" +"""Generation preparation copies the installation's Runtime; collection retains identity until local helper ownership settles.""" import fcntl import os import json -import hashlib from pathlib import Path import tempfile from types import SimpleNamespace @@ -14,23 +13,20 @@ import node_spec -class CollectionTests(unittest.TestCase): +class GenerationTests(unittest.TestCase): def setUp(self): temporary = tempfile.TemporaryDirectory() self.addCleanup(temporary.cleanup) self.root = Path(temporary.name) self.directory = self.root / "state/node/generations" self.directory.mkdir(parents=True, mode=0o700) - self.value = {"installation_id": "test-installation", "generation": 1, "provider": "docker", "native": {"image": "sha256:" + "a" * 64}, - "specification": {"resources": {"cpus": 1, "memory_mib": 1024}, "runtime": {"source_commit": "b" * 40, "image_id": "sha256:" + "a" * 64, "image_manifest_digest": "sha256:" + "c" * 64, "microsandbox_ref": "oac-runtime@sha256:" + "d" * 64, "runtime_sha256": "e" * 64, "firmware_sha256": "f" * 64}}} + self.value = {"installation_id": "test-installation", "generation": 1, "provider": "docker", "core_url": "https://core.example/api/v1", + "native": {"image": "sha256:" + "a" * 64, "seccomp_file": str(self.root / "runtime/seccomp.json")}, + "specification": {"resources": {"cpus": 1, "memory_mib": 1024}}} self.args = SimpleNamespace(installation_id="test-installation", generation=1, specification_digest=node_spec.digest("docker", self.value["specification"])) - self.release = self.root / "releases" / ("b" * 40) - self.value["native"]["seccomp_file"] = str(self.release / "runtime/seccomp.json") - self.release.mkdir(parents=True) - (self.release / "artifact").write_bytes(b"immutable bytes") node_generations.atomic_json(self.root / "provider.json", self.value) self.initialize_lease() - for patch in (mock.patch.object(node_generations, "owned_root", return_value=(self.root, {})), + for patch in (mock.patch.object(node_generations, "owned_root", return_value=(self.root, {"core_url": "https://core.example"})), mock.patch.object(installer, "checked", return_value="")): patch.start() self.addCleanup(patch.stop) @@ -40,7 +36,37 @@ def initialize_lease(self): node_generations.marker_identity(self.args), initialize=True): pass - def test_busy_helper_refuses_all_mutations_then_same_inode_collects(self): + def next_generation(self, resources): + specification = {"resources": resources} + configuration = {"installation_id": "test-installation", "provider": "docker", "core_url": "https://core.example", "generation": 2, "specification": specification, "specification_digest": node_spec.digest("docker", specification)} + self.args.generation, self.args.specification_digest = 2, configuration["specification_digest"] + return configuration + + def test_prepare_publishes_the_installation_runtime_under_the_authorized_specification(self): + configuration = self.next_generation({"cpus": 2, "memory_mib": 2048}) + with mock.patch.object(node_spec, "fetch", return_value=configuration), mock.patch.object(node_generations, "image_available", return_value=True): + node_generations.prepare(self.args, installer) + published = json.loads((self.directory / "2.json").read_text()) + self.assertEqual(published, dict(self.value, generation=2, specification=configuration["specification"])) + node_generations.prepare(self.args, installer) + self.assertEqual(json.loads((self.directory / "2.json").read_text()), published) + self.assertEqual(json.loads((self.root / "provider.json").read_text()), self.value) + + def test_prepare_refuses_an_unauthorized_dropped_or_unavailable_generation(self): + configuration = self.next_generation({"cpus": 2, "memory_mib": 2048}) + with mock.patch.object(node_spec, "fetch", return_value=dict(configuration, specification_digest="0" * 64)): + with self.assertRaisesRegex(installer.InstallError, "differs from its authorization"): + node_generations.prepare(self.args, installer) + with mock.patch.object(node_spec, "fetch", return_value=configuration), mock.patch.object(node_generations, "image_available", return_value=False): + with self.assertRaisesRegex(installer.InstallError, "Runtime is missing"): + node_generations.prepare(self.args, installer) + self.assertFalse((self.directory / "2.json").exists()) + node_generations.atomic_json(self.directory / "2.dropped", node_generations.marker_identity(self.args)) + with mock.patch.object(node_spec, "fetch", return_value=configuration): + with self.assertRaisesRegex(installer.InstallError, "dropped generation"): + node_generations.prepare(self.args, installer) + + def test_busy_helper_refuses_collection_then_same_inode_collects(self): lease = self.directory / "1.lease" descriptor = os.open(lease, os.O_CREAT | os.O_RDWR, 0o600) self.addCleanup(os.close, descriptor) @@ -48,31 +74,25 @@ def test_busy_helper_refuses_all_mutations_then_same_inode_collects(self): fcntl.flock(descriptor, fcntl.LOCK_SH) with self.assertRaisesRegex(installer.InstallError, "helper is still active"): node_generations.collect(self.args, installer) - installer.checked.assert_not_called() - self.assertEqual((self.release / "artifact").read_bytes(), b"immutable bytes") self.assertFalse((self.directory / "1.dropped").exists()) fcntl.flock(descriptor, fcntl.LOCK_UN) node_generations.collect(self.args, installer) - self.assertFalse(self.release.exists()) self.assertTrue((self.directory / "1.dropped").exists()) + self.assertEqual(node_generations.retained_configs(self.root, installer), {}) node_generations.collect(self.args, installer) - installer.checked.assert_not_called() final = lease.stat() self.assertEqual((original.st_dev, original.st_ino), (final.st_dev, final.st_ino)) - def test_collection_holds_exclusive_lease_during_deletion(self): - lease = self.directory / "1.lease" - original = node_generations.shutil.rmtree - def remove(path): - descriptor = os.open(lease, os.O_RDWR) - try: - with self.assertRaises(BlockingIOError): - fcntl.flock(descriptor, fcntl.LOCK_SH | fcntl.LOCK_NB) - finally: - os.close(descriptor) - return original(path) - with mock.patch.object(node_generations.shutil, "rmtree", side_effect=remove): - node_generations.collect(self.args, installer) + def test_collection_leaves_the_installations_runtime_and_other_generations(self): + other = dict(self.value, generation=2) + node_generations.atomic_json(self.directory / "2.json", other) + seccomp = self.root / "runtime/seccomp.json" + seccomp.parent.mkdir() + seccomp.write_text("{}") + node_generations.collect(self.args, installer) + installer.checked.assert_not_called() + self.assertTrue(seccomp.exists()) + self.assertEqual(node_generations.retained_configs(self.root, installer), {2: other}) def test_symlink_and_shared_inode_are_not_collection_authority(self): lease = self.directory / "1.lease" @@ -85,38 +105,11 @@ def test_symlink_and_shared_inode_are_not_collection_authority(self): try: with self.assertRaises((OSError, installer.InstallError)): node_generations.collect(self.args, installer) - installer.checked.assert_not_called() - self.assertTrue((self.release / "artifact").exists()) + self.assertFalse((self.directory / "1.dropped").exists()) finally: lease.unlink() - def test_native_inventory_failure_and_unknown_output_retain_every_byte(self): - self.micro_fixture() - for failure in (installer.InstallError("daemon unavailable"), "not a verified image ID"): - installer.checked.side_effect = failure if isinstance(failure, Exception) else None - installer.checked.return_value = failure - with self.assertRaises(installer.InstallError): - node_generations.collect(self.args, installer) - self.assertTrue((self.release / "artifact").exists()) - self.assertFalse((self.directory / "1.dropped").exists()) - self.assertFalse(json.loads((self.directory / "1.collecting").read_text())["native_complete"]) - - def test_restart_after_partial_file_cleanup_skips_native_and_finishes(self): - def interrupted(path): - (path / "artifact").unlink() - raise OSError("interrupted file cleanup") - with mock.patch.object(node_generations.shutil, "rmtree", side_effect=interrupted): - with self.assertRaises(OSError): - node_generations.collect(self.args, installer) - self.assertTrue(json.loads((self.directory / "1.collecting").read_text())["native_complete"]) - self.assertFalse((self.directory / "1.dropped").exists()) - installer.checked.reset_mock() - node_generations.collect(self.args, installer) - installer.checked.assert_not_called() - self.assertFalse(self.release.exists()) - self.assertEqual(node_generations.retained_configs(self.root, installer), {}) - - def test_restart_after_files_removed_before_final_marker(self): + def test_restart_after_an_interrupted_final_marker_finishes(self): original = node_generations.atomic_json def interrupt(path, value): if path.suffix == ".dropped": @@ -125,124 +118,22 @@ def interrupt(path, value): with mock.patch.object(node_generations, "atomic_json", side_effect=interrupt): with self.assertRaises(OSError): node_generations.collect(self.args, installer) - self.assertFalse(self.release.exists()) - installer.checked.reset_mock() + self.assertFalse((self.directory / "1.dropped").exists()) node_generations.collect(self.args, installer) - installer.checked.assert_not_called() self.assertTrue((self.directory / "1.dropped").exists()) - def test_invalid_journal_never_authorizes_deletion_or_hides_configuration(self): - for suffix in (".collecting", ".dropped", ".preparing"): - path = self.directory / ("1" + suffix) - value = node_generations.marker_identity(self.args) - if suffix == ".collecting": value["native_complete"] = True - if suffix == ".preparing": value["import_started"] = False - for field, invalid in (("installation_id", "foreign"), ("generation", 2), ("specification_digest", "a" * 64)): - node_generations.atomic_json(path, dict(value, **{field: invalid})) - with self.assertRaises(installer.InstallError): - node_generations.collect(self.args, installer) - self.assertTrue(self.release.exists()) - path.unlink() - installer.checked.assert_not_called() - - def test_shared_image_and_release_survive_until_last_generation(self): - other = dict(self.value, generation=2) - node_generations.atomic_json(self.directory / "2.json", other) - node_generations.collect(self.args, installer) - installer.checked.assert_not_called() - self.assertTrue((self.release / "artifact").exists()) - self.args.generation = 2 - self.initialize_lease() - node_generations.collect(self.args, installer) - self.assertFalse(self.release.exists()) - - def test_never_imported_generation_can_collect_missing_executable(self): - runtime, _, _ = self.micro_fixture() - runtime.unlink() - node_generations.atomic_json(self.directory / "1.preparing", dict(node_generations.marker_identity(self.args), import_started=False)) - node_generations.collect(self.args, installer) - installer.checked.assert_not_called() - self.assertFalse(self.release.exists()) - - def micro_fixture(self): - self.value["provider"] = "microsandbox" - home = self.root / "micro-store" - home.mkdir(mode=0o700) - node_generations.atomic_json(home / "oac-installation.json", {"installation_id": self.args.installation_id}) - runtime = self.release / "msb" - runtime.write_bytes(b"verified native executable") - runtime.chmod(0o700) - image = self.value["specification"]["runtime"]["microsandbox_ref"] - self.value["specification"]["runtime"]["runtime_sha256"] = hashlib.sha256(runtime.read_bytes()).hexdigest() - self.value["native"] = {"helper_path": str(self.release / "helper"), "runtime_home": str(home), "checkpoint_root": str(self.root / "checkpoints"), "runtime_path": str(runtime), "firmware_path": str(self.release / "firmware")} - self.args.specification_digest = node_spec.digest("microsandbox", self.value["specification"]) - node_generations.atomic_json(self.root / "provider.json", self.value) - # This fixture changes provider before any helper exists. - for suffix in (".lease", ".lease-identity"): - (self.directory / ("1" + suffix)).unlink() - self.initialize_lease() - return runtime, image, home - - def test_micro_interrupted_native_removal_and_missing_runtime_fail_closed(self): - runtime, image, home = self.micro_fixture() - present = [True] - def native(command, *_args, **_kwargs): - if command[1:3] == ["image", "list"]: return image + "\n" if present[0] else "" - if command[1:3] == ["image", "remove"]: - present[0] = False - raise installer.InstallError("interrupted native removal") - if command[1:3] == ["sandbox", "list"]: return "[]" - raise AssertionError(command) - installer.checked.side_effect = native - with self.assertRaises(installer.InstallError): node_generations.collect(self.args, installer) - raw = runtime.read_bytes() - runtime.unlink() - installer.checked.reset_mock() - with self.assertRaisesRegex(installer.InstallError, "executable"): - node_generations.collect(self.args, installer) - installer.checked.assert_not_called() - self.assertTrue(self.release.exists()) - runtime.write_bytes(raw) - runtime.chmod(0o700) - node_generations.collect(self.args, installer) - self.assertFalse(self.release.exists()) - self.assertTrue((home / "oac-installation.json").exists()) - - def test_docker_collection_preserves_another_installations_idle_serving_image(self): - second = self.root.parent / (self.root.name + "-second") - second.mkdir(mode=0o700) - self.addCleanup(lambda: node_generations.shutil.rmtree(second)) - # The second installation's enrolled generation still backs an idle - # serving pin. No container is needed to protect its daemon-level image. - other = dict(self.value, installation_id="second-installation") - node_generations.atomic_json(second / "provider.json", other) - before = (second / "provider.json").read_bytes() - host_images = {self.value["native"]["image"]} - def docker(command, *_args, **_kwargs): - if "rm" in command or "prune" in command: host_images.clear() - raise AssertionError("generation GC must not manage shared Docker images") - installer.checked.side_effect = docker - node_generations.collect(self.args, installer) - installer.checked.assert_not_called() - self.assertEqual(host_images, {other["native"]["image"]}) - self.assertEqual((second / "provider.json").read_bytes(), before) - self.assertFalse(self.release.exists()) - self.assertEqual(node_generations.retained_configs(self.root, installer), {}) + def test_invalid_journal_never_authorizes_collection(self): + path = self.directory / "1.dropped" + for field, invalid in (("installation_id", "foreign"), ("generation", 2), ("specification_digest", "a" * 64)): + node_generations.atomic_json(path, dict(node_generations.marker_identity(self.args), **{field: invalid})) + with self.assertRaises(installer.InstallError): + node_generations.collect(self.args, installer) + path.unlink() def test_fifo_journal_is_refused_without_blocking(self): - path = self.directory / "1.collecting" - os.mkfifo(path, 0o600) - with self.assertRaises(installer.InstallError): node_generations.collect(self.args, installer) - installer.checked.assert_not_called() - self.assertTrue(self.release.exists()) - - def test_release_path_conflict_is_refused_before_native_mutation(self): - foreign = self.root / "foreign" - self.release.rename(foreign) - self.release.symlink_to(foreign, target_is_directory=True) - with self.assertRaises(installer.InstallError): node_generations.collect(self.args, installer) - installer.checked.assert_not_called() - self.assertTrue((foreign / "artifact").exists()) + os.mkfifo(self.directory / "1.dropped", 0o600) + with self.assertRaises(installer.InstallError): + node_generations.collect(self.args, installer) if __name__ == "__main__": diff --git a/deploy/node/test_node_helper_transfer.py b/deploy/node/test_node_helper_transfer.py index f0a7a92aa..920a89067 100644 --- a/deploy/node/test_node_helper_transfer.py +++ b/deploy/node/test_node_helper_transfer.py @@ -32,7 +32,6 @@ def test_captured_archive_survives_removal_of_private_source(self): self.assertEqual(stat.S_IMODE(helper.stat().st_mode), 0o600) with zipfile.ZipFile(helper) as archive: self.assertEqual(set(archive.namelist()), {"__main__.py", "node_spec.py", "distribution.py", "node_generations.py", "install_display.py", "node_output.py", "provider_assets.py"}) - self.assertEqual(json.loads((root / "preparation.json").read_text()), {"source_url": "https://core.example"}) @unittest.skipUnless(hasattr(os, "fork") and os.geteuid() == 0, "requires a disposable Linux root test environment") def test_real_service_uid_cannot_read_private_download_but_can_install_snapshot(self): diff --git a/deploy/node/test_node_install.py b/deploy/node/test_node_install.py index 1d7190074..79f05d471 100644 --- a/deploy/node/test_node_install.py +++ b/deploy/node/test_node_install.py @@ -86,7 +86,7 @@ def setUp(self): mock.patch.object(installer, "open_request", side_effect=self.configuration_response), mock.patch.object(installer.distribution.urllib.request, "build_opener", return_value=mock.Mock(open=self.artifact_response)), mock.patch.object(installer, "micro_home", return_value=self.home / "m"), - mock.patch.object(installer, "fetch", side_effect=lambda source, name: io.BytesIO(self.payloads[name.split("/", 2)[2] if name.startswith("releases/") else name])), + mock.patch.object(installer, "fetch", side_effect=lambda source, name: io.BytesIO(self.payloads[name])), mock.patch.object(installer, "checked", side_effect=self.checked), mock.patch.object(installer.distribution, "docker_command", side_effect=self.docker_command), mock.patch.object(installer.subprocess, "run", return_value=subprocess.CompletedProcess([], 0, b"statically linked", b""))): @@ -100,7 +100,7 @@ def configuration_response(self, request, **kwargs): resources = {"cpus": self.cpus, "memory_mib": 6144} if self.args.provider == "microsandbox": resources.update(root_disk_mib=10240, environment_disk_mib=12288) - spec = {"resources": resources, "runtime": node_spec.release(self.manifest)} + spec = {"resources": resources} configuration = {"installation_id": self.args.installation_id, "provider": self.args.provider, "core_url": self.args.core_url, "generation": 1, "specification": spec, "max_active": 2, "max_retained": 8, "specification_digest": node_spec.digest(self.args.provider, spec)} @@ -109,7 +109,7 @@ def configuration_response(self, request, **kwargs): def artifact_response(self, request, **kwargs): url = getattr(request, "full_url", request) # The fixture manifest records a release URL; nodes still download from their console. - self.assertTrue(url.startswith(self.args.source_url + "/node-install/releases/" + self.manifest["source_commit"] + "/artifacts/"), url) + self.assertTrue(url.startswith(self.args.source_url + "/node-install/artifacts/"), url) for name, item in self.manifest["artifacts"].items(): if url.endswith("/" + item["filename"]): return Response(self.payloads[name]) @@ -179,263 +179,6 @@ def docker_command(self, arguments, **kwargs): def install(self): installer.prepare_service_node(self.args, "synthetic-once-token", installer.node_generations.helper_archive(self.args, installer)) - def install_current_program_with_retained_runtime(self, provider): - self.args.provider = provider - old_source, new_source = self.manifest["source_commit"], "f" * 40 - program = copy.deepcopy(self.manifest) - program["source_commit"] = new_source - payloads = dict(self.payloads) - payloads[installer.provider_assets.artifacts("docker", ("node",))[0]] = b"new protocol program from current release" - for name, artifact in program["artifacts"].items(): - artifact["filename"] = artifact["filename"].replace(old_source, new_source) - artifact["size"] = len(payloads[name]) - artifact["sha256"] = hashlib.sha256(payloads[name]).hexdigest() - payloads["manifest.json"] = json.dumps(program).encode() - payloads["SHA256SUMS"] = "".join(hashlib.sha256(raw).hexdigest() + " " + name + "\n" - for name, raw in payloads.items() if name != "SHA256SUMS").encode() - fetched = [] - def fetch(_source, name): - fetched.append(name) - if name.startswith("releases/" + old_source + "/"): - return io.BytesIO(self.payloads[name.split("/", 2)[2]]) - self.assertIn(name, ("manifest.json", "SHA256SUMS")) - return io.BytesIO(payloads[name]) - def artifact_response(request, **_kwargs): - url = getattr(request, "full_url", request) - for manifest, files in ((program, payloads), (self.manifest, self.payloads)): - prefix = self.args.source_url + "/node-install/releases/" + manifest["source_commit"] + "/artifacts/" - for name, item in manifest["artifacts"].items(): - if url == prefix + item["filename"]: - # Only the node executable comes from the host release. - self.assertEqual(manifest["source_commit"], new_source if name == installer.provider_assets.artifacts("docker", ("node",))[0] else old_source) - return Response(files[name]) - raise AssertionError("Unexpected immutable artifact URL " + url) - with mock.patch.object(installer, "fetch", side_effect=fetch), mock.patch.object(installer.distribution.urllib.request, "build_opener", return_value=mock.Mock(open=artifact_response)): - self.install() - self.assertEqual((self.root / installer.provider_assets.artifacts("docker", ("node",))[0]).read_bytes(), payloads[installer.provider_assets.artifacts("docker", ("node",))[0]]) - config = json.loads((self.root / "provider.json").read_text()) - self.assertEqual(config["specification"]["runtime"], node_spec.release(self.manifest)) - self.assertEqual(json.loads((self.root / "registered.json").read_text())["source_commit"], old_source) - self.assertIn("releases/" + old_source + "/runtime/seccomp.json", fetched) - if provider == "microsandbox": - for name in installer.MICRO: - self.assertEqual((self.root / name).read_bytes(), self.payloads[name]) - - def test_current_program_pairs_with_core_selected_retained_docker_runtime(self): - self.install_current_program_with_retained_runtime("docker") - - def test_current_program_pairs_with_core_selected_retained_micro_runtime(self): - self.install_current_program_with_retained_runtime("microsandbox") - - def test_missing_retained_runtime_never_falls_back_to_current_runtime(self): - program = copy.deepcopy(self.manifest) - program["source_commit"] = "f" * 40 - with mock.patch.object(installer, "metadata", side_effect=[(program, {}), installer.InstallError("Retained release unavailable")]) as metadata: - with self.assertRaisesRegex(installer.InstallError, "Retained release unavailable"): - self.install() - self.assertEqual(metadata.call_args_list[1].kwargs, {"prefix": "releases/" + self.manifest["source_commit"] + "/"}) - self.assertFalse((self.root / "installation.json").exists()) - self.assertFalse(any("register" in command or "load" in command or "enable" in command for command, _ in self.calls)) - - def recovery_args(self): - self.args.generation = 1 - self.args.specification_digest = json.loads((self.root / "state/node/identity.json").read_text())["identity"]["specification_digest"] - return self.args - - def prepare_successor_runtime(self): - self.manifest["source_commit"] = "f" * 40 - self.refresh_manifest() - config = json.loads(self.configuration_response(None).read()) - config["generation"] = 2 - self.args.generation = 2 - self.args.specification_digest = config["specification_digest"] - with mock.patch.object(installer.node_spec, "fetch", return_value=config): - installer.node_generations.prepare(self.args, installer) - return installer.private_json(self.root / "state/node/generations/2.json") - - def test_original_runtime_gc_interruption_preserves_restart_identity_and_successor(self): - self.args.provider = "microsandbox" - self.install() - successor = self.prepare_successor_runtime() - keep = (installer.provider_assets.artifacts("docker", ("node",))[0], "generation-preparer.pyz", "provider.json", "registered.json", - "installation.json", "preparation.json", "runtime-artifacts.json", "state/node/identity.json") - before = {name: (self.root / name).read_bytes() for name in keep} - original = installer.private_json(self.root / "provider.json") - self.args.generation = 1 - self.args.specification_digest = node_spec.digest("microsandbox", original["specification"]) - real_unlink = Path.unlink - def interrupted(path, *args, **kwargs): - if path == self.root / installer.MICRO[1]: - raise OSError("interrupted original Runtime cleanup") - return real_unlink(path, *args, **kwargs) - with mock.patch.object(Path, "unlink", new=interrupted): - with self.assertRaises(OSError): installer.node_generations.collect(self.args, installer) - self.assertFalse((self.root / installer.MICRO[0]).exists()) - self.assertTrue((self.root / installer.MICRO[1]).exists()) - self.assertTrue(installer.private_json(self.root / "state/node/generations/1.collecting")["native_complete"]) - self.assertEqual(installer.private_json(self.root / "provider.json"), original) - self.assertEqual(installer.node_generations.retained_configs(self.root, installer)[2], successor) - count = len(self.calls) - installer.node_generations.collect(self.args, installer) - self.assertEqual(len(self.calls), count) - for name in ("runtime/seccomp.json", "images/runtime.tar.gz", "images/runtime.tar") + installer.MICRO: - self.assertFalse((self.root / name).exists(), name) - for name, raw in before.items(): self.assertEqual((self.root / name).read_bytes(), raw, name) - self.assertEqual(installer.node_generations.retained_configs(self.root, installer), {2: successor}) - self.assertTrue(installer.node_generations.image_available(successor, installer)) - - def test_original_runtime_gc_refuses_changed_sibling_before_deleting_any_file(self): - self.args.provider = "microsandbox" - self.install() - self.prepare_successor_runtime() - original = installer.private_json(self.root / "provider.json") - self.args.generation = 1 - self.args.specification_digest = node_spec.digest("microsandbox", original["specification"]) - firmware = self.root / installer.MICRO[2] - firmware.write_bytes(b"changed private file") - before = {name: (self.root / name).read_bytes() for name in ("runtime/seccomp.json",) + installer.MICRO} - count = len(self.calls) - with self.assertRaisesRegex(installer.InstallError, "refusing deletion"): - installer.node_generations.collect(self.args, installer) - self.assertEqual(len(self.calls), count) - for name, raw in before.items(): self.assertEqual((self.root / name).read_bytes(), raw, name) - - def test_original_runtime_gc_missing_ownership_refuses_and_preserves_unknown_files(self): - self.install() - self.prepare_successor_runtime() - (self.root / "runtime-artifacts.json").unlink() - unknown = self.root / "native/bin/operator-owned-file" - unknown.write_bytes(b"unowned") - self.args.generation = 1 - original = installer.private_json(self.root / "provider.json") - self.args.specification_digest = node_spec.digest("docker", original["specification"]) - seccomp = (self.root / "runtime/seccomp.json").read_bytes() - with self.assertRaisesRegex(installer.InstallError, "ownership is missing"): - installer.node_generations.collect(self.args, installer) - self.assertEqual((self.root / "runtime/seccomp.json").read_bytes(), seccomp) - self.assertEqual(unknown.read_bytes(), b"unowned") - - def test_shared_original_firmware_survives_until_its_last_generation(self): - self.args.provider = "microsandbox" - self.install() - successor = self.prepare_successor_runtime() - successor["native"]["firmware_path"] = str(self.root / installer.MICRO[2]) - installer.node_generations.atomic_json(self.root / "state/node/generations/2.json", successor) - original = installer.private_json(self.root / "provider.json") - self.args.generation = 1 - self.args.specification_digest = node_spec.digest("microsandbox", original["specification"]) - installer.node_generations.collect(self.args, installer) - self.assertFalse((self.root / installer.MICRO[0]).exists()) - self.assertFalse((self.root / installer.MICRO[1]).exists()) - self.assertTrue((self.root / installer.MICRO[2]).exists()) - self.assertTrue(installer.node_generations.image_available(successor, installer)) - self.args.generation = 2 - self.args.specification_digest = node_spec.digest("microsandbox", successor["specification"]) - with installer.node_generations.collection_lease(self.root, 2, installer, installer.node_generations.marker_identity(self.args), initialize=True): - pass - def inventory(command, *_args, **_kwargs): - if command[1:3] == ["image", "list"]: return "" - if command[1:3] == ["sandbox", "list"]: return "[]" - raise AssertionError(command) - with mock.patch.object(installer, "checked", side_effect=inventory): - installer.node_generations.collect(self.args, installer) - self.assertFalse((self.root / installer.MICRO[2]).exists()) - self.assertTrue((self.root / "provider.json").exists()) - self.assertTrue((self.root / installer.provider_assets.artifacts("docker", ("node",))[0]).exists()) - - def test_original_runtime_gc_preserves_exact_shared_native_file_references(self): - self.args.provider = "microsandbox" - self.install() - original = installer.private_json(self.root / "provider.json") - other = json.loads(json.dumps(original)) - other["generation"] = 2 - directory = self.root / "state/node/generations" - directory.mkdir(mode=0o700, exist_ok=True) - installer.node_generations.atomic_json(directory / "2.json", other) - before = {name: (self.root / name).read_bytes() for name in ("runtime/seccomp.json",) + installer.MICRO} - installer.node_generations.collect(self.recovery_args(), installer) - for name, raw in before.items(): self.assertEqual((self.root / name).read_bytes(), raw, name) - self.assertEqual(installer.node_generations.retained_configs(self.root, installer), {2: other}) - - def test_interrupted_new_generation_recovers_partial_download_at_same_identity(self): - self.args.provider = "microsandbox" - self.install() - original = (self.root / "provider.json").read_bytes() - self.manifest["source_commit"] = "f" * 40 - self.refresh_manifest() - config = json.loads(self.configuration_response(None).read()) - config["generation"] = 2 - self.args.generation = 2 - self.args.specification_digest = config["specification_digest"] - real_obtain = installer.distribution.obtain_artifact - def interrupted(manifest, name, path): - if name == installer.MICRO[1]: - raise OSError("interrupted partial download") - return real_obtain(manifest, name, path) - with mock.patch.object(installer.node_spec, "fetch", return_value=config): - with mock.patch.object(installer.distribution, "obtain_artifact", side_effect=interrupted): - with self.assertRaises(OSError): - installer.node_generations.prepare(self.args, installer) - directory = self.root / "state/node/generations" - saved = json.loads((directory / "2.preparing").read_text())["configuration"] - self.assertFalse((directory / "2.json").exists()) - self.assertEqual(saved["specification"], config["specification"]) - self.assertFalse(json.loads((directory / "2.preparing").read_text())["import_started"]) - helper = Path(saved["native"]["helper_path"]) - inode = helper.stat().st_ino - self.assertFalse(Path(saved["native"]["runtime_path"]).exists()) - installer.node_generations.prepare(self.args, installer) - self.assertEqual(helper.stat().st_ino, inode) - self.assertEqual(json.loads((directory / "2.json").read_text()), saved) - self.assertFalse((directory / "2.preparing").exists()) - self.assertEqual((self.root / "provider.json").read_bytes(), original) - - def test_restart_repairs_only_missing_micro_bytes_at_original_paths(self): - self.args.provider = "microsandbox" - self.install() - before = (self.root / "provider.json").read_bytes() - helper = self.root / installer.MICRO[0] - runtime = self.root / installer.MICRO[1] - old_inode = runtime.stat().st_ino - helper.unlink() - installer.node_generations.prepare(self.recovery_args(), installer) - self.assertEqual(helper.read_bytes(), self.payloads[installer.MICRO[0]]) - self.assertEqual(runtime.stat().st_ino, old_inode) - self.assertEqual((self.root / "provider.json").read_bytes(), before) - self.assertEqual(json.loads((self.root / "state/node/generations/1.json").read_text()), json.loads(before)) - - def test_restart_refuses_conflicting_sibling_before_repairing_missing_file(self): - self.args.provider = "microsandbox" - self.install() - helper = self.root / installer.MICRO[0] - runtime = self.root / installer.MICRO[1] - helper.unlink() - runtime.write_bytes(b"conflicting retained runtime") - with self.assertRaisesRegex(installer.InstallError, "artifact checksum differs"): - installer.node_generations.prepare(self.recovery_args(), installer) - self.assertFalse(helper.exists()) - self.assertEqual(runtime.read_bytes(), b"conflicting retained runtime") - - def test_live_helper_or_collection_excludes_restart_repair(self): - self.args.provider = "microsandbox" - self.install() - helper = self.root / installer.MICRO[0] - helper.unlink() - directory = self.root / "state/node/generations" - directory.mkdir(mode=0o700, exist_ok=True) - descriptor = os.open(directory / "1.lease", os.O_CREAT | os.O_RDWR, 0o600) - try: - for mode in (fcntl.LOCK_SH, fcntl.LOCK_EX): - fcntl.flock(descriptor, mode | fcntl.LOCK_NB) - with self.assertRaisesRegex(installer.InstallError, "helper is still active"): - installer.node_generations.prepare(self.recovery_args(), installer) - self.assertFalse(helper.exists()) - fcntl.flock(descriptor, fcntl.LOCK_UN) - finally: - os.close(descriptor) - installer.node_generations.prepare(self.args, installer) - self.assertEqual(helper.read_bytes(), self.payloads[installer.MICRO[0]]) - def test_docker_installs_matched_payload_registers_and_starts_persistent_service(self): system = self.sudo_host() installer.install_system(self.args, "synthetic-once-token") @@ -488,8 +231,10 @@ def test_microsandbox_imports_image_and_allows_only_explicit_private_core_endpoi self.args.provider = "microsandbox" self.install() config = json.loads((self.root / "provider.json").read_text())["native"] - # Resources, the image and artifact hashes are read from the specification. - self.assertEqual(set(config), {"helper_path", "runtime_path", "firmware_path", "runtime_home", "checkpoint_root", "network"}) + # The image and artifact hashes come from the installation's verified manifest. + self.assertEqual(set(config), {"helper_path", "runtime_path", "firmware_path", "image", "runtime_sha256", "firmware_sha256", "runtime_home", "checkpoint_root", "network"}) + self.assertEqual((config["image"], config["runtime_sha256"], config["firmware_sha256"]), + (self.manifest["runtime_ref"], self.manifest["microsandbox"]["runtime_sha256"], self.manifest["microsandbox"]["firmware_sha256"])) rules = config["network"]["rules"] self.assertIn({"action": "allow", "direction": "egress", "destination": "172.29.144.1", "protocol": "tcp", "port": "24443"}, rules) self.assertNotIn("private", [rule["destination"] for rule in rules]) @@ -630,12 +375,12 @@ def test_different_commit_provider_or_core_cannot_overwrite_retained_installatio for field, value in (("provider", "microsandbox"), ("core_url", "https://other.example")): before = getattr(self.args, field) setattr(self.args, field, value) - with self.assertRaisesRegex((installer.InstallError, node_spec.SpecificationError), "differs|prepares the changed"): + with self.assertRaisesRegex((installer.InstallError, node_spec.SpecificationError), "differs|prepares the changed|added with another Core address"): self.install() setattr(self.args, field, before) self.manifest["source_commit"] = "f" * 40 self.refresh_manifest() - with self.assertRaisesRegex((installer.InstallError, node_spec.SpecificationError), "differs|prepares the changed"): + with self.assertRaisesRegex((installer.InstallError, node_spec.SpecificationError), "differs|prepares the changed|added with another Core address"): self.install() self.assertEqual((self.root / "provider.json").read_bytes(), original) @@ -654,7 +399,7 @@ def test_installed_payload_and_config_are_not_overwritten(self): self.install() target = self.root / installer.provider_assets.artifacts("docker", ("node",))[0] target.write_bytes(b"existing-different-payload") - with self.assertRaisesRegex(installer.InstallError, "Remove node.*Add node"): + with self.assertRaises(installer.distribution.DistributionError): self.install() self.assertEqual(target.read_bytes(), b"existing-different-payload") @@ -1217,13 +962,14 @@ def test_a_registered_node_keeps_its_state_when_the_configuration_changes(self): self.install() self.assertEqual((self.root / "provider.json").read_bytes(), saved) - def test_a_registered_node_given_a_newer_release_says_to_remove_and_add_it_again(self): + def test_a_registered_node_of_another_installation_says_to_remove_and_add_it_again(self): self.install() program = self.root / installer.provider_assets.artifacts("docker", ("node",))[0] saved = program.read_bytes() - self.payloads[program.relative_to(self.root).as_posix()] = b"program of the upgraded Core" + self.manifest["source_commit"] = "f" * 40 + self.payloads[program.relative_to(self.root).as_posix()] = b"program of another installation" self.refresh_manifest() - with self.assertRaisesRegex(installer.InstallError, "Remove node on the Nodes page.*Add node"): + with self.assertRaisesRegex(installer.InstallError, "added with another Core address.*Remove node on the Nodes page.*Add node"): self.install() self.assertEqual(program.read_bytes(), saved) diff --git a/deploy/node/test_node_spec.py b/deploy/node/test_node_spec.py index e4f8da322..e8e39d16e 100644 --- a/deploy/node/test_node_spec.py +++ b/deploy/node/test_node_spec.py @@ -14,11 +14,7 @@ class SpecificationTests(unittest.TestCase): def setUp(self): self.args = argparse.Namespace(core_url="https://core.example", installation_id="94be54a1-138c-4f30-bc87-b13686272dbe", provider=None) - self.spec = {"resources": {"cpus": 2, "memory_mib": 4096}, "runtime": { - "source_commit": "a" * 40, "image_id": "sha256:" + "b" * 64, - "image_manifest_digest": "sha256:" + "c" * 64, - "microsandbox_ref": "oac-runtime@sha256:" + "d" * 64, - "runtime_sha256": "e" * 64, "firmware_sha256": "f" * 64}} + self.spec = {"resources": {"cpus": 2, "memory_mib": 4096}} self.data = {"installation_id": self.args.installation_id, "provider": "docker", "generation": 3, "specification": self.spec, "specification_digest": node_spec.digest("docker", self.spec), "core_url": self.args.core_url, "max_active": 2, "max_retained": 8} @@ -112,7 +108,6 @@ def test_capacity_requires_approved_bounded_integers(self): def test_digest_is_independent_of_response_object_key_order(self): reordered = copy.deepcopy(self.spec) - reordered["runtime"] = dict(reversed(list(reordered["runtime"].items()))) reordered["resources"] = dict(reversed(list(reordered["resources"].items()))) self.assertEqual(node_spec.digest("docker", reordered), node_spec.digest("docker", self.spec)) diff --git a/deploy/test_install_source.py b/deploy/test_install_source.py index bef257254..af326e433 100644 --- a/deploy/test_install_source.py +++ b/deploy/test_install_source.py @@ -68,7 +68,7 @@ def commands(self): return [args[args.index('--project-directory') + 2:] if '--project-directory' in args else args[2:] for args, _ in self.calls] def snapshot(self): - return {name: (self.destination / name).read_bytes() for name in ('compose.yaml', 'compose.override.yaml', '.env', 'oac', 'node-payload/active.json')} + return {name: (self.destination / name).read_bytes() for name in ('compose.yaml', 'compose.override.yaml', '.env', 'oac', 'node-payload/manifest.json')} def test_complete_compose_preserves_secrets_and_serves_whole_payload(self): self.install() @@ -84,8 +84,7 @@ def test_complete_compose_preserves_secrets_and_serves_whole_payload(self): self.assertTrue(mounts['/node-payload']['read_only']) self.assertEqual(services['init']['environment']['OAC_REVISION'], self.revision) self.assertEqual(services['web']['environment']['OAC_PUBLIC_URL'], 'http://127.0.0.1:19873') - self.assertEqual(json.loads((self.destination / 'node-payload/active.json').read_text()), {'source_commit': self.revision}) - self.assertTrue((self.destination / 'node-payload/releases' / self.revision / 'node-install.pyz').is_file()) + self.assertTrue((self.destination / 'node-payload/node-install.pyz').is_file()) self.assertEqual(os.stat(self.destination / '.env').st_mode & 0o777, 0o600) self.assertIn('--pull', self.calls[-2][0]) self.assertIn('never', self.calls[-2][0]) diff --git a/docs/configuration.md b/docs/configuration.md index 2e2f1a133..84e2cc249 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -56,7 +56,7 @@ Runtime settings live in Core's database. Change them in Web; scripts use the sa | Setting | Where in Web | Core API | Notes | | --- | --- | --- | --- | | Sandbox backend: Docker, microsandbox or E2B | **Settings** → **Sandbox and scheduling**: the setup wizard, ending with **Save configuration** | `/core/v1/sandbox/deployment` | One backend per installation, chosen after the first sign-in. Another backend needs **Reset deployment** first; see [change the sandbox configuration](./getting-started/nodes.md#change-the-sandbox-configuration) | -| Sandbox size, Runtime release, E2B key and template build | **Settings** → **Sandbox and scheduling** → **Change resources** | `/core/v1/sandbox/deployment` | Web proposes the [default size the Provider declares](./sandbox-provider.md#register-the-provider-kind). Existing sandboxes keep their size and release. The E2B key is write-only and encrypted | +| Sandbox size, E2B key and template build | **Settings** → **Sandbox and scheduling** → **Change resources** | `/core/v1/sandbox/deployment` | Web proposes the [default size the Provider declares](./sandbox-provider.md#register-the-provider-kind). Existing sandboxes keep their size. The E2B key is write-only and encrypted | | Nodes and their capacity | **Nodes**: **Add node**, **Edit node** and removal | `/core/v1/sandbox/enrollment-tokens`, `/core/v1/sandbox/nodes` | See [Node capacity](#node-capacity) and [Add and manage nodes](./getting-started/nodes.md) | | Idle suspension | **Settings** → **Sandbox and scheduling** → **Automatic suspension** → **Idle before suspension (seconds)** → **Save** | `PATCH /core/v1/sandbox/deployment` | Deployment-wide; see [suspension policy](../contracts/agents-api/sandbox-deployment.md#suspension-policy) | | Independent workspace storage and tenant capacity | **Settings → Workspace storage** | `/core/v1/workspace-storage` | See [Set up workspace storage](./getting-started/workspace-storage.md); existing workspaces retain their configuration | @@ -107,12 +107,12 @@ The named Docker volume `_data` contains these paths. Docker manages Li ## Docker node configuration -The node installer writes Docker’s host settings into the `native` object of the node’s configuration file, which only the Docker adapter reads. Deployment resources, the Runtime release and capacity remain in [Core’s database](#runtime-settings-web). +The node installer writes Docker’s host settings into the `native` object of the node’s configuration file, which only the Docker adapter reads. Deployment resources and capacity remain in [Core’s database](#runtime-settings-web). | Field | Installer value | Meaning | | --- | --- | --- | | `host` | `unix:///var/run/docker.sock` | Explicit Docker Engine socket | -| `image` | The Runtime image’s local ID after loading | The release’s `image_id` or `image_manifest_digest`. The host’s image store decides which digest names the loaded image, so the value is node-local; the adapter accepts only these two | +| `image` | The Runtime image’s local ID after loading | The installation’s Runtime image. The host’s image store decides whether its local ID is the manifest’s `image_id` or `image_manifest_digest`, so the value is node-local | | `network` | `oac-node-` | Runtime container network | | `seccomp_file` | `/runtime/seccomp.json` | Matched distribution’s seccomp profile | | `nested_sandbox` | `true` | Enables the Docker adapter’s init process and proc-mask configuration | diff --git a/docs/getting-started/nodes.md b/docs/getting-started/nodes.md index b513200d1..4f3218c58 100644 --- a/docs/getting-started/nodes.md +++ b/docs/getting-started/nodes.md @@ -70,7 +70,7 @@ The node service keeps the proxy settings of the latest run, so it reaches Core | Docker | The Runtime image, imported once, and a network `oac-node-` | | microsandbox network policy | Sandboxes may reach Core, DNS on the host and public addresses, and nothing else: no inbound connections and no private networks | -Root only prepares the account, the group and the unit; everything else, the Docker network included, runs as `oac-node`, in its own session without a terminal. The installer never installs Docker, KVM or packages, never starts Docker, never changes device permissions, sudoers, firewall or SELinux settings, and never touches other accounts. +Root only prepares the account, the group and the unit; everything else, the Docker network included, runs as `oac-node`, in its own session without a terminal. The installer never installs Docker, KVM or packages, never starts Docker, never changes device permissions, sudoers, firewall or SELinux settings, and never touches other accounts. The node connects out to Core; Core needs no SSH or Docker TCP access to the host. > [!CAUTION] > Docker mode is root-equivalent. Membership in the `docker` group lets `oac-node`, and so anything that controls the node, act as root on the host, and containers share the host kernel. Add Docker nodes only on hosts dedicated to sandboxes. microsandbox nodes need only the `kvm` group and run each sandbox in its own microVM. @@ -80,12 +80,12 @@ Root only prepares the account, the group and the unit; everything else, the Doc **One Core per host.** Nodes share the `oac-node` account, so a host serves one Core; a command from a second Core is refused. -**Node software.** A node keeps the host program of the console that added it. To move a node to another release, [remove it](#remove-a-node) and add it again. +**Node software.** A node runs the host program and Runtime of the installation that added it. Another installation's command is refused on that host until you [remove the node](#remove-a-node) and add it again. ## Rerun, expiry and slow links -- Rerunning the same command is safe. Once the node is registered, a rerun uses the node's own credential, changes nothing that already matches and needs no token. It changes nothing and says why when the node's retained identity belongs to another Core address or installation, when the sandbox configuration has changed since the node registered (the node prepares it itself; see [Change the sandbox configuration](#change-the-sandbox-configuration)), or when Core was upgraded and serves a newer node program: choose **Remove node** on **Nodes**, uninstall the node on the host, then add it again with **Add node**. -- A run that fails before the node registers leaves nothing that Core knows. The next command, even after a configuration change or a Core upgrade, replaces the earlier attempt's configuration and any downloaded file that no longer matches, keeps the files that match and continues. +- Rerunning the same command is safe. Once the node is registered, a rerun uses the node's own credential, changes nothing that already matches and needs no token. It changes nothing and says why when the node's retained identity or node program belongs to another Core address or installation, or when the sandbox configuration has changed since the node registered (the node prepares it itself; see [Change the sandbox configuration](#change-the-sandbox-configuration)): choose **Remove node** on **Nodes**, uninstall the node on the host, then add it again with **Add node**. +- A run that fails before the node registers leaves nothing that Core knows. The next command, even after a configuration change, replaces the earlier attempt's configuration and any downloaded file that no longer matches, keeps the files that match and continues. - A command that already expired, or was used on another host, fails at once with `The enrollment command expired or was already used`: generate a new command and run it within an hour. - If the command expires during a slow download, registration fails with `The enrollment command expired or was already used`. The downloaded files are kept: generate a new command in Web and run it, and the download continues where it stopped. - Every file shows its progress. Failed or interrupted downloads keep what arrived, and a rerun resumes each partial file where it stopped (`Resuming …`), reuses verified complete files and downloads the rest. If the server cannot resume, the file starts again from the beginning. A transfer that brings less than 64 KiB in a minute stops; check the network and rerun the command. @@ -102,8 +102,7 @@ Run `sudo journalctl -u oac-node-.service`. In a root shell, om Sandbox settings apply to the whole installation; nodes follow them. -- **Size, Runtime release, E2B key or template build.** Open **Settings** → **Sandbox and scheduling** → **Change resources**, edit and save. Existing sandboxes keep their configuration. Each node prepares the new one while it keeps serving the old one, and **Nodes** shows its progress: **Preparing target**, **Ready for target** or **Preparation failed** with the [reason](#readiness-codes). Core places new Sessions on nodes ready for the new configuration first, and on nodes still serving an older one when those have no room. E2B changes apply at once. -- **The installation's Runtime.** When the saved Runtime differs from the one the installation serves, **Settings** → **Sandbox and scheduling** shows a **Runtime** row with both source commits. **Use this installation's Runtime** opens **Change resources** on its review with that Runtime, and **Save configuration** starts the same node preparation. Other changes keep the saved Runtime. When the console serves no Runtime manifest, enter the release under **Advanced settings**. +- **Size, E2B key or template build.** Open **Settings** → **Sandbox and scheduling** → **Change resources**, edit and save. Existing sandboxes keep their configuration. Each node prepares the new one while it keeps serving the old one, and **Nodes** shows its progress: **Preparing target**, **Ready for target** or **Preparation failed** with the [reason](#readiness-codes). Core places new Sessions on nodes ready for the new configuration first, and on nodes still serving an older one when those have no room. E2B changes apply at once. - **Capacity and idle suspension.** On **Nodes**, use **Edit node** for a node's active and retained limits. In **Settings** → **Sandbox and scheduling**, use **Automatic suspension** → **Idle before suspension (seconds)** → **Save** for the deployment-wide [idle policy](../../contracts/agents-api/sandbox-deployment.md#suspension-policy). - **Backend.** On the same page, choose **Reset deployment**. Auto reset archives idle hosted Sessions at once and lets running work finish until the deadline you set; force cancels it now. Offline nodes must come back so Core can confirm their cleanup. When the reset completes, Core has retired every node and unused command: set up the new backend, then add nodes again. **Cancel reset** stops the remaining work; archived Sessions stay archived. @@ -124,35 +123,6 @@ Uninstall first asks Core, at the address the node enrolled with, whether the no It never deletes Docker images or volumes. It keeps the Runtime image and prints the `sudo docker image rm` command. For microsandbox, once Core has confirmed the removal, it also deletes the node's store under `/var/lib/oac-node/.oac/m/` (about 1.5 GB), and with it the created account. With `--force` Core isn't asked, so the store is kept and the output prints the command that deletes it (`sudo -u oac-node rm -rf `); microVMs may still use it, so check `pgrep -u oac-node` first, then rerun uninstall to finish removing the account. The output lists what was deleted and what was kept. Uninstall can be rerun until it completes. -## Register a node manually - -Use manual registration when you manage the node's files and service yourself instead of running Web's command. A manually registered node serves only the configuration it registered with: after a size or Runtime change, **Nodes** shows it as **Node software incompatible**, and it keeps serving the old configuration until you remove it and register the host again. - -1. Take `oac-node` from the same release as Core. -2. Get an enrollment token: the token in a command from **Add node**, or `POST /core/v1/sandbox/enrollment-tokens` with the Core key. It is single-use and carries the node's approved capacity; the response's `expires_at` says when it expires. Save it in a `0600` file on the host. -3. Read the node configuration with the token, which does not consume it: `GET /api/v1/sandbox-node/configuration` with `Authorization: Bearer `. -4. Write a private provider file. Copy `provider`, `installation_id`, `core_url`, `generation` and `specification` from the response, and add a `native` object with the host settings of that provider. The adapter reads sandbox size, the Runtime image and artifact hashes from `specification`: - - Docker: the [Docker node configuration](../configuration.md#docker-node-configuration) fields, with `host` an explicit Unix socket, `image` the local ID of the imported Runtime image and `seccomp_file` absolute. - - microsandbox: absolute `helper_path`, `runtime_path` and `firmware_path`, a `network` policy, an explicit `checkpoint_root` for private checkpoint archives, and `runtime_home`: a private directory, which the helper creates with mode `0700` when it is missing. microsandbox places Unix sockets under it, so keep its path within 48 bytes; the installer refuses a longer one for its own nodes. - Before the first microsandbox registration, initialize the empty `checkpoint_root` as the node service account using the same release's source checkout: `PYTHONPATH=deploy/node python3 -c 'from node_install import prepare_checkpoint_root; prepare_checkpoint_root("/absolute/private/checkpoints")'`. This reuses the installer's exclusive UUID-marker initialization and ownership checks. Initialize a shared store once; other nodes use the existing marker. Never replace the marker or initialize over restored or nonempty unowned storage. -5. Register, then run the node under the host's service supervisor, with real absolute paths: - - ```sh - oac-node register \ - --config /var/lib/oac/provider.json \ - --state-dir /var/lib/oac/node \ - --core-url https://core.example \ - --name worker-1 \ - --enrollment-token-file /var/lib/oac/enrollment-token - oac-node run \ - --config /var/lib/oac/provider.json \ - --state-dir /var/lib/oac/node - ``` - -`oac-node run` exits with status 78 once Core rejects its credential, after the node is removed; configure the supervisor not to restart it then (systemd: `RestartPreventExitStatus=78`). - -The node connects out to Core; Core needs no SSH or Docker TCP access to the host. Registration writes the node's identity to the state directory before contacting Core, so a lost response can be retried under the same identity. Keep the state directory on persistent storage, private to the node's account and used by one process at a time. Never copy it to another directory or host: Core refuses a second connection for a node while the first is open. The provider file can't change the node's capacity or sandbox configuration. Core compares its digest at registration and on every connection, and a node whose file differs takes no work until the approved configuration is restored. - ## When a node host fails A restarted node service keeps its identity and finds its existing sandboxes again. Core does not take over an active or unconfirmed writer merely because its node is unreachable or its sandbox is missing: those resources show as **Node disconnected** or **Sandbox resource missing** while ownership remains unresolved. Restart a crashed or hung host rather than removing its node: after the node reconnects, a running sandbox that the node confirms gone is released under the [confirmed compute loss](../sandbox-provider.md#confirmed-compute-loss) rules, and the Session's next input continues on new compute; any other missing sandbox keeps **Sandbox resource missing**. A host that never returns keeps its Sessions' compute ownership. A settled retained checkpoint can follow the [checkpoint transfer contract](../sandbox-provider.md#checkpoint-transfer), and confirmed release can permit [cold replacement](../sandbox-provider.md#cold-replacement-after-checkpoint-retention). A lost node state directory is a recovery incident: restore it from its [backup](./operations.md#back-up) together with the database and the provider storage, rather than registering the host again over existing resources. @@ -163,22 +133,18 @@ Connection failures and installer messages are in [Troubleshooting](./troublesho ### Readiness codes -When a node is online but its sandbox provider is not ready, **Nodes** and **Overview** show it as **Provider not ready**. Where the reason appears depends on how the node was added: - -- **Nodes added with Web's command** report a failed check of the current sandbox configuration as **Preparation failed** in the node's target status, with the reason in a help tip (`rollout.diagnostic` in `GET /core/v1/sandbox/nodes`). The tip beside **Provider not ready** only says *Sandbox provider unavailable*. -- **Manually registered nodes** show the reason in the help tip beside **Provider not ready** (`diagnostic`). +When a node is online but its sandbox provider is not ready, **Nodes** and **Overview** show it as **Provider not ready**. A node reports a failed check of the current sandbox configuration as **Preparation failed** in the node's target status, with the reason in a help tip (`rollout.diagnostic` in `GET /core/v1/sandbox/nodes`). The tip beside **Provider not ready** only says *Sandbox provider unavailable*. -Each code is a Provider-neutral class; the local error behind it stays on the node. A manually registered node logs it. For a node added with Web's command, rerun the command on the host: the installer checks the host requirements first and names what to fix ([Installer messages](./troubleshooting.md#installer-messages)). +Each code is a Provider-neutral class; the local error behind it stays on the node. Rerun the command on the host: the installer checks the host requirements first and names what to fix ([Installer messages](./troubleshooting.md#installer-messages)). -A node reports only its first failed check, in this order: the Docker daemon or KVM, Docker's limit support, host capacity, then the installed Runtime files. An unreachable Docker daemon therefore hides a missing image. The next heartbeat, about ten seconds after a fix, clears or replaces the code. An offline node keeps its last code, which Web hides until the node reconnects. +A node reports only its first failed check, in this order: the Docker daemon or KVM, Docker's limit support, host capacity, then the installed Runtime files. An unreachable Docker daemon therefore hides a missing image. The next preparation attempt after a fix clears or replaces the code. | Code | Help tip | Cause | Fix | | --- | --- | --- | --- | -| `provider_unavailable` | Sandbox provider unavailable | The provider's service is unreachable or fails a request, such as a stopped Docker daemon, or a failure without a class | Rerun the node's command, or read a manually registered node's log | -| `host_unsupported` | Host unsupported | The host lacks a capability the provider requires, such as Docker CPU and memory limits (cgroup v2) or read-write access to `/dev/kvm` | Rerun the node's command, or read a manually registered node's log | +| `provider_unavailable` | Sandbox provider unavailable | The provider's service is unreachable or fails a request, such as a stopped Docker daemon, or a failure without a class | Rerun the node's command | +| `host_unsupported` | Host unsupported | The host lacks a capability the provider requires, such as Docker CPU and memory limits (cgroup v2) or read-write access to `/dev/kvm` | Rerun the node's command | | `capacity_insufficient` | Host too small | The host has fewer CPUs or less memory than one sandbox | Use a larger host, or change the sandbox size | -| `runtime_image_unavailable` | Runtime image missing | The provider does not have the pinned Runtime image | A node added with Web's command downloads it again by itself; otherwise load the image from the matching release | -| `artifacts_unavailable` | Provider files missing | A pinned provider file, such as the microsandbox Runtime, firmware or helper, is missing or fails its SHA-256 check | A node added with Web's command downloads the missing files by itself; otherwise restore them from the matching release | -| `runtime_download_failed` | Runtime download failed | While preparing a new configuration, the node could not download or verify the Runtime files | Check the node's HTTPS access to the console and the release. The node retries with growing delays, up to 30 minutes apart | +| `runtime_image_unavailable` | Runtime image missing | The provider does not have the installation's Runtime image | Rerun the node's command to download it again; otherwise load the image from the matching release | +| `artifacts_unavailable` | Provider files missing | A pinned provider file, such as the microsandbox Runtime, firmware or helper, is missing or fails its SHA-256 check | Rerun the node's command to download the missing files; otherwise restore them from the matching release | A new group membership applies only to a new process. Restart the node service: `sudo systemctl restart oac-node-.service`. diff --git a/docs/getting-started/troubleshooting.md b/docs/getting-started/troubleshooting.md index ebe07bb43..f631fd636 100644 --- a/docs/getting-started/troubleshooting.md +++ b/docs/getting-started/troubleshooting.md @@ -42,7 +42,7 @@ Add node shows no command: see [Before you add a node](./nodes.md#before-you-add A microsandbox sandbox cannot reach a model provider or MCP server at a private address, and the Turn fails once the Harness stops retrying. Sandbox DNS queries go to the nameservers listed in the host's `/etc/resolv.conf`, and DNS rebind protection discards any answer that contains a private address. A name those nameservers resolve to a private address therefore does not resolve in the sandbox at all, even for a public service such as a model API behind a cloud VPC resolver. -Installer-added nodes have no setting to change this. Make those nameservers return the service's public addresses. microsandbox does not read the host's `/etc/hosts`, so an entry there helps only when `/etc/resolv.conf` points at a local stub resolver that serves it, such as systemd-resolved at `127.0.0.53`. Otherwise use a Docker node, which does not filter sandbox traffic, or [register the node manually](./nodes.md#register-a-node-manually) with a `network` policy that also allows the private range in a rule without a port, because rebind protection ignores port-scoped rules. +Nodes have no setting to change this. Make those nameservers return the service's public addresses. microsandbox does not read the host's `/etc/hosts`, so an entry there helps only when `/etc/resolv.conf` points at a local stub resolver that serves it, such as systemd-resolved at `127.0.0.53`. Otherwise use a Docker node, which does not filter sandbox traffic. ### Connection failures @@ -63,7 +63,7 @@ A registered node that can't connect to Core says why in its [log](./nodes.md#lo | `The enrollment command expired or was already used` | Generate a new command in Web and run it within an hour | | `Core rejected the node configuration read (HTTP …)` | Check the Core URL and the reverse proxy in front of it, then rerun | | `Not enough disk space under …` or `The disk is full` | Free the space it names, then rerun; the download resumes | -| This host's node program differs from the one this console serves | Core was reinstalled after the node was added. Choose **Remove node**, uninstall it on the host, then add it again with **Add node** | +| This host's node was added with another Core address, release or provider | Choose **Remove node**, uninstall it on the host, then add it again with **Add node** | | Another node installation is running, or this node is preparing a new sandbox configuration | Retry once **Nodes** no longer shows **Preparing target** | | A sandbox provider check failed before registering, such as `This host lacks a capability …` | Fix what it names (see [Readiness codes](./nodes.md#readiness-codes)) and rerun; the token was not used | | Core's public URL changed after this command was generated | Generate a new command in Web and run it | diff --git a/docs/sandbox-provider.md b/docs/sandbox-provider.md index 7120e8af4..71dd64cb7 100644 --- a/docs/sandbox-provider.md +++ b/docs/sandbox-provider.md @@ -121,14 +121,14 @@ Hosted and self-hosted Environments use the same Runtime preparation; a provider ## Register the provider kind -`sandbox/providers/registry.go` is the only registration table. Each entry binds the adapter's specification and resource validators, its `sandbox.ConfigurationAdapter`, the deployment mode (`nodes` or `direct`), its `sandbox.DeploymentPolicy` (disk limits, the Runtime input and the default size setup proposes), the operation declaration and a node-local (`BuildLocal`) or direct (`BuildDirect`) constructor. `providers.Build` and `providers.BuildDirect` construct adapters without allocating compute. There is no init-time registration or plugin loading. +`sandbox/providers/registry.go` is the only registration table. Each entry binds the adapter's specification and resource validators, its `sandbox.ConfigurationAdapter`, the deployment mode (`nodes` or `direct`), its `sandbox.DeploymentPolicy` (disk limits and the default size setup proposes), the operation declaration and a node-local (`BuildLocal`) or direct (`BuildDirect`) constructor. `providers.Build` and `providers.BuildDirect` construct adapters without allocating compute. There is no init-time registration or plugin loading. A new provider takes these steps: 1. Implement the operation contracts in the adapter package, with native contract tests. 2. Add its specification and resource validators. 3. Implement `sandbox.ConfigurationAdapter` over a typed native configuration. `DecodeInput` strictly parses the separate public `configuration` and write-only `credential` objects of a request. `Encode` produces whitelisted public selectors, read-only observations and separate secret bytes, and never passes request JSON through. `Decode` restores stored selectors, and keeps access to owned resources, without remote admission or new template validation. `Normalize` copies its input before changing it. `ResolveChange`, `Equal` and `WithCredential` own inheritance, identity and credential composition. `Requirements` declares whether a credential and a public Core origin are required, and which setup operations are supported: `Discovery` for `DiscoverConfiguration`, `SelectionDiscovery` for `DiscoverSelection` and `CredentialVerification` for `VerifyCredential`. `DiscoverConfiguration` validates the query and returns a safe catalog, never a mutation or an admission decision, while Core keeps authorization, input limits and deadlines. `DiscoverSelection` resolves a candidate's omitted native values before commit, and `VerifyCredential` verifies a credential's access to owned resources without mutation. Both receive the candidate's `sandbox.DirectConfig` and build any native client for that call only. A node provider accepts only an empty public object, rejects credentials and returns Unsupported for every setup operation and for credential replacement. -4. For a node adapter, export from its package the `BuildLocal` constructor, the typed `native` object it decodes and the native files it adds to the shared node artifacts. Node-local settings, such as host paths, live only in that object; resources and the Runtime release are read from the node configuration's `specification`. +4. For a node adapter, export from its package the `BuildLocal` constructor, the typed `native` object it decodes and the native files it adds to the shared node artifacts. Node-local settings, such as host paths, live only in that object; resources are read from the node configuration's `specification`, and the Runtime image comes from the same object. 5. Register its constructor, policies, configuration adapter and operation declaration in `providers/registry.go`. Its key is the provider kind, which also labels the Provider's observations, and checkpoint support reads this entry. The generated projections combine each registered mode and deployment policy with the shared field bounds in `sandbox/deployment_contract.go`: the installer reads them from `deploy/node/node_spec.py`, and the TypeScript client and Web from `packages/agents-client/src/deployment-contract.ts`, so Web reads these declarations instead of comparing provider kinds. Regenerate both with `go run ./services/core/cmd/specification-contract -write`. 6. Supply the distribution artifacts for the adapter and its helper, and offer the provider to operators through the registered configuration contract. 7. Web's setup step, deployment summary and client projection name each provider. Add the new provider to `apps/web` until the protocol declares configuration fields. @@ -142,7 +142,6 @@ A node configuration, `sandbox.NodeConfig`, holds `provider`, `generation`, `ins - A `nodes` registration has only `BuildLocal`, and a `direct` registration only `BuildDirect`; missing, mixed or unknown modes are rejected. - The specification and resource validators, the configuration adapter and a complete operation declaration are mandatory, so an incomplete registration cannot publish a partial projection. A declared default size must pass the adapter's resource validator. -- The Runtime input policy either accepts the pinned Runtime or gives the adapter's fixed reason for rejecting it, never both. - Checkpoint is admitted only for a `nodes` registration, because the common lifecycle suspends only node allocations; registration rejects a `direct` Provider that declares it. A registration carries no suspension values: Core applies its one [suspension policy](#suspension) to every Provider that declares checkpoint support. The configuration adapter must be non-nil, including its concrete value. Every `ConfigurationRequirements` field needs an explicit valid decision: `Credential` and `PublicOrigin` are `Required` or `NotRequired`, and `Discovery`, `SelectionDiscovery` and `CredentialVerification` use the shared supported or unsupported declaration with a safe reason. A new requirement field needs an explicit validation update and never inherits an existing decision. Requiring a credential does not promise the `VerifyCredential` operation. These checks establish complete registration, not correct native SDK behavior; constructor and adapter contract tests still apply. @@ -252,7 +251,7 @@ A Provider relies on these rules: - Deletion, revocation and retention expiry win over wake, up to the final database compare-and-swap. Unknown cleanup identities are kept until owned resources are confirmed absent. ### Checkpoint transfer -A checkpoint-capable generation reports `CheckpointCompatibility`: opaque `ArtifactDomain` and `ExecutionClass` tokens. Every verified `SnapshotIdentity` carries the same qualification. Core compares these values without interpreting CPU features, filesystem paths or storage implementations. A restore destination must be online and ready for the allocation's immutable deployment generation, match both tokens, and have active capacity plus a retained slot when moving from another node. The allocation, Device and Session identities remain unchanged. The Session and deployment transaction commits the destination route, placement and exact restore intent before target-side native work. No capacity or compatible destination leaves the retained snapshot owned until its configured deadline. The target must already hold that exact generation; Core does not automatically prepare historical generations on new nodes. During upgrades, retain eligible nodes and their generation providers until their checkpoint retention obligations end. +A checkpoint-capable generation reports `CheckpointCompatibility`: opaque `ArtifactDomain` and `ExecutionClass` tokens. Every verified `SnapshotIdentity` carries the same qualification. Core compares these values without interpreting CPU features, filesystem paths or storage implementations. A restore destination must be online and ready for the allocation's immutable deployment generation, match both tokens, and have active capacity plus a retained slot when moving from another node. The allocation, Device and Session identities remain unchanged. The Session and deployment transaction commits the destination route, placement and exact restore intent before target-side native work. No capacity or compatible destination leaves the retained snapshot owned until its configured deadline. The target must already hold that exact generation; Core does not automatically prepare historical generations on new nodes. A node keeps a superseded generation's provider until its checkpoint retention obligations end. `Suspend` may report `SourceStopped` only after publishing a verified full archive durably, stopping the exact source compute and settling source-local capture and cleanup obligations. Core persists the snapshot before marking it suspended; native absence alone is insufficient. `Suspend.ObserveOnly` may finish archive publication and source cleanup for an already verified snapshot of the same operation, but cannot capture another snapshot or stop a source that has resumed running. If a previously published archive is missing or damaged, `ObserveOnly` may return its exact durable ownership receipt with `Status: unknown` and `SourceStopped: false` for cleanup; that receipt proves neither current archive usability nor source absence. `Resume` still verifies the archive before execution. After the source-settlement barrier, source-node unavailability does not prevent restoration. Deletion and ordinary expiry can transfer an idle suspended allocation's cleanup route to an online node in the same artifact domain; deletion does not require execution-class compatibility. The cleanup destination must already serve the same immutable generation and have retained capacity. Without such a node, cleanup stays pending and ownership is not released; operators must restore an eligible node or its capacity. An unknown running or restoring writer never moves merely because its node is unreachable. diff --git a/docs/web/console-api-usage.md b/docs/web/console-api-usage.md index 4c7767c1c..02899aa06 100644 --- a/docs/web/console-api-usage.md +++ b/docs/web/console-api-usage.md @@ -8,7 +8,7 @@ This page lists the Core routes each console page calls and how the console boun | Interface | Paths | Authentication | Console use | | --- | --- | --- | --- | -| Console server | `/console/auth`, `/console/auth/{login,logout}`, `/console/config`, `/node-install/manifest.json` | The Core key at sign-in, then the console session cookie; `/node-install/manifest.json` needs no sign-in | Sign-in and sign-out; the node installer and node artifacts for Add node; the distribution's Runtime release for Docker and microsandbox setup and for the Runtime row in Settings → Sandbox and scheduling. See [Console server](./console-server.md) | +| Console server | `/console/auth`, `/console/auth/{login,logout}`, `/console/config` | The Core key at sign-in, then the console session cookie | Sign-in and sign-out; the node installer and node artifacts for Add node. See [Console server](./console-server.md) | | Administrator API | `/core/v1/**` outside `/core/v1/sandbox` | The Core key, added by the console server | Projects, keys, resource reads and deletion, diagnostics, executor credentials and installation commands, provenance, summaries, Core metrics, the installation, default models | | Sandbox administration | `/core/v1/sandbox/**` | The Core key, added by the console server | Sandbox configuration, Nodes, the fleet and capacity summary on Overview, Runtime observations of every project on Sandbox metrics | | Agents API | `/v1/**` | Project API key | Not used. The console shows developers how to call it (see [Provenance and monitoring](#provenance-and-monitoring)) | @@ -103,10 +103,10 @@ The list carries each Harness's configuration, so the console does not read `GET | Operation | Route | Console use | | --- | --- | --- | -| Deployment | `GET`, `POST`, `PUT /core/v1/sandbox/deployment` | Read the provider, the read-only `core_url` (`OAC_PUBLIC_URL`, shown in the setup review and never sent), reset state, installation ID and specification; a 409 `sandbox_configuration_error` (E2B with a loopback `public_url`) shows the shared client's fixed safe address-configuration message in the setup wizard, with Managed in Settings leading to Settings → Service and deployment, and leaves nothing to confirm; initialize the deployment with `resources` and the Docker or microsandbox `runtime` release, or with the E2B account and no `resources` (Core adopts the template build's CPU and memory); change its settings with the expected generation. E2B's `metadata.template_build` (status, CPU, memory, disk) shows in Settings → Sandbox and scheduling and Sandbox metrics, and sizes each sandbox when `specification.resources` is missing; microsandbox's `suspension` (idle and retention seconds) shows in Settings → Sandbox and scheduling | +| Deployment | `GET`, `POST`, `PUT /core/v1/sandbox/deployment` | Read the provider, the read-only `core_url` (`OAC_PUBLIC_URL`, shown in the setup review and never sent), reset state, installation ID and specification; a 409 `sandbox_configuration_error` (E2B with a loopback `public_url`) shows the shared client's fixed safe address-configuration message in the setup wizard, with Managed in Settings leading to Settings → Service and deployment, and leaves nothing to confirm; initialize the deployment with Docker or microsandbox `resources`, or with the E2B account and no `resources` (Core adopts the template build's CPU and memory); change its settings with the expected generation. E2B's `metadata.template_build` (status, CPU, memory, disk) shows in Settings → Sandbox and scheduling and Sandbox metrics, and sizes each sandbox when `specification.resources` is missing; microsandbox's `suspension` (idle and retention seconds) shows in Settings → Sandbox and scheduling | | E2B discovery | `POST /core/v1/sandbox/providers/e2b/discovery` | The setup wizard lists the templates the entered E2B key can see, then the selected template's ready builds. The key travels only in these request bodies and the deployment write | | Reset | `POST`, `DELETE /core/v1/sandbox/deployment/reset` | Explicitly clear hosted resources, or cancel the remaining clear at the observed generation; show Core's remaining and offline projection | -| Nodes | `GET /core/v1/sandbox/nodes` | Nodes page; fleet on Overview; each hosted Runtime's node on Sandbox metrics. An online node's `diagnostic` (a [readiness code](../getting-started/nodes.md#readiness-codes); any other value reads as `provider_unavailable`) marks it degraded and names the reason and fix in the help tip beside its status on the Nodes list and on the node's page. A node whose `core_url` (the address it enrolled with) differs from the deployment's `core_url` is named on the Nodes page as bound to an old address, to be removed and added again, and its status there and on its page reads Old address instead of its health. **Add node** follows only the node whose `enrollment_id` equals its command's | +| Nodes | `GET /core/v1/sandbox/nodes` | Nodes page; fleet on Overview; each hosted Runtime's node on Sandbox metrics. An online node whose `provider_ready` is false is degraded; the help tip beside its status on the Nodes list and on the node's page says *Sandbox provider unavailable*, and the target preparation reason (`rollout.diagnostic`, a [readiness code](../getting-started/nodes.md#readiness-codes)) shows in its target status. A node whose `core_url` (the address it enrolled with) differs from the deployment's `core_url` is named on the Nodes page as bound to an old address, to be removed and added again, and its status there and on its page reads Old address instead of its health. **Add node** follows only the node whose `enrollment_id` equals its command's | | Node detail | `GET /core/v1/sandbox/nodes/{node_id}?range=1h\|6h\|24h` | A node's page on Nodes: the host's CPU busy share, memory and free disk from its last heartbeat, and CPU and memory history over 1 hour, 6 hours or 24 hours, refreshed every 30 s while visible. **Edit node** reads `host.effective_cpu_cores` and `host.total_memory_bytes` to show the host beside each sandbox's size and at most how many of those fit | | Allocations | `GET /core/v1/sandbox/nodes/{node_id}/allocations` | Nodes page; Sandbox metrics. A node's page shows from `compute_phase_changed_at` how long each allocation has been in its compute phase; a null time shows a dash. It does not infer snapshot expiry from the phase timestamp or the deployment's `suspension.retention_seconds` | | Enrollment | `POST /core/v1/sandbox/enrollment-tokens` | **Add node**: the administrator sets the node's sandbox limits (`max_active`; `max_retained` only for microsandbox, equal to `max_active` for Docker) before Core issues a single-use token inside a command that verifies the installer checksum, with the command's `enrollment_id`, which the node it registers reports. The command runs the installer with sudo (a system service) and passes the token on standard input; root runs it directly. No ordinary-user installation or removal entry is exposed, and the log hint always names the system service. The command downloads the installer from the installation's `public_url`. No token is requested until the installation is read, when it cannot be read, when it is `local_only` (or its `public_url` is not a valid HTTP or HTTPS origin), or when `/console/config` lists `node_artifacts` without the deployment's provider. The dialog reads both again on opening and when the window regains focus | @@ -114,7 +114,7 @@ The list carries each Harness's configuration, so the console does not read `GET | Remove node | `DELETE /core/v1/sandbox/nodes/{node_id}` | Confirmed node removal; the row goes only after Core acknowledges the deletion, and a Clean up the host dialog then gives the host's uninstall command (requiring root or sudo; for a node enrolled with another address than the deployment's, also with `--force`, which skips the installer's confirmation with Core) | | Runtime observations | `GET /core/v1/sandbox/runtime-observations` | Sandbox metrics: hosted Runtimes of every project, each labelled with its project; an E2B sandbox's dialog adds its `observation.disk` as used / limit (null elsewhere) | -An E2B deployment has no nodes; its API key is write-only. Overview and Sandbox metrics count its running and starting sandboxes from the deployment's `resources.allocations` and `resources.pending`, while the hosted Runtime rows come from Runtime observations. The two sources refresh independently, so the console does not infer retention or cleanup from their difference. The Runtime release sent for Docker and microsandbox comes from the console's own `GET /node-install/manifest.json`; without it the administrator enters the release under advanced settings. Settings → Sandbox and scheduling reads the manifest only for a deployment with a saved `specification.runtime` and shows the Runtime row only when the two releases differ; Use this installation's Runtime sends the manifest's release through the same `PUT`. +An E2B deployment has no nodes; its API key is write-only. Overview and Sandbox metrics count its running and starting sandboxes from the deployment's `resources.allocations` and `resources.pending`, while the hosted Runtime rows come from Runtime observations. The two sources refresh independently, so the console does not infer retention or cleanup from their difference. ## Writes diff --git a/docs/web/console-server.md b/docs/web/console-server.md index 06e039081..50f096fa5 100644 --- a/docs/web/console-server.md +++ b/docs/web/console-server.md @@ -92,11 +92,11 @@ Sign-in errors: 400 for a malformed body, 401 `Invalid Core key`, 405 for a meth | --- | --- | | `node_installer` | Whether the console serves a node installation payload | | `node_installer_sha256` | SHA-256 of that payload's `node-install.pyz`; Add node commands verify it before running the installer | -| `node_artifacts` | The providers (`docker`, `microsandbox`) whose node artifacts the payload holds, locally or as a pinned release download. Read on every request, so artifacts added by rerunning the installer appear without a restart | +| `node_artifacts` | The providers (`docker`, `microsandbox`) whose node artifacts the payload holds, locally or as a release download. Read on every request, so artifacts added by rerunning the installer appear without a restart | ## Node installation payload -With `OAC_WEB_NODE_PAYLOAD_DIR` set, the console serves the matched distribution's node payload at `/node-install/` without sign-in: `node-install.pyz`, `manifest.json`, `SHA256SUMS`, `runtime/seccomp.json`, and the node artifacts the manifest declares under `artifacts/`. An artifact missing locally redirects (307) to its pinned release download. Node install and uninstall commands download from `/node-install/`, so the reverse proxy must send that path to the console. Nodes verify every checksum themselves. +With `OAC_WEB_NODE_PAYLOAD_DIR` set, the console serves the matched distribution's node payload at `/node-install/` without sign-in: `node-install.pyz`, `manifest.json`, `SHA256SUMS`, `runtime/seccomp.json`, and the node artifacts the manifest declares under `artifacts/`. An artifact missing locally redirects (307) to its release download. Node install and uninstall commands download from `/node-install/`, so the reverse proxy must send that path to the console. Nodes verify every checksum themselves. ## Public address diff --git a/docs/zh/configuration.md b/docs/zh/configuration.md index 08418b96e..6e6e8c6f6 100644 --- a/docs/zh/configuration.md +++ b/docs/zh/configuration.md @@ -1,7 +1,7 @@ --- title: "配置参考" source: docs/configuration.md -source_hash: 6830daa88320510b51e74909169d4e49c7fa9017b344ba12a1456097cad29991 +source_hash: 48515be4243404bb45d5fb93ffe9836580e37f01efb495c1e06e2722d6183165 --- Core 安装的每项设置、其归属位置以及修改方式。安装程序写入初始值；之后请使用本参考。每项设置都恰好只有一个归属位置，分属以下三类： @@ -60,7 +60,7 @@ Core 安装的每项设置、其归属位置以及修改方式。安装程序写 | 设置 | Web 中的位置 | Core API | 注意事项 | | --- | --- | --- | --- | | 沙箱提供商：Docker、microsandbox 或 E2B | **设置** → **沙箱与调度**：设置向导，最后点击 **保存配置** | `/core/v1/sandbox/deployment` | 每个安装只能使用一个沙箱提供商，在首次登录后选择。要改用其他提供商，必须先执行 **重置部署**；请参阅[更改沙箱配置](getting-started/nodes.md#change-the-sandbox-configuration) | -| 沙箱大小、Runtime 发行版、E2B 密钥和模板构建 | **设置** → **沙箱与调度** → **修改资源** | `/core/v1/sandbox/deployment` | Web 会推荐 [提供商声明的默认大小](sandbox-provider.md#register-the-provider-kind)。现有沙箱会保留其大小和发行版。E2B 密钥仅可写入，并且已加密 | +| 沙箱大小、E2B 密钥和模板构建 | **设置** → **沙箱与调度** → **修改资源** | `/core/v1/sandbox/deployment` | Web 会推荐 [提供商声明的默认大小](sandbox-provider.md#register-the-provider-kind)。现有沙箱会保留其大小。E2B 密钥仅可写入，并且已加密 | | 节点及其容量 | **节点**：**添加节点**、**编辑节点** 和移除 | `/core/v1/sandbox/enrollment-tokens`、`/core/v1/sandbox/nodes` | 请参阅[节点容量](#node-capacity)和[添加和管理节点](getting-started/nodes.md) | | 空闲挂起 | **设置** → **沙箱与调度** → **自动挂起** → **空闲多久后挂起（秒）** → **保存** | `PATCH /core/v1/sandbox/deployment` | 整个部署统一生效；参阅[挂起策略](../../contracts/agents-api/zh/sandbox-deployment.md#suspension-policy) | | 独立工作区存储和租户容量 | **设置 → 工作区存储** | `/core/v1/workspace-storage` | 参阅[设置工作区存储](getting-started/workspace-storage.md)；现有工作区保留自身配置 | @@ -111,12 +111,12 @@ Session 创建可以在异步工作区准入之前成功。容量耗尽时，Cor ## Docker 节点配置 {#docker-node-configuration} -节点安装程序会将 Docker 的主机设置写入节点配置文件的 `native` 对象，只有 Docker 适配器读取它。部署资源、Runtime 发行版本和容量仍存储在 [Core 的数据库](#runtime-settings-web)中。 +节点安装程序会将 Docker 的主机设置写入节点配置文件的 `native` 对象，只有 Docker 适配器读取它。部署资源和容量仍存储在 [Core 的数据库](#runtime-settings-web)中。 | 字段 | 安装程序设置的值 | 含义 | | --- | --- | --- | | `host` | `unix:///var/run/docker.sock` | 显式 Docker Engine 套接字 | -| `image` | 加载后 Runtime 镜像的本地 ID | 发行版本的 `image_id` 或 `image_manifest_digest`。主机的镜像存储决定由哪个 digest 指代已加载的镜像，因此该值属于节点本地；适配器只接受这两个值 | +| `image` | 加载后 Runtime 镜像的本地 ID | 本安装的 Runtime 镜像。主机的镜像存储决定其本地 ID 是清单中的 `image_id` 还是 `image_manifest_digest`，因此该值属于节点本地 | | `network` | `oac-node-` | Runtime 容器网络 | | `seccomp_file` | `/runtime/seccomp.json` | 所匹配发行版的 seccomp 配置文件 | | `nested_sandbox` | `true` | 启用 Docker 适配器的 init 进程和 proc-mask 配置 | diff --git a/docs/zh/getting-started/nodes.md b/docs/zh/getting-started/nodes.md index d2ca38fe8..6f1c7f677 100644 --- a/docs/zh/getting-started/nodes.md +++ b/docs/zh/getting-started/nodes.md @@ -1,7 +1,7 @@ --- title: "添加和管理节点" source: docs/getting-started/nodes.md -source_hash: 6581acc433887f6006933cbacdf4eee0810d1d1e96871b31c968612ced3b74de +source_hash: 60f79b32a3c11ee369cf66e2ddde8cfe2652f204996c5adf37eacfe6729be25b --- 节点是一台 Linux 主机，为 Core 托管 Session 运行沙箱。本指南面向在沙箱提供商（参阅[选择沙箱提供商](./sandbox-backends.md)）为 microsandbox 或 Docker 时添加、修改和移除节点的运维人员；E2B 不需要节点。 @@ -72,7 +72,7 @@ Core 主机与其他主机一样加入：要在它上面运行沙箱，将它添 | Docker | 一次性导入 Runtime 镜像，以及网络 `oac-node-` | | microsandbox 网络策略 | 沙箱仅能访问 Core、主机上的 DNS 和公开地址：无入站连接、不可访问私有网络 | -root 只准备账号、组和服务单元；其他操作（包括 Docker 网络）都以 `oac-node` 在没有终端的独立会话中执行。安装程序不安装 Docker、KVM 或软件包，不启动 Docker，不修改设备权限、sudoers、防火墙或 SELinux 设置，也不修改其他账号。 +root 只准备账号、组和服务单元；其他操作（包括 Docker 网络）都以 `oac-node` 在没有终端的独立会话中执行。安装程序不安装 Docker、KVM 或软件包，不启动 Docker，不修改设备权限、sudoers、防火墙或 SELinux 设置，也不修改其他账号。节点向外连接 Core；Core 不需要通过 SSH 或 Docker TCP 访问主机。 > [!CAUTION] > Docker 模式等同于 root 权限。加入 `docker` 组后，`oac-node` 以及任何能控制节点的主体，都可以在主机上执行 root 操作，并且容器共享主机内核。只在沙箱专用主机上添加 Docker 节点。microsandbox 节点只需要 `kvm` 组，每个沙箱运行在独立 microVM 中。 @@ -82,12 +82,12 @@ root 只准备账号、组和服务单元；其他操作（包括 Docker 网络 **每台主机只服务一个 Core。** 节点共享 `oac-node` 账号，因此主机只服务一个 Core；第二个 Core 的命令会被拒绝。 -**节点程序。** 节点保留添加它的控制台所提供的主机程序。要让节点使用另一个版本，请[移除节点](#remove-a-node)后重新添加。 +**节点程序。** 节点运行添加它的安装所提供的主机程序和 Runtime。另一个安装的命令在该主机上会被拒绝，直到你[移除节点](#remove-a-node)后重新添加。 ## 重新运行、过期与慢速网络 {#rerun-expiry-and-slow-links} -- 重新执行同一命令是安全的。节点注册后，重新运行使用节点自身凭据，不改变已匹配的内容，也无需令牌。以下情况程序不做任何修改并说明原因：保留的节点身份属于其他 Core 地址或安装；节点注册后沙箱配置发生变化（节点会自行准备，参阅[修改沙箱配置](#change-the-sandbox-configuration)）；或 Core 已升级、提供了更新的节点程序，此时在 **节点** 选择 **移除节点**，在主机上卸载该节点，再通过 **添加节点** 重新添加。 -- 节点注册前失败的运行不会在 Core 留下任何记录。下一条命令即使在配置变更或 Core 升级之后执行，也会替换上次尝试的配置和不再匹配的已下载文件，保留匹配的文件并继续。 +- 重新执行同一命令是安全的。节点注册后，重新运行使用节点自身凭据，不改变已匹配的内容，也无需令牌。以下情况程序不做任何修改并说明原因：保留的节点身份或节点程序属于其他 Core 地址或安装；或节点注册后沙箱配置发生变化（节点会自行准备，参阅[修改沙箱配置](#change-the-sandbox-configuration)），此时在 **节点** 选择 **移除节点**，在主机上卸载该节点，再通过 **添加节点** 重新添加。 +- 节点注册前失败的运行不会在 Core 留下任何记录。下一条命令即使在配置变更之后执行，也会替换上次尝试的配置和不再匹配的已下载文件，保留匹配的文件并继续。 - 已过期或已在其他主机使用的命令立即失败，显示 `The enrollment command expired or was already used`：生成新命令并在 1 小时内执行。 - 慢速下载期间命令过期时，注册失败并显示 `The enrollment command expired or was already used`。下载文件保留：在 Web 生成新命令后执行，下载从中断处继续。 - 每个文件都会显示进度。失败或中断时保留已收到的部分；重新执行时，每个未完成的文件从中断处继续（`Resuming …`），复用已校验的完整文件，其余照常下载。服务器不支持续传时，该文件从头开始。每分钟下载不足 64 KiB 时会停止，请检查网络后重新执行命令。 @@ -104,8 +104,7 @@ root 只准备账号、组和服务单元；其他操作（包括 Docker 网络 沙箱设置适用于整个安装，节点跟随这些设置。 -- **规格、Runtime 版本、E2B 密钥或模板构建。** 打开 **设置** → **沙箱与调度** → **修改资源**，修改并保存。已有沙箱保持其配置。每个节点继续服务旧配置，同时准备新配置；**节点** 展示进度：**正在准备目标配置**、**目标配置已就绪**，或带[原因](#readiness-codes)的 **目标配置准备失败**。Core 优先在已准备好新配置的节点上放置新 Session；这些节点没有容量时，会使用仍服务旧配置的节点。E2B 变更立即生效。 -- **安装提供的 Runtime。** 已保存的 Runtime 与安装提供的不同时，**设置** → **沙箱与调度** 会显示 **Runtime** 行，列出两者的源码提交。**使用本安装的 Runtime** 以该 Runtime 打开 **修改资源** 的确认页，**保存配置** 后节点按上面的方式准备。其他修改保留已保存的 Runtime。控制台没有提供 Runtime 清单时，在 **高级设置** 中填写版本。 +- **规格、E2B 密钥或模板构建。** 打开 **设置** → **沙箱与调度** → **修改资源**，修改并保存。已有沙箱保持其配置。每个节点继续服务旧配置，同时准备新配置；**节点** 展示进度：**正在准备目标配置**、**目标配置已就绪**，或带[原因](#readiness-codes)的 **目标配置准备失败**。Core 优先在已准备好新配置的节点上放置新 Session；这些节点没有容量时，会使用仍服务旧配置的节点。E2B 变更立即生效。 - **容量与空闲挂起。** 在 **节点** 中通过 **编辑节点** 修改节点的活跃和保留上限。在 **设置** → **沙箱与调度** 中，通过 **自动挂起** → **空闲多久后挂起（秒）** → **保存** 修改整个部署的[空闲策略](../../../contracts/agents-api/zh/sandbox-deployment.md#suspension-policy)。 - **沙箱提供商。** 同一页面选择 **重置部署**。自动重置立即归档空闲托管 Session，并让运行中的工作在设定截止时间前完成；强制重置立即取消工作。离线节点必须恢复连接，让 Core 确认清理。重置完成时，Core 已撤销所有节点和未使用命令：配置新的沙箱提供商后重新添加节点。**取消重置** 停止剩余工作；已归档 Session 仍保持归档。 @@ -126,35 +125,6 @@ microsandbox 的检查点执行兼容性包含主机内核和 CPU profile。因 程序不删除 Docker 镜像或卷。保留 Runtime 镜像并输出 `sudo docker image rm` 命令。microsandbox 在 Core 确认节点已移除后，还会删除节点位于 `/var/lib/oac-node/.oac/m/` 的存储（约 1.5 GB），随后删除所创建的账号。使用 `--force` 时不询问 Core，因此保留存储，并输出删除命令（`sudo -u oac-node rm -rf `）；microVM 可能仍使用存储，请先检查 `pgrep -u oac-node`，再重新卸载以完成账号移除。输出会列出已删除和已保留的内容。可以重复卸载直到完成。 -## 手动注册节点 {#register-a-node-manually} - -如果自行管理节点文件和服务，而不执行 Web 命令，请使用手动注册。手动注册的节点只服务注册时的配置：规格或 Runtime 修改后，**节点** 显示 **节点程序不兼容**；它继续服务旧配置，直到移除节点并重新注册主机。 - -1. 使用与 Core 同一发行版本的 `oac-node`。 -2. 获取注册令牌：使用 **添加节点** 命令中的令牌，或通过 Core 密钥调用 `POST /core/v1/sandbox/enrollment-tokens`。令牌一次性使用，包含批准的节点容量；响应的 `expires_at` 给出过期时间。在主机上存入权限为 `0600` 的文件。 -3. 使用令牌读取节点配置，不会消耗令牌：`GET /api/v1/sandbox-node/configuration`，带 `Authorization: Bearer `。 -4. 写入私有提供商文件。从响应复制 `provider`、`installation_id`、`core_url`、`generation` 和 `specification`，并添加 `native` 对象，写入该提供商的主机设置。适配器从 `specification` 读取沙箱规格、Runtime 镜像和产物哈希： - - Docker：[Docker 节点配置](../configuration.md#docker-node-configuration)中的字段，其中 `host` 是显式 Unix 套接字，`image` 是导入的 Runtime 镜像的本地 ID，`seccomp_file` 是绝对路径。 - - microsandbox：绝对路径 `helper_path`、`runtime_path` 和 `firmware_path`；`network` 策略；私有检查点归档的显式 `checkpoint_root`；以及 `runtime_home` 私有目录。目录缺失时辅助程序以 `0700` 创建。microsandbox 在其中放置 Unix 套接字，因此路径不要超过 48 字节；安装程序对自管节点拒绝更长路径。 - 首次注册 microsandbox 前，以 node 服务账户在相同发行版本的源码 checkout 中初始化空的 `checkpoint_root`：`PYTHONPATH=deploy/node python3 -c 'from node_install import prepare_checkpoint_root; prepare_checkpoint_root("/absolute/private/checkpoints")'`。它复用安装程序的 UUID marker 排他初始化与所有权检查。共享 store 只初始化一次，其他 node 使用已有 marker。不得替换 marker，也不得在恢复后的存储或非空且无所有权证明的目录上重新初始化。 -5. 使用真实绝对路径注册，然后通过主机服务管理器运行节点： - - ```sh - oac-node register \ - --config /var/lib/oac/provider.json \ - --state-dir /var/lib/oac/node \ - --core-url https://core.example \ - --name worker-1 \ - --enrollment-token-file /var/lib/oac/enrollment-token - oac-node run \ - --config /var/lib/oac/provider.json \ - --state-dir /var/lib/oac/node - ``` - -节点移除后，Core 拒绝其凭据时，`oac-node run` 以状态 78 退出；配置管理器不要在该情况下重启（systemd：`RestartPreventExitStatus=78`）。 - -节点向外连接 Core；Core 不需要通过 SSH 或 Docker TCP 访问主机。注册在联系 Core 前先将节点身份写入状态目录，因此响应丢失时可以复用同一身份重试。状态目录放在持久存储上，仅节点账号可访问，每次只由一个进程使用。不要复制到其他目录或主机：节点已有连接打开时，Core 拒绝第二个连接。提供商文件不能修改节点容量或沙箱配置。Core 在注册及每次连接时比较摘要；文件不匹配的节点在恢复批准配置前不接收工作。 - ## 节点主机故障时 {#when-a-node-host-fails} 重启节点服务会保留身份并重新发现已有沙箱。Core 不会仅因节点不可达或沙箱缺失就接管活动或未确认的 writer：所有权未结清时，这些资源显示 **节点已断开** 或 **沙箱资源缺失**。主机崩溃或挂起时，应重启主机而不是移除其节点：节点重新连接后，节点确认已不存在的运行中沙箱会按[已确认的计算资源丢失](../sandbox-provider.md#confirmed-compute-loss)规则释放，Session 的下一个 input 在新计算资源上续接；其他缺失的沙箱保持 **沙箱资源缺失**。主机始终未恢复时，其 Session 的计算资源归属保持不变。已结清的保留检查点可以遵循[检查点转移契约](../sandbox-provider.md#checkpoint-transfer)，确认释放后可以按条件进行[冷替换](../sandbox-provider.md#cold-replacement-after-checkpoint-retention)。丢失节点状态目录属于恢复事件：从[备份](operations.md#back-up)恢复，并同时恢复数据库和提供商存储；不要在已有资源上重新注册主机。 @@ -165,22 +135,18 @@ microsandbox 的检查点执行兼容性包含主机内核和 CPU profile。因 ### 就绪状态码 {#readiness-codes} -节点在线但沙箱提供商未就绪时，**节点** 和 **概览** 显示 **沙箱提供商未就绪**。原因展示位置取决于节点添加方式： - -- **使用 Web 命令添加的节点**：当前沙箱配置检查失败时，在目标状态显示 **目标配置准备失败**，帮助提示中给出原因（`GET /core/v1/sandbox/nodes` 的 `rollout.diagnostic`）。**沙箱提供商未就绪** 旁的提示仅显示 *沙箱提供商不可用*。 -- **手动注册的节点**：在 **沙箱提供商未就绪** 旁的帮助提示展示原因（`diagnostic`）。 +节点在线但沙箱提供商未就绪时，**节点** 和 **概览** 显示 **沙箱提供商未就绪**。节点在当前沙箱配置检查失败时，在目标状态显示 **目标配置准备失败**，帮助提示中给出原因（`GET /core/v1/sandbox/nodes` 的 `rollout.diagnostic`）。**沙箱提供商未就绪** 旁的提示仅显示 *沙箱提供商不可用*。 -每个状态码都是与提供商无关的类别；其背后的本地错误留在节点上。手动注册的节点会把它写入日志。对于使用 Web 命令添加的节点，请在主机上重新运行该命令：安装程序会先检查主机要求，并指出需要修复的问题（[安装程序消息](./troubleshooting.md#installer-messages)）。 +每个状态码都是与提供商无关的类别；其背后的本地错误留在节点上。请在主机上重新运行该命令：安装程序会先检查主机要求，并指出需要修复的问题（[安装程序消息](./troubleshooting.md#installer-messages)）。 -节点按如下顺序仅报告首个失败检查：Docker 守护进程或 KVM、Docker 限制支持、主机容量、已安装的 Runtime 文件。因此无法访问 Docker 守护进程时，会隐藏镜像缺失问题。修复后约十秒的下一次心跳会清除或替换状态码。离线节点保留最后状态码，Web 在节点重连前隐藏它。 +节点按如下顺序仅报告首个失败检查：Docker 守护进程或 KVM、Docker 限制支持、主机容量、已安装的 Runtime 文件。因此无法访问 Docker 守护进程时，会隐藏镜像缺失问题。修复后的下一次准备尝试会清除或替换状态码。 | 状态码 | 帮助提示 | 原因 | 解决方法 | | --- | --- | --- | --- | -| `provider_unavailable` | 沙箱提供商不可用 | 提供商的服务不可达或请求失败（例如 Docker 守护进程已停止），或失败没有类别 | 重新运行节点的命令，或阅读手动注册节点的日志 | -| `host_unsupported` | 主机不满足要求 | 主机缺少提供商所需的能力，例如 Docker 的 CPU 与内存限制（cgroup v2）或对 `/dev/kvm` 的读写权限 | 重新运行节点的命令，或阅读手动注册节点的日志 | +| `provider_unavailable` | 沙箱提供商不可用 | 提供商的服务不可达或请求失败（例如 Docker 守护进程已停止），或失败没有类别 | 重新运行节点的命令 | +| `host_unsupported` | 主机不满足要求 | 主机缺少提供商所需的能力，例如 Docker 的 CPU 与内存限制（cgroup v2）或对 `/dev/kvm` 的读写权限 | 重新运行节点的命令 | | `capacity_insufficient` | 主机资源不足 | 主机 CPU 或内存不足以运行一个沙箱 | 使用更大主机或修改沙箱规格 | -| `runtime_image_unavailable` | 缺少 Runtime 镜像 | 提供商中没有固定版本的 Runtime 镜像 | Web 命令添加的节点自动重新下载；其他节点加载匹配发行版镜像 | -| `artifacts_unavailable` | 沙箱提供商文件缺失 | 固定版本的提供商文件（例如 microsandbox 的 Runtime、固件或辅助程序）缺失或 SHA-256 检查失败 | Web 命令添加的节点自动下载缺失文件；其他节点从匹配发行版恢复 | -| `runtime_download_failed` | Runtime 下载失败 | 准备新配置时无法下载或验证 Runtime 文件 | 检查节点到控制台和发行下载地址的 HTTPS 访问。节点以递增间隔重试，最长间隔 30 分钟 | +| `runtime_image_unavailable` | 缺少 Runtime 镜像 | 提供商中没有本安装的 Runtime 镜像 | 重新运行节点的命令以再次下载；否则加载匹配发行版镜像 | +| `artifacts_unavailable` | 沙箱提供商文件缺失 | 固定版本的提供商文件（例如 microsandbox 的 Runtime、固件或辅助程序）缺失或 SHA-256 检查失败 | 重新运行节点的命令以下载缺失文件；否则从匹配发行版恢复 | 新用户组成员关系仅对新进程生效。重启节点服务：`sudo systemctl restart oac-node-.service`。 diff --git a/docs/zh/getting-started/troubleshooting.md b/docs/zh/getting-started/troubleshooting.md index 5cff44127..094fc2d47 100644 --- a/docs/zh/getting-started/troubleshooting.md +++ b/docs/zh/getting-started/troubleshooting.md @@ -1,7 +1,7 @@ --- title: "故障排查" source: docs/getting-started/troubleshooting.md -source_hash: 468c8cbd13ee2a736db07f9239f56a49afad2916036a375260b59e377fc3a1e3 +source_hash: b57b7719fd48f61d1a10077eaca44efc89a58b7d2d68d4a5ce96bf537ba6680e --- 按症状查找安装、访问、节点和 Session 问题的原因与解决方法。本页面向 Core 安装的运维人员。 @@ -44,7 +44,7 @@ source_hash: 468c8cbd13ee2a736db07f9239f56a49afad2916036a375260b59e377fc3a1e3 microsandbox 沙箱无法访问私有地址上的模型提供商或 MCP 服务器，Harness 停止重试后 Turn 失败。沙箱的 DNS 查询发往主机 `/etc/resolv.conf` 中列出的域名服务器，而 DNS rebind 保护会丢弃任何包含私有地址的应答。因此，这些域名服务器解析为私有地址的名称在沙箱内根本无法解析，即使是云 VPC 解析器之后的公共模型 API 这类公共服务也是如此。 -安装程序添加的节点没有更改此行为的设置。请让这些域名服务器返回该服务的公共地址。microsandbox 不读取主机的 `/etc/hosts`，因此只有当 `/etc/resolv.conf` 指向提供该条目的本地存根解析器（例如 `127.0.0.53` 上的 systemd-resolved）时，那里的条目才有用。否则，请使用不过滤沙箱流量的 Docker 节点，或[手动注册节点](./nodes.md#register-a-node-manually)，并设置同样在不带端口的规则中放行该私有网段的 `network` 策略，因为 rebind 保护会忽略限定端口的规则。 +节点没有更改此行为的设置。请让这些域名服务器返回该服务的公共地址。microsandbox 不读取主机的 `/etc/hosts`，因此只有当 `/etc/resolv.conf` 指向提供该条目的本地存根解析器（例如 `127.0.0.53` 上的 systemd-resolved）时，那里的条目才有用。否则，请使用不过滤沙箱流量的 Docker 节点。 ### 连接失败 {#connection-failures} @@ -65,7 +65,7 @@ microsandbox 沙箱无法访问私有地址上的模型提供商或 MCP 服务 | `The enrollment command expired or was already used` | 在 Web 生成新命令并在 1 小时内执行 | | `Core rejected the node configuration read (HTTP …)` | 检查 Core URL 及其前面的反向代理后重试 | | `Not enough disk space under …` 或 `The disk is full` | 按提示释放空间后重试，下载会继续 | -| This host's node program differs from the one this console serves | 节点添加后 Core 已重新安装。选择 **移除节点**，在主机上卸载，再通过 **添加节点** 重新添加 | +| This host's node was added with another Core address, release or provider | 该节点由另一个 Core 地址、版本或提供商添加。选择 **移除节点**，在主机上卸载，再通过 **添加节点** 重新添加 | | Another node installation is running, or this node is preparing a new sandbox configuration | 待 **节点** 不再显示 **正在准备目标配置** 后重试 | | 注册前的沙箱提供商检查失败，例如 `This host lacks a capability …` | 修复提示的问题（参阅[就绪状态码](./nodes.md#readiness-codes)）后重试；令牌未被使用 | | Core's public URL changed after this command was generated | 在 Web 生成并执行新命令 | diff --git a/docs/zh/sandbox-provider.md b/docs/zh/sandbox-provider.md index e2a4fa51a..62948b727 100644 --- a/docs/zh/sandbox-provider.md +++ b/docs/zh/sandbox-provider.md @@ -1,7 +1,7 @@ --- title: "沙箱提供商协议" source: docs/sandbox-provider.md -source_hash: 90b575c1ba7ab3e5ff9525c81643f93472e6f4f1cee214f96d2c174437258068 +source_hash: 19dcc4268e7d7ed547c1de470ba1bf0f771a35e909a4f869daaeefeddffeea7c --- 沙箱提供商协议连接 Core 与为 Core 管理的（`openai_hosted`）Environment 提供计算资源的 adapter，并由该 adapter 运行启动 Runtime daemon 的有界引导流程。本页面向 adapter 作者：说明提供商必须实现什么，以及可以依赖 Core 做什么。Docker、microsandbox 和 E2B 实现了该协议。 @@ -123,14 +123,14 @@ Checkpoint 支持增加 `Compute` generation、name、ID 和 `SnapshotIdentity` ## 注册提供商 kind {#register-the-provider-kind} -`sandbox/providers/registry.go` 是唯一注册表。每项绑定 adapter 的 specification 与 resource validator、`sandbox.ConfigurationAdapter`、部署模式（`nodes` 或 `direct`）、`sandbox.DeploymentPolicy`（磁盘限制、Runtime 输入，以及 setup 推荐的默认大小）、operation 声明，以及 node-local（`BuildLocal`）或 direct（`BuildDirect`）constructor。`providers.Build` 和 `providers.BuildDirect` 构造 adapter，不分配计算资源。没有 init 时注册或 plugin 加载。 +`sandbox/providers/registry.go` 是唯一注册表。每项绑定 adapter 的 specification 与 resource validator、`sandbox.ConfigurationAdapter`、部署模式（`nodes` 或 `direct`）、`sandbox.DeploymentPolicy`（磁盘限制，以及 setup 推荐的默认大小）、operation 声明，以及 node-local（`BuildLocal`）或 direct（`BuildDirect`）constructor。`providers.Build` 和 `providers.BuildDirect` 构造 adapter，不分配计算资源。没有 init 时注册或 plugin 加载。 新提供商执行以下步骤： 1. 在 adapter 包中实现 operation 契约，并编写原生契约测试。 2. 添加 specification 和 resource validator。 3. 基于类型化原生配置实现 `sandbox.ConfigurationAdapter`。`DecodeInput` 严格解析请求中独立的公开 `configuration` 与只写 `credential` 对象。`Encode` 生成白名单公开 selector、只读观测和独立 secret bytes，不透传请求 JSON。`Decode` 恢复已存储 selector 并保留对所属资源的访问，不做远程 admission 或新模板验证。`Normalize` 修改前复制输入。`ResolveChange`、`Equal` 和 `WithCredential` 负责继承、身份与凭据组合。`Requirements` 声明是否需要凭据和公开 Core origin，以及支持哪些 setup 操作：`Discovery` 对应 `DiscoverConfiguration`，`SelectionDiscovery` 对应 `DiscoverSelection`，`CredentialVerification` 对应 `VerifyCredential`。`DiscoverConfiguration` 验证 query 并返回安全 catalog，不做 mutation 或 admission decision；Core 保留授权、输入限制与 deadline。`DiscoverSelection` 在提交前解析候选项省略的原生值，`VerifyCredential` 验证凭据对所属资源的访问，不修改资源。两者都接收候选项的 `sandbox.DirectConfig`，原生 client 只为该次调用构造。node 提供商仅接受空公开对象，拒绝凭据，对每项 setup 操作和 credential replacement 返回 Unsupported。 -4. node adapter 从自己的包中导出 `BuildLocal` constructor、它解码的类型化 `native` 对象，以及它在共享 node artifact 之外添加的原生文件。node-local 设置（如主机路径）只存放在该对象中；resources 和 Runtime 版本从 node 配置的 `specification` 读取。 +4. node adapter 从自己的包中导出 `BuildLocal` constructor、它解码的类型化 `native` 对象，以及它在共享 node artifact 之外添加的原生文件。node-local 设置（如主机路径）只存放在该对象中；resources 从 node 配置的 `specification` 读取，Runtime 镜像来自同一对象。 5. 在 `providers/registry.go` 中注册 constructor、policy、configuration adapter 和 operation 声明。其键即提供商 kind，也用于标记该提供商的观测；checkpoint 支持读取此项。生成的投影组合每个已注册的部署模式和 deployment policy 与 `sandbox/deployment_contract.go` 中的共享 field bound：installer 从 `deploy/node/node_spec.py` 读取，TypeScript 客户端和 Web 从 `packages/agents-client/src/deployment-contract.ts` 读取，因此 Web 读取这些声明，而不比较提供商 kind。通过 `go run ./services/core/cmd/specification-contract -write` 重新生成两者。 6. 提供 adapter 和 helper 的发行产物，通过已注册 configuration 契约把该提供商交付给运维人员。 7. Web 的 setup 步骤、部署摘要和客户端投影会指名每个提供商。在协议声明配置字段之前，把新提供商加入 `apps/web`。 @@ -143,7 +143,6 @@ node 配置 `sandbox.NodeConfig` 包含 `provider`、`generation`、`installatio - `nodes` 注册仅有 `BuildLocal`，`direct` 注册仅有 `BuildDirect`；缺失、混合或未知 mode 被拒绝。 - specification 和 resource validator、configuration adapter 与完整 operation 声明都是必需项，因此不完整注册不能发布部分投影。声明的默认大小必须通过 adapter 的 resource validator。 -- Runtime input policy 要么接受固定 Runtime，要么给出 adapter 拒绝它的固定原因，不能两者兼有。 - Checkpoint 仅准入 `nodes` 注册，因为公共 lifecycle 只暂停 node allocation；声明 checkpoint 的 `direct` 提供商会被注册拒绝。注册不携带 suspension 数值：Core 对每个声明 checkpoint 支持的提供商应用同一个 [suspension policy](#suspension)。 configuration adapter 必须非 nil，包括其具体值。每个 `ConfigurationRequirements` 字段都需要明确有效的决定：`Credential` 和 `PublicOrigin` 为 `Required` 或 `NotRequired`，`Discovery`、`SelectionDiscovery` 和 `CredentialVerification` 使用共享 supported 或 unsupported 声明并携带安全 reason。新增 requirement field 需要明确更新验证，不继承已有决定。要求凭据不承诺支持 `VerifyCredential` 操作。这些检查证明注册完整，不证明原生 SDK 行为正确；constructor 和 adapter 契约测试仍然适用。 @@ -254,7 +253,7 @@ Core 在初始化完成、没有 root 或 Subagent Turn 处于排队、进行中 ### 检查点转移 {#checkpoint-transfer} -支持检查点的 generation 报告 `CheckpointCompatibility`，包含不透明的 `ArtifactDomain` 与 `ExecutionClass` token。每个已验证 `SnapshotIdentity` 携带同样的资格。Core 只比较这些值，不解释 CPU 特性、文件系统路径或存储实现。恢复目标必须在线、对 allocation 的不可变 deployment generation 已就绪、两个 token 均匹配，并有 active 容量；跨 node 时还需 retained slot。allocation、Device 和 Session 身份保持不变。Session 与 deployment 事务先提交目标路由、placement 和精确 restore intent，再执行目标侧原生操作。没有容量或兼容目标时，保留快照的所有权一直持续到配置期限。目标必须已持有该精确 generation；Core 不会在新节点自动准备历史 generation。升级期间应保留符合条件的节点及其 generation 提供商，直到检查点保留义务结束。 +支持检查点的 generation 报告 `CheckpointCompatibility`，包含不透明的 `ArtifactDomain` 与 `ExecutionClass` token。每个已验证 `SnapshotIdentity` 携带同样的资格。Core 只比较这些值，不解释 CPU 特性、文件系统路径或存储实现。恢复目标必须在线、对 allocation 的不可变 deployment generation 已就绪、两个 token 均匹配，并有 active 容量；跨 node 时还需 retained slot。allocation、Device 和 Session 身份保持不变。Session 与 deployment 事务先提交目标路由、placement 和精确 restore intent，再执行目标侧原生操作。没有容量或兼容目标时，保留快照的所有权一直持续到配置期限。目标必须已持有该精确 generation；Core 不会在新节点自动准备历史 generation。节点会保留被取代的 generation 的提供商，直到检查点保留义务结束。 `Suspend` 只有在持久发布已验证的完整归档、停止精确 source compute，并结清 source 本地 capture 与 cleanup 义务后，才能报告 `SourceStopped`。Core 先持久化快照，再标记 suspended；仅原生资源不存在并不足够。`Suspend.ObserveOnly` 可以为同一 operation 已验证的快照完成归档发布和源清理，但不能重新捕获快照，也不能停止已经恢复 running 的源。此前已发布的归档缺失或损坏时，`ObserveOnly` 可以返回其精确持久 ownership receipt，标记 `Status: unknown` 和 `SourceStopped: false`，供清理使用；该 receipt 不证明归档当前可用，也不证明源不存在。`Resume` 仍在执行前验证归档。完成源结清屏障后，source node 不可用不再阻止恢复。删除和正常到期可以将空闲 suspended allocation 的清理路由转给同一 artifact domain 内的在线 node；删除不要求 execution class 匹配。清理目标必须已提供同一不可变 generation，且有 retained 容量。没有这样的 node 时，清理保持 pending，不释放所有权；运维需恢复符合条件的 node 或其容量。未知 running 或 restoring writer 不会仅因 node 不可达而迁移。 diff --git a/docs/zh/web/console-api-usage.md b/docs/zh/web/console-api-usage.md index 9bbf7a2a7..ea0df6348 100644 --- a/docs/zh/web/console-api-usage.md +++ b/docs/zh/web/console-api-usage.md @@ -1,7 +1,7 @@ --- title: "控制台 API 使用" source: docs/web/console-api-usage.md -source_hash: 6a1378a61a173f9ce037c553ac5d6b94cc7140a4ae5acbd0d0885b36b0e14388 +source_hash: 8529f1216a0e6987b099247270583f1ed9f3b4482dd8593231bab3389dfd5520 --- 本页列出各控制台页面调用的 Core 路由，以及控制台如何限定读取范围。本页面向控制台开发者；运维人员请阅读 [Web 控制台](index.md)。[Core 管理 API](../../../contracts/agents-api/zh/admin-api.md) 定义了路由、响应结构、分页和审计记录；[API 命名空间和凭据](../api/index.md) 定义了本文使用的术语。 @@ -10,7 +10,7 @@ source_hash: 6a1378a61a173f9ce037c553ac5d6b94cc7140a4ae5acbd0d0885b36b0e14388 | 接口 | 路径 | 身份验证 | 控制台用途 | | --- | --- | --- | --- | -| Console server | `/console/auth`、`/console/auth/{login,logout}`、`/console/config`、`/node-install/manifest.json` | 登录时使用 Core 密钥，随后使用控制台会话 Cookie；`/node-install/manifest.json` 无需登录 | 登录和退出；**添加节点** 所用的节点安装程序和节点构件；用于 Docker 和 microsandbox 设置以及 **设置** → **沙箱与调度** 中 Runtime 行的发行版 Runtime release。参见 [控制台服务器](console-server.md) | +| Console server | `/console/auth`、`/console/auth/{login,logout}`、`/console/config` | 登录时使用 Core 密钥，随后使用控制台会话 Cookie | 登录和退出；**添加节点** 所用的节点安装程序和节点构件。参见 [控制台服务器](console-server.md) | | Administrator API | `/core/v1/**`，不包括 `/core/v1/sandbox` | Core 密钥，由控制台服务器添加 | 项目、密钥、资源读取和删除、诊断、执行器凭据和安装命令、来源信息、汇总、Core 指标、安装信息、默认模型 | | Sandbox administration | `/core/v1/sandbox/**` | Core 密钥，由控制台服务器添加 | 沙箱配置；**节点**；**概览** 上的机群与容量摘要；**沙箱监控** 中每个项目的 Runtime 观测 | | Agents API | `/v1/**` | 项目 API 密钥 | 不使用。控制台会向开发者说明如何调用它（参见 [Provenance and monitoring](#provenance-and-monitoring)） | @@ -105,10 +105,10 @@ source_hash: 6a1378a61a173f9ce037c553ac5d6b94cc7140a4ae5acbd0d0885b36b0e14388 | 操作 | 路由 | 控制台用途 | | --- | --- | --- | -| 部署 | `GET`、`POST`、`PUT /core/v1/sandbox/deployment` | 读取提供商、只读 `core_url`（即 `OAC_PUBLIC_URL`，会显示在设置审核中且绝不发送）、重置状态、安装 ID 和规范；409 `sandbox_configuration_error`（E2B 搭配回环地址形式的 `public_url`）会在设置向导中显示共享客户端固定的安全地址配置消息，并通过 **在设置中管理** 前往 **设置 → 服务与部署**，且无需确认；使用 `resources` 以及 Docker 或 microsandbox 的 `runtime` release 初始化部署，或者使用 E2B 账户且不提供 `resources`（Core 采用模板构建的 CPU 和内存）；使用预期的 generation 更改设置。E2B 的 `metadata.template_build`（状态、CPU、内存、磁盘）会显示在 **设置 → 沙箱与调度** 和 **沙箱监控** 中；当缺少 `specification.resources` 时，它还会确定每个 Sandbox 的大小；microsandbox 的 `suspension`（空闲和保留秒数）会显示在 **设置 → 沙箱与调度** 中 | +| 部署 | `GET`、`POST`、`PUT /core/v1/sandbox/deployment` | 读取提供商、只读 `core_url`（即 `OAC_PUBLIC_URL`，会显示在设置审核中且绝不发送）、重置状态、安装 ID 和规范；409 `sandbox_configuration_error`（E2B 搭配回环地址形式的 `public_url`）会在设置向导中显示共享客户端固定的安全地址配置消息，并通过 **在设置中管理** 前往 **设置 → 服务与部署**，且无需确认；使用 Docker 或 microsandbox 的 `resources` 初始化部署，或者使用 E2B 账户且不提供 `resources`（Core 采用模板构建的 CPU 和内存）；使用预期的 generation 更改设置。E2B 的 `metadata.template_build`（状态、CPU、内存、磁盘）会显示在 **设置 → 沙箱与调度** 和 **沙箱监控** 中；当缺少 `specification.resources` 时，它还会确定每个 Sandbox 的大小；microsandbox 的 `suspension`（空闲和保留秒数）会显示在 **设置 → 沙箱与调度** 中 | | E2B 发现 | `POST /core/v1/sandbox/providers/e2b/discovery` | 设置向导先列出输入的 E2B 密钥可见的模板，再列出所选模板的可用构建。该密钥只会通过这些请求体和部署写入请求传输 | | 重置 | `POST`、`DELETE /core/v1/sandbox/deployment/reset` | 显式清除托管资源，或在观测到的 generation 处取消剩余清除；显示 Core 的剩余资源和离线预测 | -| 节点 | `GET /core/v1/sandbox/nodes` | **节点** 页面；**概览** 上的机群；**沙箱监控** 中每个托管 Runtime 所在的节点。在线节点的 `diagnostic`（一个[就绪状态码](../getting-started/nodes.md#readiness-codes)；任何其他值均读取为 `provider_unavailable`）会将其标记为降级，并在 **节点** 列表及节点页面中，紧邻状态的帮助提示里说明原因和修复方法。如果节点的 `core_url`（其注册时使用的地址）与部署的 `core_url` 不同，**节点** 页面会将其标记为绑定到旧地址，需要移除后重新添加；此时它在该页面和节点页面中的状态会显示 **地址已过期**，而不是健康状态。**添加节点** 仅跟踪 `enrollment_id` 与其命令所含 `enrollment_id` 相等的节点 | +| 节点 | `GET /core/v1/sandbox/nodes` | **节点** 页面；**概览** 上的机群；**沙箱监控** 中每个托管 Runtime 所在的节点。`provider_ready` 为 false 的在线节点为降级；**节点** 列表及节点页面中紧邻其状态的帮助提示显示 *沙箱提供商不可用*，目标准备原因（`rollout.diagnostic`，一个[就绪状态码](../getting-started/nodes.md#readiness-codes)）显示在其目标状态中。如果节点的 `core_url`（其注册时使用的地址）与部署的 `core_url` 不同，**节点** 页面会将其标记为绑定到旧地址，需要移除后重新添加；此时它在该页面和节点页面中的状态会显示 **地址已过期**，而不是健康状态。**添加节点** 仅跟踪 `enrollment_id` 与其命令所含 `enrollment_id` 相等的节点 | | 节点详情 | `GET /core/v1/sandbox/nodes/{node_id}?range=1h\|6h\|24h` | **节点** 中的节点页面：主机最近一次心跳报告的 CPU 忙碌占比、内存使用量和可用磁盘，以及 1 小时、6 小时或 24 小时内 CPU 与内存的历史记录；页面可见时每 30 秒刷新一次。**编辑节点** 读取 `host.effective_cpu_cores` 和 `host.total_memory_bytes`，用于在每个 Sandbox 大小旁显示主机容量，以及最多可容纳多少个该大小的 Sandbox | | 分配 | `GET /core/v1/sandbox/nodes/{node_id}/allocations` | **节点** 页面；**沙箱监控**。节点页面根据 `compute_phase_changed_at` 显示每个分配处于计算阶段的时间；时间为 null 时显示短横线。它不会根据阶段时间或部署的 `suspension.retention_seconds` 推算快照到期时间 | | 注册 | `POST /core/v1/sandbox/enrollment-tokens` | **添加节点**：管理员先设置节点的 Sandbox 限制（`max_active`；`max_retained` 仅适用于 microsandbox，在 Docker 下等于 `max_active`），然后 Core 才会把一次性令牌放入命令中；该命令会验证安装程序校验和，并包含命令的 `enrollment_id`，节点注册时会报告此 ID。命令使用 sudo 运行安装程序（作为系统服务），并通过标准输入传递令牌；以 root 运行时则直接执行。界面不提供普通用户安装或移除入口，日志提示始终指明系统服务。命令从安装的 `public_url` 下载安装程序。只有成功读取安装信息后才会请求令牌；如果安装信息无法读取、安装为 `local_only`（或其 `public_url` 不是有效的 HTTP 或 HTTPS 来源），或者 `/console/config` 列出的 `node_artifacts` 不包含部署的提供商，则不会请求令牌。对话框在打开时和窗口重新获得焦点时，会再次读取这两项信息 | @@ -116,7 +116,7 @@ source_hash: 6a1378a61a173f9ce037c553ac5d6b94cc7140a4ae5acbd0d0885b36b0e14388 | 移除节点 | `DELETE /core/v1/sandbox/nodes/{node_id}` | 确认后移除节点；只有 Core 确认删除后该行才会消失，随后 **清理主机** 对话框会提供主机的卸载命令（需要 root 或 sudo；对于使用不同于部署地址的地址注册的节点，还需要使用 `--force`，以跳过安装程序与 Core 的确认） | | Runtime 观测 | `GET /core/v1/sandbox/runtime-observations` | **沙箱监控**：每个项目的托管 Runtime，每项均以所属项目为标签；E2B Sandbox 对话框还会将 `observation.disk` 显示为已用量／限制值（其他位置为 null） | -E2B 部署没有节点；其 API 密钥为只写。**概览** 和 **沙箱监控** 根据部署的 `resources.allocations` 和 `resources.pending` 统计其正在运行和正在启动的 Sandbox，而托管 Runtime 行来自 Runtime 观测。两个来源独立刷新，因此控制台不会根据两者之间的差异推断保留或清理状态。用于 Docker 和 microsandbox 的 Runtime 版本来自控制台自身的 `GET /node-install/manifest.json`；如果无法获取该信息，管理员需在高级设置中输入 release。**设置** → **沙箱与调度** 只为已保存 `specification.runtime` 的部署读取该清单，并且只在两个 release 不同时显示 Runtime 行；**使用本安装的 Runtime** 通过同一个 `PUT` 发送清单中的 release。 +E2B 部署没有节点；其 API 密钥为只写。**概览** 和 **沙箱监控** 根据部署的 `resources.allocations` 和 `resources.pending` 统计其正在运行和正在启动的 Sandbox，而托管 Runtime 行来自 Runtime 观测。两个来源独立刷新，因此控制台不会根据两者之间的差异推断保留或清理状态。 ## 写入 {#writes} diff --git a/docs/zh/web/console-server.md b/docs/zh/web/console-server.md index 889c21149..dd2518afa 100644 --- a/docs/zh/web/console-server.md +++ b/docs/zh/web/console-server.md @@ -1,7 +1,7 @@ --- title: "控制台服务器" source: docs/web/console-server.md -source_hash: 6e2f860b4f43159b1dcb691ea0343745c82e170b8fb9670247036753a9acd733 +source_hash: 2c8b3bfed09d29c70cfee4b3381a248befd722c8336c4964a079e20c073f1a4a --- 控制台服务器（`oac-web`）提供 Web，使用 Core 密钥认证管理员，并将浏览器的 `/core/v1` 请求转发到 Core。本参考面向不使用安装程序运行 Web 的运维人员，以及评审其安全边界的人员。 @@ -94,11 +94,11 @@ flowchart LR | --- | --- | | `node_installer` | 控制台是否提供节点安装文件 | | `node_installer_sha256` | 文件中 `node-install.pyz` 的 SHA-256；**添加节点** 命令执行安装程序前验证它 | -| `node_artifacts` | 文件中包含节点资产的提供商（`docker`、`microsandbox`），资产在本地或通过固定发行下载提供。每次请求都读取，因此重新运行安装程序新增的资产无需重启即可出现 | +| `node_artifacts` | 文件中包含节点资产的提供商（`docker`、`microsandbox`），资产在本地或通过发行下载提供。每次请求都读取，因此重新运行安装程序新增的资产无需重启即可出现 | ## 节点安装文件 {#node-installation-payload} -设置 `OAC_WEB_NODE_PAYLOAD_DIR` 后，控制台在 `/node-install/` 无需登录地提供匹配发行版的节点文件：`node-install.pyz`、`manifest.json`、`SHA256SUMS`、`runtime/seccomp.json`，以及清单声明的 `artifacts/` 下节点资产。本地缺失的资产重定向（307）到固定发行下载地址。节点安装和卸载命令从 `/node-install/` 下载，因此反向代理必须将该路径发给控制台。节点自行验证每个校验和。 +设置 `OAC_WEB_NODE_PAYLOAD_DIR` 后，控制台在 `/node-install/` 无需登录地提供匹配发行版的节点文件：`node-install.pyz`、`manifest.json`、`SHA256SUMS`、`runtime/seccomp.json`，以及清单声明的 `artifacts/` 下节点资产。本地缺失的资产重定向（307）到其发行下载地址。节点安装和卸载命令从 `/node-install/` 下载，因此反向代理必须将该路径发给控制台。节点自行验证每个校验和。 ## 公开地址 {#public-address} diff --git a/packages/agents-client/src/deployment-contract.ts b/packages/agents-client/src/deployment-contract.ts index 66f8bd5db..67c1254dc 100644 --- a/packages/agents-client/src/deployment-contract.ts +++ b/packages/agents-client/src/deployment-contract.ts @@ -1,3 +1,3 @@ // Code generated by services/core/cmd/specification-contract from sandbox/deployment_contract.go; DO NOT EDIT. -export const deploymentContract = {"workspace_fields":["attachment","user_xattr","capacity_quota"],"resources":[{"name":"cpus","min":1,"max":255,"omit_zero":false},{"name":"memory_mib","min":512,"max":1048576,"omit_zero":false},{"name":"root_disk_mib","min":0,"max":4294967295,"omit_zero":true},{"name":"environment_disk_mib","min":0,"max":4294967295,"omit_zero":true}],"runtime":[{"name":"source_commit","pattern":"[0-9a-f]{40}"},{"name":"image_id","pattern":"sha256:[0-9a-f]{64}"},{"name":"image_manifest_digest","pattern":"sha256:[0-9a-f]{64}"},{"name":"microsandbox_ref","pattern":"oac-runtime@sha256:[0-9a-f]{64}"},{"name":"runtime_sha256","pattern":"[0-9a-f]{64}"},{"name":"firmware_sha256","pattern":"[0-9a-f]{64}"}],"providers":{"docker":{"mode":"nodes","disk":false,"runtime":true,"default_resources":{"cpus":2,"memory_mib":2048}},"e2b":{"mode":"direct","disk":false,"runtime":false,"default_resources":null},"microsandbox":{"mode":"nodes","workspace":{"attachment":"host_directory","user_xattr":true},"disk":true,"runtime":true,"default_resources":{"cpus":2,"memory_mib":4096,"root_disk_mib":8192,"environment_disk_mib":8192}}},"minimum_disk":1024} as const; +export const deploymentContract = {"workspace_fields":["attachment","user_xattr","capacity_quota"],"resources":[{"name":"cpus","min":1,"max":255,"omit_zero":false},{"name":"memory_mib","min":512,"max":1048576,"omit_zero":false},{"name":"root_disk_mib","min":0,"max":4294967295,"omit_zero":true},{"name":"environment_disk_mib","min":0,"max":4294967295,"omit_zero":true}],"providers":{"docker":{"mode":"nodes","disk":false,"default_resources":{"cpus":2,"memory_mib":2048}},"e2b":{"mode":"direct","disk":false,"default_resources":null},"microsandbox":{"mode":"nodes","workspace":{"attachment":"host_directory","user_xattr":true},"disk":true,"default_resources":{"cpus":2,"memory_mib":4096,"root_disk_mib":8192,"environment_disk_mib":8192}}},"minimum_disk":1024} as const; diff --git a/packages/agents-client/src/generated/core-api.ts b/packages/agents-client/src/generated/core-api.ts index bf70a525e..94892ac69 100644 --- a/packages/agents-client/src/generated/core-api.ts +++ b/packages/agents-client/src/generated/core-api.ts @@ -190,10 +190,9 @@ export interface DeploymentResources { export const deploymentResourcesFields = ["allocations", "pending"] as const; export interface DeploymentSpec { resources: SandboxResources; - runtime?: RuntimeRelease; workspace?: Declaration; } -export const deploymentSpecFields = ["resources", "runtime", "workspace"] as const; +export const deploymentSpecFields = ["resources", "workspace"] as const; export const deploymentSpecRequired = ["resources"] as const; export interface DeploymentView { configuration?: Record; @@ -417,7 +416,6 @@ export interface Node { core_url: string; cpu_count: number | null; created_at: string; - diagnostic?: NodeDiagnosticCode; enrollment_id: string | null; id: string; last_seen_at: string | null; @@ -433,8 +431,7 @@ export interface Node { running: number; snapshots: number; } -export const nodeFields = ["active", "available_disk_bytes", "available_memory_bytes", "cleanup_pending", "core_url", "cpu_count", "created_at", "diagnostic", "enrollment_id", "id", "last_seen_at", "max_active", "max_retained", "name", "online", "provider", "provider_ready", "reserved", "retained", "rollout", "running", "snapshots"] as const; -export const nodeRequired = ["active", "available_disk_bytes", "available_memory_bytes", "cleanup_pending", "core_url", "cpu_count", "created_at", "enrollment_id", "id", "last_seen_at", "max_active", "max_retained", "name", "online", "provider", "provider_ready", "reserved", "retained", "rollout", "running", "snapshots"] as const; +export const nodeFields = ["active", "available_disk_bytes", "available_memory_bytes", "cleanup_pending", "core_url", "cpu_count", "created_at", "enrollment_id", "id", "last_seen_at", "max_active", "max_retained", "name", "online", "provider", "provider_ready", "reserved", "retained", "rollout", "running", "snapshots"] as const; export interface NodeAllocation { compute_phase: string; compute_phase_changed_at: string | null; @@ -460,7 +457,6 @@ export interface NodeDetail { core_url: string; cpu_count: number | null; created_at: string; - diagnostic?: NodeDiagnosticCode; enrollment_id: string | null; history: HostHistory; host: NodeHost; @@ -478,9 +474,8 @@ export interface NodeDetail { running: number; snapshots: number; } -export const nodeDetailFields = ["active", "available_disk_bytes", "available_memory_bytes", "cleanup_pending", "core_url", "cpu_count", "created_at", "diagnostic", "enrollment_id", "history", "host", "id", "last_seen_at", "max_active", "max_retained", "name", "online", "provider", "provider_ready", "reserved", "retained", "rollout", "running", "snapshots"] as const; -export const nodeDetailRequired = ["active", "available_disk_bytes", "available_memory_bytes", "cleanup_pending", "core_url", "cpu_count", "created_at", "enrollment_id", "history", "host", "id", "last_seen_at", "max_active", "max_retained", "name", "online", "provider", "provider_ready", "reserved", "retained", "rollout", "running", "snapshots"] as const; -export const nodeDiagnosticCodeValues = ["provider_unavailable", "host_unsupported", "artifacts_unavailable", "runtime_download_failed", "runtime_image_unavailable", "capacity_insufficient"] as const; +export const nodeDetailFields = ["active", "available_disk_bytes", "available_memory_bytes", "cleanup_pending", "core_url", "cpu_count", "created_at", "enrollment_id", "history", "host", "id", "last_seen_at", "max_active", "max_retained", "name", "online", "provider", "provider_ready", "reserved", "retained", "rollout", "running", "snapshots"] as const; +export const nodeDiagnosticCodeValues = ["provider_unavailable", "host_unsupported", "artifacts_unavailable", "runtime_image_unavailable", "capacity_insufficient"] as const; export type NodeDiagnosticCode = (typeof nodeDiagnosticCodeValues)[number]; export interface NodeHost { available_disk_bytes: number | null; @@ -498,7 +493,7 @@ export interface NodeRollout { } export const nodeRolloutFields = ["diagnostic", "ready_generation", "state"] as const; export const nodeRolloutRequired = ["ready_generation", "state"] as const; -export const nodeRolloutStateValues = ["ready", "preparing", "failed", "update_required", "unknown"] as const; +export const nodeRolloutStateValues = ["ready", "preparing", "failed", "unknown"] as const; export type NodeRolloutState = (typeof nodeRolloutStateValues)[number]; export interface NodeUpdate { max_active: number; @@ -649,9 +644,8 @@ export interface RolloutNodes { preparing: number; ready: number; unknown: number; - update_required: number; } -export const rolloutNodesFields = ["failed", "preparing", "ready", "unknown", "update_required"] as const; +export const rolloutNodesFields = ["failed", "preparing", "ready", "unknown"] as const; export const rolloutStateValues = ["settled", "preparing"] as const; export type RolloutState = (typeof rolloutStateValues)[number]; export interface RuntimeCPUObservation { @@ -788,15 +782,6 @@ export const runtimeObservationReasonValues = ["runtime_mode_not_observable", "a export type RuntimeObservationReason = (typeof runtimeObservationReasonValues)[number]; export const runtimeObservationStatusValues = ["observed", "unsupported", "unavailable"] as const; export type RuntimeObservationStatus = (typeof runtimeObservationStatusValues)[number]; -export interface RuntimeRelease { - firmware_sha256: string; - image_id: string; - image_manifest_digest: string; - microsandbox_ref: string; - runtime_sha256: string; - source_commit: string; -} -export const runtimeReleaseFields = ["firmware_sha256", "image_id", "image_manifest_digest", "microsandbox_ref", "runtime_sha256", "source_commit"] as const; export interface SandboxAllocationList { data: NodeAllocation[]; } @@ -807,9 +792,8 @@ export interface SandboxDeploymentChangeInput { expected_generation: number; provider?: string; resources?: SandboxResources; - runtime?: RuntimeRelease; } -export const sandboxDeploymentChangeInputFields = ["configuration", "credential", "expected_generation", "provider", "resources", "runtime"] as const; +export const sandboxDeploymentChangeInputFields = ["configuration", "credential", "expected_generation", "provider", "resources"] as const; export const sandboxDeploymentChangeInputRequired = ["expected_generation"] as const; export interface SandboxDeploymentInput { configuration?: Record; @@ -817,9 +801,8 @@ export interface SandboxDeploymentInput { expected_generation: number; provider?: string; resources?: SandboxResources; - runtime?: RuntimeRelease; } -export const sandboxDeploymentInputFields = ["configuration", "credential", "expected_generation", "provider", "resources", "runtime"] as const; +export const sandboxDeploymentInputFields = ["configuration", "credential", "expected_generation", "provider", "resources"] as const; export const sandboxDeploymentInputRequired = ["expected_generation"] as const; export interface SandboxEnrollmentToken { enrollment_id: string; diff --git a/packages/agents-client/src/sandbox-client.test.ts b/packages/agents-client/src/sandbox-client.test.ts index 5489e6413..b40d14918 100644 --- a/packages/agents-client/src/sandbox-client.test.ts +++ b/packages/agents-client/src/sandbox-client.test.ts @@ -9,7 +9,7 @@ function response(value: unknown, status = 200) { return new Response(JSON.strin // Shapes as Core serializes them (deployment.Node, deployment.NodeDetail, deployment.NodeAllocation, deployment.View). const created = "2026-09-25T08:00:00.123456789Z"; -/** A ready node: Core omits `diagnostic`. */ +/** A ready node. */ const node = { rollout: { state: "ready", ready_generation: 1 }, id: "3b0c1f4e-8a2d-4c6b-9e7f-1a2b3c4d5e6f", name: "core-01", provider: "docker", online: true, provider_ready: true, @@ -17,9 +17,9 @@ const node = { last_seen_at: "2026-09-25T09:00:00Z", max_active: 2, max_retained: 2, active: 1, reserved: 0, retained: 1, cleanup_pending: 0, created_at: created, core_url: "https://core.example", enrollment_id: "9d2e6b1a-4c3f-4e8d-a7b6-5c4d3e2f1a0b", }; -/** Never heard from, enrolled before Core recorded enrollment IDs, with a fixed readiness code. */ +/** Never heard from and enrolled before Core recorded enrollment IDs. */ const unready = { - ...node, rollout: { state: "unknown", ready_generation: 1 }, id: "7f6e5d4c-3b2a-4190-8f7e-6d5c4b3a2918", online: false, provider_ready: false, diagnostic: "host_unsupported", + ...node, rollout: { state: "unknown", ready_generation: 1 }, id: "7f6e5d4c-3b2a-4190-8f7e-6d5c4b3a2918", online: false, provider_ready: false, cpu_count: null, available_memory_bytes: null, available_disk_bytes: null, running: 0, last_seen_at: null, active: 0, retained: 0, enrollment_id: null, }; const detail = { @@ -41,16 +41,15 @@ const allocation = { session_id: "2c3d4e5f-6a7b-4c8d-9e0f-1a2b3c4d5e6f", environment_id: "3d4e5f6a-7b8c-4d9e-8f1a-2b3c4d5e6f7a", state: "running", compute_phase: "running", compute_phase_changed_at: null, diagnostic: "", initialization: "ready", created_at: created, }; -const runtime = { source_commit: "a".repeat(40), image_id: "sha256:" + "b".repeat(64), image_manifest_digest: "sha256:" + "c".repeat(64), microsandbox_ref: "oac-runtime@sha256:" + "d".repeat(64), runtime_sha256: "e".repeat(64), firmware_sha256: "f".repeat(64) }; const unconfigured = { credential_configured: false, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: null }, installation_id: "", provider: "", core_url: "https://core.example", reset: null, owner_epoch: 0, generation: 0, mode: "", resources: { allocations: 0, pending: 0 }, suspension: null }; const docker = { configuration: {}, metadata: {}, - ...unconfigured, installation_id: "94be54a1-138c-4f30-bc87-b13686272dbe", provider: "docker", rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, update_required: 0, unknown: 0 } }, owner_epoch: 1, generation: 1, mode: "nodes", - resources: { allocations: 2, pending: 1 }, specification: { resources: { cpus: 2, memory_mib: 2048 }, runtime }, specification_digest: "0".repeat(64), + ...unconfigured, installation_id: "94be54a1-138c-4f30-bc87-b13686272dbe", provider: "docker", rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, owner_epoch: 1, generation: 1, mode: "nodes", + resources: { allocations: 2, pending: 1 }, specification: { resources: { cpus: 2, memory_mib: 2048 } }, specification_digest: "0".repeat(64), }; const microsandbox = { - ...docker, provider: "microsandbox", specification: { resources: { cpus: 2, memory_mib: 2048, root_disk_mib: 8192, environment_disk_mib: 8192 }, runtime }, + ...docker, provider: "microsandbox", specification: { resources: { cpus: 2, memory_mib: 2048, root_disk_mib: 8192, environment_disk_mib: 8192 } }, suspension: { idle_seconds: 300, retention_seconds: 86400 }, }; const externalWorkspace = { ...deploymentContract.providers.microsandbox.workspace, capacity_quota: false }; @@ -93,7 +92,7 @@ describe("strict sandbox administration projections", () => { it.each([ ["nodes", "an unknown member", { data: [{ ...node, credential: "node-secret" }] }], ["nodes", "a missing enrollment_id", { data: [unenrolled] }], - ["nodes", "an empty diagnostic", { data: [{ ...node, diagnostic: "" }] }], + ["nodes", "a node diagnostic", { data: [{ ...node, diagnostic: "host_unsupported" }] }], ["nodes", "a string count", { data: [{ ...node, cpu_count: "16" }] }], ["nodes", "a numeric timestamp", { data: [{ ...node, last_seen_at: 0 }] }], ["nodes", "a null list", { data: null }], @@ -161,12 +160,6 @@ describe("Core sandbox credential boundaries", () => { expect(new Headers(fetch.mock.calls[0]?.[1]?.headers).has("Authorization")).toBe(false); expect(fetch.mock.calls[0]?.[1]?.signal).toBe(controller.signal); }); - it("returns the node's fixed readiness diagnostic unchanged and reads an unknown code as provider_unavailable", async () => { - const fetch = vi.fn().mockResolvedValue(response({ data: [unready, { ...unready, diagnostic: "future_code" }, node] })); - const { data } = await new SandboxAdminClient({ baseUrl: "/core/v1/sandbox", fetch }).listNodes(); - expect(data.map((entry) => entry.diagnostic)).toEqual(["host_unsupported", "provider_unavailable", undefined]); - expectTypeOf().toEqualTypeOf(); - }); it("requires each node's enrollment ID: a string, or null for nodes enrolled before Core recorded it", async () => { const fetch = vi.fn().mockResolvedValue(response({ data: [node, unready] })); const { data } = await new SandboxAdminClient({ fetch }).listNodes(); @@ -214,7 +207,7 @@ describe("Core sandbox credential boundaries", () => { it("forwards one specification with generation and preserves backend conflict details", async () => { const fetch = vi.fn().mockResolvedValue(response({ error: { code: "sandbox_specification_mismatch", message: "Node specification differs" } }, 409)); const admin = new SandboxAdminClient({ token: "admin-only", fetch }); - const input = { provider: "docker" as const, resources: { cpus: 2, memory_mib: 2048 }, runtime: { source_commit: "a".repeat(40), image_id: "sha256:" + "b".repeat(64), image_manifest_digest: "sha256:" + "c".repeat(64), microsandbox_ref: "oac-runtime@sha256:" + "d".repeat(64), runtime_sha256: "e".repeat(64), firmware_sha256: "f".repeat(64) }, expected_generation: 3 }; + const input = { provider: "docker" as const, resources: { cpus: 2, memory_mib: 2048 }, expected_generation: 3 }; await expect(admin.updateDeployment(input)).rejects.toMatchObject({ status: 409, code: "sandbox_specification_mismatch" }); expect(fetch).toHaveBeenCalledTimes(1); expect(JSON.parse(String(fetch.mock.calls[0]?.[1]?.body))).toEqual(input); @@ -410,14 +403,14 @@ it("keeps omitted-key public-URL errors actionable without reflecting a stored k expect(fetch).toHaveBeenCalledTimes(1); }); -it.each(["unknown", "preparing", "failed", "update_required"])("does not erase old serving readiness for target %s", async (state) => { - const value = { ...node, rollout: { state, ready_generation: 1, ...(state === "failed" ? { diagnostic: "runtime_download_failed" } : {}) } }; +it.each(["unknown", "preparing", "failed"])("does not erase old serving readiness for target %s", async (state) => { + const value = { ...node, rollout: { state, ready_generation: 1, ...(state === "failed" ? { diagnostic: "capacity_insufficient" } : {}) } }; const client = new SandboxAdminClient({ fetch: async () => response({ data: [value] }) }); expect((await client.listNodes()).data[0]).toEqual(value); }); - it("preserves artifact transfer diagnostics on node and target without exposing transport details", async () => { - const value = { ...unready, online: true, diagnostic: "runtime_download_failed", rollout: { state: "failed", ready_generation: 1, diagnostic: "runtime_download_failed" } }; + it("preserves the target diagnostic without exposing transport details", async () => { + const value = { ...unready, online: true, rollout: { state: "failed", ready_generation: 1, diagnostic: "capacity_insufficient" } }; const client = new SandboxAdminClient({ fetch: async () => response({ data: [value] }) }); expect((await client.listNodes()).data[0]).toEqual(value); }); @@ -427,11 +420,10 @@ describe("shared node diagnostic contract", () => { expect([...sandboxNodeDiagnostics].sort()).toEqual([...nodeDiagnosticFixture].sort()); expect(normalizeSandboxNodeDiagnostic("future_code")).toBe("provider_unavailable"); }); - it.each(nodeDiagnosticFixture)("preserves %s through node and rollout projections", async (diagnostic) => { - const value = { ...node, provider_ready: false, diagnostic, rollout: { state: "failed", ready_generation: 1, diagnostic } }; + it.each(nodeDiagnosticFixture)("preserves %s through the rollout projection", async (diagnostic) => { + const value = { ...node, provider_ready: false, rollout: { state: "failed", ready_generation: 1, diagnostic } }; const client = new SandboxAdminClient({ baseUrl: "https://core.example", token: "test", fetch: async () => response({ data: [value] }) }); const result = await client.listNodes(); - expect(result.data[0]?.diagnostic).toBe(diagnostic); expect(result.data[0]?.rollout.diagnostic).toBe(diagnostic); }); }); diff --git a/packages/agents-client/src/sandbox-client.ts b/packages/agents-client/src/sandbox-client.ts index 9b8a9bca3..045093f6f 100644 --- a/packages/agents-client/src/sandbox-client.ts +++ b/packages/agents-client/src/sandbox-client.ts @@ -5,12 +5,12 @@ import { hasOwn, isNonnegativeInteger, isOneOf, isRecord, onlyFields, sameResour import { attachmentKindValues, deploymentResourcesFields, deploymentSpecFields, deploymentSpecRequired, deploymentViewFields, deploymentModeValues, deploymentViewRequired, - hostHistoryFields, hostHistoryPointFields, nodeAllocationDiagnosticValues, nodeAllocationFields, nodeDetailFields, nodeDetailRequired, - nodeDiagnosticCodeValues, nodeFields, nodeHostFields, nodeRequired, nodeRolloutFields, nodeRolloutRequired, nodeRolloutStateValues, resetModeValues, - resetFields, resetOfflineNodeFields, resetRemainingFields, rolloutFields, rolloutNodesFields, rolloutStateValues, runtimeReleaseFields, + hostHistoryFields, hostHistoryPointFields, nodeAllocationDiagnosticValues, nodeAllocationFields, nodeDetailFields, + nodeDiagnosticCodeValues, nodeFields, nodeHostFields, nodeRolloutFields, nodeRolloutRequired, nodeRolloutStateValues, resetModeValues, + resetFields, resetOfflineNodeFields, resetRemainingFields, rolloutFields, rolloutNodesFields, rolloutStateValues, sandboxAllocationListFields, sandboxNodeListFields, sandboxResourcesFields, sandboxResourcesRequired, suspensionFields, type DeploymentSpec, type DeploymentView, type HostHistoryPoint, type Node, type NodeAllocation, type NodeDetail, type NodeDiagnosticCode, - type NodeHost, type NodeRollout, type NodeUpdate, type Reset, type ResetOfflineNode, type ResetRequest, type Rollout, type RuntimeRelease, + type NodeHost, type NodeRollout, type NodeUpdate, type Reset, type ResetOfflineNode, type ResetRequest, type Rollout, type SandboxIdlePolicyInput, type SandboxDeploymentInput, type SandboxEnrollmentToken, type SandboxResources as SandboxResourcesResource, } from "./generated/core-api"; import type { ReadOptions } from "./types"; @@ -29,19 +29,18 @@ export function normalizeSandboxNodeDiagnostic(value: string): SandboxNodeDiagno export type SandboxProvider = keyof typeof deploymentContract.providers; /** Client-generated, never a Core code: a deployment write whose rejection could echo the key and is withheld. */ export const sandboxConfigurationUnconfirmed = "sandbox_configuration_unconfirmed"; -const sandboxConfigurationParams = new Set(["runtime", ...deploymentContract.resources.map(({ name }) => `resources.${name}`)]); +const sandboxConfigurationParams = new Set([...deploymentContract.resources.map(({ name }) => `resources.${name}`)]); // Generated types keep their schema names in ./generated/core-api; these are the client's names for them. /** CPU and MiB limits for each sandbox, not node concurrency. */ export type SandboxResources = SandboxResourcesResource; -export type SandboxRuntimeRelease = RuntimeRelease; export type SandboxSpecification = DeploymentSpec; export type SandboxRollout = Rollout; /** `state` is target preparation, which never invalidates a qualified older serving pin; `ready_generation` is that durable pin and alone does not imply connection readiness. */ export type SandboxNodeRollout = NodeRollout; export type StartSandboxReset = ResetRequest; export type SandboxReset = Reset; -/** `provider_ready` is the last provider report, so combine it with `online`; `diagnostic` is absent while the provider is ready. A `core_url` other than the installation public URL means the node receives no new sandboxes and must be re-added. */ +/** `provider_ready` is true while the node is online and its serving generation is ready. A `core_url` other than the installation public URL means the node receives no new sandboxes and must be re-added. */ export type SandboxNode = Node; /** A one-time node enrollment command; `enrollment_id` is its public handle, never a credential. */ export type SandboxEnrollment = SandboxEnrollmentToken; @@ -54,7 +53,7 @@ export type SandboxAllocation = NodeAllocation; // The schema types each Provider's configuration, credential and metadata as free-form objects; the client names E2B's. /** * Core derives the deployment's address from the installation public URL; a `core_url` member is rejected. - * `expected_generation` is required, zero for first setup. Docker and microsandbox require `resources` and `runtime`; + * `expected_generation` is required, zero for first setup. Docker and microsandbox require `resources`; * E2B may omit `resources` to adopt its template build's CPU and memory, and always runs that build. */ export type InitializeSandboxDeployment = Omit & { @@ -107,11 +106,6 @@ function projectSpecification(value: unknown): SandboxSpecification { if (!isOneOf(attachmentKindValues, workspace.attachment) || typeof workspace.user_xattr !== "boolean" || typeof workspace.capacity_quota !== "boolean") return invalidSandboxResponse(); result.workspace = { attachment: workspace.attachment, user_xattr: workspace.user_xattr, capacity_quota: workspace.capacity_quota }; } - if (hasOwn(specification, "runtime")) { - const runtime = members(specification.runtime, runtimeReleaseFields); - valid(strings(runtime, runtimeReleaseFields)); - result.runtime = { ...runtime } as unknown as SandboxRuntimeRelease; - } return result; } /** The adapter's public projection has no credential member. */ @@ -181,26 +175,23 @@ function projectDeployment(value: unknown): SandboxDeployment { } as unknown as SandboxDeployment; } -/** Core omits an empty `diagnostic`, so a present one is a code; an unknown code reads as provider_unavailable. */ function projectNode(node: Record): SandboxNode { - const { diagnostic, rollout, ...rest } = node; + const { rollout, ...rest } = node; const fields = { ...rest, rollout: projectNodeRollout(rollout, node.online) }; valid(strings(node, ["id", "name", "provider", "core_url"]) && typeof node.online === "boolean" && typeof node.provider_ready === "boolean" && [node.cpu_count, node.available_memory_bytes, node.available_disk_bytes].every(nullable(isNonnegativeInteger)) && [node.running, node.snapshots, node.max_active, node.max_retained, node.active, node.reserved, node.retained, node.cleanup_pending].every(isNonnegativeInteger) && - nullable(timestamp)(node.last_seen_at) && timestamp(node.created_at) && (node.enrollment_id === null || typeof node.enrollment_id === "string") && - (diagnostic === undefined || (typeof diagnostic === "string" && diagnostic !== ""))); - if (diagnostic === undefined) return { ...fields } as unknown as SandboxNode; - return { ...fields, diagnostic: normalizeSandboxNodeDiagnostic(diagnostic as string) } as unknown as SandboxNode; + nullable(timestamp)(node.last_seen_at) && timestamp(node.created_at) && (node.enrollment_id === null || typeof node.enrollment_id === "string")); + return fields as unknown as SandboxNode; } function projectNodeList(value: unknown): { data: SandboxNode[] } { const list = members(value, sandboxNodeListFields); valid(Array.isArray(list.data)); - return { data: (list.data as unknown[]).map((entry) => projectNode(members(entry, nodeFields, nodeRequired))) }; + return { data: (list.data as unknown[]).map((entry) => projectNode(members(entry, nodeFields))) }; } /** A never-observed host is all null; the history lists every bucket of the range. */ function projectNodeDetail(value: unknown, nodeId: string): SandboxNodeDetail { - const { host, history, ...fields } = members(value, nodeDetailFields, nodeDetailRequired); + const { host, history, ...fields } = members(value, nodeDetailFields); const node = projectNode(fields); const observed = members(host, nodeHostFields); const buckets = members(history, hostHistoryFields); diff --git a/services/core/cmd/oac/init.go b/services/core/cmd/oac/init.go index 3ed11fc77..b56dc13d1 100644 --- a/services/core/cmd/oac/init.go +++ b/services/core/cmd/oac/init.go @@ -209,10 +209,6 @@ func initialize(root string, release releaseIdentity, fetch func() (map[string][ } } if previous.SourceCommit == release.revision { - // The pointer follows the receipt, so a crash between the two heals here. - if err := writePointer(root, release); err != nil { - return err - } if err := syncCoreKeyDigest(root); err != nil { return err } @@ -229,7 +225,7 @@ func initialize(root string, release releaseIdentity, fetch func() (map[string][ logger.InfoContext(ctx, "Bundled node installation metadata verified", "file_count", len(files)) nextStep("publish_node_metadata") names := []string{} - prefix := "node-payload/releases/" + release.revision + "/" + prefix := "node-payload/" for _, name := range releaseMembers { if err := writeOwned(filepath.Join(root, prefix+name), files[name]); err != nil { return err @@ -294,24 +290,14 @@ func initialize(root string, release releaseIdentity, fetch func() (map[string][ receipt.Files[name] = checksum } raw, _ = json.Marshal(receipt) - // The receipt is the commit point; the pointer follows it. if err := writeOwned(marker, raw); err != nil { return err } - if err := writePointer(root, release); err != nil { - return err - } logger.InfoContext(ctx, "Installation initialized", "sign_in_key_command", "docker compose exec web oac-web core-key") return nil }) } -// writePointer selects the installation's release for Web and node installs. -func writePointer(root string, release releaseIdentity) error { - active, _ := json.Marshal(map[string]string{"source_commit": release.revision}) - return writeOwned(filepath.Join(root, "node-payload", "active.json"), active) -} - func randomBytes(n int) []byte { value := make([]byte, n) if _, err := rand.Read(value); err != nil { diff --git a/services/core/cmd/oac/init_test.go b/services/core/cmd/oac/init_test.go index b914c55e0..64a2681e9 100644 --- a/services/core/cmd/oac/init_test.go +++ b/services/core/cmd/oac/init_test.go @@ -136,22 +136,17 @@ func TestInitializeRefusesChangedOrMissingFiles(t *testing.T) { } } -func TestInitializeHealsTheReleasePointerAfterACrash(t *testing.T) { - root, release, files := initFixture(t) - if err := initialize(root, release, fixed(files)); err != nil { - t.Fatal(err) - } - pointer := filepath.Join(root, "node-payload", "active.json") - if err := os.WriteFile(pointer, []byte("{}"), 0o600); err != nil { +func readReceipt(t *testing.T, root string) installReceipt { + t.Helper() + var receipt installReceipt + raw, err := os.ReadFile(filepath.Join(root, "installation.json")) + if err != nil { t.Fatal(err) } - if err := initialize(root, release, refuseDownload(t)); err != nil { + if err := json.Unmarshal(raw, &receipt); err != nil { t.Fatal(err) } - want := `{"source_commit":"` + release.revision + `"}` - if healed, _ := os.ReadFile(pointer); string(healed) != want { - t.Fatalf("pointer = %s", healed) - } + return receipt } func TestInitializeRefusesDataOfAnotherRelease(t *testing.T) { diff --git a/services/core/cmd/sandbox-node/generations.go b/services/core/cmd/sandbox-node/generations.go index 136aabac9..6274e89b0 100644 --- a/services/core/cmd/sandbox-node/generations.go +++ b/services/core/cmd/sandbox-node/generations.go @@ -40,14 +40,9 @@ func runGenerations(ctx context.Context, registry *providerconfig.Registry, conf if err != nil { return err } - pending, err := filepath.Glob(filepath.Join(stateDir, "generations", "*.preparing")) - if err != nil { - return err - } - paths = append(append([]string{configFile}, paths...), pending...) + paths = append([]string{configFile}, paths...) values := map[uint64]node.GenerationProvider{} recovery := []sandbox.GenerationReference{} - collection := []sandbox.GenerationReference{} seen := map[uint64]sandbox.NodeConfig{} closeValues := func() { for _, v := range values { @@ -57,19 +52,7 @@ func runGenerations(ctx context.Context, registry *providerconfig.Registry, conf } } for _, path := range paths { - var config sandbox.NodeConfig - if strings.HasSuffix(path, ".preparing") { - journal, readErr := readGenerationJournal(path) - err = readErr - if err == nil && journal.Configuration == nil { - err = sandbox.ErrOwnership - } - if err == nil { - config = *journal.Configuration - } - } else { - config, err = providerconfig.Load(path) - } + config, err := providerconfig.Load(path) if err != nil { closeValues() return err @@ -78,16 +61,16 @@ func runGenerations(ctx context.Context, registry *providerconfig.Registry, conf closeValues() return sandbox.ErrOwnership } - if path != configFile && strings.TrimSuffix(strings.TrimSuffix(filepath.Base(path), ".json"), ".preparing") != strconv.FormatUint(config.Generation, 10) { + if path != configFile && strings.TrimSuffix(filepath.Base(path), ".json") != strconv.FormatUint(config.Generation, 10) { closeValues() return sandbox.ErrOwnership } - state, err := generationLocalState(config, stateDir) + dropped, err := generationDropped(config, stateDir) if err != nil { closeValues() return err } - if state == "dropped" { + if dropped { continue } if previous, ok := seen[config.Generation]; ok { @@ -98,14 +81,6 @@ func runGenerations(ctx context.Context, registry *providerconfig.Registry, conf continue } seen[config.Generation] = config - if state == "preparing" { - recovery = append(recovery, sandbox.GenerationReference{Generation: config.Generation, SpecificationDigest: config.Specification.Digest(config.Provider)}) - continue - } - if state == "collecting" { - collection = append(collection, sandbox.GenerationReference{Generation: config.Generation, SpecificationDigest: config.Specification.Digest(config.Provider)}) - continue - } value, err := buildGeneration(registry, config, stateDir) if errors.Is(err, os.ErrNotExist) { recovery = append(recovery, sandbox.GenerationReference{Generation: config.Generation, SpecificationDigest: config.Specification.Digest(config.Provider)}) @@ -135,7 +110,7 @@ func runGenerations(ctx context.Context, registry *providerconfig.Registry, conf for _, v := range values { initial = append(initial, v) } - manager, err := node.NewGenerationManager(ctx, node.GenerationManagerOptions{Initial: initial, Recover: recovery, Collect: collection, + manager, err := node.NewGenerationManager(ctx, node.GenerationManagerOptions{Initial: initial, Recover: recovery, Prepare: func(ctx context.Context, generation uint64, digest string) (node.GenerationProvider, error) { if err := runHelper(ctx, "prepare", generation, digest); err != nil { return node.GenerationProvider{}, err @@ -176,12 +151,9 @@ func buildGeneration(registry *providerconfig.Registry, config sandbox.NodeConfi } type generationJournal struct { - InstallationID string `json:"installation_id"` - Generation uint64 `json:"generation"` - SpecificationDigest string `json:"specification_digest"` - NativeComplete json.RawMessage `json:"native_complete,omitempty"` - ImportStarted json.RawMessage `json:"import_started,omitempty"` - Configuration *sandbox.NodeConfig `json:"configuration,omitempty"` + InstallationID string `json:"installation_id"` + Generation uint64 `json:"generation"` + SpecificationDigest string `json:"specification_digest"` } func readGenerationJournal(path string) (generationJournal, error) { @@ -211,62 +183,43 @@ func readGenerationJournal(path string) (generationJournal, error) { return journal, nil } -// The preparation configuration is an immutable plan, never a usable provider. -// A generation's Core-visible envelope is fixed; native is the adapter's -// node-local state, which preparation may resolve and the adapter validates -// when it builds. +// sameGenerationPlan compares the Core-visible envelope of two copies of one +// generation; native is the adapter's node-local state. func sameGenerationPlan(final, plan sandbox.NodeConfig) bool { final.Native, plan.Native = nil, nil return reflect.DeepEqual(final, plan) } -// Pending-only entries stay recovery/collection-only. No journal is readiness -// or authority to collect without a fresh current-connection Core grant. -func generationLocalState(config sandbox.NodeConfig, stateDir string) (string, error) { +// generationDropped reports whether the permanent dropped receipt names this +// generation. No receipt is readiness or authority to collect without a fresh +// current-connection Core grant. +func generationDropped(config sandbox.NodeConfig, stateDir string) (bool, error) { directory := filepath.Join(stateDir, "generations") info, err := os.Lstat(directory) if os.IsNotExist(err) { - return "", nil + return false, nil } if err != nil { - return "", err + return false, err } owner, ok := info.Sys().(*syscall.Stat_t) resolved, err := filepath.EvalSymlinks(directory) if err != nil || resolved != directory || !info.IsDir() || info.Mode().Perm() != 0700 || !ok || owner.Uid != uint32(os.Getuid()) { - return "", sandbox.ErrOwnership + return false, sandbox.ErrOwnership } - state := "" - for _, suffix := range []string{".preparing", ".collecting", ".dropped"} { - path := filepath.Join(directory, strconv.FormatUint(config.Generation, 10)+suffix) - journal, err := readGenerationJournal(path) - if errors.Is(err, os.ErrNotExist) { - continue - } - if err != nil || journal.InstallationID != config.InstallationID || journal.Generation != config.Generation || journal.SpecificationDigest != config.Specification.Digest(config.Provider) { - return "", sandbox.ErrOwnership - } - if suffix == ".preparing" { - if len(journal.NativeComplete) != 0 || (string(journal.ImportStarted) != "true" && string(journal.ImportStarted) != "false") || journal.Configuration == nil || !sameGenerationPlan(config, *journal.Configuration) { - return "", sandbox.ErrOwnership - } - } else if len(journal.ImportStarted) != 0 || journal.Configuration != nil || suffix == ".collecting" && string(journal.NativeComplete) != "true" && string(journal.NativeComplete) != "false" || suffix == ".dropped" && len(journal.NativeComplete) != 0 { - return "", sandbox.ErrOwnership - } - state = strings.TrimPrefix(suffix, ".") + journal, err := readGenerationJournal(filepath.Join(directory, strconv.FormatUint(config.Generation, 10)+".dropped")) + if errors.Is(err, os.ErrNotExist) { + return false, nil } - return state, nil + if err != nil || journal.InstallationID != config.InstallationID || journal.Generation != config.Generation || journal.SpecificationDigest != config.Specification.Digest(config.Provider) { + return false, sandbox.ErrOwnership + } + return true, nil } -// Exit 65 is reserved by the private preparer for artifact transfer/provenance. -// Arbitrary provider output never selects a wire diagnostic. func generationHelperError(ctx context.Context, err error) error { if ctx.Err() != nil { return ctx.Err() } - var exit *exec.ExitError - if errors.As(err, &exit) && exit.ExitCode() == 65 { - return sandbox.ErrRuntimeDownloadFailed - } return errors.New("local generation operation did not complete") } diff --git a/services/core/cmd/sandbox-node/generations_test.go b/services/core/cmd/sandbox-node/generations_test.go index ae08f7ad0..d830b300c 100644 --- a/services/core/cmd/sandbox-node/generations_test.go +++ b/services/core/cmd/sandbox-node/generations_test.go @@ -14,143 +14,95 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) -func TestGenerationJournalRestartIdentity(t *testing.T) { +func TestGenerationDroppedReceiptIdentity(t *testing.T) { config := sandbox.NodeConfig{InstallationID: "installation", Generation: 9, Provider: "docker"} stateDir := t.TempDir() directory := filepath.Join(stateDir, "generations") if err := os.Mkdir(directory, 0700); err != nil { t.Fatal(err) } + path := filepath.Join(directory, "9.dropped") journal := map[string]any{"installation_id": config.InstallationID, "generation": config.Generation, "specification_digest": config.Specification.Digest(config.Provider)} - for _, suffix := range []string{".collecting", ".dropped"} { - t.Run(suffix, func(t *testing.T) { - path := filepath.Join(directory, "9"+suffix) - if suffix == ".collecting" { - journal["native_complete"] = true - } else { - delete(journal, "native_complete") - } - write := func(value map[string]any) { - raw, _ := json.Marshal(value) - if err := os.WriteFile(path, raw, 0600); err != nil { - t.Fatal(err) - } - } - write(journal) - state, err := generationLocalState(config, stateDir) - if err != nil || state != suffix[1:] { - t.Fatal(state, err) - } - for _, field := range []string{"installation_id", "generation", "specification_digest"} { - previous := journal[field] - journal[field] = "foreign" - write(journal) - if _, err = generationLocalState(config, stateDir); err == nil { - t.Fatal("accepted mismatched", field) - } - journal[field] = previous - } - journal["native_complete"] = nil - write(journal) - if _, err = generationLocalState(config, stateDir); err == nil { - t.Fatal("accepted null journal phase") - } - if suffix == ".collecting" { - journal["native_complete"] = true - } else { - delete(journal, "native_complete") - } - write(journal) - if err = os.Chmod(path, 0644); err != nil { - t.Fatal(err) - } - if _, err = generationLocalState(config, stateDir); err == nil { - t.Fatal("accepted non-private marker") - } - if err = os.Remove(path); err != nil { - t.Fatal(err) - } - foreign := filepath.Join(directory, "foreign") - raw, _ := json.Marshal(journal) - if err = os.WriteFile(foreign, raw, 0600); err != nil { - t.Fatal(err) - } - for _, link := range []func(string, string) error{os.Symlink, os.Link} { - if err = link(foreign, path); err != nil { - t.Fatal(err) - } - if _, err = generationLocalState(config, stateDir); err == nil { - t.Fatal("accepted linked journal") - } - if err = os.Remove(path); err != nil { - t.Fatal(err) - } - } - if err = syscall.Mkfifo(path, 0600); err != nil { - t.Fatal(err) - } - if _, err = generationLocalState(config, stateDir); err == nil { - t.Fatal("accepted FIFO journal") - } - if err = os.Remove(path); err != nil { - t.Fatal(err) - } - }) - } -} - -func TestGenerationHelperUsesOnlyTypedExit(t *testing.T) { - for _, code := range []string{"1", "65", "78"} { - err := exec.Command("sh", "-c", "exit "+code).Run() - got := generationHelperError(t.Context(), err) - if errors.Is(got, sandbox.ErrRuntimeDownloadFailed) != (code == "65") { - t.Fatal(code, got) + write := func(value map[string]any) { + raw, _ := json.Marshal(value) + if err := os.WriteFile(path, raw, 0600); err != nil { + t.Fatal(err) } - ctx, cancel := context.WithCancel(t.Context()) - cancel() - if !errors.Is(generationHelperError(ctx, err), context.Canceled) { - t.Fatal("cancellation was lost") + } + if dropped, err := generationDropped(config, stateDir); err != nil || dropped { + t.Fatal("absent receipt", dropped, err) + } + write(journal) + if dropped, err := generationDropped(config, stateDir); err != nil || !dropped { + t.Fatal(dropped, err) + } + for _, field := range []string{"installation_id", "generation", "specification_digest"} { + previous := journal[field] + journal[field] = "foreign" + write(journal) + if _, err := generationDropped(config, stateDir); err == nil { + t.Fatal("accepted mismatched", field) } + journal[field] = previous } - if errors.Is(generationHelperError(t.Context(), errors.New("runtime_download_failed")), sandbox.ErrRuntimeDownloadFailed) { - t.Fatal("parsed arbitrary provider text") + journal["native_complete"] = true + write(journal) + if _, err := generationDropped(config, stateDir); err == nil { + t.Fatal("accepted an unknown journal field") } -} - -func TestUnresolvedPreparationRemainsRecoveryOnly(t *testing.T) { - config := sandbox.NodeConfig{InstallationID: "installation", Generation: 2, Provider: "docker"} - stateDir := t.TempDir() - directory := filepath.Join(stateDir, "generations") - if err := os.Mkdir(directory, 0700); err != nil { + delete(journal, "native_complete") + write(journal) + if err := os.Chmod(path, 0644); err != nil { t.Fatal(err) } - path := filepath.Join(directory, "2.preparing") - for _, started := range []bool{false, true} { - raw, _ := json.Marshal(map[string]any{"installation_id": config.InstallationID, "generation": config.Generation, "specification_digest": config.Specification.Digest(config.Provider), "import_started": started, "configuration": config}) - if err := os.WriteFile(path, raw, 0600); err != nil { + if _, err := generationDropped(config, stateDir); err == nil { + t.Fatal("accepted non-private marker") + } + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + foreign := filepath.Join(directory, "foreign") + raw, _ := json.Marshal(journal) + if err := os.WriteFile(foreign, raw, 0600); err != nil { + t.Fatal(err) + } + for _, link := range []func(string, string) error{os.Symlink, os.Link} { + if err := link(foreign, path); err != nil { t.Fatal(err) } - journal, err := readGenerationJournal(path) - if err != nil || journal.Configuration == nil { - t.Fatal("pending-only discovery", err) + if _, err := generationDropped(config, stateDir); err == nil { + t.Fatal("accepted linked journal") } - state, err := generationLocalState(*journal.Configuration, stateDir) - if err != nil || state != "preparing" { - t.Fatal(state, err) - } - if _, err := os.Stat(filepath.Join(directory, "2.json")); !os.IsNotExist(err) { - t.Fatal("unresolved plan published") + if err := os.Remove(path); err != nil { + t.Fatal(err) } } + if err := syscall.Mkfifo(path, 0600); err != nil { + t.Fatal(err) + } + if _, err := generationDropped(config, stateDir); err == nil { + t.Fatal("accepted FIFO journal") + } +} + +func TestGenerationHelperKeepsCancellation(t *testing.T) { + err := exec.Command("sh", "-c", "exit 1").Run() + if got := generationHelperError(t.Context(), err); got == nil || errors.Is(got, context.Canceled) { + t.Fatal(got) + } + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if !errors.Is(generationHelperError(ctx, err), context.Canceled) { + t.Fatal("cancellation was lost") + } } // Preparation may resolve native state, such as the digest a containerd image // store names the loaded Runtime image by; the envelope stays fixed. func TestGenerationPlanFixesOnlyTheEnvelope(t *testing.T) { - release := sandbox.RuntimeRelease{ImageID: "sha256:" + strings.Repeat("b", 64), ImageManifestDigest: "sha256:" + strings.Repeat("c", 64)} - plan := sandbox.NodeConfig{InstallationID: "installation", Generation: 2, Provider: "docker", Specification: sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 2, MemoryMiB: 2048}, Runtime: &release}, Native: json.RawMessage(`{"image":"` + release.ImageID + `"}`)} + plan := sandbox.NodeConfig{InstallationID: "installation", Generation: 2, Provider: "docker", Specification: sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 2, MemoryMiB: 2048}}, Native: json.RawMessage(`{"image":"sha256:` + strings.Repeat("b", 64) + `"}`)} final := plan - final.Native = json.RawMessage(`{"image":"` + release.ImageManifestDigest + `"}`) + final.Native = json.RawMessage(`{"image":"sha256:` + strings.Repeat("c", 64) + `"}`) if !sameGenerationPlan(final, plan) { t.Fatal("refused a resolved native image") } @@ -186,7 +138,7 @@ func TestGenerationStateDirectoryOwnership(t *testing.T) { } stateDir = link } - _, err := generationLocalState(sandbox.NodeConfig{Generation: 1}, stateDir) + _, err := generationDropped(sandbox.NodeConfig{Generation: 1}, stateDir) if mode == "private" && err != nil || mode != "private" && !errors.Is(err, sandbox.ErrOwnership) { t.Fatalf("directory ownership: %v", err) } diff --git a/services/core/cmd/sandbox-node/main.go b/services/core/cmd/sandbox-node/main.go index 4e955be99..5b6695c6b 100644 --- a/services/core/cmd/sandbox-node/main.go +++ b/services/core/cmd/sandbox-node/main.go @@ -44,10 +44,6 @@ func exitCode(err error) int { } func run(ctx context.Context, args []string) error { - if len(args) == 1 && args[0] == "protocol-version" { - fmt.Println(node.ProtocolVersion) - return nil - } if len(args) == 0 || (args[0] != "register" && args[0] != "run") { return errors.New("usage: oac-node register|run --config PATH --state-dir PATH") } @@ -69,14 +65,10 @@ func run(ctx context.Context, args []string) error { registry := providerconfig.Builtin() if args[0] == "run" { helper := filepath.Join(filepath.Dir(*configFile), "generation-preparer.pyz") - if info, err := os.Lstat(helper); err == nil { - if !info.Mode().IsRegular() || info.Mode().Perm() != 0600 { - return errors.New("generation preparer must be a private regular file") - } - return runGenerations(ctx, registry, *configFile, *stateDir) - } else if !os.IsNotExist(err) { - return err + if info, err := os.Lstat(helper); err != nil || !info.Mode().IsRegular() || info.Mode().Perm() != 0600 { + return errors.New("generation preparer must be a private regular file") } + return runGenerations(ctx, registry, *configFile, *stateDir) } config, err := providerconfig.Load(*configFile) if err != nil { @@ -92,63 +84,41 @@ func run(ctx context.Context, args []string) error { return errors.New("node configuration requires one local provider backend") } expected := node.Identity{SpecificationDigest: built.SpecificationDigest, DeploymentGeneration: config.Generation, InstallationID: built.InstallationID, Provider: config.Provider, BackendFingerprint: built.BackendFingerprint} - probe := func(ctx context.Context) (node.Health, error) { - checkpoint, err := built.Probe(ctx) - status := sandbox.GenerationStatus{Generation: config.Generation, SpecificationDigest: built.SpecificationDigest, State: "ready", Checkpoint: checkpoint} - if err != nil { - status.State, status.Diagnostic, status.Checkpoint = "failed", sandbox.NodeDiagnostic(err), nil - } - return node.Health{ProviderReady: err == nil, Generations: []sandbox.GenerationStatus{status}}, err + if !filepath.IsAbs(*tokenFile) { + return errors.New("enrollment-token-file must be absolute") } - if args[0] == "register" { - if !filepath.IsAbs(*tokenFile) { - return errors.New("enrollment-token-file must be absolute") - } - fd, err := syscall.Open(*tokenFile, syscall.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_CLOEXEC, 0) - if err != nil { - return errors.New("cannot open enrollment token") - } - file := os.NewFile(uintptr(fd), "enrollment token") - st, err := file.Stat() - if err != nil || !st.Mode().IsRegular() || st.Mode().Perm() != 0600 { - file.Close() - return errors.New("enrollment token must be a private regular file (0600)") - } - raw, err := io.ReadAll(io.LimitReader(file, 4097)) + fd, err := syscall.Open(*tokenFile, syscall.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_CLOEXEC, 0) + if err != nil { + return errors.New("cannot open enrollment token") + } + file := os.NewFile(uintptr(fd), "enrollment token") + st, err := file.Stat() + if err != nil || !st.Mode().IsRegular() || st.Mode().Perm() != 0600 { file.Close() - if err != nil { - return errors.New("cannot read enrollment token") - } - token := strings.TrimSpace(string(raw)) - if token == "" || len(token) > 4096 { - return errors.New("invalid enrollment token") - } - if _, err = node.InitIdentity(*stateDir, *coreURL, expected); err != nil { - return err - } - probeCtx, stopProbe := context.WithTimeout(ctx, 5*time.Second) - _, err = probe(probeCtx) - stopProbe() - if err != nil { - return fmt.Errorf("local provider readiness check failed (%s): %w", sandbox.NodeDiagnostic(err), err) - } - stored, err := node.Enroll(ctx, *coreURL, *stateDir, token, node.EnrollmentRequest{Name: *name}) - if err != nil { - return err - } - return json.NewEncoder(os.Stdout).Encode(node.EnrollmentResponse{MaxActive: stored.Identity.MaxActive, MaxRetained: stored.Identity.MaxRetained, SpecificationDigest: stored.Identity.SpecificationDigest, DeploymentGeneration: stored.Identity.DeploymentGeneration, NodeID: stored.Identity.NodeID, InstallationID: stored.Identity.InstallationID, Provider: stored.Identity.Provider}) + return errors.New("enrollment token must be a private regular file (0600)") } - stored, err := node.RefreshIdentity(ctx, *stateDir) + raw, err := io.ReadAll(io.LimitReader(file, 4097)) + file.Close() if err != nil { + return errors.New("cannot read enrollment token") + } + token := strings.TrimSpace(string(raw)) + if token == "" || len(token) > 4096 { + return errors.New("invalid enrollment token") + } + if _, err = node.InitIdentity(*stateDir, *coreURL, expected); err != nil { return err } - expected.NodeID = stored.Identity.NodeID - if expected.SpecificationDigest != stored.Identity.SpecificationDigest || expected.DeploymentGeneration != stored.Identity.DeploymentGeneration || expected.InstallationID != stored.Identity.InstallationID || expected.Provider != stored.Identity.Provider || expected.BackendFingerprint != stored.Identity.BackendFingerprint { - return errors.New("provider configuration differs from retained node identity") + probeCtx, stopProbe := context.WithTimeout(ctx, 5*time.Second) + _, err = built.Probe(probeCtx) + stopProbe() + if err != nil { + return fmt.Errorf("local provider readiness check failed (%s): %w", sandbox.NodeDiagnostic(err), err) } - if *coreURL != "" && *coreURL != stored.CoreURL { - return errors.New("run uses the retained Core URL") + stored, err := node.Enroll(ctx, *coreURL, *stateDir, token, node.EnrollmentRequest{Name: *name}) + if err != nil { + return err } - return node.Run(ctx, node.AgentConfig{CoreURL: stored.CoreURL, StateDirectory: *stateDir, Identity: stored.Identity, Credential: stored.Credential, Provider: built.Provider, Probe: probe}) + return json.NewEncoder(os.Stdout).Encode(node.EnrollmentResponse{MaxActive: stored.Identity.MaxActive, MaxRetained: stored.Identity.MaxRetained, SpecificationDigest: stored.Identity.SpecificationDigest, DeploymentGeneration: stored.Identity.DeploymentGeneration, NodeID: stored.Identity.NodeID, InstallationID: stored.Identity.InstallationID, Provider: stored.Identity.Provider}) } diff --git a/services/core/cmd/sandbox-node/main_test.go b/services/core/cmd/sandbox-node/main_test.go index 4da4298bc..b83e08a41 100644 --- a/services/core/cmd/sandbox-node/main_test.go +++ b/services/core/cmd/sandbox-node/main_test.go @@ -58,13 +58,3 @@ func refreshIdentity(t *testing.T, coreURL string) error { _, err := node.RefreshIdentity(t.Context(), dir) return err } - -func TestProtocolVersionRequiresNoProviderOrIdentity(t *testing.T) { - // This is a binary capability check, not a provider readiness probe. - if err := run(t.Context(), []string{"protocol-version"}); err != nil { - t.Fatal(err) - } - if err := run(t.Context(), []string{"protocol-version", "--config", "/missing"}); err == nil { - t.Fatal("extra protocol capability arguments accepted") - } -} diff --git a/services/core/cmd/server/managed_nodes.go b/services/core/cmd/server/managed_nodes.go index c514c7e3d..03168abf3 100644 --- a/services/core/cmd/server/managed_nodes.go +++ b/services/core/cmd/server/managed_nodes.go @@ -30,7 +30,7 @@ func configureManagedNodes(nodes *deployment.Service, reader deployment.Reader, if err := owner(ctx); err != nil { return err } - return nodes.HeartbeatGenerations(ctx, n.NodeID, connection, epoch, nodeHealthRecord(health), health.Generations) + return nodes.Heartbeat(ctx, n.NodeID, connection, epoch, nodeHealthRecord(health), health.Generations) }, Retention: func(ctx context.Context, n node.Identity, connection string, epoch uint64, refs []sandbox.GenerationReference) (sandbox.NodeDeployment, []sandbox.GenerationRetention, error) { if err := owner(ctx); err != nil { @@ -63,12 +63,6 @@ func configureManagedNodes(nodes *deployment.Service, reader deployment.Reader, Disconnected: func(ctx context.Context, n node.Identity, connection string, epoch uint64) { _ = nodes.DisconnectNode(ctx, n.NodeID, connection, epoch) }, - Heartbeat: func(ctx context.Context, n node.Identity, connection string, epoch uint64, health node.Health) error { - if err := owner(ctx); err != nil { - return err - } - return nodes.Heartbeat(ctx, n.NodeID, connection, epoch, nodeHealthRecord(health), health.Generations) - }, }) result.setup = &managedSetup{processPaths: config.ProviderPaths, registry: registry, deployment: nodes, allocations: reader, hub: result.hub, installationID: config.InstallationID, runtimeAPI: config.PublicOrigin.RuntimeAPI()} result.runtime = execution.NewDeferredRuntimeProvider(config.InstallationID, result.setup.load, result.setup.prepare) @@ -77,5 +71,5 @@ func configureManagedNodes(nodes *deployment.Service, reader deployment.Reader, } func nodeHealthRecord(health node.Health) deployment.NodeHealth { - return deployment.NodeHealth{Host: &deployment.NodeHost{EffectiveCPUCores: health.EffectiveCPUCores, CPUUtilization: health.CPUUtilization, TotalMemoryBytes: health.TotalMemoryBytes, AvailableMemoryBytes: health.AvailableMemoryBytes, AvailableDiskBytes: health.AvailableDiskBytes, ObservedAt: &health.ObservedAt}, ProviderReady: health.ProviderReady, Diagnostic: sandbox.NodeDiagnosticCode(health.Diagnostic), CPUCount: health.CPUCount, AvailableMemoryBytes: health.AvailableMemoryBytes, AvailableDiskBytes: health.AvailableDiskBytes} + return deployment.NodeHealth{Host: &deployment.NodeHost{EffectiveCPUCores: health.EffectiveCPUCores, CPUUtilization: health.CPUUtilization, TotalMemoryBytes: health.TotalMemoryBytes, AvailableMemoryBytes: health.AvailableMemoryBytes, AvailableDiskBytes: health.AvailableDiskBytes, ObservedAt: &health.ObservedAt}, CPUCount: health.CPUCount, AvailableMemoryBytes: health.AvailableMemoryBytes, AvailableDiskBytes: health.AvailableDiskBytes} } diff --git a/services/core/internal/api/core_store_validation_test.go b/services/core/internal/api/core_store_validation_test.go index bd950b480..eafc6d5d1 100644 --- a/services/core/internal/api/core_store_validation_test.go +++ b/services/core/internal/api/core_store_validation_test.go @@ -33,7 +33,6 @@ func TestCoreStoreValidationFieldsAndPublicFallback(t *testing.T) { upperCapacityErr := nodes.UpdateNode(context.Background(), managementProjectID, deployment.NodeUpdate{Name: "node", MaxActive: 1000001, MaxRetained: 8}) capacityErr := nodes.UpdateNode(context.Background(), managementProjectID, deployment.NodeUpdate{Name: "node", MaxActive: 0}) _, resourceErr := nodes.SetupForSelection(managementProjectID, sandbox.Selection{Provider: "docker", DeploymentSpec: sandbox.DeploymentSpec{}}) - _, runtimeErr := nodes.SetupForSelection(managementProjectID, sandbox.Selection{Provider: "docker", DeploymentSpec: sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 1, MemoryMiB: 512}}}) for _, tc := range []struct { err error code, param string @@ -46,7 +45,6 @@ func TestCoreStoreValidationFieldsAndPublicFallback(t *testing.T) { {upperCapacityErr, "invalid_node_capacity", "max_active", map[string]any{"min": float64(1), "max": float64(1000000)}, "Invalid resource identifier or request limits.", "invalid_request", writeDeploymentError}, {capacityErr, "invalid_node_capacity", "max_active", map[string]any{"min": float64(1), "max": float64(1000000)}, "Invalid resource identifier or request limits.", "invalid_request", writeDeploymentError}, {resourceErr, "invalid_sandbox_configuration", "resources.cpus", map[string]any{"min": float64(1), "max": float64(255)}, "invalid sandbox configuration: cpus must be 1..255 and memory_mib must be 512..1048576", "invalid_sandbox_configuration", writeDeploymentError}, - {runtimeErr, "invalid_sandbox_configuration", "runtime", nil, "invalid sandbox configuration: managed nodes require a pinned Runtime release", "invalid_sandbox_configuration", writeDeploymentError}, } { if tc.err == nil { t.Fatal("missing validator error") diff --git a/services/core/internal/api/errors_deployment.go b/services/core/internal/api/errors_deployment.go index 76646108d..553518e8a 100644 --- a/services/core/internal/api/errors_deployment.go +++ b/services/core/internal/api/errors_deployment.go @@ -93,7 +93,7 @@ func writeSandboxError(w http.ResponseWriter, err error) bool { case errors.Is(err, deployment.ErrNodeAddressMismatch): writeError(w, http.StatusConflict, "sandbox_node_address_mismatch", "This node uses a different Core address than the installation public URL. Generate a new command on the Nodes page and run it on the host.") case errors.Is(err, deployment.ErrSpecificationMismatch): - writeError(w, http.StatusConflict, "sandbox_specification_mismatch", "The node resource limits or Runtime release do not match the active deployment. Restore its installed configuration or remove and enroll the node again after a drained deployment change.") + writeError(w, http.StatusConflict, "sandbox_specification_mismatch", "The node resource limits do not match the active deployment. Restore its installed configuration or remove and enroll the node again after a drained deployment change.") case errors.Is(err, deployment.ErrNodeExists): writeError(w, http.StatusConflict, "idempotency_conflict", "This idempotency key was used with different input.") case errors.Is(err, deployment.ErrConflict): diff --git a/services/core/internal/api/sandbox_deployment_setup.go b/services/core/internal/api/sandbox_deployment_setup.go index 030fb8251..6e0ba893e 100644 --- a/services/core/internal/api/sandbox_deployment_setup.go +++ b/services/core/internal/api/sandbox_deployment_setup.go @@ -16,11 +16,10 @@ type SandboxDeploymentInput struct { ExpectedGeneration *uint64 `json:"expected_generation" binding:"required"` // Per-sandbox limits, required for Docker and microsandbox. E2B may omit // them; Core then uses the validated template build's cpus and memory_mib. - Resources sandbox.Resources `json:"resources"` - Runtime *sandbox.RuntimeRelease `json:"runtime,omitempty"` - Provider string `json:"provider"` - Configuration json.RawMessage `json:"configuration" swaggertype:"object"` - Credential json.RawMessage `json:"credential,omitempty" swaggertype:"object"` + Resources sandbox.Resources `json:"resources"` + Provider string `json:"provider"` + Configuration json.RawMessage `json:"configuration" swaggertype:"object"` + Credential json.RawMessage `json:"credential,omitempty" swaggertype:"object"` } type SandboxDeploymentChangeInput struct { @@ -33,7 +32,7 @@ func (h *Handler) selection(v SandboxDeploymentInput) (sandbox.Selection, error) if err != nil { return sandbox.Selection{}, err } - return sandbox.Selection{ExpectedGeneration: *v.ExpectedGeneration, Provider: v.Provider, DeploymentSpec: sandbox.DeploymentSpec{Resources: v.Resources, Runtime: v.Runtime}, Configuration: c}, nil + return sandbox.Selection{ExpectedGeneration: *v.ExpectedGeneration, Provider: v.Provider, DeploymentSpec: sandbox.DeploymentSpec{Resources: v.Resources}, Configuration: c}, nil } // DeploymentChanges sets up and updates the sandbox deployment through the @@ -51,7 +50,7 @@ type DeploymentReset interface { } // @Summary Initialize the deployment sandbox provider -// @Description Selects a provider, enforced resource limits and pinned Runtime release. Core derives the deployment's core_url from the installation public URL and rejects a core_url member with 400. E2B returns 409 sandbox_configuration_error while the public URL is loopback. E2B credentials are write-only. E2B may omit resources to adopt the validated template build's CPU and memory, returned in specification.resources. Requires explicit expected_generation, including zero at first setup. Stale retries reject before provider validation. An identical selection at the current generation is a no-op; a differing selection rejects. This does not create compute or execute work. +// @Description Selects a provider and its enforced resource limits. Core derives the deployment's core_url from the installation public URL and rejects a core_url member with 400. E2B returns 409 sandbox_configuration_error while the public URL is loopback. E2B credentials are write-only. E2B may omit resources to adopt the validated template build's CPU and memory, returned in specification.resources. Requires explicit expected_generation, including zero at first setup. Stale retries reject before provider validation. An identical selection at the current generation is a no-op; a differing selection rejects. This does not create compute or execute work. // @Tags Sandbox Manager // @Produce json // @Security DeploymentAdminAuth @@ -66,7 +65,7 @@ func (h *Handler) initializeSandboxDeployment(w http.ResponseWriter, r *http.Req return } var input SandboxDeploymentInput - if decodeInputObject(raw, &input, "provider", "configuration", "credential", "resources", "runtime", "expected_generation") != nil || input.ExpectedGeneration == nil { + if decodeInputObject(raw, &input, "provider", "configuration", "credential", "resources", "expected_generation") != nil || input.ExpectedGeneration == nil { writeDeploymentError(w, r, deployment.ErrInvalidInput) return } @@ -99,7 +98,7 @@ func (h *Handler) updateSandboxDeployment(w http.ResponseWriter, r *http.Request return } var input SandboxDeploymentChangeInput - if decodeInputObject(raw, &input, "provider", "configuration", "credential", "resources", "runtime", "expected_generation") != nil || input.ExpectedGeneration == nil { + if decodeInputObject(raw, &input, "provider", "configuration", "credential", "resources", "expected_generation") != nil || input.ExpectedGeneration == nil { writeDeploymentError(w, r, deployment.ErrInvalidInput) return } diff --git a/services/core/internal/db/queries/node_generations.sql b/services/core/internal/db/queries/node_generations.sql index 436268027..5ef5bb1fc 100644 --- a/services/core/internal/db/queries/node_generations.sql +++ b/services/core/internal/db/queries/node_generations.sql @@ -19,12 +19,6 @@ ON CONFLICT(node_id,generation) DO UPDATE SET specification_digest=EXCLUDED.spec UPDATE runtime_nodes SET ready_generation=$2 WHERE id=$1 AND removed_at IS NULL AND $2=(SELECT generation FROM runtime_deployment) AND (ready_generation IS NULL OR ready_generation<$2); --- name: RefreshNodeServingReadiness :exec -UPDATE runtime_nodes n SET protocol_version=$2, provider_ready=EXISTS( - SELECT 1 FROM runtime_node_generation_status g WHERE g.node_id=n.id AND g.generation=n.ready_generation - AND g.connection_id=n.connection_id AND g.owner_epoch=n.connected_epoch AND g.state='ready') -WHERE n.id=$1 AND n.removed_at IS NULL; - -- name: NodeGenerationReady :one SELECT EXISTS(SELECT 1 FROM runtime_node_generation_status g JOIN runtime_nodes n ON n.id=g.node_id CROSS JOIN runtime_deployment d WHERE n.id=sqlc.arg(node_id) AND g.generation=sqlc.arg(generation) AND n.removed_at IS NULL diff --git a/services/core/internal/db/queries/runtime_nodes.sql b/services/core/internal/db/queries/runtime_nodes.sql index e90b407d2..d8c48bc24 100644 --- a/services/core/internal/db/queries/runtime_nodes.sql +++ b/services/core/internal/db/queries/runtime_nodes.sql @@ -10,7 +10,7 @@ SELECT * FROM runtime_nodes WHERE id=$1 AND removed_at IS NULL; -- name: ListRuntimeNodes :many SELECT n.*, (n.connection_id IS NOT NULL AND n.connected_epoch=d.owner_epoch AND n.last_seen_at > clock_timestamp()-interval '45 seconds')::boolean AS online, - d.provider_kind, d.generation AS target_generation, + d.provider_kind, EXISTS(SELECT 1 FROM runtime_node_generation_status g WHERE g.node_id=n.id AND g.generation=n.ready_generation AND g.connection_id=n.connection_id AND g.owner_epoch=d.owner_epoch AND g.state='ready')::boolean AS serving_ready, COALESCE((SELECT g.state FROM runtime_node_generation_status g WHERE g.node_id=n.id AND g.generation=d.generation AND g.connection_id=n.connection_id AND g.owner_epoch=d.owner_epoch),'')::text AS target_state, COALESCE((SELECT g.diagnostic FROM runtime_node_generation_status g WHERE g.node_id=n.id AND g.generation=d.generation AND g.connection_id=n.connection_id AND g.owner_epoch=d.owner_epoch),'')::text AS target_diagnostic, @@ -31,12 +31,12 @@ UPDATE runtime_nodes SET name=$2,max_active=$3,max_retained=$4 WHERE id=$1 AND r UPDATE runtime_nodes SET removed_at=clock_timestamp(),connection_id=NULL,ready_generation=NULL WHERE id=$1 AND removed_at IS NULL; -- name: ConnectRuntimeNode :execrows -UPDATE runtime_nodes SET connection_id=$2,provider_ready=false,connected_epoch=d.owner_epoch,last_seen_at=clock_timestamp() +UPDATE runtime_nodes SET connection_id=$2,connected_epoch=d.owner_epoch,last_seen_at=clock_timestamp() FROM runtime_deployment d WHERE runtime_nodes.id=$1 AND removed_at IS NULL AND runtime_nodes.installation_id=d.installation_id AND d.owner_epoch=sqlc.arg(owner_epoch) AND runtime_nodes.connected_epoch <= sqlc.arg(owner_epoch); -- name: HeartbeatRuntimeNode :execrows -UPDATE runtime_nodes SET last_seen_at=clock_timestamp(),provider_ready=$3,health=sqlc.arg(health)::jsonb FROM runtime_deployment d +UPDATE runtime_nodes SET last_seen_at=clock_timestamp(),health=sqlc.arg(health)::jsonb FROM runtime_deployment d WHERE runtime_nodes.id=$1 AND connection_id=$2 AND removed_at IS NULL AND connected_epoch=d.owner_epoch AND d.owner_epoch=sqlc.arg(owner_epoch); -- name: LockRuntimeNodePresence :one diff --git a/services/core/internal/db/queries/sandbox_deployment_setup.sql b/services/core/internal/db/queries/sandbox_deployment_setup.sql index 6365c7e0f..fb59854f3 100644 --- a/services/core/internal/db/queries/sandbox_deployment_setup.sql +++ b/services/core/internal/db/queries/sandbox_deployment_setup.sql @@ -13,7 +13,7 @@ UPDATE runtime_deployment SET provider_metadata=$1,updated_at=clock_timestamp() WHERE singleton=true AND provider_kind<>''; -- name: RetireSandboxNodes :exec -UPDATE runtime_nodes SET removed_at=clock_timestamp(),connection_id=NULL,provider_ready=false,ready_generation=NULL WHERE removed_at IS NULL; +UPDATE runtime_nodes SET removed_at=clock_timestamp(),connection_id=NULL,ready_generation=NULL WHERE removed_at IS NULL; -- name: RetireSandboxEnrollments :exec UPDATE runtime_node_enrollments SET expires_at=clock_timestamp() WHERE consumed_at IS NULL; diff --git a/services/core/internal/db/queries/sandbox_reset.sql b/services/core/internal/db/queries/sandbox_reset.sql index 5e5132e96..335321ac4 100644 --- a/services/core/internal/db/queries/sandbox_reset.sql +++ b/services/core/internal/db/queries/sandbox_reset.sql @@ -90,7 +90,6 @@ held AS ( ), rollout_nodes AS ( SELECT CASE WHEN n.connection_id IS NULL OR n.connected_epoch <> d.owner_epoch OR n.last_seen_at IS NULL OR n.last_seen_at <= observed.as_of - interval '45 seconds' THEN 'unknown' - WHEN n.protocol_version=1 AND n.deployment_generation <> d.generation THEN 'update_required' ELSE COALESCE(g.state,'unknown') END AS state FROM runtime_nodes n CROSS JOIN deployment d CROSS JOIN observed LEFT JOIN runtime_node_generation_status g ON g.node_id=n.id AND g.generation=d.generation AND g.connection_id=n.connection_id AND g.owner_epoch=d.owner_epoch @@ -114,7 +113,6 @@ SELECT sqlc.embed(d), 'ready',(SELECT count(*) FROM rollout_nodes WHERE state='ready'), 'preparing',(SELECT count(*) FROM rollout_nodes WHERE state='preparing'), 'failed',(SELECT count(*) FROM rollout_nodes WHERE state='failed'), - 'update_required',(SELECT count(*) FROM rollout_nodes WHERE state='update_required'), 'unknown',(SELECT count(*) FROM rollout_nodes WHERE state='unknown')) ELSE NULL END )::jsonb AS rollout FROM runtime_deployment d diff --git a/services/core/internal/db/sqlc/models.go b/services/core/internal/db/sqlc/models.go index 4a12508fd..16d2f1c41 100644 --- a/services/core/internal/db/sqlc/models.go +++ b/services/core/internal/db/sqlc/models.go @@ -311,7 +311,6 @@ type RuntimeNode struct { MaxActive int32 `json:"max_active"` MaxRetained int32 `json:"max_retained"` ConnectionID pgtype.UUID `json:"connection_id"` - ProviderReady bool `json:"provider_ready"` Health []byte `json:"health"` ConnectedEpoch int64 `json:"connected_epoch"` LastSeenAt pgtype.Timestamptz `json:"last_seen_at"` @@ -322,7 +321,6 @@ type RuntimeNode struct { CoreUrl string `json:"core_url"` EnrollmentID pgtype.UUID `json:"enrollment_id"` ReadyGeneration pgtype.Int8 `json:"ready_generation"` - ProtocolVersion int32 `json:"protocol_version"` } type RuntimeNodeEnrollment struct { diff --git a/services/core/internal/db/sqlc/node_generations.sql.go b/services/core/internal/db/sqlc/node_generations.sql.go index f52041660..a26ca6842 100644 --- a/services/core/internal/db/sqlc/node_generations.sql.go +++ b/services/core/internal/db/sqlc/node_generations.sql.go @@ -132,23 +132,6 @@ func (q *Queries) PromoteNodeServingGeneration(ctx context.Context, arg PromoteN return err } -const refreshNodeServingReadiness = `-- name: RefreshNodeServingReadiness :exec -UPDATE runtime_nodes n SET protocol_version=$2, provider_ready=EXISTS( - SELECT 1 FROM runtime_node_generation_status g WHERE g.node_id=n.id AND g.generation=n.ready_generation - AND g.connection_id=n.connection_id AND g.owner_epoch=n.connected_epoch AND g.state='ready') -WHERE n.id=$1 AND n.removed_at IS NULL -` - -type RefreshNodeServingReadinessParams struct { - ID pgtype.UUID `json:"id"` - ProtocolVersion int32 `json:"protocol_version"` -} - -func (q *Queries) RefreshNodeServingReadiness(ctx context.Context, arg RefreshNodeServingReadinessParams) error { - _, err := q.db.Exec(ctx, refreshNodeServingReadiness, arg.ID, arg.ProtocolVersion) - return err -} - const upsertNodeGenerationStatus = `-- name: UpsertNodeGenerationStatus :exec INSERT INTO runtime_node_generation_status(node_id,generation,specification_digest,connection_id,owner_epoch,state,diagnostic,checkpoint) VALUES($1,$2,$3,$4,$5,$6,$7,$8) diff --git a/services/core/internal/db/sqlc/runtime_nodes.sql.go b/services/core/internal/db/sqlc/runtime_nodes.sql.go index c07025aaa..d6635736f 100644 --- a/services/core/internal/db/sqlc/runtime_nodes.sql.go +++ b/services/core/internal/db/sqlc/runtime_nodes.sql.go @@ -12,7 +12,7 @@ import ( ) const connectRuntimeNode = `-- name: ConnectRuntimeNode :execrows -UPDATE runtime_nodes SET connection_id=$2,provider_ready=false,connected_epoch=d.owner_epoch,last_seen_at=clock_timestamp() +UPDATE runtime_nodes SET connection_id=$2,connected_epoch=d.owner_epoch,last_seen_at=clock_timestamp() FROM runtime_deployment d WHERE runtime_nodes.id=$1 AND removed_at IS NULL AND runtime_nodes.installation_id=d.installation_id AND d.owner_epoch=$3 AND runtime_nodes.connected_epoch <= $3 ` @@ -169,7 +169,7 @@ func (q *Queries) GetRuntimeEnrollment(ctx context.Context, tokenSha256 string) } const getRuntimeNode = `-- name: GetRuntimeNode :one -SELECT id, installation_id, name, backend_fingerprint, credential_sha256, max_active, max_retained, connection_id, provider_ready, health, connected_epoch, last_seen_at, created_at, removed_at, specification_digest, deployment_generation, core_url, enrollment_id, ready_generation, protocol_version FROM runtime_nodes WHERE id=$1 AND removed_at IS NULL +SELECT id, installation_id, name, backend_fingerprint, credential_sha256, max_active, max_retained, connection_id, health, connected_epoch, last_seen_at, created_at, removed_at, specification_digest, deployment_generation, core_url, enrollment_id, ready_generation FROM runtime_nodes WHERE id=$1 AND removed_at IS NULL ` func (q *Queries) GetRuntimeNode(ctx context.Context, id pgtype.UUID) (RuntimeNode, error) { @@ -184,7 +184,6 @@ func (q *Queries) GetRuntimeNode(ctx context.Context, id pgtype.UUID) (RuntimeNo &i.MaxActive, &i.MaxRetained, &i.ConnectionID, - &i.ProviderReady, &i.Health, &i.ConnectedEpoch, &i.LastSeenAt, @@ -195,7 +194,6 @@ func (q *Queries) GetRuntimeNode(ctx context.Context, id pgtype.UUID) (RuntimeNo &i.CoreUrl, &i.EnrollmentID, &i.ReadyGeneration, - &i.ProtocolVersion, ) return i, err } @@ -239,23 +237,21 @@ func (q *Queries) GetRuntimePlacement(ctx context.Context, environmentID pgtype. } const heartbeatRuntimeNode = `-- name: HeartbeatRuntimeNode :execrows -UPDATE runtime_nodes SET last_seen_at=clock_timestamp(),provider_ready=$3,health=$4::jsonb FROM runtime_deployment d -WHERE runtime_nodes.id=$1 AND connection_id=$2 AND removed_at IS NULL AND connected_epoch=d.owner_epoch AND d.owner_epoch=$5 +UPDATE runtime_nodes SET last_seen_at=clock_timestamp(),health=$3::jsonb FROM runtime_deployment d +WHERE runtime_nodes.id=$1 AND connection_id=$2 AND removed_at IS NULL AND connected_epoch=d.owner_epoch AND d.owner_epoch=$4 ` type HeartbeatRuntimeNodeParams struct { - ID pgtype.UUID `json:"id"` - ConnectionID pgtype.UUID `json:"connection_id"` - ProviderReady bool `json:"provider_ready"` - Health []byte `json:"health"` - OwnerEpoch int64 `json:"owner_epoch"` + ID pgtype.UUID `json:"id"` + ConnectionID pgtype.UUID `json:"connection_id"` + Health []byte `json:"health"` + OwnerEpoch int64 `json:"owner_epoch"` } func (q *Queries) HeartbeatRuntimeNode(ctx context.Context, arg HeartbeatRuntimeNodeParams) (int64, error) { result, err := q.db.Exec(ctx, heartbeatRuntimeNode, arg.ID, arg.ConnectionID, - arg.ProviderReady, arg.Health, arg.OwnerEpoch, ) @@ -267,7 +263,7 @@ func (q *Queries) HeartbeatRuntimeNode(ctx context.Context, arg HeartbeatRuntime const insertRuntimeNode = `-- name: InsertRuntimeNode :one INSERT INTO runtime_nodes(id,installation_id,name,backend_fingerprint,credential_sha256,max_active,max_retained,specification_digest,deployment_generation,core_url,enrollment_id,ready_generation) -VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$9) RETURNING id, installation_id, name, backend_fingerprint, credential_sha256, max_active, max_retained, connection_id, provider_ready, health, connected_epoch, last_seen_at, created_at, removed_at, specification_digest, deployment_generation, core_url, enrollment_id, ready_generation, protocol_version +VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$9) RETURNING id, installation_id, name, backend_fingerprint, credential_sha256, max_active, max_retained, connection_id, health, connected_epoch, last_seen_at, created_at, removed_at, specification_digest, deployment_generation, core_url, enrollment_id, ready_generation ` type InsertRuntimeNodeParams struct { @@ -308,7 +304,6 @@ func (q *Queries) InsertRuntimeNode(ctx context.Context, arg InsertRuntimeNodePa &i.MaxActive, &i.MaxRetained, &i.ConnectionID, - &i.ProviderReady, &i.Health, &i.ConnectedEpoch, &i.LastSeenAt, @@ -319,7 +314,6 @@ func (q *Queries) InsertRuntimeNode(ctx context.Context, arg InsertRuntimeNodePa &i.CoreUrl, &i.EnrollmentID, &i.ReadyGeneration, - &i.ProtocolVersion, ) return i, err } @@ -379,8 +373,8 @@ func (q *Queries) ListNodeRuntimeAllocations(ctx context.Context, nodeID pgtype. } const listRuntimeNodes = `-- name: ListRuntimeNodes :many -SELECT n.id, n.installation_id, n.name, n.backend_fingerprint, n.credential_sha256, n.max_active, n.max_retained, n.connection_id, n.provider_ready, n.health, n.connected_epoch, n.last_seen_at, n.created_at, n.removed_at, n.specification_digest, n.deployment_generation, n.core_url, n.enrollment_id, n.ready_generation, n.protocol_version, (n.connection_id IS NOT NULL AND n.connected_epoch=d.owner_epoch AND n.last_seen_at > clock_timestamp()-interval '45 seconds')::boolean AS online, - d.provider_kind, d.generation AS target_generation, +SELECT n.id, n.installation_id, n.name, n.backend_fingerprint, n.credential_sha256, n.max_active, n.max_retained, n.connection_id, n.health, n.connected_epoch, n.last_seen_at, n.created_at, n.removed_at, n.specification_digest, n.deployment_generation, n.core_url, n.enrollment_id, n.ready_generation, (n.connection_id IS NOT NULL AND n.connected_epoch=d.owner_epoch AND n.last_seen_at > clock_timestamp()-interval '45 seconds')::boolean AS online, + d.provider_kind, EXISTS(SELECT 1 FROM runtime_node_generation_status g WHERE g.node_id=n.id AND g.generation=n.ready_generation AND g.connection_id=n.connection_id AND g.owner_epoch=d.owner_epoch AND g.state='ready')::boolean AS serving_ready, COALESCE((SELECT g.state FROM runtime_node_generation_status g WHERE g.node_id=n.id AND g.generation=d.generation AND g.connection_id=n.connection_id AND g.owner_epoch=d.owner_epoch),'')::text AS target_state, COALESCE((SELECT g.diagnostic FROM runtime_node_generation_status g WHERE g.node_id=n.id AND g.generation=d.generation AND g.connection_id=n.connection_id AND g.owner_epoch=d.owner_epoch),'')::text AS target_diagnostic, @@ -404,7 +398,6 @@ type ListRuntimeNodesRow struct { MaxActive int32 `json:"max_active"` MaxRetained int32 `json:"max_retained"` ConnectionID pgtype.UUID `json:"connection_id"` - ProviderReady bool `json:"provider_ready"` Health []byte `json:"health"` ConnectedEpoch int64 `json:"connected_epoch"` LastSeenAt pgtype.Timestamptz `json:"last_seen_at"` @@ -415,10 +408,8 @@ type ListRuntimeNodesRow struct { CoreUrl string `json:"core_url"` EnrollmentID pgtype.UUID `json:"enrollment_id"` ReadyGeneration pgtype.Int8 `json:"ready_generation"` - ProtocolVersion int32 `json:"protocol_version"` Online bool `json:"online"` ProviderKind string `json:"provider_kind"` - TargetGeneration int64 `json:"target_generation"` ServingReady bool `json:"serving_ready"` TargetState string `json:"target_state"` TargetDiagnostic string `json:"target_diagnostic"` @@ -448,7 +439,6 @@ func (q *Queries) ListRuntimeNodes(ctx context.Context, nodeID pgtype.UUID) ([]L &i.MaxActive, &i.MaxRetained, &i.ConnectionID, - &i.ProviderReady, &i.Health, &i.ConnectedEpoch, &i.LastSeenAt, @@ -459,10 +449,8 @@ func (q *Queries) ListRuntimeNodes(ctx context.Context, nodeID pgtype.UUID) ([]L &i.CoreUrl, &i.EnrollmentID, &i.ReadyGeneration, - &i.ProtocolVersion, &i.Online, &i.ProviderKind, - &i.TargetGeneration, &i.ServingReady, &i.TargetState, &i.TargetDiagnostic, @@ -582,7 +570,7 @@ func (q *Queries) SetRuntimeObservation(ctx context.Context, arg SetRuntimeObser } const updateRuntimeNode = `-- name: UpdateRuntimeNode :one -UPDATE runtime_nodes SET name=$2,max_active=$3,max_retained=$4 WHERE id=$1 AND removed_at IS NULL RETURNING id, installation_id, name, backend_fingerprint, credential_sha256, max_active, max_retained, connection_id, provider_ready, health, connected_epoch, last_seen_at, created_at, removed_at, specification_digest, deployment_generation, core_url, enrollment_id, ready_generation, protocol_version +UPDATE runtime_nodes SET name=$2,max_active=$3,max_retained=$4 WHERE id=$1 AND removed_at IS NULL RETURNING id, installation_id, name, backend_fingerprint, credential_sha256, max_active, max_retained, connection_id, health, connected_epoch, last_seen_at, created_at, removed_at, specification_digest, deployment_generation, core_url, enrollment_id, ready_generation ` type UpdateRuntimeNodeParams struct { @@ -609,7 +597,6 @@ func (q *Queries) UpdateRuntimeNode(ctx context.Context, arg UpdateRuntimeNodePa &i.MaxActive, &i.MaxRetained, &i.ConnectionID, - &i.ProviderReady, &i.Health, &i.ConnectedEpoch, &i.LastSeenAt, @@ -620,7 +607,6 @@ func (q *Queries) UpdateRuntimeNode(ctx context.Context, arg UpdateRuntimeNodePa &i.CoreUrl, &i.EnrollmentID, &i.ReadyGeneration, - &i.ProtocolVersion, ) return i, err } diff --git a/services/core/internal/db/sqlc/sandbox_deployment_setup.sql.go b/services/core/internal/db/sqlc/sandbox_deployment_setup.sql.go index 9b169c9bf..706812ea6 100644 --- a/services/core/internal/db/sqlc/sandbox_deployment_setup.sql.go +++ b/services/core/internal/db/sqlc/sandbox_deployment_setup.sql.go @@ -82,7 +82,7 @@ func (q *Queries) RetireSandboxEnrollments(ctx context.Context) error { } const retireSandboxNodes = `-- name: RetireSandboxNodes :exec -UPDATE runtime_nodes SET removed_at=clock_timestamp(),connection_id=NULL,provider_ready=false,ready_generation=NULL WHERE removed_at IS NULL +UPDATE runtime_nodes SET removed_at=clock_timestamp(),connection_id=NULL,ready_generation=NULL WHERE removed_at IS NULL ` func (q *Queries) RetireSandboxNodes(ctx context.Context) error { diff --git a/services/core/internal/db/sqlc/sandbox_reset.sql.go b/services/core/internal/db/sqlc/sandbox_reset.sql.go index 4b2ac0e79..302fe1620 100644 --- a/services/core/internal/db/sqlc/sandbox_reset.sql.go +++ b/services/core/internal/db/sqlc/sandbox_reset.sql.go @@ -83,7 +83,6 @@ held AS ( ), rollout_nodes AS ( SELECT CASE WHEN n.connection_id IS NULL OR n.connected_epoch <> d.owner_epoch OR n.last_seen_at IS NULL OR n.last_seen_at <= observed.as_of - interval '45 seconds' THEN 'unknown' - WHEN n.protocol_version=1 AND n.deployment_generation <> d.generation THEN 'update_required' ELSE COALESCE(g.state,'unknown') END AS state FROM runtime_nodes n CROSS JOIN deployment d CROSS JOIN observed LEFT JOIN runtime_node_generation_status g ON g.node_id=n.id AND g.generation=d.generation AND g.connection_id=n.connection_id AND g.owner_epoch=d.owner_epoch @@ -107,7 +106,6 @@ SELECT d.singleton, d.installation_id, d.backend_fingerprint, d.updated_at, d.pr 'ready',(SELECT count(*) FROM rollout_nodes WHERE state='ready'), 'preparing',(SELECT count(*) FROM rollout_nodes WHERE state='preparing'), 'failed',(SELECT count(*) FROM rollout_nodes WHERE state='failed'), - 'update_required',(SELECT count(*) FROM rollout_nodes WHERE state='update_required'), 'unknown',(SELECT count(*) FROM rollout_nodes WHERE state='unknown')) ELSE NULL END )::jsonb AS rollout FROM runtime_deployment d diff --git a/services/core/internal/deployment/fakes_test.go b/services/core/internal/deployment/fakes_test.go index 814aef4d3..9510e5d4a 100644 --- a/services/core/internal/deployment/fakes_test.go +++ b/services/core/internal/deployment/fakes_test.go @@ -231,7 +231,6 @@ type fakeNodeTx struct { deleteGenerationStatus func(string, uint64) error upsertGenerationStatus func(GenerationStatusRecord) error promoteServingGeneration func(string, uint64) error - refreshServingReadiness func(string, int) error } func (f *fakeNodeTx) LoadDeployment() (Record, error) { @@ -339,13 +338,6 @@ func (f *fakeNodeTx) PromoteServingGeneration(nodeID string, generation uint64) return f.promoteServingGeneration(nodeID, generation) } -func (f *fakeNodeTx) RefreshServingReadiness(nodeID string, protocol int) error { - if f.refreshServingReadiness == nil { - unexpected(f.t, "RefreshServingReadiness") - } - return f.refreshServingReadiness(nodeID, protocol) -} - type fakeDeploymentTx struct { t testing.TB loadDeployment func() (Record, error) diff --git a/services/core/internal/deployment/health.go b/services/core/internal/deployment/health.go index 3b5c9492b..946fcc4ab 100644 --- a/services/core/internal/deployment/health.go +++ b/services/core/internal/deployment/health.go @@ -8,14 +8,8 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) -// normalizeHealth validates a node's reported health. A diagnostic explains -// unreadiness only: unknown values, including arbitrary text, become -// provider_unavailable, a ready provider has none and empty stays empty. +// normalizeHealth validates a node's reported health. func normalizeHealth(health NodeHealth) (NodeHealth, error) { - health.Diagnostic = sandbox.NodeDiagnosticCode(sandbox.NormalizeNodeDiagnostic(string(health.Diagnostic))) - if health.ProviderReady { - health.Diagnostic = "" - } for _, value := range []*int64{health.CPUCount, health.AvailableMemoryBytes, health.AvailableDiskBytes} { if value != nil && *value < 0 { return health, ErrInvalidInput @@ -79,10 +73,6 @@ func nodeRollout(n NodeRecord) NodeRollout { if !n.Online { return out } - if n.ProtocolVersion == 1 && n.DeploymentGeneration != n.TargetGeneration { - out.State = NodeRolloutUpdateRequired - return out - } switch NodeRolloutState(n.TargetState) { case NodeRolloutReady, NodeRolloutPreparing, NodeRolloutFailed: out.State = NodeRolloutState(n.TargetState) diff --git a/services/core/internal/deployment/node.go b/services/core/internal/deployment/node.go index 0bd2e3b31..1216cf4a5 100644 --- a/services/core/internal/deployment/node.go +++ b/services/core/internal/deployment/node.go @@ -44,18 +44,17 @@ type Enrollment struct { } type NodeHealth struct { - Host *NodeHost `json:"-"` - // Fixed reason for the last reported unreadiness; absent while the provider is ready. Clients treat an unknown value as provider_unavailable. - Diagnostic sandbox.NodeDiagnosticCode `json:"diagnostic,omitempty"` - ProviderReady bool `json:"provider_ready" binding:"required"` - CPUCount *int64 `json:"cpu_count" extensions:"x-nullable" binding:"required"` - AvailableMemoryBytes *int64 `json:"available_memory_bytes" extensions:"x-nullable" binding:"required"` - AvailableDiskBytes *int64 `json:"available_disk_bytes" extensions:"x-nullable" binding:"required"` + Host *NodeHost `json:"-"` + CPUCount *int64 `json:"cpu_count" extensions:"x-nullable" binding:"required"` + AvailableMemoryBytes *int64 `json:"available_memory_bytes" extensions:"x-nullable" binding:"required"` + AvailableDiskBytes *int64 `json:"available_disk_bytes" extensions:"x-nullable" binding:"required"` } type Node struct { Rollout NodeRollout `json:"rollout" binding:"required"` NodeHealth + // True while the node is online and its serving generation is ready. + ProviderReady bool `json:"provider_ready" binding:"required"` Running int64 `json:"running" binding:"required"` Snapshots int64 `json:"snapshots" binding:"required"` ID string `json:"id" binding:"required"` diff --git a/services/core/internal/deployment/nodes.go b/services/core/internal/deployment/nodes.go index 8dfe2049d..1ae61160c 100644 --- a/services/core/internal/deployment/nodes.go +++ b/services/core/internal/deployment/nodes.go @@ -32,8 +32,7 @@ func (s *Service) ListNodes(ctx context.Context) ([]Node, error) { func nodeView(n NodeRecord) Node { health := n.Health health.Host = nil - health.ProviderReady = n.Online && n.ServingReady - return Node{Rollout: nodeRollout(n), NodeHealth: health, Running: n.Running, Snapshots: n.Snapshots, ID: n.ID, Name: n.Name, CoreURL: n.CoreURL, EnrollmentID: n.EnrollmentID, Provider: n.Provider, Online: n.Online, LastSeenAt: n.LastSeenAt, MaxActive: n.MaxActive, MaxRetained: n.MaxRetained, Active: n.Active, Reserved: n.Reserved, Retained: n.Retained, CleanupPending: n.CleanupPending, CreatedAt: n.CreatedAt} + return Node{Rollout: nodeRollout(n), NodeHealth: health, ProviderReady: n.Online && n.ServingReady, Running: n.Running, Snapshots: n.Snapshots, ID: n.ID, Name: n.Name, CoreURL: n.CoreURL, EnrollmentID: n.EnrollmentID, Provider: n.Provider, Online: n.Online, LastSeenAt: n.LastSeenAt, MaxActive: n.MaxActive, MaxRetained: n.MaxRetained, Active: n.Active, Reserved: n.Reserved, Retained: n.Retained, CleanupPending: n.CleanupPending, CreatedAt: n.CreatedAt} } // NodeDetail returns one node with its last host observation and its host @@ -495,22 +494,12 @@ func (s *Service) NodeRetention(ctx context.Context, nodeID, connectionID string return deployment, grants, nil } -// Heartbeat records a protocol 1 node's health, which reports readiness of its -// enrollment generation only. +// Heartbeat records a node's health and the preparation state of the +// generations it reports. func (s *Service) Heartbeat(ctx context.Context, nodeID, connectionID string, epoch uint64, health NodeHealth, statuses []sandbox.GenerationStatus) error { - return s.heartbeat(ctx, nodeID, connectionID, epoch, health, statuses, 1) -} - -// HeartbeatGenerations records a node's health and the preparation state of -// the generations it reports. -func (s *Service) HeartbeatGenerations(ctx context.Context, nodeID, connectionID string, epoch uint64, health NodeHealth, statuses []sandbox.GenerationStatus) error { if len(statuses) > maxReportedGenerations { return ErrInvalidInput } - return s.heartbeat(ctx, nodeID, connectionID, epoch, health, statuses, 2) -} - -func (s *Service) heartbeat(ctx context.Context, nodeID, connectionID string, epoch uint64, health NodeHealth, statuses []sandbox.GenerationStatus, protocol int) error { id, err := parseID(nodeID) if err != nil { return err @@ -539,23 +528,11 @@ func (s *Service) heartbeat(ctx context.Context, nodeID, connectionID string, ep if !current { return ErrNodeCredential } - if protocol == 1 && len(statuses) > 0 { - if len(statuses) != 1 || statuses[0].Generation != n.DeploymentGeneration || statuses[0].SpecificationDigest != n.SpecificationDigest || (statuses[0].State == "ready") != health.ProviderReady { - return ErrInvalidInput - } - } - if protocol == 1 && len(statuses) == 0 { - state := "failed" - if health.ProviderReady { - state = "ready" - } - statuses = []sandbox.GenerationStatus{{Generation: n.DeploymentGeneration, SpecificationDigest: n.SpecificationDigest, State: state, Diagnostic: string(health.Diagnostic)}} - } - return s.recordGenerations(tx, d, n, statuses, protocol) + return s.recordGenerations(tx, d, n, statuses) }) } -func (s *Service) recordGenerations(tx NodeTx, d Record, n StoredNode, statuses []sandbox.GenerationStatus, protocol int) error { +func (s *Service) recordGenerations(tx NodeTx, d Record, n StoredNode, statuses []sandbox.GenerationStatus) error { adapter, err := s.registry.Lookup(d.Provider) if err != nil { return err @@ -595,7 +572,7 @@ func (s *Service) recordGenerations(tx NodeTx, d Record, n StoredNode, statuses } } } - return tx.RefreshServingReadiness(n.ID, protocol) + return nil } // ConnectNode records a node connection for the owner epoch. diff --git a/services/core/internal/deployment/rules_test.go b/services/core/internal/deployment/rules_test.go index cddba64ee..97b97da8d 100644 --- a/services/core/internal/deployment/rules_test.go +++ b/services/core/internal/deployment/rules_test.go @@ -142,23 +142,20 @@ func TestNormalizeHealth(t *testing.T) { negative, zero, one := int64(-1), float64(0), float64(1) now := time.Now() for _, c := range []struct { - name string - in NodeHealth - diagnostic sandbox.NodeDiagnosticCode - invalid bool + name string + in NodeHealth + invalid bool }{ - {"ready clears the diagnostic", NodeHealth{ProviderReady: true, Diagnostic: sandbox.NodeProviderUnavailable}, "", false}, - {"unknown text becomes provider_unavailable", NodeHealth{Diagnostic: "disk on fire"}, sandbox.NodeProviderUnavailable, false}, - {"empty stays empty", NodeHealth{}, "", false}, - {"negative CPU count", NodeHealth{CPUCount: &negative}, "", true}, - {"negative memory", NodeHealth{AvailableMemoryBytes: &negative}, "", true}, - {"negative disk", NodeHealth{AvailableDiskBytes: &negative}, "", true}, - {"host without observation", NodeHealth{Host: &NodeHost{CPUUtilization: &one}}, "", true}, - {"valid host", NodeHealth{Host: &NodeHost{ObservedAt: &now, CPUUtilization: &one, EffectiveCPUCores: &one}}, "", false}, - {"zero effective cores", NodeHealth{Host: &NodeHost{ObservedAt: &now, EffectiveCPUCores: &zero}}, "", true}, + {"empty health", NodeHealth{}, false}, + {"negative CPU count", NodeHealth{CPUCount: &negative}, true}, + {"negative memory", NodeHealth{AvailableMemoryBytes: &negative}, true}, + {"negative disk", NodeHealth{AvailableDiskBytes: &negative}, true}, + {"host without observation", NodeHealth{Host: &NodeHost{CPUUtilization: &one}}, true}, + {"valid host", NodeHealth{Host: &NodeHost{ObservedAt: &now, CPUUtilization: &one, EffectiveCPUCores: &one}}, false}, + {"zero effective cores", NodeHealth{Host: &NodeHost{ObservedAt: &now, EffectiveCPUCores: &zero}}, true}, } { got, err := normalizeHealth(c.in) - if c.invalid != errors.Is(err, ErrInvalidInput) || (!c.invalid && (err != nil || got.Diagnostic != c.diagnostic)) { + if c.invalid != errors.Is(err, ErrInvalidInput) || (!c.invalid && err != nil) { t.Errorf("%s: normalizeHealth = %+v, %v", c.name, got, err) } } @@ -230,15 +227,11 @@ func TestNodeRollout(t *testing.T) { diagnostic sandbox.NodeDiagnosticCode }{ {"offline", NodeRecord{TargetState: "ready", ReadyGeneration: &ready}, "unknown", ""}, - {"protocol 1 on another generation", NodeRecord{Online: true, ProtocolVersion: 1, DeploymentGeneration: 1, TargetGeneration: 2, TargetState: "ready", ReadyGeneration: &ready}, "update_required", ""}, - {"protocol 1 on the target", NodeRecord{Online: true, ProtocolVersion: 1, DeploymentGeneration: 2, TargetGeneration: 2, TargetState: "ready", ReadyGeneration: &ready}, "ready", ""}, - {"preparing", NodeRecord{Online: true, ProtocolVersion: 2, TargetState: "preparing", ReadyGeneration: &ready}, "preparing", ""}, - {"failed with an unknown diagnostic", NodeRecord{Online: true, ProtocolVersion: 2, TargetState: "failed", TargetDiagnostic: "boom", ReadyGeneration: &ready}, "failed", sandbox.NodeProviderUnavailable}, - {"failed without diagnostic", NodeRecord{Online: true, ProtocolVersion: 2, TargetState: "failed", ReadyGeneration: &ready}, "failed", ""}, - {"ready ignores a diagnostic", NodeRecord{Online: true, ProtocolVersion: 2, TargetState: "ready", TargetDiagnostic: "boom", ReadyGeneration: &ready}, "ready", ""}, - {"unknown target state", NodeRecord{Online: true, ProtocolVersion: 2, TargetState: "other", ReadyGeneration: &ready}, "unknown", ""}, - {"protocol 1 failed on the target", NodeRecord{Online: true, ProtocolVersion: 1, DeploymentGeneration: 2, TargetGeneration: 2, TargetState: "failed", TargetDiagnostic: "host_unsupported", ReadyGeneration: &ready}, "failed", "host_unsupported"}, - {"protocol 1 without a target state", NodeRecord{Online: true, ProtocolVersion: 1, DeploymentGeneration: 2, TargetGeneration: 2, ReadyGeneration: &ready}, "unknown", ""}, + {"preparing", NodeRecord{Online: true, TargetState: "preparing", ReadyGeneration: &ready}, "preparing", ""}, + {"failed with an unknown diagnostic", NodeRecord{Online: true, TargetState: "failed", TargetDiagnostic: "boom", ReadyGeneration: &ready}, "failed", sandbox.NodeProviderUnavailable}, + {"failed without diagnostic", NodeRecord{Online: true, TargetState: "failed", ReadyGeneration: &ready}, "failed", ""}, + {"ready ignores a diagnostic", NodeRecord{Online: true, TargetState: "ready", TargetDiagnostic: "boom", ReadyGeneration: &ready}, "ready", ""}, + {"unknown target state", NodeRecord{Online: true, TargetState: "other", ReadyGeneration: &ready}, "unknown", ""}, } { got := nodeRollout(c.n) if got.State != c.state || got.Diagnostic != c.diagnostic || got.ReadyGeneration != &ready { diff --git a/services/core/internal/deployment/service_test.go b/services/core/internal/deployment/service_test.go index 1c2c85878..387ffb117 100644 --- a/services/core/internal/deployment/service_test.go +++ b/services/core/internal/deployment/service_test.go @@ -57,7 +57,6 @@ func operations(t *testing.T, publicURL string, tx *fakeDeploymentTx) *Execution // testSpecification is a valid deployment specification for provider. func testSpecification(provider string) sandbox.DeploymentSpec { s := sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 2, MemoryMiB: 2048}} - s.Runtime = &sandbox.RuntimeRelease{SourceCommit: strings.Repeat("a", 40), ImageID: "sha256:" + strings.Repeat("b", 64), ImageManifestDigest: "sha256:" + strings.Repeat("c", 64), MicrosandboxRef: "oac-runtime@sha256:" + strings.Repeat("d", 64), RuntimeSHA256: strings.Repeat("e", 64), FirmwareSHA256: strings.Repeat("f", 64)} if provider == "microsandbox" { s.Resources.RootDiskMiB = 8192 s.Resources.EnvironmentDiskMiB = 8192 @@ -151,10 +150,6 @@ func TestSandboxValidationWrapperPreservesClassification(t *testing.T) { if errors.Is(err, sandbox.ErrInvalid) { t.Fatal("wrapper changed sentinel identity") } - _, err = service.SetupForSelection(installation, sandbox.Selection{Provider: "e2b", DeploymentSpec: sandbox.DeploymentSpec{Runtime: &sandbox.RuntimeRelease{}}}) - if !errors.As(err, &configuration) || configuration.Validation == nil || configuration.Validation.Param != "runtime" || err.Error() != "invalid sandbox configuration: E2B Runtime is selected by its immutable template build" { - t.Fatal("pending E2B validation order changed", err) - } } // A stored provider the registry no longer holds fails every read that consults it. diff --git a/services/core/internal/deployment/storage.go b/services/core/internal/deployment/storage.go index 28ea28e05..7e987fc4e 100644 --- a/services/core/internal/deployment/storage.go +++ b/services/core/internal/deployment/storage.go @@ -324,7 +324,6 @@ type NodeTx interface { DeleteGenerationStatus(nodeID string, generation uint64) error UpsertGenerationStatus(status GenerationStatusRecord) error PromoteServingGeneration(nodeID string, generation uint64) error - RefreshServingReadiness(nodeID string, protocol int) error } // DeploymentTx is one leased deployment change. @@ -463,9 +462,6 @@ type NodeRecord struct { LastSeenAt *time.Time Health NodeHealth Online, ServingReady bool - ProtocolVersion int - DeploymentGeneration uint64 - TargetGeneration uint64 ReadyGeneration *uint64 TargetState string TargetDiagnostic string diff --git a/services/core/internal/deployment/view.go b/services/core/internal/deployment/view.go index 23bef5045..88cf94ef7 100644 --- a/services/core/internal/deployment/view.go +++ b/services/core/internal/deployment/view.go @@ -10,11 +10,10 @@ import ( type NodeRolloutState string const ( - NodeRolloutReady NodeRolloutState = "ready" - NodeRolloutPreparing NodeRolloutState = "preparing" - NodeRolloutFailed NodeRolloutState = "failed" - NodeRolloutUpdateRequired NodeRolloutState = "update_required" - NodeRolloutUnknown NodeRolloutState = "unknown" + NodeRolloutReady NodeRolloutState = "ready" + NodeRolloutPreparing NodeRolloutState = "preparing" + NodeRolloutFailed NodeRolloutState = "failed" + NodeRolloutUnknown NodeRolloutState = "unknown" ) type RolloutState string @@ -67,11 +66,10 @@ type NodeRollout struct { } type RolloutNodes struct { - Ready int64 `json:"ready" binding:"required"` - Preparing int64 `json:"preparing" binding:"required"` - Failed int64 `json:"failed" binding:"required"` - UpdateRequired int64 `json:"update_required" binding:"required"` - Unknown int64 `json:"unknown" binding:"required"` + Ready int64 `json:"ready" binding:"required"` + Preparing int64 `json:"preparing" binding:"required"` + Failed int64 `json:"failed" binding:"required"` + Unknown int64 `json:"unknown" binding:"required"` } type Rollout struct { diff --git a/services/core/internal/execution/runtime_preparation_capacity_test.go b/services/core/internal/execution/runtime_preparation_capacity_test.go index 9e19a286b..45142170d 100644 --- a/services/core/internal/execution/runtime_preparation_capacity_test.go +++ b/services/core/internal/execution/runtime_preparation_capacity_test.go @@ -212,7 +212,7 @@ func TestUnsettledFilesystemPreparationYieldsOnlyCompute(t *testing.T) { if err := r.deployments.ConnectNode(t.Context(), newer.NodeID, connection, epoch); err != nil { t.Fatal(err) } - if err := r.deployments.Heartbeat(t.Context(), newer.NodeID, connection, epoch, deployment.NodeHealth{ProviderReady: true}, []sandbox.GenerationStatus{{Generation: view.Generation, SpecificationDigest: view.SpecificationDigest, State: "ready", Checkpoint: &sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"}}}); err != nil { + if err := r.deployments.Heartbeat(t.Context(), newer.NodeID, connection, epoch, deployment.NodeHealth{}, []sandbox.GenerationStatus{{Generation: view.Generation, SpecificationDigest: view.SpecificationDigest, State: "ready", Checkpoint: &sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"}}}); err != nil { t.Fatal(err) } demands, _, err := r.reader.PlacementDemand(t.Context(), deployment.PlacementDemandCursor{}) diff --git a/services/core/internal/execution/runtime_replacement_test.go b/services/core/internal/execution/runtime_replacement_test.go index f1ce2eda6..1d066d5c3 100644 --- a/services/core/internal/execution/runtime_replacement_test.go +++ b/services/core/internal/execution/runtime_replacement_test.go @@ -330,7 +330,7 @@ func workspaceNodeSetup(t *testing.T, external bool) func(Owner, *deployment.Ser if err := owner.Deployment.Claim(t.Context(), installation); err != nil { t.Fatal(err) } - specification := sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 2, MemoryMiB: 2048, RootDiskMiB: 8192}, Workspace: &workspacefs.Declaration{Attachment: workspacefs.AttachmentHostDirectory, UserXAttr: true}, Runtime: &sandbox.RuntimeRelease{SourceCommit: strings.Repeat("a", 40), ImageID: "sha256:" + strings.Repeat("b", 64), ImageManifestDigest: "sha256:" + strings.Repeat("c", 64), MicrosandboxRef: "oac-runtime@sha256:" + strings.Repeat("d", 64), RuntimeSHA256: strings.Repeat("e", 64), FirmwareSHA256: strings.Repeat("f", 64)}} + specification := sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 2, MemoryMiB: 2048, RootDiskMiB: 8192}, Workspace: &workspacefs.Declaration{Attachment: workspacefs.AttachmentHostDirectory, UserXAttr: true}} if !external { specification.Workspace = nil specification.Resources.EnvironmentDiskMiB = 8192 @@ -355,7 +355,7 @@ func workspaceNodeSetup(t *testing.T, external bool) func(Owner, *deployment.Ser if err = service.ConnectNode(t.Context(), node.NodeID, connection, epoch); err != nil { t.Fatal(err) } - if err = service.Heartbeat(t.Context(), node.NodeID, connection, epoch, deployment.NodeHealth{ProviderReady: true}, []sandbox.GenerationStatus{{Generation: view.Generation, SpecificationDigest: view.SpecificationDigest, State: "ready", Checkpoint: &sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"}}}); err != nil { + if err = service.Heartbeat(t.Context(), node.NodeID, connection, epoch, deployment.NodeHealth{}, []sandbox.GenerationStatus{{Generation: view.Generation, SpecificationDigest: view.SpecificationDigest, State: "ready", Checkpoint: &sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"}}}); err != nil { t.Fatal(err) } return installation, node.NodeID diff --git a/services/core/internal/execution/runtime_restore_attempt_test.go b/services/core/internal/execution/runtime_restore_attempt_test.go index c18833796..ce584d995 100644 --- a/services/core/internal/execution/runtime_restore_attempt_test.go +++ b/services/core/internal/execution/runtime_restore_attempt_test.go @@ -130,7 +130,7 @@ func TestCheckpointTransferRoutesBeforeTargetIO(t *testing.T) { t.Fatal(err) } compat := sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"} - if err = r.deployments.Heartbeat(t.Context(), targetNode, connection, view.OwnerEpoch, deployment.NodeHealth{ProviderReady: true}, []sandbox.GenerationStatus{{Generation: view.Generation, SpecificationDigest: view.SpecificationDigest, State: "ready", Checkpoint: &compat}}); err != nil { + if err = r.deployments.Heartbeat(t.Context(), targetNode, connection, view.OwnerEpoch, deployment.NodeHealth{}, []sandbox.GenerationStatus{{Generation: view.Generation, SpecificationDigest: view.SpecificationDigest, State: "ready", Checkpoint: &compat}}); err != nil { t.Fatal(err) } state := runtimeCompute{Current: sandbox.Compute{ID: "source", Name: "source", Generation: 1}, Snapshot: &sandbox.SnapshotIdentity{ID: "snapshot", Compatibility: compat}} diff --git a/services/core/internal/persistence/postgres/deploymentpg/checkpoint_transfer_test.go b/services/core/internal/persistence/postgres/deploymentpg/checkpoint_transfer_test.go index 8d3713ef3..c13ad8399 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/checkpoint_transfer_test.go +++ b/services/core/internal/persistence/postgres/deploymentpg/checkpoint_transfer_test.go @@ -26,7 +26,7 @@ func readyCheckpointNode(t *testing.T, f fixture, node deployment.Enrollment, vi if err := f.service.ConnectNode(t.Context(), node.NodeID, connection, epoch); err != nil { t.Fatal(err) } - if err := f.service.HeartbeatGenerations(t.Context(), node.NodeID, connection, epoch, deployment.NodeHealth{}, []sandbox.GenerationStatus{{Generation: view.Generation, SpecificationDigest: view.SpecificationDigest, State: "ready", Checkpoint: &compatibility}}); err != nil { + if err := f.service.Heartbeat(t.Context(), node.NodeID, connection, epoch, deployment.NodeHealth{}, []sandbox.GenerationStatus{{Generation: view.Generation, SpecificationDigest: view.SpecificationDigest, State: "ready", Checkpoint: &compatibility}}); err != nil { t.Fatal(err) } } diff --git a/services/core/internal/persistence/postgres/deploymentpg/fixture_test.go b/services/core/internal/persistence/postgres/deploymentpg/fixture_test.go index 11dfeafe5..a50c40d3d 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/fixture_test.go +++ b/services/core/internal/persistence/postgres/deploymentpg/fixture_test.go @@ -92,10 +92,6 @@ func admin(t *testing.T) context.Context { // testSpecification is a valid deployment specification for provider. func testSpecification(provider string) sandbox.DeploymentSpec { s := sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 2, MemoryMiB: 2048}} - if provider == "e2b" { - return s - } - s.Runtime = &sandbox.RuntimeRelease{SourceCommit: strings.Repeat("a", 40), ImageID: "sha256:" + strings.Repeat("b", 64), ImageManifestDigest: "sha256:" + strings.Repeat("c", 64), MicrosandboxRef: "oac-runtime@sha256:" + strings.Repeat("d", 64), RuntimeSHA256: strings.Repeat("e", 64), FirmwareSHA256: strings.Repeat("f", 64)} if provider == "microsandbox" { s.Resources.RootDiskMiB = 8192 s.Resources.EnvironmentDiskMiB = 8192 @@ -145,22 +141,29 @@ func (f fixture) connect(t *testing.T, nodeID string) string { if err := f.service.ConnectNode(t.Context(), nodeID, connection, epoch); err != nil { t.Fatal(err) } - if err := f.heartbeat(t.Context(), nodeID, connection, epoch, deployment.NodeHealth{ProviderReady: true}, nil); err != nil { + if err := f.heartbeat(t.Context(), nodeID, connection, epoch, deployment.NodeHealth{}, true); err != nil { t.Fatal(err) } return connection } // heartbeat reports the fixture's verified generation alongside static readiness. -func (f fixture) heartbeat(ctx context.Context, nodeID, connection string, epoch uint64, health deployment.NodeHealth, _ []sandbox.GenerationStatus) error { +func (f fixture) heartbeat(ctx context.Context, nodeID, connection string, epoch uint64, health deployment.NodeHealth, ready bool) error { var generation uint64 var digest, provider string if err := f.pool.QueryRow(ctx, "SELECT n.deployment_generation, n.specification_digest, d.provider_kind FROM runtime_nodes n CROSS JOIN runtime_deployment d WHERE n.id=$1", nodeID).Scan(&generation, &digest, &provider); err != nil { return err } var statuses []sandbox.GenerationStatus - if health.ProviderReady && provider == "microsandbox" { - statuses = []sandbox.GenerationStatus{{Generation: generation, SpecificationDigest: digest, State: "ready", Checkpoint: &sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"}}} + { + status := sandbox.GenerationStatus{Generation: generation, SpecificationDigest: digest, State: "ready"} + if !ready { + status.State = "failed" + } + if provider == "microsandbox" && ready { + status.Checkpoint = &sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"} + } + statuses = []sandbox.GenerationStatus{status} } return f.service.Heartbeat(ctx, nodeID, connection, epoch, health, statuses) } diff --git a/services/core/internal/persistence/postgres/deploymentpg/generations_test.go b/services/core/internal/persistence/postgres/deploymentpg/generations_test.go index b0463ac7d..e25f7e389 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/generations_test.go +++ b/services/core/internal/persistence/postgres/deploymentpg/generations_test.go @@ -83,11 +83,6 @@ func TestE2BChangeClassifierOmittedKeyAndExplicitSameKey(t *testing.T) { if _, unchanged, err = changes.ClassifyChange(t.Context(), id, input); err != nil || unchanged { t.Fatal("explicit same key skipped verification", err) } - invalid := input - invalid.Runtime = &sandbox.RuntimeRelease{} - if _, _, err := changes.ClassifyChange(t.Context(), id, invalid); err == nil { - t.Fatal("omitted resources erased forbidden Runtime input") - } input.ExpectedGeneration = 0 _, _, err = changes.ClassifyChange(t.Context(), id, input) var stale *deployment.GenerationStaleError diff --git a/services/core/internal/persistence/postgres/deploymentpg/host_history_test.go b/services/core/internal/persistence/postgres/deploymentpg/host_history_test.go index 4b7c09e2a..ed92fad46 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/host_history_test.go +++ b/services/core/internal/persistence/postgres/deploymentpg/host_history_test.go @@ -35,8 +35,8 @@ func TestNodeHostHistorySamplingAndDetail(t *testing.T) { f, node, connection, epoch := hostHistoryNode(t) now := time.Now().UTC() host := &deployment.NodeHost{EffectiveCPUCores: hostHistoryPtr(2.0), CPUUtilization: hostHistoryPtr(0.35), TotalMemoryBytes: hostHistoryPtr(int64(4096)), AvailableMemoryBytes: hostHistoryPtr(int64(1024)), AvailableDiskBytes: hostHistoryPtr(int64(8192)), ObservedAt: &now} - health := deployment.NodeHealth{ProviderReady: true, Host: host} - if err := f.heartbeat(t.Context(), node, connection, epoch, health, nil); err != nil { + health := deployment.NodeHealth{Host: host} + if err := f.heartbeat(t.Context(), node, connection, epoch, health, true); err != nil { t.Fatal(err) } for i, want := range []int64{1, 0} { @@ -113,7 +113,7 @@ func TestNodeHostHistorySamplingAndDetail(t *testing.T) { // A disconnected node cannot create another history row, even with a fresh last observation. next := now.Add(time.Millisecond) host.ObservedAt = &next - if err := f.heartbeat(t.Context(), node, connection, epoch, health, nil); err != nil { + if err := f.heartbeat(t.Context(), node, connection, epoch, health, true); err != nil { t.Fatal(err) } if err := f.service.DisconnectNode(t.Context(), node, connection, epoch); err != nil { @@ -136,7 +136,7 @@ func TestNodeHostHistorySamplingAndDetail(t *testing.T) { func TestNodeHostHistoryFencingAndUnknown(t *testing.T) { f, node, conn, epoch := hostHistoryNode(t) for _, at := range []time.Time{time.Now().Add(-time.Minute), time.Now().Add(time.Hour)} { - if err := f.heartbeat(t.Context(), node, conn, epoch, deployment.NodeHealth{Host: &deployment.NodeHost{ObservedAt: &at}}, nil); err != nil { + if err := f.heartbeat(t.Context(), node, conn, epoch, deployment.NodeHealth{Host: &deployment.NodeHost{ObservedAt: &at}}, true); err != nil { t.Fatal(err) } if n, err := f.adapter.SampleHostHistory(t.Context()); err != nil || n != 0 { @@ -145,10 +145,10 @@ func TestNodeHostHistoryFencingAndUnknown(t *testing.T) { } now := time.Now().UTC() health := deployment.NodeHealth{Host: &deployment.NodeHost{ObservedAt: &now}} - if err := f.heartbeat(t.Context(), node, uuid.NewString(), epoch, health, nil); !errors.Is(err, deployment.ErrNodeCredential) { + if err := f.heartbeat(t.Context(), node, uuid.NewString(), epoch, health, true); !errors.Is(err, deployment.ErrNodeCredential) { t.Fatal(err) } - if err := f.heartbeat(t.Context(), node, conn, epoch, health, nil); err != nil { + if err := f.heartbeat(t.Context(), node, conn, epoch, health, true); err != nil { t.Fatal(err) } if n, err := f.adapter.SampleHostHistory(t.Context()); err != nil || n != 1 { @@ -164,7 +164,7 @@ func TestNodeHostHistoryFencingAndUnknown(t *testing.T) { {ObservedAt: &now, TotalMemoryBytes: hostHistoryPtr(int64(10)), AvailableMemoryBytes: hostHistoryPtr(int64(11))}, {ObservedAt: &now, AvailableDiskBytes: hostHistoryPtr(int64(-1))}, {ObservedAt: &now, EffectiveCPUCores: hostHistoryPtr(0.0)}, {}, } { - if err := f.heartbeat(t.Context(), node, conn, epoch, deployment.NodeHealth{Host: host}, nil); !errors.Is(err, deployment.ErrInvalidInput) { + if err := f.heartbeat(t.Context(), node, conn, epoch, deployment.NodeHealth{Host: host}, true); !errors.Is(err, deployment.ErrInvalidInput) { t.Fatal(host, err) } } diff --git a/services/core/internal/persistence/postgres/deploymentpg/presence_test.go b/services/core/internal/persistence/postgres/deploymentpg/presence_test.go index 83957fb31..adba97ccd 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/presence_test.go +++ b/services/core/internal/persistence/postgres/deploymentpg/presence_test.go @@ -246,7 +246,7 @@ func TestNodeStaleEpochCannotReplaceCurrentConnection(t *testing.T) { if err := f.service.DisconnectNode(t.Context(), node.NodeID, connection, epoch); err != nil { t.Fatal(err) } - if err := f.heartbeat(t.Context(), node.NodeID, connection, epoch, deployment.NodeHealth{ProviderReady: false}, nil); !errors.Is(err, deployment.ErrNodeCredential) { + if err := f.heartbeat(t.Context(), node.NodeID, connection, epoch, deployment.NodeHealth{}, false); !errors.Is(err, deployment.ErrNodeCredential) { t.Fatal("old Core rewrote health", err) } nodes, err := f.service.ListNodes(t.Context()) @@ -255,46 +255,6 @@ func TestNodeStaleEpochCannotReplaceCurrentConnection(t *testing.T) { } } -func TestNodeDiagnosticReachesListAndDetail(t *testing.T) { - f := newFixture(t) - changes, _ := f.execution(t) - _, view := f.initialize(t, changes, sandbox.Selection{Provider: "docker", DeploymentSpec: testSpecification("docker")}) - node := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 1}) - connection := f.connect(t, node.NodeID) - epoch, err := f.adapter.OwnerEpoch(t.Context()) - if err != nil { - t.Fatal(err) - } - for _, tc := range []struct { - reported, want string - ready bool - }{ - {reported: "host_unsupported", want: "host_unsupported"}, - {reported: "capacity_insufficient", want: "capacity_insufficient"}, - // Older nodes send provider_unavailable or nothing; unknown text is never stored. - {reported: "provider_unavailable", want: "provider_unavailable"}, - {reported: "", want: ""}, - {reported: "dial unix /var/run/docker.sock: permission denied", want: "provider_unavailable"}, - {reported: "artifacts_unavailable", want: "", ready: true}, - } { - if err := f.heartbeat(t.Context(), node.NodeID, connection, epoch, deployment.NodeHealth{ProviderReady: tc.ready, Diagnostic: sandbox.NodeDiagnosticCode(tc.reported)}, nil); err != nil { - t.Fatal(tc.reported, err) - } - list, err := f.service.ListNodes(t.Context()) - if err != nil || len(list) != 1 || string(list[0].Diagnostic) != tc.want { - t.Fatal(tc.reported, list, err) - } - detail, err := f.service.NodeDetail(t.Context(), node.NodeID, "1h") - if err != nil || string(detail.Diagnostic) != tc.want { - t.Fatal(tc.reported, detail.Diagnostic, err) - } - var stored string - if err := f.pool.QueryRow(t.Context(), "SELECT health::text FROM runtime_nodes WHERE id=$1", node.NodeID).Scan(&stored); err != nil || strings.Contains(stored, "/var/run") { - t.Fatal("raw diagnostic stored", stored, err) - } - } -} - func TestNodeStatusUsesAuthenticatedFreshPresence(t *testing.T) { f := newFixture(t) changes, _ := f.execution(t) @@ -321,11 +281,11 @@ func TestNodeStatusUsesAuthenticatedFreshPresence(t *testing.T) { if err != nil { t.Fatal(err) } - if err := f.heartbeat(t.Context(), node.NodeID, connection, epoch, deployment.NodeHealth{ProviderReady: false}, nil); err != nil { + if err := f.heartbeat(t.Context(), node.NodeID, connection, epoch, deployment.NodeHealth{}, false); err != nil { t.Fatal(err) } status(true, false) - if err := f.heartbeat(t.Context(), node.NodeID, connection, epoch, deployment.NodeHealth{ProviderReady: true}, nil); err != nil { + if err := f.heartbeat(t.Context(), node.NodeID, connection, epoch, deployment.NodeHealth{}, true); err != nil { t.Fatal(err) } status(true, true) diff --git a/services/core/internal/persistence/postgres/deploymentpg/records.go b/services/core/internal/persistence/postgres/deploymentpg/records.go index 9124c05b9..a17f2a5a9 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/records.go +++ b/services/core/internal/persistence/postgres/deploymentpg/records.go @@ -111,8 +111,8 @@ func nodeRecords(rows []sqlc.ListRuntimeNodesRow) ([]deployment.NodeRecord, erro } health.NodeHealth.Host = health.Host record := deployment.NodeRecord{ID: uuidString(n.ID), Name: n.Name, Provider: n.ProviderKind, CoreURL: n.CoreUrl, CreatedAt: n.CreatedAt.Time, LastSeenAt: timestamp(n.LastSeenAt), - Health: health.NodeHealth, Online: n.Online, ServingReady: n.ServingReady, ProtocolVersion: int(n.ProtocolVersion), DeploymentGeneration: uint64(n.DeploymentGeneration), - TargetGeneration: uint64(n.TargetGeneration), TargetState: n.TargetState, TargetDiagnostic: n.TargetDiagnostic, MaxActive: int(n.MaxActive), MaxRetained: int(n.MaxRetained), + Health: health.NodeHealth, Online: n.Online, ServingReady: n.ServingReady, + TargetState: n.TargetState, TargetDiagnostic: n.TargetDiagnostic, MaxActive: int(n.MaxActive), MaxRetained: int(n.MaxRetained), Active: n.Active, Retained: n.Retained, Reserved: n.Reserved, CleanupPending: n.CleanupPending, Running: n.Running, Snapshots: n.Snapshots} if n.EnrollmentID.Valid { id := uuidString(n.EnrollmentID) diff --git a/services/core/internal/persistence/postgres/deploymentpg/tx.go b/services/core/internal/persistence/postgres/deploymentpg/tx.go index fde99c754..fb290d7db 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/tx.go +++ b/services/core/internal/persistence/postgres/deploymentpg/tx.go @@ -216,7 +216,7 @@ func (t *nodeTx) HeartbeatNode(heartbeat deployment.Heartbeat) (bool, error) { if err != nil { return false, err } - changed, err := t.q.HeartbeatRuntimeNode(t.ctx, sqlc.HeartbeatRuntimeNodeParams{ID: id, ConnectionID: connection, OwnerEpoch: int64(heartbeat.Epoch), ProviderReady: heartbeat.Health.ProviderReady, Health: raw}) + changed, err := t.q.HeartbeatRuntimeNode(t.ctx, sqlc.HeartbeatRuntimeNodeParams{ID: id, ConnectionID: connection, OwnerEpoch: int64(heartbeat.Epoch), Health: raw}) return changed == 1, translate(err) } @@ -255,14 +255,6 @@ func (t *nodeTx) PromoteServingGeneration(nodeID string, generation uint64) erro return t.q.PromoteNodeServingGeneration(t.ctx, sqlc.PromoteNodeServingGenerationParams{ID: id, ReadyGeneration: pgtype.Int8{Int64: int64(generation), Valid: true}}) } -func (t *nodeTx) RefreshServingReadiness(nodeID string, protocol int) error { - id, err := parseID(nodeID) - if err != nil { - return err - } - return t.q.RefreshNodeServingReadiness(t.ctx, sqlc.RefreshNodeServingReadinessParams{ID: id, ProtocolVersion: int32(protocol)}) -} - // deploymentTx is one leased deployment change. type deploymentTx struct { unit diff --git a/services/core/internal/persistence/postgres/sessionpg/creation_test.go b/services/core/internal/persistence/postgres/sessionpg/creation_test.go index bc2b5326b..6467210b3 100644 --- a/services/core/internal/persistence/postgres/sessionpg/creation_test.go +++ b/services/core/internal/persistence/postgres/sessionpg/creation_test.go @@ -472,9 +472,7 @@ func TestHostedCreationAdmitsUnderDeploymentLockAndLeavesComputeUnreserved(t *te ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() tenant := uuid.NewString() - nodes, err := json.Marshal(sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 2, MemoryMiB: 2048}, Runtime: &sandbox.RuntimeRelease{SourceCommit: strings.Repeat("a", 40), - ImageID: "sha256:" + strings.Repeat("b", 64), ImageManifestDigest: "sha256:" + strings.Repeat("c", 64), MicrosandboxRef: "oac-runtime@sha256:" + strings.Repeat("d", 64), - RuntimeSHA256: strings.Repeat("e", 64), FirmwareSHA256: strings.Repeat("f", 64)}}) + nodes, err := json.Marshal(sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 2, MemoryMiB: 2048}}) if err != nil { t.Fatal(err) } diff --git a/services/core/internal/sandbox/deployment.go b/services/core/internal/sandbox/deployment.go index 2e44bbd74..904885c43 100644 --- a/services/core/internal/sandbox/deployment.go +++ b/services/core/internal/sandbox/deployment.go @@ -6,7 +6,6 @@ import ( "encoding/json" "fmt" "reflect" - "regexp" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/workspacefs" ) @@ -69,31 +68,8 @@ func (r Resources) validatePolicy(provider string, rules DeploymentPolicy) error return nil } -// RuntimeRelease preserves the identities of one verified distribution. Docker -// may address the same archive by config ID or OCI manifest digest; microsandbox -// has its own imported OCI identity. These are not interchangeable hashes. -type RuntimeRelease struct { - SourceCommit string `json:"source_commit" binding:"required"` - ImageID string `json:"image_id" binding:"required"` - ImageManifestDigest string `json:"image_manifest_digest" binding:"required"` - MicrosandboxRef string `json:"microsandbox_ref" binding:"required"` - RuntimeSHA256 string `json:"runtime_sha256" binding:"required"` - FirmwareSHA256 string `json:"firmware_sha256" binding:"required"` -} - -func (r RuntimeRelease) Validate() error { - values := reflect.ValueOf(r) - for i, rule := range runtimeContract { - if !regexp.MustCompile("^(?:" + rule.Pattern + ")$").MatchString(values.Field(i).String()) { - return &ValidationError{Param: "runtime", Message: fmt.Sprintf("%s: Runtime must reference one immutable distribution", ErrInvalid)} - } - } - return nil -} - type DeploymentSpec struct { - Resources Resources `json:"resources" binding:"required"` - Runtime *RuntimeRelease `json:"runtime,omitempty"` + Resources Resources `json:"resources" binding:"required"` // Workspace is a derived immutable capability receipt, never filesystem configuration. Workspace *workspacefs.Declaration `json:"workspace,omitempty" readonly:"true"` } @@ -142,18 +118,5 @@ func (r Resources) ValidateWorkspacePolicy(provider string, rules DeploymentPoli return checked.validatePolicy(provider, rules) } func (s DeploymentSpec) ValidateWorkspacePolicy(provider string, policy DeploymentPolicy, declaration *workspacefs.Declaration) error { - if err := s.Resources.ValidateWorkspacePolicy(provider, policy, declaration); err != nil { - return err - } - - if !policy.Runtime { - if s.Runtime != nil { - return &ValidationError{Param: "runtime", Message: fmt.Sprintf("%s: %s", ErrInvalid, policy.RuntimeError)} - } - return nil - } - if s.Runtime == nil { - return &ValidationError{Param: "runtime", Message: fmt.Sprintf("%s: managed nodes require a pinned Runtime release", ErrInvalid)} - } - return s.Runtime.Validate() + return s.Resources.ValidateWorkspacePolicy(provider, policy, declaration) } diff --git a/services/core/internal/sandbox/deployment_contract.go b/services/core/internal/sandbox/deployment_contract.go index d1bfaf248..4a0a94418 100644 --- a/services/core/internal/sandbox/deployment_contract.go +++ b/services/core/internal/sandbox/deployment_contract.go @@ -8,7 +8,7 @@ import ( "strings" ) -// This contract owns numeric bounds, release patterns and canonical field order. +// This contract owns numeric bounds and canonical field order. // The node installer (node_spec.py) and the TypeScript client // (deployment-contract.ts) consume its generated projections. const minimumDiskMiB uint32 = 1024 @@ -20,10 +20,6 @@ type resourceRule struct { OmitZero bool `json:"omit_zero"` Message string `json:"-"` } -type runtimeRule struct { - Name string `json:"name"` - Pattern string `json:"pattern"` -} var resourceContract = []resourceRule{ {"cpus", 1, 255, false, "cpus must be 1..255 and memory_mib must be 512..1048576"}, @@ -31,14 +27,6 @@ var resourceContract = []resourceRule{ {"root_disk_mib", 0, ^uint32(0), true, ""}, {"environment_disk_mib", 0, ^uint32(0), true, ""}, } -var runtimeContract = []runtimeRule{ - {"source_commit", "[0-9a-f]{40}"}, - {"image_id", "sha256:[0-9a-f]{64}"}, - {"image_manifest_digest", "sha256:[0-9a-f]{64}"}, - {"microsandbox_ref", "oac-runtime@sha256:[0-9a-f]{64}"}, - {"runtime_sha256", "[0-9a-f]{64}"}, - {"firmware_sha256", "[0-9a-f]{64}"}, -} // ProviderProjection is one registered Provider in the generated projections. type ProviderProjection struct { @@ -54,7 +42,7 @@ func DeploymentContract(providers map[string]ProviderProjection) (python, typesc value any names []string }{ - {Resources{}, resourceNames()}, {RuntimeRelease{}, runtimeNames()}, + {Resources{}, resourceNames()}, } { typ := reflect.TypeOf(item.value) if typ.NumField() != len(item.names) { @@ -74,10 +62,9 @@ func DeploymentContract(providers map[string]ProviderProjection) (python, typesc raw, _ := json.Marshal(struct { WorkspaceFields []string `json:"workspace_fields"` Resources []resourceRule `json:"resources"` - Runtime []runtimeRule `json:"runtime"` Providers map[string]ProviderProjection `json:"providers"` MinimumDisk uint32 `json:"minimum_disk"` - }{workspaceFields, resourceContract, runtimeContract, providers, minimumDiskMiB}) + }{workspaceFields, resourceContract, providers, minimumDiskMiB}) return "# BEGIN GENERATED DEPLOYMENT CONTRACT\n# Generated from sandbox/deployment_contract.go; do not edit.\n_CONTRACT = json.loads(" + fmt.Sprintf("%q", string(raw)) + ")\n# END GENERATED DEPLOYMENT CONTRACT", "// Code generated by services/core/cmd/specification-contract from sandbox/deployment_contract.go; DO NOT EDIT.\n\nexport const deploymentContract = " + string(raw) + " as const;\n" } @@ -88,10 +75,3 @@ func resourceNames() []string { } return names } -func runtimeNames() []string { - var names []string - for _, r := range runtimeContract { - names = append(names, r.Name) - } - return names -} diff --git a/services/core/internal/sandbox/docker/node.go b/services/core/internal/sandbox/docker/node.go index b8ddc04fb..4b3fcff0e 100644 --- a/services/core/internal/sandbox/docker/node.go +++ b/services/core/internal/sandbox/docker/node.go @@ -15,10 +15,8 @@ import ( ) // Native is the node configuration's native object for Docker. Image is the -// local ID of the release's Runtime image: the host's image store decides -// whether the config digest (image_id) or the manifest digest -// (image_manifest_digest) names the loaded image, so the installer records the -// one this host resolves. +// local ID of the installation's Runtime image, as this host's image store +// resolves it. type Native struct { Host string `json:"host"` Image string `json:"image"` @@ -29,13 +27,9 @@ type Native struct { func decodeNative(config sandbox.NodeConfig) (Native, error) { var entry Native - if sandbox.DecodeConfigurationObject(config.Native, &entry, "host", "image", "network", "seccomp_file", "nested_sandbox") != nil { + if sandbox.DecodeConfigurationObject(config.Native, &entry, "host", "image", "network", "seccomp_file", "nested_sandbox") != nil || entry.Image == "" { return entry, errors.New("invalid managed Docker node configuration") } - release := config.Specification.Runtime - if entry.Image != release.ImageID && entry.Image != release.ImageManifestDigest { - return entry, errors.New("Docker Runtime image differs from the deployment release") - } return entry, nil } @@ -77,7 +71,7 @@ func BuildNode(config sandbox.NodeConfig, _ sandbox.LocalOptions, result *sandbo // The probe reports its first failed check as its readiness class: the Docker // daemon, then limit support, then host capacity for one sandbox of the -// deployment specification, then the pinned Runtime image. Unclassified +// deployment specification, then the Runtime image. Unclassified // failures stay provider_unavailable. The returned text is local; only its // class is reported. func dockerProbe(c *client.Client, image string, resources sandbox.Resources) func(context.Context) error { @@ -102,7 +96,7 @@ func dockerProbe(c *client.Client, image string, resources sandbox.Resources) fu return sandbox.ErrRuntimeImageUnavailable } else if err != nil { // The daemon did not answer; the image may still be present. - return fmt.Errorf("%w: cannot inspect the pinned Runtime image", sandbox.ErrProviderUnavailable) + return fmt.Errorf("%w: cannot inspect the Runtime image", sandbox.ErrProviderUnavailable) } return nil } diff --git a/services/core/internal/sandbox/docker/node_test.go b/services/core/internal/sandbox/docker/node_test.go index 0d5f576c2..61667bb14 100644 --- a/services/core/internal/sandbox/docker/node_test.go +++ b/services/core/internal/sandbox/docker/node_test.go @@ -12,17 +12,13 @@ import ( "github.com/moby/moby/client" ) -// The host's image store decides which release digest names the loaded image. -func TestNativeImageIsAReleaseIdentity(t *testing.T) { - release := sandbox.RuntimeRelease{ImageID: "sha256:" + strings.Repeat("b", 64), ImageManifestDigest: "sha256:" + strings.Repeat("c", 64)} - config := sandbox.NodeConfig{Specification: sandbox.DeploymentSpec{Runtime: &release}} - for _, image := range []string{release.ImageID, release.ImageManifestDigest} { - config.Native = json.RawMessage(`{"image":"` + image + `"}`) - if entry, err := decodeNative(config); err != nil || entry.Image != image { - t.Fatalf("rejected release image %s: %v", image, err) - } +func TestNativeRequiresTheInstalledImage(t *testing.T) { + image := "sha256:" + strings.Repeat("b", 64) + config := sandbox.NodeConfig{Native: json.RawMessage(`{"image":"` + image + `"}`)} + if entry, err := decodeNative(config); err != nil || entry.Image != image { + t.Fatalf("rejected the installed image: %v", err) } - for _, native := range []string{`{"image":"sha256:` + strings.Repeat("a", 64) + `"}`, `{"image":"` + release.ImageID + `","cpus":2}`, `{"image":null}`} { + for _, native := range []string{`{"image":"` + image + `","cpus":2}`, `{"image":null}`, `{}`} { config.Native = json.RawMessage(native) if _, err := decodeNative(config); err == nil { t.Fatalf("accepted native %s", native) diff --git a/services/core/internal/sandbox/docker/selection.go b/services/core/internal/sandbox/docker/selection.go index 3a7ec247a..80c83f758 100644 --- a/services/core/internal/sandbox/docker/selection.go +++ b/services/core/internal/sandbox/docker/selection.go @@ -5,7 +5,7 @@ import ( ) func Policy() sandbox.DeploymentPolicy { - return sandbox.DeploymentPolicy{Runtime: true, DefaultResources: &sandbox.Resources{CPUs: 2, MemoryMiB: 2048}} + return sandbox.DeploymentPolicy{DefaultResources: &sandbox.Resources{CPUs: 2, MemoryMiB: 2048}} } func ValidateResources(r sandbox.Resources) error { return r.ValidatePolicy("docker", Policy()) } diff --git a/services/core/internal/sandbox/e2b/deployment.go b/services/core/internal/sandbox/e2b/deployment.go index 83c0cdc6c..8fcd9c285 100644 --- a/services/core/internal/sandbox/e2b/deployment.go +++ b/services/core/internal/sandbox/e2b/deployment.go @@ -12,7 +12,7 @@ import ( ) func Policy() sandbox.DeploymentPolicy { - return sandbox.DeploymentPolicy{RuntimeError: "E2B Runtime is selected by its immutable template build"} + return sandbox.DeploymentPolicy{} } func ValidateResources(r sandbox.Resources) error { return r.ValidatePolicy("e2b", Policy()) } @@ -44,12 +44,10 @@ func NormalizeSelection(s sandbox.Selection) (sandbox.Selection, error) { if s.Workspace != nil { return s, sandbox.ErrInvalid } - if s.Resources == (sandbox.Resources{}) { - if s.Runtime != nil { - return s, &sandbox.ValidationError{Param: "runtime", Message: sandbox.ErrInvalid.Error() + ": " + Policy().RuntimeError} + if s.Resources != (sandbox.Resources{}) { + if err := ValidateSpecification(s.DeploymentSpec); err != nil { + return s, err } - } else if err := ValidateSpecification(s.DeploymentSpec); err != nil { - return s, err } if err := ValidateConfiguration(configuration(s)); err != nil { return s, err diff --git a/services/core/internal/sandbox/microsandbox/deployment.go b/services/core/internal/sandbox/microsandbox/deployment.go index c69d53c4d..3329198f7 100644 --- a/services/core/internal/sandbox/microsandbox/deployment.go +++ b/services/core/internal/sandbox/microsandbox/deployment.go @@ -6,7 +6,7 @@ import ( ) func Policy() sandbox.DeploymentPolicy { - return sandbox.DeploymentPolicy{Workspace: &workspacefs.Requirements{Attachment: workspacefs.AttachmentHostDirectory, UserXAttr: true}, Disk: true, Runtime: true, + return sandbox.DeploymentPolicy{Workspace: &workspacefs.Requirements{Attachment: workspacefs.AttachmentHostDirectory, UserXAttr: true}, Disk: true, DefaultResources: &sandbox.Resources{CPUs: 2, MemoryMiB: 4096, RootDiskMiB: 8192, EnvironmentDiskMiB: 8192}} } diff --git a/services/core/internal/sandbox/microsandbox/node.go b/services/core/internal/sandbox/microsandbox/node.go index 2b68552dc..4dd9df035 100644 --- a/services/core/internal/sandbox/microsandbox/node.go +++ b/services/core/internal/sandbox/microsandbox/node.go @@ -20,15 +20,19 @@ var NodeArtifacts = []providerassets.Artifact{ } // Native is the node configuration's native object for microsandbox: the -// helper, Runtime and firmware paths, the private store and the network policy. -// The helper owns local paths; no ambient backend is selected. Resources, the -// image and the artifact hashes come from the deployment specification. +// helper, Runtime and firmware paths, the installation's verified Runtime image +// and artifact hashes, the private store and the network policy. The helper owns +// local paths; no ambient backend is selected. Resources come from the +// deployment specification. type Native struct { HelperPath string `json:"helper_path"` RuntimeHome string `json:"runtime_home"` CheckpointRoot string `json:"checkpoint_root"` RuntimePath string `json:"runtime_path"` FirmwarePath string `json:"firmware_path"` + Image string `json:"image"` + RuntimeSHA256 string `json:"runtime_sha256"` + FirmwareSHA256 string `json:"firmware_sha256"` Network Network `json:"network"` } @@ -54,11 +58,11 @@ func configureMicrosandbox(entry Native, spec sandbox.DeploymentSpec, caller *Pr for _, rule := range entry.Network.Rules { network.Rules = append(network.Rules, NetworkRule{Action: rule.Action, Direction: rule.Direction, Destination: rule.Destination, Protocol: rule.Protocol, Port: rule.Port}) } - release, resources := spec.Runtime, spec.Resources + resources := spec.Resources config := Config{ ExternalWorkspace: spec.Workspace != nil, InstallationID: result.InstallationID, HelperPath: entry.HelperPath, RuntimeHome: entry.RuntimeHome, CheckpointRoot: entry.CheckpointRoot, RuntimePath: entry.RuntimePath, FirmwarePath: entry.FirmwarePath, - RuntimeSHA256: release.RuntimeSHA256, FirmwareSHA256: release.FirmwareSHA256, Image: release.MicrosandboxRef, + RuntimeSHA256: entry.RuntimeSHA256, FirmwareSHA256: entry.FirmwareSHA256, Image: entry.Image, MemoryMiB: resources.MemoryMiB, CPUs: uint8(resources.CPUs), RootDiskMiB: resources.RootDiskMiB, EnvironmentDiskMiB: resources.EnvironmentDiskMiB, Network: network, } provider, err := NewWithCaller(config, caller) @@ -87,7 +91,7 @@ func configureMicrosandbox(entry Native, spec sandbox.DeploymentSpec, caller *Pr func BuildNode(c sandbox.NodeConfig, options sandbox.LocalOptions, result *sandbox.Built) (func(), error) { closeProvider := func() {} var entry Native - if sandbox.DecodeConfigurationObject(c.Native, &entry, "helper_path", "runtime_home", "runtime_path", "firmware_path", "network", "checkpoint_root") != nil { + if sandbox.DecodeConfigurationObject(c.Native, &entry, "helper_path", "runtime_home", "runtime_path", "firmware_path", "image", "runtime_sha256", "firmware_sha256", "network", "checkpoint_root") != nil { return closeProvider, errors.New("invalid managed microsandbox node configuration") } caller := &ProcessCaller{} diff --git a/services/core/internal/sandbox/microsandbox/node_test.go b/services/core/internal/sandbox/microsandbox/node_test.go index 8e542fda8..2a8b91cc8 100644 --- a/services/core/internal/sandbox/microsandbox/node_test.go +++ b/services/core/internal/sandbox/microsandbox/node_test.go @@ -21,7 +21,10 @@ func TestMicrosandboxConstructionSelectsGenerationReadiness(t *testing.T) { } useKVM(t, true) dir := t.TempDir() - native := Native{HelperPath: filepath.Join(dir, "helper"), RuntimeHome: dir, CheckpointRoot: dir, RuntimePath: filepath.Join(dir, "msb"), FirmwarePath: filepath.Join(dir, "firmware"), Network: Network{DefaultEgress: "allow", DefaultIngress: "deny"}} + script := []byte("#!/bin/sh\nprintf '%s' '{}'\n") + digest := sha256.Sum256(script) + native := Native{HelperPath: filepath.Join(dir, "helper"), RuntimeHome: dir, CheckpointRoot: dir, RuntimePath: filepath.Join(dir, "msb"), FirmwarePath: filepath.Join(dir, "firmware"), + Image: "oac-runtime@sha256:" + strings.Repeat("d", 64), RuntimeSHA256: hex.EncodeToString(digest[:]), FirmwareSHA256: hex.EncodeToString(digest[:]), Network: Network{DefaultEgress: "allow", DefaultIngress: "deny"}} if err := os.Chmod(dir, 0700); err != nil { t.Fatal(err) } @@ -32,10 +35,7 @@ func TestMicrosandboxConstructionSelectsGenerationReadiness(t *testing.T) { if err != nil { t.Fatal(err) } - script := []byte("#!/bin/sh\nprintf '%s' '{}'\n") - digest := sha256.Sum256(script) - release := sandbox.RuntimeRelease{MicrosandboxRef: "oac-runtime@sha256:" + strings.Repeat("d", 64), RuntimeSHA256: hex.EncodeToString(digest[:]), FirmwareSHA256: hex.EncodeToString(digest[:])} - config := sandbox.NodeConfig{Provider: "microsandbox", Generation: 9, Specification: sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 2, MemoryMiB: 2048, RootDiskMiB: 8192, EnvironmentDiskMiB: 8192}, Runtime: &release}, Native: raw} + config := sandbox.NodeConfig{Provider: "microsandbox", Generation: 9, Specification: sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 2, MemoryMiB: 2048, RootDiskMiB: 8192, EnvironmentDiskMiB: 8192}}, Native: raw} for _, path := range []string{native.RuntimePath, native.FirmwarePath, native.HelperPath} { if err := os.WriteFile(path, script, 0700); err != nil { t.Fatal(err) diff --git a/services/core/internal/sandbox/microsandbox/process.go b/services/core/internal/sandbox/microsandbox/process.go index ce3218740..47898ad60 100644 --- a/services/core/internal/sandbox/microsandbox/process.go +++ b/services/core/internal/sandbox/microsandbox/process.go @@ -142,10 +142,8 @@ func (p *ProcessCaller) acquireLease() (*os.File, error) { if err := p.verifyLeaseIdentity(stat); err != nil { return fail(err) } - for _, suffix := range []string{".preparing", ".collecting", ".dropped"} { - if _, err := os.Lstat(strings.TrimSuffix(p.LeasePath, ".lease") + suffix); !os.IsNotExist(err) { - return fail(errors.New("generation is not finalized and usable")) - } + if _, err := os.Lstat(strings.TrimSuffix(p.LeasePath, ".lease") + ".dropped"); !os.IsNotExist(err) { + return fail(errors.New("generation is not finalized and usable")) } if err := p.verifyFinalizedGeneration(); err != nil { return fail(err) diff --git a/services/core/internal/sandbox/microsandbox/process_test.go b/services/core/internal/sandbox/microsandbox/process_test.go index 68021cd2d..46713fe55 100644 --- a/services/core/internal/sandbox/microsandbox/process_test.go +++ b/services/core/internal/sandbox/microsandbox/process_test.go @@ -392,7 +392,7 @@ func TestHelperRequiresFinalConfigWithoutAnyUnfinishedJournal(t *testing.T) { path := filepath.Join(t.TempDir(), "1.lease") writeTestLease(t, path) caller := &ProcessCaller{LeasePath: path, LeaseIdentity: testLeaseIdentity()} - for _, suffix := range []string{".preparing", ".collecting", ".dropped"} { + for _, suffix := range []string{".dropped"} { marker := strings.TrimSuffix(path, ".lease") + suffix if err := os.WriteFile(marker, []byte("{}"), 0600); err != nil { t.Fatal(err) diff --git a/services/core/internal/sandbox/node/agent.go b/services/core/internal/sandbox/node/agent.go index 36804c5cc..7390a8114 100644 --- a/services/core/internal/sandbox/node/agent.go +++ b/services/core/internal/sandbox/node/agent.go @@ -27,21 +27,17 @@ type AgentConfig struct { StateDirectory string Identity Identity Credential string - Provider sandbox.SandboxProvider - Probe func(context.Context) (Health, error) // Dialer is optional, primarily for an operator-supplied TLS trust configuration. Dialer *websocket.Dialer } type agent struct { - config AgentConfig - stored StoredIdentity - mu sync.Mutex - current *agentConnection - queue chan work - active atomic.Int32 - ready atomic.Bool - healthSeen atomic.Bool + config AgentConfig + stored StoredIdentity + mu sync.Mutex + current *agentConnection + queue chan work + active atomic.Int32 // failureLogged is set when Run logged a connection failure and cleared when // the next accepted connection is logged. Only Run's goroutine uses it. failureLogged bool @@ -78,20 +74,14 @@ type work struct { connection *agentConnection provider sandbox.SandboxProvider release func() - ready bool queued time.Time } // Run owns one persistent node identity and reconnects its transport only. It // never resends a Provider request. The bounded worker outlives each connection. func Run(ctx context.Context, config AgentConfig) error { - if config.Generations == nil && (config.Provider == nil || config.Probe == nil) { - return sandbox.ErrInvalid - } if config.Generations == nil { - if err := sandbox.ValidateProvider(config.Provider); err != nil { - return err - } + return sandbox.ErrInvalid } release, err := lockDirectory(config.StateDirectory) if err != nil { @@ -163,33 +153,10 @@ func Run(ctx context.Context, config AgentConfig) error { } } } -func (a *agent) health(ctx context.Context, host *hostHealthSampler) (Health, error) { - if a.config.Generations != nil { - health := Health{Generations: a.config.Generations.Statuses(), ObservedAt: time.Now().UTC(), ActiveOperations: int(a.active.Load())} - host.fill(&health, a.config.StateDirectory) - return health, nil - } - probeCtx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - h, e := a.config.Probe(probeCtx) - h.ProviderReady = e == nil - // Only the fixed code leaves this process; the probe error may name host paths. - h.Diagnostic = sandbox.NodeDiagnostic(e) - if e != nil && ctx.Err() != nil { - // A closing connection cancelled the probe; that says nothing about the provider. - h.Diagnostic = string(sandbox.NodeProviderUnavailable) - } else { - wasReady := a.ready.Swap(h.ProviderReady) - seen := a.healthSeen.Swap(true) - if e != nil && probeCtx.Err() == nil && (!seen || wasReady) { - // The local error stays in this host's journal; it may name host paths. - log.Ctx(ctx).Warn("sandbox node provider unavailable; check local runtime configuration and permissions", "node_id", a.config.Identity.NodeID, "diagnostic", h.Diagnostic, "error", e) - } - } - h.ObservedAt = time.Now().UTC() - h.ActiveOperations = int(a.active.Load()) - host.fill(&h, a.config.StateDirectory) - return h, e +func (a *agent) health(host *hostHealthSampler) Health { + health := Health{Generations: a.config.Generations.Statuses(), ObservedAt: time.Now().UTC(), ActiveOperations: int(a.active.Load())} + host.fill(&health, a.config.StateDirectory) + return health } func (a *agent) connect(ctx context.Context) error { endpointURL, err := endpoint(a.config.CoreURL, "/api/v1/sandbox-node/connect") @@ -221,17 +188,17 @@ func (a *agent) connect(ctx context.Context) error { defer conn.Close() conn.SetReadLimit(MaxFrameBytes) host := new(hostHealthSampler) - health, _ := a.health(ctx, host) + health := a.health(host) _ = conn.SetReadDeadline(time.Now().Add(15 * time.Second)) version := ProtocolVersion - if err = writeFrame(conn, frame{Version: version, Type: "hello", GenerationManagement: a.config.Generations != nil, Identity: &a.config.Identity, Health: &health}); err != nil { + if err = writeFrame(conn, frame{Version: version, Type: "hello", Identity: &a.config.Identity, Health: &health}); err != nil { return err } welcome, err := readFrame(conn) if err != nil { return notAccepted{err} } - if (a.config.Generations == nil && welcome.Deployment != nil) || welcome.Version != version || welcome.Type != "welcome" || !validID(welcome.ConnectionID) || welcome.OwnerEpoch == 0 { + if welcome.Deployment == nil || welcome.Version != version || welcome.Type != "welcome" || !validID(welcome.ConnectionID) || welcome.OwnerEpoch == 0 { return notAccepted{sandbox.ErrInvalid} } a.mu.Lock() @@ -259,19 +226,13 @@ func (a *agent) connect(ctx context.Context) error { detach := context.AfterFunc(connectionCtx, current.close) defer detach() go a.heartbeats(connectionCtx, current, host) - var controlDone chan struct{} - controlWork := make(chan []sandbox.GenerationRetention, 1) - if a.config.Generations != nil { - if welcome.Deployment == nil { - return sandbox.ErrInvalid - } - if err := a.config.Generations.Deployment(*welcome.Deployment); err != nil { - return err - } - controlDone = make(chan struct{}) - go func() { defer close(controlDone); a.garbageCollection(connectionCtx, current, controlWork) }() - defer func() { cancel(); <-controlDone }() + if err := a.config.Generations.Deployment(*welcome.Deployment); err != nil { + return err } + controlWork := make(chan []sandbox.GenerationRetention, 1) + controlDone := make(chan struct{}) + go func() { defer close(controlDone); a.garbageCollection(connectionCtx, current, controlWork) }() + defer func() { cancel(); <-controlDone }() sequence := uint64(0) for { _ = conn.SetReadDeadline(time.Now().Add(35 * time.Second)) @@ -283,20 +244,18 @@ func (a *agent) connect(ctx context.Context) error { return sandbox.ErrInvalid } if f.Type == "heartbeat_ack" && f.ConnectionID == current.id { - if f.OwnerEpoch != current.epoch || (a.config.Generations == nil && f.Deployment != nil) { + if f.OwnerEpoch != current.epoch { return sandbox.ErrOwnership } - if a.config.Generations != nil { - if f.Deployment == nil { - return sandbox.ErrInvalid - } - if err := a.config.Generations.Deployment(*f.Deployment); err != nil { - return err - } + if f.Deployment == nil { + return sandbox.ErrInvalid + } + if err := a.config.Generations.Deployment(*f.Deployment); err != nil { + return err } continue } - if f.Type == "retention_ack" && a.config.Generations != nil { + if f.Type == "retention_ack" { if err := a.acceptRetention(current, f, controlWork); err != nil { return err } @@ -306,9 +265,6 @@ func (a *agent) connect(ctx context.Context) error { return sandbox.ErrInvalid } q := *f.Request - if a.config.Generations == nil && q.DeploymentGeneration != a.config.Identity.DeploymentGeneration { - return sandbox.ErrOwnership - } if q.ConnectionID != current.id || q.OwnerEpoch != current.epoch || q.Sequence != sequence+1 { return sandbox.ErrOwnership } @@ -317,20 +273,16 @@ func (a *agent) connect(ctx context.Context) error { _ = current.write(frame{Type: "response", Response: &response{ID: q.ID, ConnectionID: current.id, ErrorCode: "invalid"}}) continue } - task := work{request: q, connection: current, provider: a.config.Provider, ready: a.ready.Load(), queued: time.Now()} - if a.config.Generations != nil { - task.provider, task.ready, task.release, err = a.config.Generations.Acquire(q.DeploymentGeneration) - if err != nil { - _ = current.write(frame{Type: "response", Response: &response{ID: q.ID, ConnectionID: current.id, ErrorCode: "unconfirmed"}}) - continue - } + task := work{request: q, connection: current, queued: time.Now()} + task.provider, _, task.release, err = a.config.Generations.Acquire(q.DeploymentGeneration) + if err != nil { + _ = current.write(frame{Type: "response", Response: &response{ID: q.ID, ConnectionID: current.id, ErrorCode: "unconfirmed"}}) + continue } select { case a.queue <- task: default: - if task.release != nil { - task.release() - } + task.release() return ErrUnavailable } } @@ -382,12 +334,12 @@ func (a *agent) heartbeats(ctx context.Context, c *agentConnection, host *hostHe case <-c.done: return case <-ticker.C: - h, _ := a.health(ctx, host) + h := a.health(host) if c.write(frame{Type: "heartbeat", ConnectionID: c.id, OwnerEpoch: c.epoch, Health: &h}) != nil { c.close() return } - if a.config.Generations != nil && a.requestRetention(c) != nil { + if a.requestRetention(c) != nil { c.close() return } @@ -399,9 +351,7 @@ func (a *agent) worker(ctx context.Context) { for { select { case task := <-a.queue: - if task.release != nil { - task.release() - } + task.release() default: return } @@ -417,9 +367,7 @@ func (a *agent) worker(ctx context.Context) { } } func (a *agent) executeWork(ctx context.Context, task work) { - if task.release != nil { - defer task.release() - } + defer task.release() a.mu.Lock() current := a.current a.mu.Unlock() @@ -431,10 +379,7 @@ func (a *agent) executeWork(ctx context.Context, task work) { return default: } - ready := a.ready.Load() - if a.config.Generations != nil { - ready = a.config.Generations.Ready(task.request.DeploymentGeneration) - } + ready := a.config.Generations.Ready(task.request.DeploymentGeneration) if !time.Now().Before(task.request.deadline) || requiresReady(task.request) && !ready { _ = task.connection.write(frame{Type: "response", Response: &response{ID: task.request.ID, ConnectionID: task.connection.id, ErrorCode: "unconfirmed"}}) return diff --git a/services/core/internal/sandbox/node/capacity_test.go b/services/core/internal/sandbox/node/capacity_test.go index c408cb82a..3a3aee5b2 100644 --- a/services/core/internal/sandbox/node/capacity_test.go +++ b/services/core/internal/sandbox/node/capacity_test.go @@ -52,7 +52,7 @@ func TestHubCapacityCacheCannotOverrideAdminChangesOrBlockReconnect(t *testing.T id := identity() approved := id var mu sync.Mutex - hub := NewHub(HubOptions{Authenticate: func(context.Context, string, string) (Identity, error) { + hub := newTestHub(HubOptions{Authenticate: func(context.Context, string, string) (Identity, error) { mu.Lock() defer mu.Unlock() return approved, nil @@ -74,7 +74,7 @@ func TestHubCapacityCacheCannotOverrideAdminChangesOrBlockReconnect(t *testing.T mu.Lock() approved.MaxActive, approved.MaxRetained = 2, 8 mu.Unlock() - if err := writeFrame(conn, frame{Type: "heartbeat", ConnectionID: peer.id, OwnerEpoch: peer.epoch, Health: &Health{ProviderReady: true, ObservedAt: time.Now().UTC()}}); err != nil { + if err := writeFrame(conn, frame{Type: "heartbeat", ConnectionID: peer.id, OwnerEpoch: peer.epoch, Health: &Health{ObservedAt: time.Now().UTC()}}); err != nil { t.Fatal(err) } _ = conn.SetReadDeadline(time.Now().Add(time.Second)) diff --git a/services/core/internal/sandbox/node/checkpoint_generation_test.go b/services/core/internal/sandbox/node/checkpoint_generation_test.go index 5f5bd6251..21b40bf4c 100644 --- a/services/core/internal/sandbox/node/checkpoint_generation_test.go +++ b/services/core/internal/sandbox/node/checkpoint_generation_test.go @@ -6,51 +6,11 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/microsandbox" "github.com/google/uuid" "net/http/httptest" - "strings" "testing" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) -func TestStaticNodeReportsExactCheckpointGeneration(t *testing.T) { - id := identity() - compatibility := &sandbox.CheckpointCompatibility{ArtifactDomain: "private-store", ExecutionClass: "native-class"} - status := sandbox.GenerationStatus{Generation: id.DeploymentGeneration, SpecificationDigest: id.SpecificationDigest, State: "ready", Checkpoint: compatibility} - seen := false - hub := NewHub(HubOptions{Heartbeat: func(_ context.Context, _ Identity, _ string, _ uint64, health Health) error { - seen = true - if len(health.Generations) != 1 || *health.Generations[0].Checkpoint != *compatibility { - t.Fatal("qualification lost") - } - return nil - }}) - p := &peer{identity: id} - health := Health{ProviderReady: true, Generations: []sandbox.GenerationStatus{status}} - if err := hub.recordHealth(t.Context(), p, health); err != nil || !seen { - t.Fatal(err, seen) - } - for _, mutation := range []string{"generation", "digest", "readiness", "extra"} { - t.Run(mutation, func(t *testing.T) { - changed := health - changed.Generations = append([]sandbox.GenerationStatus(nil), health.Generations...) - switch mutation { - case "generation": - changed.Generations[0].Generation++ - case "digest": - changed.Generations[0].SpecificationDigest = strings.Repeat("f", 64) - case "readiness": - changed.ProviderReady = false - case "extra": - changed.Generations = append(changed.Generations, status) - } - seen = false - if err := hub.recordHealth(t.Context(), p, changed); err == nil || seen { - t.Fatal("invalid static qualification reached persistence") - } - }) - } -} - type closedRestoreProvider struct{ fakeProvider } func (*closedRestoreProvider) ProviderOperations() providercontract.Operations { @@ -62,7 +22,7 @@ func (*closedRestoreProvider) Resume(_ context.Context, q sandbox.ResumeRequest) func TestClosedRestoreEvidenceSurvivesNodeTransport(t *testing.T) { id := identity() var credential string - hub := NewHub(HubOptions{Authenticate: func(_ context.Context, node, token string) (Identity, error) { + hub := newTestHub(HubOptions{Authenticate: func(_ context.Context, node, token string) (Identity, error) { if node != id.NodeID || token != credential { return Identity{}, ErrAuthentication } @@ -80,7 +40,7 @@ func TestClosedRestoreEvidenceSurvivesNodeTransport(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) done := make(chan error, 1) go func() { - done <- Run(ctx, AgentConfig{CoreURL: server.URL, StateDirectory: directory, Identity: id, Credential: credential, Provider: &closedRestoreProvider{}, Probe: probe}) + done <- runProvider(ctx, AgentConfig{CoreURL: server.URL, StateDirectory: directory, Identity: id, Credential: credential}, &closedRestoreProvider{}, probe) }() defer func() { cancel() diff --git a/services/core/internal/sandbox/node/connection_test.go b/services/core/internal/sandbox/node/connection_test.go index eb451d8af..b88a00792 100644 --- a/services/core/internal/sandbox/node/connection_test.go +++ b/services/core/internal/sandbox/node/connection_test.go @@ -53,7 +53,7 @@ func connectRawNode(t *testing.T, origin string, id Identity) *websocket.Conn { if err != nil { t.Fatal(err) } - if err = writeFrame(conn, frame{Type: "hello", Identity: &id, Health: &Health{ProviderReady: true, ObservedAt: time.Now().UTC()}}); err != nil { + if err = writeFrame(conn, frame{Type: "hello", Identity: &id, Health: &Health{ObservedAt: time.Now().UTC()}}); err != nil { conn.Close() t.Fatal(err) } @@ -67,7 +67,7 @@ func connectRawNode(t *testing.T, origin string, id Identity) *websocket.Conn { func TestHubReservesIdentityAcrossConcurrentOpeningHandshakes(t *testing.T) { id := identity() var connected atomic.Int32 - hub := NewHub(HubOptions{Authenticate: func(context.Context, string, string) (Identity, error) { return id, nil }, OwnerEpoch: func(context.Context) (uint64, error) { return 1, nil }, Connected: func(context.Context, Identity, string, uint64) error { connected.Add(1); return nil }}) + hub := newTestHub(HubOptions{Authenticate: func(context.Context, string, string) (Identity, error) { return id, nil }, OwnerEpoch: func(context.Context) (uint64, error) { return 1, nil }, Connected: func(context.Context, Identity, string, uint64) error { connected.Add(1); return nil }}) server := httptest.NewServer(hub) defer server.Close() defer hub.Close() @@ -127,7 +127,7 @@ func TestCopiedIdentityCannotReplaceNodeWithInflightCreate(t *testing.T) { var credential string var authenticated, connected atomic.Int32 duplicateAttempt := make(chan struct{}, 1) - hub := NewHub(HubOptions{Authenticate: func(ctx context.Context, node, token string) (Identity, error) { + hub := newTestHub(HubOptions{Authenticate: func(ctx context.Context, node, token string) (Identity, error) { if node != id.NodeID || token != credential { return Identity{}, ErrAuthentication } @@ -160,7 +160,7 @@ func TestCopiedIdentityCannotReplaceNodeWithInflightCreate(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) go func() { - done <- Run(ctx, AgentConfig{CoreURL: server.URL, StateDirectory: dir, Identity: id, Credential: credential, Provider: p, Probe: probe}) + done <- runProvider(ctx, AgentConfig{CoreURL: server.URL, StateDirectory: dir, Identity: id, Credential: credential}, p, probe) }() return cancel, done } @@ -232,7 +232,7 @@ func TestReservationRemainsUntilFencedDisconnectFinishes(t *testing.T) { active := "" connections := 0 var callbackErr error - hub := NewHub(HubOptions{Authenticate: func(context.Context, string, string) (Identity, error) { return id, nil }, OwnerEpoch: func(context.Context) (uint64, error) { return 9, nil }, Connected: func(ctx context.Context, id Identity, connection string, epoch uint64) error { + hub := newTestHub(HubOptions{Authenticate: func(context.Context, string, string) (Identity, error) { return id, nil }, OwnerEpoch: func(context.Context) (uint64, error) { return 9, nil }, Connected: func(ctx context.Context, id Identity, connection string, epoch uint64) error { mu.Lock() defer mu.Unlock() if epoch != 9 { @@ -323,7 +323,7 @@ func TestNodeLogsEachConnectionCauseOnce(t *testing.T) { return } defer conn.Close() - if _, err = readFrame(conn); err != nil || writeFrame(conn, frame{Type: "welcome", ConnectionID: uuid.NewString(), OwnerEpoch: 1}) != nil { + if _, err = readFrame(conn); err != nil || writeFrame(conn, frame{Type: "welcome", ConnectionID: uuid.NewString(), OwnerEpoch: 1, Deployment: testDeployment()}) != nil { return } for err == nil { @@ -344,7 +344,7 @@ func TestNodeLogsEachConnectionCauseOnce(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) go func() { - done <- Run(ctx, AgentConfig{CoreURL: server.URL, StateDirectory: dir, Identity: id, Credential: stored.Credential, Provider: &fakeProvider{}, Probe: probe}) + done <- runProvider(ctx, AgentConfig{CoreURL: server.URL, StateDirectory: dir, Identity: id, Credential: stored.Credential}, &fakeProvider{}, probe) }() wait(t, func() bool { return refused.Load() >= 2 }) accept.Store(true) @@ -366,7 +366,7 @@ func TestNodeConnectionFailuresNameTheirCause(t *testing.T) { defer untrusted.Close() // Core closes the socket without a frame when it refuses a hello, here one // whose identity differs from the registered node. - hub := NewHub(HubOptions{ + hub := newTestHub(HubOptions{ Authenticate: func(_ context.Context, node, _ string) (Identity, error) { other := identity() other.NodeID = node @@ -382,7 +382,7 @@ func TestNodeConnectionFailuresNameTheirCause(t *testing.T) { {untrusted.URL, "tls: failed to verify certificate: x509: certificate signed by unknown authority"}, {core.URL, "Core did not accept the node: websocket: close 1006 (abnormal closure): unexpected EOF"}, } { - a := agent{config: AgentConfig{CoreURL: c.origin, StateDirectory: t.TempDir(), Identity: identity(), Credential: "credential", Probe: probe}} + a := agent{config: AgentConfig{CoreURL: c.origin, StateDirectory: t.TempDir(), Identity: identity(), Credential: "credential", Generations: generationFixture(0)}} if cause := connectionCause(a.connect(t.Context())); cause != c.cause { t.Errorf("%s: cause %q, want %q", c.origin, cause, c.cause) } diff --git a/services/core/internal/sandbox/node/creation_settlement_test.go b/services/core/internal/sandbox/node/creation_settlement_test.go index 09d8daec5..4f5641ac6 100644 --- a/services/core/internal/sandbox/node/creation_settlement_test.go +++ b/services/core/internal/sandbox/node/creation_settlement_test.go @@ -43,7 +43,7 @@ func TestNodeCarriesCreationSettlementWithoutConvertingFailureToSuccess(t *testi } { t.Run(test.name, func(t *testing.T) { id := identity() - hub := NewHub(HubOptions{ + hub := newTestHub(HubOptions{ Authenticate: func(context.Context, string, string) (Identity, error) { return id, nil }, OwnerEpoch: func(context.Context) (uint64, error) { return 1, nil }, }) @@ -58,8 +58,8 @@ func TestNodeCarriesCreationSettlementWithoutConvertingFailureToSuccess(t *testi ctx, cancel := context.WithCancel(t.Context()) done := make(chan error, 1) go func() { - done <- Run(ctx, AgentConfig{CoreURL: server.URL, StateDirectory: dir, Identity: id, Credential: stored.Credential, - Provider: &settledProvider{info: test.info, err: test.err}, Probe: probe}) + done <- runProvider(ctx, AgentConfig{CoreURL: server.URL, StateDirectory: dir, Identity: id, Credential: stored.Credential}, + &settledProvider{info: test.info, err: test.err}, probe) }() defer func() { cancel() diff --git a/services/core/internal/sandbox/node/docker_live_test.go b/services/core/internal/sandbox/node/docker_live_test.go index bec3e408e..7b2e33e1a 100644 --- a/services/core/internal/sandbox/node/docker_live_test.go +++ b/services/core/internal/sandbox/node/docker_live_test.go @@ -39,7 +39,7 @@ func TestDockerNodeTransportLifecycle(t *testing.T) { t.Fatal(err) } var credential string - hub := NewHub(HubOptions{Authenticate: func(ctx context.Context, node, token string) (Identity, error) { + hub := newTestHub(HubOptions{Authenticate: func(ctx context.Context, node, token string) (Identity, error) { if node != id.NodeID || token != credential { return Identity{}, ErrAuthentication } @@ -58,10 +58,10 @@ func TestDockerNodeTransportLifecycle(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) go func() { - done <- Run(ctx, AgentConfig{CoreURL: server.URL, StateDirectory: dir, Identity: id, Credential: credential, Provider: provider, Probe: func(ctx context.Context) (Health, error) { + done <- runProvider(ctx, AgentConfig{CoreURL: server.URL, StateDirectory: dir, Identity: id, Credential: credential}, provider, func(ctx context.Context) (*sandbox.CheckpointCompatibility, error) { _, err := dockerClient.Ping(ctx, client.PingOptions{}) - return Health{}, err - }}) + return nil, err + }) }() return cancel, done } diff --git a/services/core/internal/sandbox/node/generation_connection_test.go b/services/core/internal/sandbox/node/generation_connection_test.go index 1c3e59d6c..2052f0913 100644 --- a/services/core/internal/sandbox/node/generation_connection_test.go +++ b/services/core/internal/sandbox/node/generation_connection_test.go @@ -20,7 +20,7 @@ func TestGenerationWireRoutesOldOwnershipAndCurrentTargetSeparately(t *testing.T original := manager.values[1].value.Provider.(*fakeProvider) current := manager.values[17].value.Provider.(*fakeProvider) ninth := manager.values[9].value.Provider.(*fakeProvider) - hub := NewHub(HubOptions{ + hub := newTestHub(HubOptions{ Authenticate: func(context.Context, string, string) (Identity, error) { return id, nil }, OwnerEpoch: func(context.Context) (uint64, error) { return 7, nil }, Generations: func(_ context.Context, _ Identity, _ string, _ uint64, h Health) error { diff --git a/services/core/internal/sandbox/node/generation_json.go b/services/core/internal/sandbox/node/generation_json.go index 06ae10fff..a9418fd3a 100644 --- a/services/core/internal/sandbox/node/generation_json.go +++ b/services/core/internal/sandbox/node/generation_json.go @@ -80,7 +80,6 @@ func validateGenerationJSON(raw []byte, kind string) error { switch kind { case "hello": fields = "identity health" - optional = "generation_management" case "welcome", "heartbeat_ack": fields = "connection_id owner_epoch" optional = "deployment" @@ -112,7 +111,7 @@ func validateGenerationJSON(raw []byte, kind string) error { } } if value := values["health"]; value != nil { - health, err := generationObject(value, "provider_ready observed_at active_operations cpu_utilization total_memory_bytes effective_cpu_cores", "generations diagnostic cpu_count available_memory_bytes available_disk_bytes", "cpu_utilization total_memory_bytes effective_cpu_cores cpu_count available_memory_bytes available_disk_bytes") + health, err := generationObject(value, "observed_at active_operations cpu_utilization total_memory_bytes effective_cpu_cores", "generations cpu_count available_memory_bytes available_disk_bytes", "cpu_utilization total_memory_bytes effective_cpu_cores cpu_count available_memory_bytes available_disk_bytes") if err != nil { return err } diff --git a/services/core/internal/sandbox/node/generation_json_test.go b/services/core/internal/sandbox/node/generation_json_test.go index 2f0f7e069..d8fcf99f5 100644 --- a/services/core/internal/sandbox/node/generation_json_test.go +++ b/services/core/internal/sandbox/node/generation_json_test.go @@ -2,7 +2,6 @@ package node import ( "encoding/json" - "fmt" "strings" "testing" "time" @@ -55,7 +54,7 @@ func TestGenerationHealthRejectsUnboundedOrAmbiguousNumbers(t *testing.T) { {`"cpu_utilization":null`, `"cpu_utilization":1.1`}, {`"total_memory_bytes":null`, `"total_memory_bytes":9007199254740992`}, {`"effective_cpu_cores":null`, `"effective_cpu_cores":0`}, - {`"provider_ready":false`, `"Provider_Ready":false`}, + {`"active_operations":0`, `"Active_Operations":0`}, } { if _, err := decodeFrame([]byte(strings.Replace(string(good), replacement.from, replacement.to, 1))); err == nil { t.Fatal("invalid health accepted", replacement.to) @@ -72,79 +71,55 @@ func TestNodeProtocolRejectsHistoricalVersions(t *testing.T) { } } -func TestNodeGenerationManagementIsAnExplicitCurrentCapability(t *testing.T) { - for _, managed := range []bool{false, true} { - f := frame{Version: ProtocolVersion, Type: "hello", GenerationManagement: managed, Identity: new(Identity), Health: &Health{ObservedAt: time.Now().UTC()}} - if managed { - f.Health.Generations = []sandbox.GenerationStatus{{Generation: 1, SpecificationDigest: strings.Repeat("a", 64), State: "ready"}} - } - raw, _ := json.Marshal(f) - if _, err := decodeFrame(raw); err != nil { - t.Fatalf("current mode managed=%v: %v", managed, err) - } - if managed { - f.GenerationManagement = false - raw, _ = json.Marshal(f) +func TestCheckpointGenerationHealthJSONRoundTrip(t *testing.T) { + for _, kind := range []string{"hello", "heartbeat"} { + t.Run(kind, func(t *testing.T) { + compatibility := sandbox.CheckpointCompatibility{ArtifactDomain: "private-store", ExecutionClass: "native-class"} + f := frame{Version: ProtocolVersion, Type: kind, Health: &Health{ObservedAt: time.Now().UTC(), Generations: []sandbox.GenerationStatus{{Generation: 5, SpecificationDigest: strings.Repeat("a", 64), State: "ready", Checkpoint: &compatibility}}}} + if kind == "hello" { + f.Identity = new(Identity) + } else { + f.ConnectionID = uuid.NewString() + f.OwnerEpoch = 1 + } + raw, err := json.Marshal(f) + if err != nil { + t.Fatal(err) + } + if err := validateVersionFrame(f, len(raw)); err != nil { + t.Fatal("writer rejected valid status", err) + } decoded, err := decodeFrame(raw) - if err != nil || decoded.GenerationManagement || len(decoded.Health.Generations) != 1 { - t.Fatal("static qualification implicitly enabled generation management", err) + if err != nil { + t.Fatal("decoder rejected valid status", err) } - } - } -} - -func TestCheckpointGenerationHealthJSONRoundTrip(t *testing.T) { - for _, managed := range []bool{false, true} { - for _, kind := range []string{"hello", "heartbeat"} { - t.Run(fmt.Sprintf("managed=%t/%s", managed, kind), func(t *testing.T) { - compatibility := sandbox.CheckpointCompatibility{ArtifactDomain: "private-store", ExecutionClass: "native-class"} - f := frame{Version: ProtocolVersion, Type: kind, Health: &Health{ProviderReady: true, ObservedAt: time.Now().UTC(), Generations: []sandbox.GenerationStatus{{Generation: 5, SpecificationDigest: strings.Repeat("a", 64), State: "ready", Checkpoint: &compatibility}}}} - if kind == "hello" { - f.Identity = new(Identity) - f.GenerationManagement = managed - } else { - f.ConnectionID = uuid.NewString() - f.OwnerEpoch = 1 - } - raw, err := json.Marshal(f) - if err != nil { - t.Fatal(err) - } - if err := validateVersionFrame(f, len(raw)); err != nil { - t.Fatal("writer rejected valid status", err) - } - decoded, err := decodeFrame(raw) - if err != nil { - t.Fatal("decoder rejected valid status", err) - } - if decoded.Health == nil || len(decoded.Health.Generations) != 1 || decoded.Health.Generations[0].Checkpoint == nil || *decoded.Health.Generations[0].Checkpoint != compatibility { - t.Fatal("checkpoint qualification lost", decoded) - } - original := string(raw) - for name, replacement := range map[string]string{ - "unknown nested field": strings.Replace(original, `"artifact_domain":"private-store"`, `"artifact_domain":"private-store","extra":"value"`, 1), - "unknown generation field": strings.Replace(original, `"checkpoint":`, `"extra":true,"checkpoint":`, 1), - "duplicate token": strings.Replace(original, `"artifact_domain":"private-store"`, `"artifact_domain":"private-store","artifact_domain":"other"`, 1), - "case alias": strings.Replace(original, `"artifact_domain":`, `"Artifact_Domain":`, 1), - "missing token": strings.Replace(original, `"artifact_domain":"private-store",`, "", 1), - "null token": strings.Replace(original, `"artifact_domain":"private-store"`, `"artifact_domain":null`, 1), - "null checkpoint": strings.Replace(original, `{"artifact_domain":"private-store","execution_class":"native-class"}`, `null`, 1), - "empty domain": strings.Replace(original, `"private-store"`, `""`, 1), - "empty execution": strings.Replace(original, `"native-class"`, `""`, 1), - "oversized domain": strings.Replace(original, `"private-store"`, `"`+strings.Repeat("x", 257)+`"`, 1), - "oversized execution": strings.Replace(original, `"native-class"`, `"`+strings.Repeat("x", 257)+`"`, 1), - "nonready checkpoint": strings.Replace(original, `"state":"ready"`, `"state":"preparing"`, 1), - } { - t.Run(name, func(t *testing.T) { - if replacement == original { - t.Fatal("fixture unchanged") - } - if _, err := decodeFrame([]byte(replacement)); err == nil { - t.Fatal("invalid checkpoint accepted") - } - }) - } - }) - } + if decoded.Health == nil || len(decoded.Health.Generations) != 1 || decoded.Health.Generations[0].Checkpoint == nil || *decoded.Health.Generations[0].Checkpoint != compatibility { + t.Fatal("checkpoint qualification lost", decoded) + } + original := string(raw) + for name, replacement := range map[string]string{ + "unknown nested field": strings.Replace(original, `"artifact_domain":"private-store"`, `"artifact_domain":"private-store","extra":"value"`, 1), + "unknown generation field": strings.Replace(original, `"checkpoint":`, `"extra":true,"checkpoint":`, 1), + "duplicate token": strings.Replace(original, `"artifact_domain":"private-store"`, `"artifact_domain":"private-store","artifact_domain":"other"`, 1), + "case alias": strings.Replace(original, `"artifact_domain":`, `"Artifact_Domain":`, 1), + "missing token": strings.Replace(original, `"artifact_domain":"private-store",`, "", 1), + "null token": strings.Replace(original, `"artifact_domain":"private-store"`, `"artifact_domain":null`, 1), + "null checkpoint": strings.Replace(original, `{"artifact_domain":"private-store","execution_class":"native-class"}`, `null`, 1), + "empty domain": strings.Replace(original, `"private-store"`, `""`, 1), + "empty execution": strings.Replace(original, `"native-class"`, `""`, 1), + "oversized domain": strings.Replace(original, `"private-store"`, `"`+strings.Repeat("x", 257)+`"`, 1), + "oversized execution": strings.Replace(original, `"native-class"`, `"`+strings.Repeat("x", 257)+`"`, 1), + "nonready checkpoint": strings.Replace(original, `"state":"ready"`, `"state":"preparing"`, 1), + } { + t.Run(name, func(t *testing.T) { + if replacement == original { + t.Fatal("fixture unchanged") + } + if _, err := decodeFrame([]byte(replacement)); err == nil { + t.Fatal("invalid checkpoint accepted") + } + }) + } + }) } } diff --git a/services/core/internal/sandbox/node/generation_wire.go b/services/core/internal/sandbox/node/generation_wire.go index d8c83327b..4df61cc2a 100644 --- a/services/core/internal/sandbox/node/generation_wire.go +++ b/services/core/internal/sandbox/node/generation_wire.go @@ -28,9 +28,6 @@ func validateVersionFrame(f frame, size int) error { if f.Version != ProtocolVersion { return sandbox.ErrInvalid } - if f.GenerationManagement && f.Type != "hello" { - return sandbox.ErrInvalid - } if f.Type != "request" && f.Type != "response" && size > MaxControlFrameBytes { return sandbox.ErrInvalid } @@ -112,9 +109,6 @@ func validGenerationHealth(h Health) bool { if h.ObservedAt.IsZero() || h.ObservedAt.Year() < 1970 || h.ObservedAt.Year() > 9999 || h.ActiveOperations < 0 || h.ActiveOperations > maxPending { return false } - if len(h.Diagnostic) > 64 || sandbox.NormalizeNodeDiagnostic(h.Diagnostic) != h.Diagnostic { - return false - } for _, value := range []*int64{h.CPUCount, h.TotalMemoryBytes, h.AvailableMemoryBytes, h.AvailableDiskBytes} { if value != nil && (*value < 0 || *value > 1<<53-1) { return false diff --git a/services/core/internal/sandbox/node/generations.go b/services/core/internal/sandbox/node/generations.go index 9e44ef959..89e3add8d 100644 --- a/services/core/internal/sandbox/node/generations.go +++ b/services/core/internal/sandbox/node/generations.go @@ -25,7 +25,6 @@ type GenerationProvider struct { type GenerationManagerOptions struct { Initial []GenerationProvider Recover []sandbox.GenerationReference - Collect []sandbox.GenerationReference Prepare func(context.Context, uint64, string) (GenerationProvider, error) Remove func(context.Context, GenerationProvider) error } @@ -78,13 +77,6 @@ func NewGenerationManager(ctx context.Context, options GenerationManagerOptions) } m.values[ref.Generation] = &localGeneration{value: GenerationProvider{Generation: ref.Generation, SpecificationDigest: ref.SpecificationDigest}, state: "failed", diagnostic: string(sandbox.NodeProviderUnavailable), repairing: true} } - for _, ref := range options.Collect { - if !validGeneration(ref.Generation) || !validSpecificationDigest(ref.SpecificationDigest) || m.values[ref.Generation] != nil { - cancel() - return nil, sandbox.ErrInvalid - } - m.values[ref.Generation] = &localGeneration{value: GenerationProvider{Generation: ref.Generation, SpecificationDigest: ref.SpecificationDigest}, collecting: true} - } m.workers.Add(2) go func() { defer m.workers.Done(); m.prepareLoop() }() go func() { defer m.workers.Done(); m.probeLoop() }() diff --git a/services/core/internal/sandbox/node/generations_test.go b/services/core/internal/sandbox/node/generations_test.go index f81a5e38f..b8256c771 100644 --- a/services/core/internal/sandbox/node/generations_test.go +++ b/services/core/internal/sandbox/node/generations_test.go @@ -290,81 +290,8 @@ func TestGenerationDeploymentRejectsConflictingImmutableIdentity(t *testing.T) { } } -func TestInterruptedCollectionNeverPreparesOrServesAfterRestart(t *testing.T) { - digest := strings.Repeat("a", 64) - ref := sandbox.GenerationReference{Generation: 1, SpecificationDigest: digest} - probed := make(chan struct{}, 1) - removes := 0 - manager, err := NewGenerationManager(t.Context(), GenerationManagerOptions{ - Initial: []GenerationProvider{{Generation: 2, SpecificationDigest: digest, Provider: &fakeProvider{}, Probe: func(context.Context) (*sandbox.CheckpointCompatibility, error) { - select { - case probed <- struct{}{}: - default: - } - return nil, nil - }}}, - Collect: []sandbox.GenerationReference{ref}, - Prepare: func(context.Context, uint64, string) (GenerationProvider, error) { - t.Error("collection entered preparation") - return GenerationProvider{}, sandbox.ErrInvalid - }, - Remove: func(_ context.Context, value GenerationProvider) error { - removes++ - if value.Generation != 1 { - t.Error("wrong generation") - } - if removes == 1 { - return errors.New("interrupted cleanup") - } - return nil - }, - }) - if err != nil { - t.Fatal(err) - } - defer manager.Close() - pin := uint64(2) - if err = manager.Deployment(sandbox.NodeDeployment{Generation: 2, SpecificationDigest: digest, ServingGeneration: &pin}); err != nil { - t.Fatal(err) - } - select { - case <-probed: - case <-time.After(3 * time.Second): - t.Fatal("provider loop did not run") - } - if _, _, _, err = manager.Acquire(1); err == nil { - t.Fatal("unfinished collection admitted an operation") - } - for _, status := range manager.Statuses() { - if status.Generation == 1 { - t.Fatal("unfinished collection advertised readiness", status) - } - } - if len(manager.Retained(0)) != 2 || removes != 0 { - t.Fatal("restart consumed old drop authorization") - } - cancelled, cancel := context.WithCancel(t.Context()) - cancel() - grant := sandbox.GenerationRetention{GenerationReference: ref} - if err = manager.Drop(cancelled, grant); err == nil || removes != 0 { - t.Fatal("cancelled connection collected") - } - if err = manager.Drop(t.Context(), grant); err == nil { - t.Fatal("expected interrupted cleanup") - } - if _, _, _, err = manager.Acquire(1); err == nil { - t.Fatal("failed cleanup resumed serving") - } - if err = manager.Drop(t.Context(), grant); err != nil { - t.Fatal(err) - } - if len(manager.Retained(0)) != 1 || removes != 2 { - t.Fatal("cleanup did not settle") - } -} - func TestPreparationDiagnosticPreservesTypedCause(t *testing.T) { - for _, cause := range []error{sandbox.ErrRuntimeDownloadFailed, sandbox.ErrProviderUnavailable, sandbox.ErrHostUnsupported, sandbox.ErrOwnership, context.Canceled, errors.New("raw secret provider text")} { + for _, cause := range []error{sandbox.ErrProviderUnavailable, sandbox.ErrHostUnsupported, sandbox.ErrOwnership, context.Canceled, errors.New("raw secret provider text")} { t.Run(sandbox.NodeDiagnostic(cause)+cause.Error(), func(t *testing.T) { m, err := NewGenerationManager(t.Context(), GenerationManagerOptions{ Prepare: func(context.Context, uint64, string) (GenerationProvider, error) { return GenerationProvider{}, cause }, diff --git a/services/core/internal/sandbox/node/health_connection_linux_test.go b/services/core/internal/sandbox/node/health_connection_linux_test.go index c2c8fefae..695834890 100644 --- a/services/core/internal/sandbox/node/health_connection_linux_test.go +++ b/services/core/internal/sandbox/node/health_connection_linux_test.go @@ -28,12 +28,12 @@ func TestHostHealthReconnectStartsFreshInterval(t *testing.T) { return } hellos <- *hello.Health - if err := writeFrame(conn, frame{Type: "welcome", ConnectionID: uuid.NewString(), OwnerEpoch: 1}); err != nil { + if err := writeFrame(conn, frame{Type: "welcome", ConnectionID: uuid.NewString(), OwnerEpoch: 1, Deployment: testDeployment()}); err != nil { t.Error(err) } })) defer server.Close() - a := agent{config: AgentConfig{CoreURL: server.URL, StateDirectory: t.TempDir(), Identity: identity(), Probe: probe}, stored: StoredIdentity{OwnerEpoch: 1}} + a := agent{config: AgentConfig{CoreURL: server.URL, StateDirectory: t.TempDir(), Identity: identity(), Generations: generationFixture(0)}, stored: StoredIdentity{OwnerEpoch: 1}} ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() for i := 0; i < 2; i++ { diff --git a/services/core/internal/sandbox/node/health_linux_test.go b/services/core/internal/sandbox/node/health_linux_test.go index e27dbf468..feaa7d1a4 100644 --- a/services/core/internal/sandbox/node/health_linux_test.go +++ b/services/core/internal/sandbox/node/health_linux_test.go @@ -3,7 +3,6 @@ package node import ( - "context" "encoding/json" "os" "path/filepath" @@ -187,11 +186,11 @@ func TestHealthCgroupMountPaths(t *testing.T) { func TestHostHealthReadOnly(t *testing.T) { dir := t.TempDir() sampler := new(hostHealthSampler) - a := agent{config: AgentConfig{StateDirectory: dir, Probe: probe}} + a := agent{config: AgentConfig{StateDirectory: dir, Generations: generationFixture(0)}} before := time.Now().UTC() - h, err := a.health(context.Background(), sampler) - if err != nil || h.ObservedAt.Before(before) || h.ObservedAt.After(time.Now()) || h.ObservedAt.Location() != time.UTC { - t.Fatalf("invalid sample identity: %+v, %v", h, err) + h := a.health(sampler) + if h.ObservedAt.Before(before) || h.ObservedAt.After(time.Now()) || h.ObservedAt.Location() != time.UTC { + t.Fatalf("invalid sample identity: %+v", h) } if h.CPUUtilization != nil || h.TotalMemoryBytes == nil || *h.TotalMemoryBytes <= 0 || h.AvailableMemoryBytes == nil || *h.AvailableMemoryBytes < 0 || h.CPUCount == nil || *h.CPUCount <= 0 { t.Fatalf("invalid initial host observation: %+v", h) @@ -211,15 +210,15 @@ func TestHostHealthReadOnly(t *testing.T) { t.Log(string(data)) } time.Sleep(100 * time.Millisecond) - h, err = a.health(context.Background(), sampler) - if err != nil || h.CPUUtilization == nil || *h.CPUUtilization < 0 || *h.CPUUtilization > 1 || !h.ObservedAt.After(before) { - t.Fatalf("invalid follow-up observation: %+v, %v", h, err) + h = a.health(sampler) + if h.CPUUtilization == nil || *h.CPUUtilization < 0 || *h.CPUUtilization > 1 || !h.ObservedAt.After(before) { + t.Fatalf("invalid follow-up observation: %+v", h) } if os.Getenv("OAC_TEST_NODE_HEALTH_PROBE") == "1" { data, _ := json.Marshal(h) t.Log(string(data)) } - if fresh, _ := a.health(context.Background(), new(hostHealthSampler)); fresh.CPUUtilization != nil { + if fresh := a.health(new(hostHealthSampler)); fresh.CPUUtilization != nil { t.Fatal("new connection reused a CPU baseline") } sampler.fill(&h, filepath.Join(dir, "missing")) diff --git a/services/core/internal/sandbox/node/hub.go b/services/core/internal/sandbox/node/hub.go index 4feb0e2f6..ddf5febe0 100644 --- a/services/core/internal/sandbox/node/hub.go +++ b/services/core/internal/sandbox/node/hub.go @@ -20,7 +20,6 @@ type HubOptions struct { OwnerEpoch func(context.Context) (uint64, error) Connected func(context.Context, Identity, string, uint64) error Disconnected func(context.Context, Identity, string, uint64) - Heartbeat func(context.Context, Identity, string, uint64, Health) error } type Hub struct { @@ -35,22 +34,20 @@ type Hub struct { } type peer struct { - generationManagement bool - version int - generations map[uint64]sandbox.GenerationStatus - controlSequence uint64 - identity Identity - id string - epoch uint64 - conn *websocket.Conn - send chan struct{} - mu sync.Mutex - sequence uint64 - ready bool - pending map[string]chan response - done chan struct{} - once sync.Once - cancel context.CancelFunc + version int + generations map[uint64]sandbox.GenerationStatus + controlSequence uint64 + identity Identity + id string + epoch uint64 + conn *websocket.Conn + send chan struct{} + mu sync.Mutex + sequence uint64 + pending map[string]chan response + done chan struct{} + once sync.Once + cancel context.CancelFunc } func NewHub(options HubOptions) *Hub { @@ -159,7 +156,7 @@ func (h *Hub) ServeHTTP(w http.ResponseWriter, r *http.Request) { _ = conn.Close() return } - p := &peer{generationManagement: hello.GenerationManagement, version: hello.Version, generations: map[uint64]sandbox.GenerationStatus{}, identity: identity, id: uuid.NewString(), epoch: epoch, conn: conn, cancel: cancel, send: make(chan struct{}, 1), pending: map[string]chan response{}, ready: hello.Health.ProviderReady, done: make(chan struct{})} + p := &peer{version: hello.Version, generations: map[uint64]sandbox.GenerationStatus{}, identity: identity, id: uuid.NewString(), epoch: epoch, conn: conn, cancel: cancel, send: make(chan struct{}, 1), pending: map[string]chan response{}, done: make(chan struct{})} stopPeerClose := context.AfterFunc(ctx, p.close) defer stopPeerClose() presenceAttempted := false @@ -191,14 +188,11 @@ func (h *Hub) ServeHTTP(w http.ResponseWriter, r *http.Request) { if err = h.recordHealth(ctx, p, *hello.Health); err != nil { return } - welcome := frame{Version: p.version, Type: "welcome", ConnectionID: p.id, OwnerEpoch: epoch} - if p.generationManagement { - deployment, _, err := h.retention(ctx, p, nil) - if err != nil { - return - } - welcome.Deployment = &deployment + deployment, _, err := h.retention(ctx, p, nil) + if err != nil { + return } + welcome := frame{Version: p.version, Type: "welcome", ConnectionID: p.id, OwnerEpoch: epoch, Deployment: &deployment} if ctx.Err() != nil || writeFrame(conn, welcome) != nil { return } @@ -251,22 +245,18 @@ func (h *Hub) ServeHTTP(w http.ResponseWriter, r *http.Request) { if e = p.lockSend(ctx); e != nil { return } - ack := frame{Version: p.version, Type: "heartbeat_ack", ConnectionID: p.id, OwnerEpoch: p.epoch} - if p.generationManagement { - deployment, _, readErr := h.retention(ctx, p, nil) - if readErr != nil { - p.unlockSend() - return - } - ack.Deployment = &deployment + deployment, _, readErr := h.retention(ctx, p, nil) + if readErr != nil { + p.unlockSend() + return } - e = writeFrame(conn, ack) + e = writeFrame(conn, frame{Version: p.version, Type: "heartbeat_ack", ConnectionID: p.id, OwnerEpoch: p.epoch, Deployment: &deployment}) p.unlockSend() if e != nil { return } case "retention": - if !p.generationManagement || f.Control == nil { + if f.Control == nil { return } c := f.Control @@ -329,14 +319,8 @@ func (h *Hub) call(ctx context.Context, id string, q request) (response, error) return response{}, uncertain(q.Operation, ErrUnavailable) } p.mu.Lock() - ready := p.ready - if p.generationManagement { - status, ok := p.generations[q.DeploymentGeneration] - ready = ok && status.State == "ready" - } else if q.DeploymentGeneration != p.identity.DeploymentGeneration { - p.mu.Unlock() - return response{}, uncertain(q.Operation, ErrUnavailable) - } + status, ok := p.generations[q.DeploymentGeneration] + ready := ok && status.State == "ready" p.mu.Unlock() if requiresReady(q) && !ready { return response{}, uncertain(q.Operation, ErrUnavailable) @@ -391,52 +375,36 @@ func (h *Hub) call(ctx context.Context, id string, q request) (response, error) } func (h *Hub) recordHealth(ctx context.Context, p *peer, health Health) error { - if !p.generationManagement && len(health.Generations) > 0 { - if len(health.Generations) != 1 || health.Generations[0].Generation != p.identity.DeploymentGeneration || health.Generations[0].SpecificationDigest != p.identity.SpecificationDigest || (health.Generations[0].State == "ready") != health.ProviderReady { - return sandbox.ErrInvalid - } + if h.options.Generations == nil { + return sandbox.ErrInvalid } - if p.generationManagement { - if h.options.Generations == nil { - return sandbox.ErrInvalid - } - if err := callback(ctx, func(ctx context.Context) error { return h.options.Generations(ctx, p.identity, p.id, p.epoch, health) }); err != nil { - return err - } - refs := make([]sandbox.GenerationReference, 0, len(health.Generations)) - for _, g := range health.Generations { - refs = append(refs, sandbox.GenerationReference{Generation: g.Generation, SpecificationDigest: g.SpecificationDigest}) - } - _, grants, err := h.retention(ctx, p, refs) - if err != nil { - return err - } - if len(grants) != len(refs) { + if err := callback(ctx, func(ctx context.Context) error { return h.options.Generations(ctx, p.identity, p.id, p.epoch, health) }); err != nil { + return err + } + refs := make([]sandbox.GenerationReference, 0, len(health.Generations)) + for _, g := range health.Generations { + refs = append(refs, sandbox.GenerationReference{Generation: g.Generation, SpecificationDigest: g.SpecificationDigest}) + } + _, grants, err := h.retention(ctx, p, refs) + if err != nil { + return err + } + if len(grants) != len(refs) { + return sandbox.ErrInvalid + } + for i, grant := range grants { + if grant.GenerationReference != refs[i] { return sandbox.ErrInvalid } - for i, grant := range grants { - if grant.GenerationReference != refs[i] { - return sandbox.ErrInvalid - } - } - p.mu.Lock() - for i, g := range health.Generations { - if grants[i].Keep { - p.generations[g.Generation] = g - } else { - delete(p.generations, g.Generation) - } - } - p.mu.Unlock() - return nil } - if h.options.Heartbeat != nil { - if err := callback(ctx, func(ctx context.Context) error { return h.options.Heartbeat(ctx, p.identity, p.id, p.epoch, health) }); err != nil { - return err + p.mu.Lock() + for i, g := range health.Generations { + if grants[i].Keep { + p.generations[g.Generation] = g + } else { + delete(p.generations, g.Generation) } } - p.mu.Lock() - p.ready = health.ProviderReady p.mu.Unlock() return nil } diff --git a/services/core/internal/sandbox/node/hub_lifetime_test.go b/services/core/internal/sandbox/node/hub_lifetime_test.go index 37b07c74b..26850904e 100644 --- a/services/core/internal/sandbox/node/hub_lifetime_test.go +++ b/services/core/internal/sandbox/node/hub_lifetime_test.go @@ -23,7 +23,7 @@ func beginRawNode(t *testing.T, origin string, id Identity) *websocket.Conn { t.Fatal(err) } t.Cleanup(func() { conn.Close() }) - if err = writeFrame(conn, frame{Type: "hello", Identity: &id, Health: &Health{ProviderReady: true, ObservedAt: time.Now().UTC()}}); err != nil { + if err = writeFrame(conn, frame{Type: "hello", Identity: &id, Health: &Health{ObservedAt: time.Now().UTC()}}); err != nil { t.Fatal(err) } return conn @@ -77,7 +77,7 @@ func TestHubCloseCancelsEveryOpeningCallback(t *testing.T) { close(returned) } var cleanups atomic.Int32 - hub := NewHub(HubOptions{ + hub := newTestHub(HubOptions{ Authenticate: func(ctx context.Context, _, _ string) (Identity, error) { if stage == "authenticate" { block(ctx) @@ -96,7 +96,7 @@ func TestHubCloseCancelsEveryOpeningCallback(t *testing.T) { } return nil }, - Heartbeat: func(ctx context.Context, _ Identity, _ string, _ uint64, _ Health) error { + Generations: func(ctx context.Context, _ Identity, _ string, _ uint64, _ Health) error { if stage == "heartbeat" { block(ctx) } @@ -126,7 +126,7 @@ func TestHubCloseCancelsEveryOpeningCallback(t *testing.T) { return } defer conn.Close() - _ = writeFrame(conn, frame{Type: "hello", Identity: &id, Health: &Health{ProviderReady: true, ObservedAt: time.Now().UTC()}}) + _ = writeFrame(conn, frame{Type: "hello", Identity: &id, Health: &Health{ObservedAt: time.Now().UTC()}}) if f, err := readFrame(conn); err == nil { t.Errorf("canceled opening published welcome: %s", f.Type) } @@ -167,7 +167,7 @@ func TestSlowOpeningAndClosingLeaveOtherNodesResponsive(t *testing.T) { opening, closing, finish := make(chan struct{}), make(chan struct{}), make(chan struct{}) var release sync.Once defer release.Do(func() { close(finish) }) - hub := NewHub(HubOptions{ + hub := newTestHub(HubOptions{ Authenticate: func(_ context.Context, id, _ string) (Identity, error) { if id == slow.NodeID { return slow, nil @@ -228,7 +228,7 @@ func TestHubCleansPresenceAfterPartialOpeningFailure(t *testing.T) { var mu sync.Mutex persisted := "" var connects, cleanups int - hub := NewHub(HubOptions{ + hub := newTestHub(HubOptions{ Authenticate: func(context.Context, string, string) (Identity, error) { return id, nil }, OwnerEpoch: func(context.Context) (uint64, error) { return 11, nil }, Connected: func(_ context.Context, _ Identity, connection string, _ uint64) error { @@ -241,7 +241,7 @@ func TestHubCleansPresenceAfterPartialOpeningFailure(t *testing.T) { } return nil }, - Heartbeat: func(context.Context, Identity, string, uint64, Health) error { return errors.New("heartbeat failed") }, + Generations: func(context.Context, Identity, string, uint64, Health) error { return errors.New("heartbeat failed") }, Disconnected: func(_ context.Context, _ Identity, connection string, epoch uint64) { mu.Lock() defer mu.Unlock() @@ -270,7 +270,7 @@ func TestHubCleansPresenceAfterPartialOpeningFailure(t *testing.T) { func TestHubSendQueueRespectsCallerCancellation(t *testing.T) { id := identity() - hub := NewHub(HubOptions{Authenticate: func(context.Context, string, string) (Identity, error) { return id, nil }, OwnerEpoch: func(context.Context) (uint64, error) { return 1, nil }}) + hub := newTestHub(HubOptions{Authenticate: func(context.Context, string, string) (Identity, error) { return id, nil }, OwnerEpoch: func(context.Context) (uint64, error) { return 1, nil }}) server := httptest.NewServer(hub) defer server.Close() defer hub.Close() @@ -312,7 +312,7 @@ func TestHubSendQueueRespectsCallerCancellation(t *testing.T) { func TestHubCallBoundsOwnershipBeforeDispatch(t *testing.T) { entered := make(chan struct{}) - hub := NewHub(HubOptions{OwnerEpoch: func(ctx context.Context) (uint64, error) { + hub := newTestHub(HubOptions{OwnerEpoch: func(ctx context.Context) (uint64, error) { deadline, ok := ctx.Deadline() if !ok || time.Until(deadline) > callbackTimeout { t.Error("ownership callback unbounded") @@ -347,7 +347,7 @@ func TestHubRetainsOpeningReservationUntilCanceledCallbackReturns(t *testing.T) var mu sync.Mutex persisted := "" cleanups := 0 - hub := NewHub(HubOptions{ + hub := newTestHub(HubOptions{ Authenticate: func(context.Context, string, string) (Identity, error) { return id, nil }, OwnerEpoch: func(context.Context) (uint64, error) { return 4, nil }, Connected: func(ctx context.Context, _ Identity, connection string, _ uint64) error { @@ -403,10 +403,10 @@ func TestHubDisconnectCancelsPeriodicHeartbeatCallback(t *testing.T) { id := identity() entered := make(chan struct{}) var heartbeats atomic.Int32 - hub := NewHub(HubOptions{ + hub := newTestHub(HubOptions{ Authenticate: func(context.Context, string, string) (Identity, error) { return id, nil }, OwnerEpoch: func(context.Context) (uint64, error) { return 2, nil }, - Heartbeat: func(ctx context.Context, _ Identity, _ string, _ uint64, _ Health) error { + Generations: func(ctx context.Context, _ Identity, _ string, _ uint64, _ Health) error { if heartbeats.Add(1) == 1 { return nil } @@ -427,7 +427,7 @@ func TestHubDisconnectCancelsPeriodicHeartbeatCallback(t *testing.T) { hub.mu.Lock() connection := hub.peers[id.NodeID].id hub.mu.Unlock() - if err := writeFrame(conn, frame{Type: "heartbeat", ConnectionID: connection, OwnerEpoch: 2, Health: &Health{ProviderReady: true, ObservedAt: time.Now().UTC()}}); err != nil { + if err := writeFrame(conn, frame{Type: "heartbeat", ConnectionID: connection, OwnerEpoch: 2, Health: &Health{ObservedAt: time.Now().UTC()}}); err != nil { t.Fatal(err) } select { diff --git a/services/core/internal/sandbox/node/hub_postgres_test.go b/services/core/internal/sandbox/node/hub_postgres_test.go index 38a751663..e379ff14d 100644 --- a/services/core/internal/sandbox/node/hub_postgres_test.go +++ b/services/core/internal/sandbox/node/hub_postgres_test.go @@ -56,7 +56,7 @@ func TestHubPostgresBlockedOpeningIsBounded(t *testing.T) { } pids, results := make(chan uint32, 1), make(chan error, 1) var attempts atomic.Int32 - hub := NewHub(HubOptions{ + hub := newTestHub(HubOptions{ Authenticate: func(_ context.Context, id, _ string) (Identity, error) { if id == slow.NodeID { return slow, nil diff --git a/services/core/internal/sandbox/node/node_test.go b/services/core/internal/sandbox/node/node_test.go index 946470389..9bcc5c3fb 100644 --- a/services/core/internal/sandbox/node/node_test.go +++ b/services/core/internal/sandbox/node/node_test.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "errors" - "fmt" "net/http" "net/http/httptest" "os" @@ -82,12 +81,11 @@ func wait(t *testing.T, ready func() bool) { time.Sleep(10 * time.Millisecond) } } -func probe(context.Context) (Health, error) { return Health{ProviderReady: true}, nil } func TestLostCreateResponseDoesNotReplayAndReconnectSerializesCleanup(t *testing.T) { id := identity() var credential string - hub := NewHub(HubOptions{Authenticate: func(ctx context.Context, node, token string) (Identity, error) { + hub := newTestHub(HubOptions{Authenticate: func(ctx context.Context, node, token string) (Identity, error) { if node != id.NodeID || token != credential { return Identity{}, ErrAuthentication } @@ -106,7 +104,7 @@ func TestLostCreateResponseDoesNotReplayAndReconnectSerializesCleanup(t *testing ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) go func() { - done <- Run(ctx, AgentConfig{CoreURL: server.URL, StateDirectory: dir, Identity: id, Credential: credential, Provider: p, Probe: probe}) + done <- runProvider(ctx, AgentConfig{CoreURL: server.URL, StateDirectory: dir, Identity: id, Credential: credential}, p, probe) }() defer func() { cancel() @@ -167,7 +165,7 @@ func TestLostCreateResponseDoesNotReplayAndReconnectSerializesCleanup(t *testing } func TestOfflineIsUnknownAndDockerDoesNotAdvertiseCheckpoint(t *testing.T) { - h := NewHub(HubOptions{OwnerEpoch: func(context.Context) (uint64, error) { return 1, nil }}) + h := newTestHub(HubOptions{OwnerEpoch: func(context.Context) (uint64, error) { return 1, nil }}) p := h.Proxy(uuid.NewString(), docker.Operations(), 1) if sandbox.SupportsCheckpoint(p) { t.Fatal("docker advertised checkpoint") @@ -193,7 +191,7 @@ func TestAgentRejectsDuplicateSequenceAndRetainsEpoch(t *testing.T) { return } connectionID := uuid.NewString() - _ = writeFrame(conn, frame{Type: "welcome", ConnectionID: connectionID, OwnerEpoch: 9}) + _ = writeFrame(conn, frame{Type: "welcome", ConnectionID: connectionID, OwnerEpoch: 9, Deployment: testDeployment()}) q := request{DeploymentGeneration: id.DeploymentGeneration, ID: uuid.NewString(), Sequence: 1, ConnectionID: connectionID, OwnerEpoch: 9, Operation: "info", TimeoutMillis: 60000, Reference: reference()} _ = writeFrame(conn, frame{Type: "request", Request: &q}) _, _ = readFrame(conn) @@ -209,7 +207,7 @@ func TestAgentRejectsDuplicateSequenceAndRetainsEpoch(t *testing.T) { } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - err := Run(ctx, AgentConfig{CoreURL: server.URL, StateDirectory: dir, Identity: id, Credential: stored.Credential, Provider: p, Probe: probe}) + err := runProvider(ctx, AgentConfig{CoreURL: server.URL, StateDirectory: dir, Identity: id, Credential: stored.Credential}, p, probe) if !errors.Is(err, sandbox.ErrOwnership) { t.Fatalf("duplicate sequence = %v", err) } @@ -297,26 +295,6 @@ func TestCoreURLRejectsOtherSchemesAndCredentials(t *testing.T) { } } -func TestHealthSendsOnlyFixedDiagnosticCode(t *testing.T) { - for _, tc := range []struct { - err error - want string - }{ - {fmt.Errorf("%w: open /home/operator/private/kvm", sandbox.ErrHostUnsupported), "host_unsupported"}, - {errors.New("open /home/operator/private/runtime: permission denied"), "provider_unavailable"}, - {nil, ""}, - } { - a := &agent{config: AgentConfig{StateDirectory: stateDir(t), Identity: identity(), Probe: func(context.Context) (Health, error) { - return Health{Diagnostic: "/home/operator/private"}, tc.err - }}} - h, _ := a.health(t.Context(), new(hostHealthSampler)) - raw, err := json.Marshal(frame{Type: "heartbeat", Health: &h}) - if err != nil || h.Diagnostic != tc.want || h.ProviderReady != (tc.err == nil) || strings.Contains(string(raw), "private") { - t.Fatalf("heartbeat = %s, %v", raw, err) - } - } -} - // A rejected credential stops the node for good; any other rejection names Core's // status and error code so the installer can say what to fix. func TestEnrollmentRejectionNamesCoreCode(t *testing.T) { diff --git a/services/core/internal/sandbox/node/observations_test.go b/services/core/internal/sandbox/node/observations_test.go index 23a5cbfdb..7ba468fd4 100644 --- a/services/core/internal/sandbox/node/observations_test.go +++ b/services/core/internal/sandbox/node/observations_test.go @@ -50,7 +50,9 @@ func runObservationNode(t *testing.T, hub *Hub, url string, id Identity, provide ctx, cancel := context.WithCancel(t.Context()) done := make(chan error, 1) go func() { - done <- Run(ctx, AgentConfig{CoreURL: url, StateDirectory: dir, Identity: id, Credential: stored.Credential, Provider: provider, Probe: func(context.Context) (Health, error) { return Health{}, errors.New("provider not ready for creation") }}) + done <- runProvider(ctx, AgentConfig{CoreURL: url, StateDirectory: dir, Identity: id, Credential: stored.Credential}, provider, func(context.Context) (*sandbox.CheckpointCompatibility, error) { + return nil, errors.New("provider not ready for creation") + }) }() t.Cleanup(func() { cancel() @@ -69,7 +71,7 @@ func runObservationNode(t *testing.T, hub *Hub, url string, id Identity, provide func TestObservationsRouteThroughAssignedNodeWithoutLifecycleCalls(t *testing.T) { first, second := identity(), identity() second.InstallationID = first.InstallationID - hub := NewHub(HubOptions{Authenticate: func(_ context.Context, id, _ string) (Identity, error) { + hub := newTestHub(HubOptions{Authenticate: func(_ context.Context, id, _ string) (Identity, error) { switch id { case first.NodeID: return first, nil diff --git a/services/core/internal/sandbox/node/provider_agent_test.go b/services/core/internal/sandbox/node/provider_agent_test.go new file mode 100644 index 000000000..83813db92 --- /dev/null +++ b/services/core/internal/sandbox/node/provider_agent_test.go @@ -0,0 +1,61 @@ +package node + +import ( + "context" + "errors" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" +) + +func probe(context.Context) (*sandbox.CheckpointCompatibility, error) { return nil, nil } + +// testDeployment is the deployment Core tells a node built from identity(). +func testDeployment() *sandbox.NodeDeployment { + id := identity() + return &sandbox.NodeDeployment{Generation: id.DeploymentGeneration, SpecificationDigest: id.SpecificationDigest} +} + +// newTestHub completes the generation callbacks a test does not observe: +// every reported generation is kept and the deployment targets the identity. +func newTestHub(options HubOptions) *Hub { + if options.Generations == nil { + options.Generations = func(context.Context, Identity, string, uint64, Health) error { return nil } + } + if options.Retention == nil { + options.Retention = func(_ context.Context, id Identity, _ string, _ uint64, refs []sandbox.GenerationReference) (sandbox.NodeDeployment, []sandbox.GenerationRetention, error) { + grants := make([]sandbox.GenerationRetention, 0, len(refs)) + for _, ref := range refs { + grants = append(grants, sandbox.GenerationRetention{GenerationReference: ref, Keep: true}) + } + return *testDeployment(), grants, nil + } + } + return NewHub(options) +} + +// runProvider serves provider as the identity's only generation until ctx ends. +func runProvider(ctx context.Context, config AgentConfig, provider sandbox.SandboxProvider, probe func(context.Context) (*sandbox.CheckpointCompatibility, error)) error { + manager, err := NewGenerationManager(ctx, GenerationManagerOptions{ + Initial: []GenerationProvider{{Generation: config.Identity.DeploymentGeneration, SpecificationDigest: config.Identity.SpecificationDigest, Provider: provider, Probe: probe}}, + Prepare: func(context.Context, uint64, string) (GenerationProvider, error) { + return GenerationProvider{}, errors.New("unexpected preparation") + }, + Remove: func(context.Context, GenerationProvider) error { return nil }, + }) + if err != nil { + return err + } + defer manager.Close() + // Settle the first probe now so the first connection reports its outcome. + manager.mu.Lock() + g := manager.values[config.Identity.DeploymentGeneration] + g.state = "ready" + if checkpoint, err := probe(ctx); err != nil { + g.state, g.diagnostic = "failed", sandbox.NodeDiagnostic(err) + } else { + g.checkpoint = checkpoint + } + manager.mu.Unlock() + config.Generations = manager + return Run(ctx, config) +} diff --git a/services/core/internal/sandbox/node/recovery_test.go b/services/core/internal/sandbox/node/recovery_test.go index 78d106da3..ad62151d7 100644 --- a/services/core/internal/sandbox/node/recovery_test.go +++ b/services/core/internal/sandbox/node/recovery_test.go @@ -24,8 +24,8 @@ func TestCoreRestartFencesOldConnectionAndNodeRestartKeepsIdentity(t *testing.T) options := func(epoch uint64) HubOptions { return HubOptions{Authenticate: func(context.Context, string, string) (Identity, error) { return id, nil }, OwnerEpoch: func(context.Context) (uint64, error) { return epoch, nil }} } - first := NewHub(options(4)) - second := NewHub(options(5)) + first := newTestHub(options(4)) + second := newTestHub(options(5)) defer first.Close() defer second.Close() var mu sync.Mutex @@ -42,7 +42,7 @@ func TestCoreRestartFencesOldConnectionAndNodeRestartKeepsIdentity(t *testing.T) ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) go func() { - done <- Run(ctx, AgentConfig{CoreURL: server.URL, StateDirectory: dir, Identity: id, Credential: credential, Provider: &fakeProvider{}, Probe: probe}) + done <- runProvider(ctx, AgentConfig{CoreURL: server.URL, StateDirectory: dir, Identity: id, Credential: credential}, &fakeProvider{}, probe) }() return cancel, done } @@ -90,7 +90,7 @@ func TestAgentRejectsOwnerEpochRollback(t *testing.T) { } defer conn.Close() _, _ = readFrame(conn) - _ = writeFrame(conn, frame{Type: "welcome", ConnectionID: uuid.NewString(), OwnerEpoch: 3}) + _ = writeFrame(conn, frame{Type: "welcome", ConnectionID: uuid.NewString(), OwnerEpoch: 3, Deployment: testDeployment()}) _, _ = readFrame(conn) })) defer server.Close() @@ -105,7 +105,7 @@ func TestAgentRejectsOwnerEpochRollback(t *testing.T) { } ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() - err = Run(ctx, AgentConfig{CoreURL: server.URL, StateDirectory: dir, Identity: id, Credential: stored.Credential, Provider: &fakeProvider{}, Probe: probe}) + err = runProvider(ctx, AgentConfig{CoreURL: server.URL, StateDirectory: dir, Identity: id, Credential: stored.Credential}, &fakeProvider{}, probe) if !errors.Is(err, sandbox.ErrOwnership) { t.Fatalf("stale owner = %v", err) } @@ -115,7 +115,7 @@ func TestHeartbeatAcknowledgementKeepsIdleConnectionAlive(t *testing.T) { id := identity() beats := make(chan Health, 1) var heartbeats int - hub := NewHub(HubOptions{Authenticate: func(context.Context, string, string) (Identity, error) { return id, nil }, OwnerEpoch: func(context.Context) (uint64, error) { return 1, nil }, Heartbeat: func(ctx context.Context, id Identity, connection string, epoch uint64, h Health) error { + hub := newTestHub(HubOptions{Authenticate: func(context.Context, string, string) (Identity, error) { return id, nil }, OwnerEpoch: func(context.Context) (uint64, error) { return 1, nil }, Generations: func(ctx context.Context, id Identity, connection string, epoch uint64, h Health) error { heartbeats++ if heartbeats == 1 { return nil @@ -137,7 +137,7 @@ func TestHeartbeatAcknowledgementKeepsIdleConnectionAlive(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) go func() { - done <- Run(ctx, AgentConfig{CoreURL: server.URL, StateDirectory: dir, Identity: id, Credential: stored.Credential, Provider: &fakeProvider{}, Probe: probe}) + done <- runProvider(ctx, AgentConfig{CoreURL: server.URL, StateDirectory: dir, Identity: id, Credential: stored.Credential}, &fakeProvider{}, probe) }() defer func() { cancel() @@ -147,7 +147,7 @@ func TestHeartbeatAcknowledgementKeepsIdleConnectionAlive(t *testing.T) { }() select { case h := <-beats: - if !h.ProviderReady || h.ObservedAt.IsZero() { + if len(h.Generations) != 1 || h.Generations[0].State != "ready" || h.ObservedAt.IsZero() { t.Fatal("invalid heartbeat") } case <-time.After(12 * time.Second): @@ -166,7 +166,7 @@ func TestHubExpiresSilentNode(t *testing.T) { t.Skip("real heartbeat timeout") } id := identity() - hub := NewHub(HubOptions{Authenticate: func(context.Context, string, string) (Identity, error) { return id, nil }, OwnerEpoch: func(context.Context) (uint64, error) { return 1, nil }}) + hub := newTestHub(HubOptions{Authenticate: func(context.Context, string, string) (Identity, error) { return id, nil }, OwnerEpoch: func(context.Context) (uint64, error) { return 1, nil }}) server := httptest.NewServer(hub) defer server.Close() defer hub.Close() @@ -176,7 +176,7 @@ func TestHubExpiresSilentNode(t *testing.T) { t.Fatal(err) } defer conn.Close() - if err = writeFrame(conn, frame{Type: "hello", Identity: &id, Health: &Health{ProviderReady: true, ObservedAt: time.Now().UTC()}}); err != nil { + if err = writeFrame(conn, frame{Type: "hello", Identity: &id, Health: &Health{ObservedAt: time.Now().UTC()}}); err != nil { t.Fatal(err) } if _, err = readFrame(conn); err != nil { @@ -198,7 +198,7 @@ func TestHubExpiresSilentNode(t *testing.T) { func TestDegradedNodeRetainsObservationAndCleanup(t *testing.T) { id := identity() health := make(chan Health, 1) - hub := NewHub(HubOptions{Authenticate: func(context.Context, string, string) (Identity, error) { return id, nil }, OwnerEpoch: func(context.Context) (uint64, error) { return 1, nil }, Heartbeat: func(ctx context.Context, id Identity, connection string, epoch uint64, h Health) error { + hub := newTestHub(HubOptions{Authenticate: func(context.Context, string, string) (Identity, error) { return id, nil }, OwnerEpoch: func(context.Context) (uint64, error) { return 1, nil }, Generations: func(ctx context.Context, id Identity, connection string, epoch uint64, h Health) error { select { case health <- h: default: @@ -217,7 +217,9 @@ func TestDegradedNodeRetainsObservationAndCleanup(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) go func() { - done <- Run(ctx, AgentConfig{CoreURL: server.URL, StateDirectory: dir, Identity: id, Credential: stored.Credential, Provider: p, Probe: func(context.Context) (Health, error) { return Health{}, errors.New("private backend detail") }}) + done <- runProvider(ctx, AgentConfig{CoreURL: server.URL, StateDirectory: dir, Identity: id, Credential: stored.Credential}, p, func(context.Context) (*sandbox.CheckpointCompatibility, error) { + return nil, errors.New("private backend detail") + }) }() defer func() { cancel() @@ -228,7 +230,7 @@ func TestDegradedNodeRetainsObservationAndCleanup(t *testing.T) { wait(t, func() bool { return hub.Online(id.NodeID) }) select { case h := <-health: - if h.ProviderReady || h.Diagnostic != "provider_unavailable" { + if len(h.Generations) != 1 || h.Generations[0].State != "failed" || h.Generations[0].Diagnostic != "provider_unavailable" { t.Fatalf("unsafe health: %+v", h) } case <-time.After(time.Second): diff --git a/services/core/internal/sandbox/node/timeout_test.go b/services/core/internal/sandbox/node/timeout_test.go index 15f8da3c5..ef859d0b0 100644 --- a/services/core/internal/sandbox/node/timeout_test.go +++ b/services/core/internal/sandbox/node/timeout_test.go @@ -77,7 +77,7 @@ func TestRequestBudgetBounds(t *testing.T) { func TestQueuedMutationExpiresWithoutExecution(t *testing.T) { id := identity() - hub := NewHub(HubOptions{Authenticate: func(context.Context, string, string) (Identity, error) { return id, nil }, OwnerEpoch: func(context.Context) (uint64, error) { return 1, nil }}) + hub := newTestHub(HubOptions{Authenticate: func(context.Context, string, string) (Identity, error) { return id, nil }, OwnerEpoch: func(context.Context) (uint64, error) { return 1, nil }}) server := httptest.NewServer(hub) defer server.Close() defer hub.Close() @@ -90,7 +90,7 @@ func TestQueuedMutationExpiresWithoutExecution(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) go func() { - done <- Run(ctx, AgentConfig{CoreURL: server.URL, StateDirectory: dir, Identity: id, Credential: stored.Credential, Provider: p, Probe: probe}) + done <- runProvider(ctx, AgentConfig{CoreURL: server.URL, StateDirectory: dir, Identity: id, Credential: stored.Credential}, p, probe) }() released := false defer func() { diff --git a/services/core/internal/sandbox/node/wire.go b/services/core/internal/sandbox/node/wire.go index bd7a5c5b0..a8ac9993f 100644 --- a/services/core/internal/sandbox/node/wire.go +++ b/services/core/internal/sandbox/node/wire.go @@ -40,8 +40,6 @@ type Identity struct { type Health struct { Generations []sandbox.GenerationStatus `json:"generations,omitempty"` - Diagnostic string `json:"diagnostic,omitempty"` - ProviderReady bool `json:"provider_ready"` ObservedAt time.Time `json:"observed_at"` ActiveOperations int `json:"active_operations"` CPUCount *int64 `json:"cpu_count,omitempty"` @@ -110,17 +108,16 @@ type response struct { } type frame struct { - GenerationManagement bool `json:"generation_management,omitempty"` - Deployment *sandbox.NodeDeployment `json:"deployment,omitempty"` - Control *generationControl `json:"control,omitempty"` - Version int `json:"version"` - Type string `json:"type"` - Identity *Identity `json:"identity,omitempty"` - Health *Health `json:"health,omitempty"` - ConnectionID string `json:"connection_id,omitempty"` - OwnerEpoch uint64 `json:"owner_epoch,omitempty"` - Request *request `json:"request,omitempty"` - Response *response `json:"response,omitempty"` + Deployment *sandbox.NodeDeployment `json:"deployment,omitempty"` + Control *generationControl `json:"control,omitempty"` + Version int `json:"version"` + Type string `json:"type"` + Identity *Identity `json:"identity,omitempty"` + Health *Health `json:"health,omitempty"` + ConnectionID string `json:"connection_id,omitempty"` + OwnerEpoch uint64 `json:"owner_epoch,omitempty"` + Request *request `json:"request,omitempty"` + Response *response `json:"response,omitempty"` } func validID(s string) bool { diff --git a/services/core/internal/sandbox/node_diagnostic.go b/services/core/internal/sandbox/node_diagnostic.go index 15527a4d2..2a2f29035 100644 --- a/services/core/internal/sandbox/node_diagnostic.go +++ b/services/core/internal/sandbox/node_diagnostic.go @@ -16,10 +16,8 @@ var ( ErrHostUnsupported = errors.New("host lacks a capability the sandbox provider requires") // A pinned native artifact is missing or fails its integrity check. ErrArtifactsUnavailable = errors.New("pinned provider artifacts are unavailable") - // The exact Runtime artifacts could not be transferred or verified. - ErrRuntimeDownloadFailed = errors.New("Runtime preparation failed") - // The pinned Runtime image is not available to the Provider. - ErrRuntimeImageUnavailable = errors.New("pinned Runtime image is unavailable") + // The Runtime image installed on the node is not available to the Provider. + ErrRuntimeImageUnavailable = errors.New("installed Runtime image is unavailable") // The host cannot hold one sandbox of the deployment specification. ErrCapacityInsufficient = errors.New("node cannot provide one sandbox of the deployment specification") ) @@ -38,7 +36,6 @@ const ( NodeProviderUnavailable NodeDiagnosticCode = "provider_unavailable" NodeHostUnsupported NodeDiagnosticCode = "host_unsupported" NodeArtifactsUnavailable NodeDiagnosticCode = "artifacts_unavailable" - NodeRuntimeDownloadFailed NodeDiagnosticCode = "runtime_download_failed" NodeRuntimeImageUnavailable NodeDiagnosticCode = "runtime_image_unavailable" NodeCapacityInsufficient NodeDiagnosticCode = "capacity_insufficient" ) @@ -50,7 +47,6 @@ var nodeDiagnostics = []struct { {ErrProviderUnavailable, NodeProviderUnavailable}, {ErrHostUnsupported, NodeHostUnsupported}, {ErrArtifactsUnavailable, NodeArtifactsUnavailable}, - {ErrRuntimeDownloadFailed, NodeRuntimeDownloadFailed}, {ErrRuntimeImageUnavailable, NodeRuntimeImageUnavailable}, {ErrCapacityInsufficient, NodeCapacityInsufficient}, } diff --git a/services/core/internal/sandbox/providers/configuration_flow_test.go b/services/core/internal/sandbox/providers/configuration_flow_test.go index 9fe09c5bb..1f1f095f6 100644 --- a/services/core/internal/sandbox/providers/configuration_flow_test.go +++ b/services/core/internal/sandbox/providers/configuration_flow_test.go @@ -165,8 +165,7 @@ func TestAdditionalConfigurationProviderUsesCommonAPIAndStore(t *testing.T) { if err != nil { t.Fatal(err) } - runtime := sandbox.RuntimeRelease{SourceCommit: strings.Repeat("a", 40), ImageID: "sha256:" + strings.Repeat("b", 64), ImageManifestDigest: "sha256:" + strings.Repeat("c", 64), MicrosandboxRef: "oac-runtime@sha256:" + strings.Repeat("d", 64), RuntimeSHA256: strings.Repeat("e", 64), FirmwareSHA256: strings.Repeat("f", 64)} - body, _ := json.Marshal(map[string]any{"provider": kind, "expected_generation": 0, "resources": sandbox.Resources{CPUs: 2, MemoryMiB: 2048}, "runtime": runtime, "configuration": map[string]string{"zone": "west"}}) + body, _ := json.Marshal(map[string]any{"provider": kind, "expected_generation": 0, "resources": sandbox.Resources{CPUs: 2, MemoryMiB: 2048}, "configuration": map[string]string{"zone": "west"}}) request := httptest.NewRequest("POST", "/core/v1/sandbox/deployment", bytes.NewReader(body)) request.Header.Set("Authorization", "Bearer fixture-admin") response := httptest.NewRecorder() diff --git a/services/core/internal/sandbox/providers/generation_test.go b/services/core/internal/sandbox/providers/generation_test.go index 60b42d297..9abb76f54 100644 --- a/services/core/internal/sandbox/providers/generation_test.go +++ b/services/core/internal/sandbox/providers/generation_test.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "strconv" + "strings" "syscall" "testing" "time" @@ -22,6 +23,7 @@ func generationConfig(t *testing.T) (sandbox.NodeConfig, sandboxmicro.Native) { spec := validRegistrationSpec() spec.Resources.RootDiskMiB, spec.Resources.EnvironmentDiskMiB = 8192, 8192 native := sandboxmicro.Native{HelperPath: filepath.Join(dir, "helper"), RuntimeHome: dir, CheckpointRoot: t.TempDir(), RuntimePath: filepath.Join(dir, "msb"), FirmwarePath: filepath.Join(dir, "firmware"), + Image: "oac-runtime@sha256:" + strings.Repeat("d", 64), RuntimeSHA256: strings.Repeat("e", 64), FirmwareSHA256: strings.Repeat("f", 64), Network: sandboxmicro.Network{DefaultEgress: "allow", DefaultIngress: "deny"}} raw, err := json.Marshal(native) if err != nil { @@ -152,8 +154,9 @@ func TestMicrosandboxGenerationRejectsUnpinnedImage(t *testing.T) { registry := Builtin() for _, image := range []string{"latest", "oac-runtime@sha256:bad", "oac-runtime@sha256:"} { t.Run(image, func(t *testing.T) { - config, _ := generationConfig(t) - config.Specification.Runtime.MicrosandboxRef = image + config, native := generationConfig(t) + native.Image = image + config.Native, _ = json.Marshal(native) built, closeProvider, err := registry.Build(config, sandbox.LocalOptions{GenerationStateDirectory: t.TempDir()}) closeProvider() if err == nil || built != nil { diff --git a/services/core/internal/sandbox/providers/registration.go b/services/core/internal/sandbox/providers/registration.go index 5a5668d35..86722aa1c 100644 --- a/services/core/internal/sandbox/providers/registration.go +++ b/services/core/internal/sandbox/providers/registration.go @@ -45,9 +45,6 @@ func ValidateRegistration(a Adapter) error { if a.Policy.Workspace != nil && a.Policy.Workspace.Attachment != workspacefs.AttachmentHostDirectory { return invalid("workspace attachment requirements") } - if a.Policy.Runtime == (a.Policy.RuntimeError != "") { - return invalid("Runtime input policy") - } if a.Operations == nil { return invalid("operation declaration") } diff --git a/services/core/internal/sandbox/providers/registration_test.go b/services/core/internal/sandbox/providers/registration_test.go index 592bdf313..455670bb4 100644 --- a/services/core/internal/sandbox/providers/registration_test.go +++ b/services/core/internal/sandbox/providers/registration_test.go @@ -14,12 +14,6 @@ import ( func validRegistrationSpec() sandbox.DeploymentSpec { return sandbox.DeploymentSpec{ Resources: sandbox.Resources{CPUs: 2, MemoryMiB: 2048}, - Runtime: &sandbox.RuntimeRelease{ - SourceCommit: strings.Repeat("a", 40), ImageID: "sha256:" + strings.Repeat("b", 64), - ImageManifestDigest: "sha256:" + strings.Repeat("c", 64), - MicrosandboxRef: "oac-runtime@sha256:" + strings.Repeat("d", 64), - RuntimeSHA256: strings.Repeat("e", 64), FirmwareSHA256: strings.Repeat("f", 64), - }, } } @@ -30,8 +24,6 @@ func TestRegistrationRejectsBeforeCallbacksOrConstruction(t *testing.T) { name string mutate func(*Adapter) }{ - {"missing Runtime input policy", func(a *Adapter) { a.Policy = sandbox.DeploymentPolicy{} }}, - {"contradictory Runtime input policy", func(a *Adapter) { a.Policy.RuntimeError = "Runtime rejected" }}, {"missing mode", func(a *Adapter) { a.Mode = "" }}, {"unknown mode", func(a *Adapter) { a.Mode = "private-token" }}, {"missing local constructor", func(a *Adapter) { a.BuildLocal = nil }}, diff --git a/services/core/internal/sandbox/providers/validation_test.go b/services/core/internal/sandbox/providers/validation_test.go index 3fe6eb981..821a91ce6 100644 --- a/services/core/internal/sandbox/providers/validation_test.go +++ b/services/core/internal/sandbox/providers/validation_test.go @@ -34,21 +34,6 @@ func TestDeploymentValidationFieldsPreserveMessages(t *testing.T) { } } } - for _, tc := range []struct { - provider string - runtime *sandbox.RuntimeRelease - message string - }{ - {"docker", nil, "managed nodes require a pinned Runtime release"}, - {"docker", &sandbox.RuntimeRelease{}, "Runtime must reference one immutable distribution"}, - {"e2b", &sandbox.RuntimeRelease{}, "E2B Runtime is selected by its immutable template build"}, - } { - err := registry.ValidateSpecification(tc.provider, sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 1, MemoryMiB: 512}, Runtime: tc.runtime}) - var field *sandbox.ValidationError - if !errors.As(err, &field) || field.Param != "runtime" || err.Error() != sandbox.ErrInvalid.Error()+": "+tc.message || !errors.Is(err, sandbox.ErrInvalid) { - t.Fatal(err) - } - } if err := registry.ValidateResources("docker", sandbox.Resources{CPUs: 255, MemoryMiB: 1048576}); err != nil { t.Fatal(err) } diff --git a/services/core/internal/sandbox/sandbox_provider.go b/services/core/internal/sandbox/sandbox_provider.go index 6d024b275..c53320c48 100644 --- a/services/core/internal/sandbox/sandbox_provider.go +++ b/services/core/internal/sandbox/sandbox_provider.go @@ -289,14 +289,11 @@ type ConfigurationRequirements struct { } // DeploymentPolicy is the deployment declaration. Disk declares independent -// disk limits; Runtime requires a pinned Runtime release, and RuntimeError is -// the fixed reason for rejecting one otherwise. DefaultResources is the size -// setup proposes, or nil when the Provider's configuration selects it. +// disk limits. DefaultResources is the size setup proposes, or nil when the +// Provider's configuration selects it. type DeploymentPolicy struct { Workspace *workspacefs.Requirements `json:"workspace,omitempty"` - RuntimeError string `json:"-"` Disk bool `json:"disk"` - Runtime bool `json:"runtime"` DefaultResources *Resources `json:"default_resources"` } @@ -343,8 +340,8 @@ type DirectConfig struct { // NodeConfig is a node's configuration for one deployment generation. Native // holds only the selected adapter's node-local settings, such as host paths; -// that adapter alone decodes it, strictly. Resources and the Runtime release -// are read from Specification, never copied into Native. +// that adapter alone decodes it, strictly. Resources are read from +// Specification, never copied into Native. type NodeConfig struct { Specification DeploymentSpec `json:"specification"` Generation uint64 `json:"generation"` diff --git a/services/core/internal/sandbox/testdata/deployment-contract.json b/services/core/internal/sandbox/testdata/deployment-contract.json index bd4a68738..2e76faa7e 100644 --- a/services/core/internal/sandbox/testdata/deployment-contract.json +++ b/services/core/internal/sandbox/testdata/deployment-contract.json @@ -6,19 +6,11 @@ "resources": { "cpus": 2, "memory_mib": 4096 - }, - "runtime": { - "source_commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "image_id": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "image_manifest_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", - "microsandbox_ref": "oac-runtime@sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", - "runtime_sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", - "firmware_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" } }, "valid": true, - "canonical": "{\"provider\":\"docker\",\"resources\":{\"cpus\":2,\"memory_mib\":4096},\"runtime\":{\"source_commit\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"image_id\":\"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"image_manifest_digest\":\"sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\",\"microsandbox_ref\":\"oac-runtime@sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd\",\"runtime_sha256\":\"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee\",\"firmware_sha256\":\"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff\"}}", - "digest": "1237a0d19e665b387922ad3d60143d5cd530864f0e3e4ee2c5ace57a1134aa38" + "canonical": "{\"provider\":\"docker\",\"resources\":{\"cpus\":2,\"memory_mib\":4096}}", + "digest": "6931ff71e3c92bccfbc422fb11d0e69fdf591e7bdd1aed76ac47fa4f144baca1" }, { "name": "docker-zero-disks", @@ -29,19 +21,11 @@ "memory_mib": 4096, "root_disk_mib": 0, "environment_disk_mib": 0 - }, - "runtime": { - "source_commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "image_id": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "image_manifest_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", - "microsandbox_ref": "oac-runtime@sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", - "runtime_sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", - "firmware_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" } }, "valid": true, - "canonical": "{\"provider\":\"docker\",\"resources\":{\"cpus\":2,\"memory_mib\":4096},\"runtime\":{\"source_commit\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"image_id\":\"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"image_manifest_digest\":\"sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\",\"microsandbox_ref\":\"oac-runtime@sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd\",\"runtime_sha256\":\"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee\",\"firmware_sha256\":\"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff\"}}", - "digest": "1237a0d19e665b387922ad3d60143d5cd530864f0e3e4ee2c5ace57a1134aa38" + "canonical": "{\"provider\":\"docker\",\"resources\":{\"cpus\":2,\"memory_mib\":4096}}", + "digest": "6931ff71e3c92bccfbc422fb11d0e69fdf591e7bdd1aed76ac47fa4f144baca1" }, { "name": "microsandbox", @@ -52,19 +36,11 @@ "memory_mib": 1048576, "root_disk_mib": 1024, "environment_disk_mib": 4294967295 - }, - "runtime": { - "source_commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "image_id": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "image_manifest_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", - "microsandbox_ref": "oac-runtime@sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", - "runtime_sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", - "firmware_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" } }, "valid": true, - "canonical": "{\"provider\":\"microsandbox\",\"resources\":{\"cpus\":255,\"memory_mib\":1048576,\"root_disk_mib\":1024,\"environment_disk_mib\":4294967295},\"runtime\":{\"source_commit\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"image_id\":\"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"image_manifest_digest\":\"sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\",\"microsandbox_ref\":\"oac-runtime@sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd\",\"runtime_sha256\":\"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee\",\"firmware_sha256\":\"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff\"}}", - "digest": "1bb6daadcf9e34b236a8ecba6eb4c5215b946a3f226721b2bf148f72acb5f05e" + "canonical": "{\"provider\":\"microsandbox\",\"resources\":{\"cpus\":255,\"memory_mib\":1048576,\"root_disk_mib\":1024,\"environment_disk_mib\":4294967295}}", + "digest": "bab416199b7aa94421b948fa3dfcb55a4547fa56209395b2a4dbaeacbefe6e72" }, { "name": "e2b", @@ -86,14 +62,6 @@ "resources": { "cpus": 0, "memory_mib": 4096 - }, - "runtime": { - "source_commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "image_id": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "image_manifest_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", - "microsandbox_ref": "oac-runtime@sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", - "runtime_sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", - "firmware_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" } }, "valid": false @@ -105,14 +73,6 @@ "resources": { "cpus": 256, "memory_mib": 4096 - }, - "runtime": { - "source_commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "image_id": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "image_manifest_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", - "microsandbox_ref": "oac-runtime@sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", - "runtime_sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", - "firmware_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" } }, "valid": false @@ -124,14 +84,6 @@ "resources": { "cpus": true, "memory_mib": 4096 - }, - "runtime": { - "source_commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "image_id": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "image_manifest_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", - "microsandbox_ref": "oac-runtime@sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", - "runtime_sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", - "firmware_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" } }, "valid": false @@ -143,14 +95,6 @@ "resources": { "cpus": 2, "memory_mib": 511 - }, - "runtime": { - "source_commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "image_id": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "image_manifest_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", - "microsandbox_ref": "oac-runtime@sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", - "runtime_sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", - "firmware_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" } }, "valid": false @@ -163,14 +107,6 @@ "cpus": 2, "memory_mib": 4096, "other": 1 - }, - "runtime": { - "source_commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "image_id": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "image_manifest_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", - "microsandbox_ref": "oac-runtime@sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", - "runtime_sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", - "firmware_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" } }, "valid": false @@ -183,14 +119,6 @@ "cpus": 2, "memory_mib": 4096, "root_disk_mib": 1024 - }, - "runtime": { - "source_commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "image_id": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "image_manifest_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", - "microsandbox_ref": "oac-runtime@sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", - "runtime_sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", - "firmware_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" } }, "valid": false @@ -204,20 +132,12 @@ "memory_mib": 1048576, "root_disk_mib": 1023, "environment_disk_mib": 4294967295 - }, - "runtime": { - "source_commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "image_id": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "image_manifest_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", - "microsandbox_ref": "oac-runtime@sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", - "runtime_sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", - "firmware_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" } }, "valid": false }, { - "name": "invalid-release-digest", + "name": "runtime-rejected", "provider": "docker", "specification": { "resources": { @@ -225,12 +145,7 @@ "memory_mib": 4096 }, "runtime": { - "source_commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "image_id": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "image_manifest_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", - "microsandbox_ref": "oac-runtime@sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", - "runtime_sha256": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", - "firmware_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + "source_commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" } }, "valid": false @@ -244,14 +159,6 @@ "memory_mib": 1048576, "root_disk_mib": 1024 }, - "runtime": { - "source_commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "image_id": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "image_manifest_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", - "microsandbox_ref": "oac-runtime@sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", - "runtime_sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", - "firmware_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" - }, "workspace": { "attachment": "host_directory", "user_xattr": true, @@ -259,8 +166,8 @@ } }, "valid": true, - "canonical": "{\"provider\":\"microsandbox\",\"resources\":{\"cpus\":255,\"memory_mib\":1048576,\"root_disk_mib\":1024},\"runtime\":{\"source_commit\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"image_id\":\"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"image_manifest_digest\":\"sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\",\"microsandbox_ref\":\"oac-runtime@sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd\",\"runtime_sha256\":\"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee\",\"firmware_sha256\":\"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff\"},\"workspace\":{\"attachment\":\"host_directory\",\"user_xattr\":true,\"capacity_quota\":false}}", - "digest": "ea1aa3cb9bd74682f5fb1a727dc43c63df193f71c7946c395da30451ee5ee3b0" + "canonical": "{\"provider\":\"microsandbox\",\"resources\":{\"cpus\":255,\"memory_mib\":1048576,\"root_disk_mib\":1024},\"workspace\":{\"attachment\":\"host_directory\",\"user_xattr\":true,\"capacity_quota\":false}}", + "digest": "cd31758a8ec9df729120eccd3db650d0f82cb35c4fbd1b1f5b7cf70140ae9faf" }, { "name": "microsandbox-external-requires-root", @@ -271,14 +178,6 @@ "memory_mib": 1048576, "root_disk_mib": 0 }, - "runtime": { - "source_commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "image_id": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "image_manifest_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", - "microsandbox_ref": "oac-runtime@sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", - "runtime_sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", - "firmware_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" - }, "workspace": { "attachment": "host_directory", "user_xattr": true, @@ -297,14 +196,6 @@ "root_disk_mib": 1024, "environment_disk_mib": 4096 }, - "runtime": { - "source_commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "image_id": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "image_manifest_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", - "microsandbox_ref": "oac-runtime@sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", - "runtime_sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", - "firmware_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" - }, "workspace": { "attachment": "host_directory", "user_xattr": true, @@ -322,14 +213,6 @@ "memory_mib": 1048576, "root_disk_mib": 1024 }, - "runtime": { - "source_commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "image_id": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "image_manifest_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", - "microsandbox_ref": "oac-runtime@sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", - "runtime_sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", - "firmware_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" - }, "workspace": { "attachment": "host_directory", "user_xattr": false, @@ -346,14 +229,6 @@ "cpus": 255, "memory_mib": 1048576, "root_disk_mib": 1024 - }, - "runtime": { - "source_commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "image_id": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "image_manifest_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", - "microsandbox_ref": "oac-runtime@sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", - "runtime_sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", - "firmware_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" } }, "valid": false @@ -366,14 +241,6 @@ "cpus": 255, "memory_mib": 1048576 }, - "runtime": { - "source_commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "image_id": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "image_manifest_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", - "microsandbox_ref": "oac-runtime@sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", - "runtime_sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", - "firmware_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" - }, "workspace": { "attachment": "host_directory", "user_xattr": true, @@ -391,22 +258,14 @@ "memory_mib": 1048576, "root_disk_mib": 1024 }, - "runtime": { - "source_commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "image_id": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "image_manifest_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", - "microsandbox_ref": "oac-runtime@sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", - "runtime_sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", - "firmware_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" - }, "workspace": { "attachment": "host_directory", "user_xattr": true } }, "valid": true, - "canonical": "{\"provider\":\"microsandbox\",\"resources\":{\"cpus\":255,\"memory_mib\":1048576,\"root_disk_mib\":1024},\"runtime\":{\"source_commit\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"image_id\":\"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"image_manifest_digest\":\"sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\",\"microsandbox_ref\":\"oac-runtime@sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd\",\"runtime_sha256\":\"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee\",\"firmware_sha256\":\"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff\"},\"workspace\":{\"attachment\":\"host_directory\",\"user_xattr\":true,\"capacity_quota\":false}}", - "digest": "ea1aa3cb9bd74682f5fb1a727dc43c63df193f71c7946c395da30451ee5ee3b0" + "canonical": "{\"provider\":\"microsandbox\",\"resources\":{\"cpus\":255,\"memory_mib\":1048576,\"root_disk_mib\":1024},\"workspace\":{\"attachment\":\"host_directory\",\"user_xattr\":true,\"capacity_quota\":false}}", + "digest": "cd31758a8ec9df729120eccd3db650d0f82cb35c4fbd1b1f5b7cf70140ae9faf" }, { "name": "microsandbox-external-null-false-quota", @@ -417,14 +276,6 @@ "memory_mib": 1048576, "root_disk_mib": 1024 }, - "runtime": { - "source_commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "image_id": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "image_manifest_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", - "microsandbox_ref": "oac-runtime@sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", - "runtime_sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", - "firmware_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" - }, "workspace": { "attachment": "host_directory", "user_xattr": true, @@ -432,7 +283,7 @@ } }, "valid": true, - "canonical": "{\"provider\":\"microsandbox\",\"resources\":{\"cpus\":255,\"memory_mib\":1048576,\"root_disk_mib\":1024},\"runtime\":{\"source_commit\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"image_id\":\"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"image_manifest_digest\":\"sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\",\"microsandbox_ref\":\"oac-runtime@sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd\",\"runtime_sha256\":\"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee\",\"firmware_sha256\":\"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff\"},\"workspace\":{\"attachment\":\"host_directory\",\"user_xattr\":true,\"capacity_quota\":false}}", - "digest": "ea1aa3cb9bd74682f5fb1a727dc43c63df193f71c7946c395da30451ee5ee3b0" + "canonical": "{\"provider\":\"microsandbox\",\"resources\":{\"cpus\":255,\"memory_mib\":1048576,\"root_disk_mib\":1024},\"workspace\":{\"attachment\":\"host_directory\",\"user_xattr\":true,\"capacity_quota\":false}}", + "digest": "cd31758a8ec9df729120eccd3db650d0f82cb35c4fbd1b1f5b7cf70140ae9faf" } ] diff --git a/services/core/internal/sandbox/testdata/node-diagnostics.json b/services/core/internal/sandbox/testdata/node-diagnostics.json index 4c885eec2..7d799410e 100644 --- a/services/core/internal/sandbox/testdata/node-diagnostics.json +++ b/services/core/internal/sandbox/testdata/node-diagnostics.json @@ -2,7 +2,6 @@ "provider_unavailable", "host_unsupported", "artifacts_unavailable", - "runtime_download_failed", "runtime_image_unavailable", "capacity_insufficient" ] diff --git a/services/core/migrations/000002_remove_dead_node_columns.sql b/services/core/migrations/000002_remove_dead_node_columns.sql new file mode 100644 index 000000000..32fd3dbfa --- /dev/null +++ b/services/core/migrations/000002_remove_dead_node_columns.sql @@ -0,0 +1,4 @@ +-- +goose Up +ALTER TABLE runtime_nodes DROP CONSTRAINT runtime_nodes_protocol_version_check; +ALTER TABLE runtime_nodes DROP COLUMN protocol_version; +ALTER TABLE runtime_nodes DROP COLUMN provider_ready; diff --git a/services/core/tests/integration/deployment_fixture_test.go b/services/core/tests/integration/deployment_fixture_test.go index 2073a5b60..ffce51e69 100644 --- a/services/core/tests/integration/deployment_fixture_test.go +++ b/services/core/tests/integration/deployment_fixture_test.go @@ -53,15 +53,22 @@ func deploymentExecution(t testing.TB, w *Store) *deployment.ExecutionOperations return operations } -func fixtureNodeHeartbeat(ctx context.Context, store *Store, service *deployment.Service, nodeID, connection string, epoch uint64, health deployment.NodeHealth) error { +func fixtureNodeHeartbeat(ctx context.Context, store *Store, service *deployment.Service, nodeID, connection string, epoch uint64, ready bool) error { var generation uint64 var digest, provider string if err := store.pool.QueryRow(ctx, "SELECT n.deployment_generation,n.specification_digest,d.provider_kind FROM runtime_nodes n CROSS JOIN runtime_deployment d WHERE n.id=$1", nodeID).Scan(&generation, &digest, &provider); err != nil { return err } var statuses []sandbox.GenerationStatus - if health.ProviderReady && provider == "microsandbox" { - statuses = []sandbox.GenerationStatus{{Generation: generation, SpecificationDigest: digest, State: "ready", Checkpoint: &sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"}}} + { + status := sandbox.GenerationStatus{Generation: generation, SpecificationDigest: digest, State: "ready"} + if !ready { + status.State = "failed" + } + if provider == "microsandbox" && ready { + status.Checkpoint = &sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"} + } + statuses = []sandbox.GenerationStatus{status} } - return service.Heartbeat(ctx, nodeID, connection, epoch, health, statuses) + return service.Heartbeat(ctx, nodeID, connection, epoch, deployment.NodeHealth{}, statuses) } diff --git a/services/core/tests/integration/node_diagnostic_contract_test.go b/services/core/tests/integration/node_diagnostic_contract_test.go index 88cb7ce96..da595b00c 100644 --- a/services/core/tests/integration/node_diagnostic_contract_test.go +++ b/services/core/tests/integration/node_diagnostic_contract_test.go @@ -29,7 +29,7 @@ func TestNodeDiagnosticSchemaContract(t *testing.T) { t.Errorf("%s diagnostic enum = %v, want shared fixture %v", name, values, codes) } } - for _, model := range []reflect.Type{reflect.TypeFor[deployment.NodeHealth](), reflect.TypeFor[deployment.NodeRollout]()} { + for _, model := range []reflect.Type{reflect.TypeFor[deployment.NodeRollout]()} { field, ok := model.FieldByName("Diagnostic") if !ok { t.Fatalf("%s has no Diagnostic field", model.Name()) @@ -57,7 +57,7 @@ func TestNodeDiagnosticSchemaContract(t *testing.T) { t.Fatal(err) } check("sandbox.NodeDiagnosticCode", document.Definitions["sandbox.NodeDiagnosticCode"].Enum) - for _, name := range []string{"deployment.Node", "deployment.NodeDetail", "deployment.NodeRollout"} { + for _, name := range []string{"deployment.NodeRollout"} { field := document.Definitions[name].Properties["diagnostic"] ref := field.Ref if len(field.AllOf) == 1 { diff --git a/services/core/tests/integration/runtime_node_generations_test.go b/services/core/tests/integration/runtime_node_generations_test.go index 3197a00c7..be902f790 100644 --- a/services/core/tests/integration/runtime_node_generations_test.go +++ b/services/core/tests/integration/runtime_node_generations_test.go @@ -33,7 +33,7 @@ func generationHeartbeat(t *testing.T, s *Store, node deployment.Enrollment, con if state == "ready" && view.Provider == "microsandbox" { status.Checkpoint = &sandbox.CheckpointCompatibility{ArtifactDomain: "fixture-store", ExecutionClass: "fixture-runtime"} } - err := deploymentService(t, s).HeartbeatGenerations(t.Context(), node.NodeID, connection, view.OwnerEpoch, deployment.NodeHealth{}, []sandbox.GenerationStatus{status}) + err := deploymentService(t, s).Heartbeat(t.Context(), node.NodeID, connection, view.OwnerEpoch, deployment.NodeHealth{}, []sandbox.GenerationStatus{status}) if err != nil { t.Fatal(err) } @@ -146,24 +146,24 @@ func TestNodeGenerationsCapacityFallbackAndImmutablePending(t *testing.T) { } } -func TestNodeGenerationsReconnectAndV1Fallback(t *testing.T) { +func TestNodeGenerationsReconnectKeepsServingPin(t *testing.T) { s, w, first, input := webSpecificationFixture(t, "docker") service := deploymentService(t, s) node := specificationNode(t, s, first) old := onlineManagerNode(t, s, node.NodeID) second, _ := changeNodeTarget(t, w, first, input) nodes, err := service.ListNodes(t.Context()) - if err != nil || !nodes[0].ProviderReady || nodes[0].Rollout.State != "update_required" { + if err != nil || !nodes[0].ProviderReady || nodes[0].Rollout.State != "unknown" { t.Fatal(nodes, err) } if _, err = s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput(uuid.NewString())); err != nil { - t.Fatal("v1 lost fallback", err) + t.Fatal("serving pin lost fallback", err) } connection := uuid.NewString() if err = service.ConnectNode(t.Context(), node.NodeID, connection, first.OwnerEpoch); err != nil { t.Fatal(err) } - if err = service.HeartbeatGenerations(t.Context(), node.NodeID, old, first.OwnerEpoch, deployment.NodeHealth{}, []sandbox.GenerationStatus{{Generation: second.Generation, SpecificationDigest: second.SpecificationDigest, State: "ready"}}); !errors.Is(err, deployment.ErrNodeCredential) { + if err = service.Heartbeat(t.Context(), node.NodeID, old, first.OwnerEpoch, deployment.NodeHealth{}, []sandbox.GenerationStatus{{Generation: second.Generation, SpecificationDigest: second.SpecificationDigest, State: "ready"}}); !errors.Is(err, deployment.ErrNodeCredential) { t.Fatal("old connection qualified", err) } nodes, err = service.ListNodes(t.Context()) @@ -177,11 +177,11 @@ func TestNodeGenerationsReconnectAndV1Fallback(t *testing.T) { if err := reserveSessionPlacement(t, s, w, queued); !errors.Is(err, placement.ErrNodeUnavailable) { t.Fatal("unconfirmed connection admitted", err) } - if err = service.Heartbeat(t.Context(), node.NodeID, connection, first.OwnerEpoch, deployment.NodeHealth{ProviderReady: true}, nil); err != nil { + if err = fixtureNodeHeartbeat(t.Context(), s, service, node.NodeID, connection, first.OwnerEpoch, true); err != nil { t.Fatal(err) } if _, err = s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput(uuid.NewString())); err != nil { - t.Fatal("fresh v1 readiness did not restore fallback", err) + t.Fatal("fresh pin readiness did not restore fallback", err) } } diff --git a/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go b/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go index 90fbc8c5a..0d304a63b 100644 --- a/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go +++ b/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go @@ -149,7 +149,7 @@ func (f *nodeIsolationFixture) online(id string) { if err := f.nodes.ConnectNode(f.t.Context(), id, connection, f.epoch); err != nil { f.t.Fatal(err) } - if err := fixtureNodeHeartbeat(f.t.Context(), f.store, f.nodes, id, connection, f.epoch, deployment.NodeHealth{ProviderReady: true}); err != nil { + if err := fixtureNodeHeartbeat(f.t.Context(), f.store, f.nodes, id, connection, f.epoch, true); err != nil { f.t.Fatal(err) } } @@ -166,7 +166,7 @@ func (f *nodeIsolationFixture) session(node string, initialize bool) (string, se id, connection string epoch uint64 } - rows, err := f.pool.Query(f.t.Context(), "SELECT id::text, connection_id::text, connected_epoch FROM runtime_nodes WHERE provider_ready AND id<>$1 AND connection_id IS NOT NULL AND removed_at IS NULL", node) + rows, err := f.pool.Query(f.t.Context(), "SELECT id::text, connection_id::text, connected_epoch FROM runtime_nodes WHERE id<>$1 AND connection_id IS NOT NULL AND removed_at IS NULL", node) if err != nil { f.t.Fatal(err) } @@ -184,7 +184,7 @@ func (f *nodeIsolationFixture) session(node string, initialize bool) (string, se f.t.Fatal(err) } for _, value := range others { - if err := fixtureNodeHeartbeat(f.t.Context(), f.store, f.nodes, value.id, value.connection, value.epoch, deployment.NodeHealth{ProviderReady: false}); err != nil { + if err := fixtureNodeHeartbeat(f.t.Context(), f.store, f.nodes, value.id, value.connection, value.epoch, false); err != nil { f.t.Fatal(err) } } @@ -193,7 +193,7 @@ func (f *nodeIsolationFixture) session(node string, initialize bool) (string, se _, err = f.placement.EnsurePlacement(f.t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: session.Environment.ID}, f.key) } for _, value := range others { - if err := fixtureNodeHeartbeat(context.WithoutCancel(f.t.Context()), f.store, f.nodes, value.id, value.connection, value.epoch, deployment.NodeHealth{ProviderReady: true}); err != nil { + if err := fixtureNodeHeartbeat(context.WithoutCancel(f.t.Context()), f.store, f.nodes, value.id, value.connection, value.epoch, true); err != nil { f.t.Fatal(err) } } diff --git a/services/core/tests/integration/runtime_nodes_test.go b/services/core/tests/integration/runtime_nodes_test.go index 2d83b955e..0093253d3 100644 --- a/services/core/tests/integration/runtime_nodes_test.go +++ b/services/core/tests/integration/runtime_nodes_test.go @@ -34,7 +34,7 @@ func onlineManagerNode(t *testing.T, s *Store, id string) string { if err := nodes.ConnectNode(t.Context(), id, connection, managerEpoch(t, s)); err != nil { t.Fatal(err) } - if err := fixtureNodeHeartbeat(t.Context(), s, nodes, id, connection, managerEpoch(t, s), deployment.NodeHealth{ProviderReady: true}); err != nil { + if err := fixtureNodeHeartbeat(t.Context(), s, nodes, id, connection, managerEpoch(t, s), true); err != nil { t.Fatal(err) } return connection @@ -53,7 +53,7 @@ func createSessionOnNode(t *testing.T, s, w *Store, tenant string, input session id, connection string epoch uint64 } - rows, err := s.pool.Query(t.Context(), "SELECT id::text, connection_id::text, connected_epoch FROM runtime_nodes WHERE provider_ready AND id<>$1 AND connection_id IS NOT NULL AND removed_at IS NULL", node) + rows, err := s.pool.Query(t.Context(), "SELECT id::text, connection_id::text, connected_epoch FROM runtime_nodes WHERE id<>$1 AND connection_id IS NOT NULL AND removed_at IS NULL", node) if err != nil { t.Fatal(err) } @@ -72,13 +72,13 @@ func createSessionOnNode(t *testing.T, s, w *Store, tenant string, input session } nodes := deploymentService(t, s) for _, value := range others { - if err := fixtureNodeHeartbeat(t.Context(), s, nodes, value.id, value.connection, value.epoch, deployment.NodeHealth{ProviderReady: false}); err != nil { + if err := fixtureNodeHeartbeat(t.Context(), s, nodes, value.id, value.connection, value.epoch, false); err != nil { t.Fatal(err) } } defer func() { for _, value := range others { - if err := fixtureNodeHeartbeat(context.WithoutCancel(t.Context()), s, nodes, value.id, value.connection, value.epoch, deployment.NodeHealth{ProviderReady: true}); err != nil { + if err := fixtureNodeHeartbeat(context.WithoutCancel(t.Context()), s, nodes, value.id, value.connection, value.epoch, true); err != nil { t.Fatal(err) } } @@ -268,7 +268,7 @@ func TestRuntimeNodesEnrollmentAndEpoch(t *testing.T) { if next := managerEpoch(t, s); next != epoch+1 { t.Fatal(next) } - if err := fixtureNodeHeartbeat(t.Context(), s, nodes, input.NodeID, connection, epoch, deployment.NodeHealth{ProviderReady: true}); !errors.Is(err, deployment.ErrNodeCredential) { + if err := fixtureNodeHeartbeat(t.Context(), s, nodes, input.NodeID, connection, epoch, true); !errors.Is(err, deployment.ErrNodeCredential) { t.Fatal("old epoch heartbeat revived node", err) } queued, err := s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput("stale")) diff --git a/services/core/tests/integration/sandbox_deployment_audit_http_test.go b/services/core/tests/integration/sandbox_deployment_audit_http_test.go index 1f2ee5ba7..09890735f 100644 --- a/services/core/tests/integration/sandbox_deployment_audit_http_test.go +++ b/services/core/tests/integration/sandbox_deployment_audit_http_test.go @@ -38,7 +38,7 @@ func TestSandboxDeploymentUpdateHTTPCommitsWithAdministratorAudit(t *testing.T) t.Helper() resources := input.Resources resources.CPUs = cpus - body, err := json.Marshal(map[string]any{"expected_generation": generation, "provider": "docker", "configuration": map[string]any{}, "resources": resources, "runtime": input.Runtime}) + body, err := json.Marshal(map[string]any{"expected_generation": generation, "provider": "docker", "configuration": map[string]any{}, "resources": resources}) if err != nil { t.Fatal(err) } diff --git a/services/core/tests/integration/sandbox_deployment_worker_test.go b/services/core/tests/integration/sandbox_deployment_worker_test.go index 528a351c1..39417be9f 100644 --- a/services/core/tests/integration/sandbox_deployment_worker_test.go +++ b/services/core/tests/integration/sandbox_deployment_worker_test.go @@ -72,7 +72,7 @@ func TestSandboxDeploymentWorkerActivatesWithoutRestart(t *testing.T) { if err := deployments.ConnectNode(t.Context(), nodeID, connection, epoch); err != nil { t.Fatal(err) } - if err := deployments.Heartbeat(t.Context(), nodeID, connection, epoch, deployment.NodeHealth{ProviderReady: true}, nil); err != nil { + if err := fixtureNodeHeartbeat(t.Context(), s, deployments, nodeID, connection, epoch, true); err != nil { t.Fatal(err) } } diff --git a/services/core/tests/integration/sandbox_generations_test.go b/services/core/tests/integration/sandbox_generations_test.go index 44aef4f80..8ee2ea69a 100644 --- a/services/core/tests/integration/sandbox_generations_test.go +++ b/services/core/tests/integration/sandbox_generations_test.go @@ -178,16 +178,16 @@ func TestGenerationUpdateSerializesWithAllocationAdmission(t *testing.T) { } func TestSandboxSnapshotRolloutEquivalence(t *testing.T) { - s, w, view, input := webSpecificationFixture(t, "docker") + s, _, view, _ := webSpecificationFixture(t, "docker") node := specificationNode(t, s, view) deployments := deploymentService(t, s) heartbeat := func(connection, state string) { t.Helper() - if err := deployments.HeartbeatGenerations(t.Context(), node.NodeID, connection, view.OwnerEpoch, deployment.NodeHealth{}, []sandbox.GenerationStatus{{Generation: view.Generation, SpecificationDigest: view.SpecificationDigest, State: state}}); err != nil { + if err := deployments.Heartbeat(t.Context(), node.NodeID, connection, view.OwnerEpoch, deployment.NodeHealth{}, []sandbox.GenerationStatus{{Generation: view.Generation, SpecificationDigest: view.SpecificationDigest, State: state}}); err != nil { t.Fatal(err) } } - for _, state := range []string{"ready", "preparing", "failed", "unconfirmed", "offline", "update_required"} { + for _, state := range []string{"ready", "preparing", "failed", "unconfirmed", "offline"} { t.Run(state, func(t *testing.T) { connection := onlineManagerNode(t, s, node.NodeID) want := deployment.RolloutNodes{} @@ -207,24 +207,13 @@ func TestSandboxSnapshotRolloutEquivalence(t *testing.T) { if err := deployments.ConnectNode(t.Context(), node.NodeID, connection, view.OwnerEpoch); err != nil { t.Fatal(err) } - if err := deployments.HeartbeatGenerations(t.Context(), node.NodeID, connection, view.OwnerEpoch, deployment.NodeHealth{}, nil); err != nil { + if err := deployments.Heartbeat(t.Context(), node.NodeID, connection, view.OwnerEpoch, deployment.NodeHealth{}, nil); err != nil { t.Fatal(err) } want.Unknown = 1 case "offline": runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_nodes SET last_seen_at=clock_timestamp()-interval '46 seconds',health='{"diagnostic":"provider_unavailable"}' WHERE id=$1`, node.NodeID) want.Unknown = 1 - case "update_required": - // Change the target on the same backend so the node's enrolled - // generation falls behind. - change := input - change.Resources.CPUs++ - change.ExpectedGeneration = view.Generation - next, err := deploymentExecution(t, w).Update(SandboxResetTestContext(t.Context()), view.InstallationID, change) - if err != nil || next.Generation != view.Generation+1 || next.OwnerEpoch != view.OwnerEpoch { - t.Fatal("target change replaced execution ownership", next, err) - } - want.UpdateRequired = 1 } assertSandboxSnapshotEquivalent(t, s.pool) got, err := deployments.View(t.Context()) diff --git a/services/core/tests/integration/sandbox_reset_test.go b/services/core/tests/integration/sandbox_reset_test.go index 18936ec6b..146a5c086 100644 --- a/services/core/tests/integration/sandbox_reset_test.go +++ b/services/core/tests/integration/sandbox_reset_test.go @@ -277,7 +277,7 @@ func TestSandboxResetSnapshotCountsOfflineOwnershipOnce(t *testing.T) { onlineManagerNode(t, s, d.NodeID) switch state { case "preparing": - runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_nodes SET provider_ready=false`) + runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_nodes SET ready_generation=NULL`) case "stale": runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_nodes SET last_seen_at=clock_timestamp()-interval '46 seconds'`) case "epoch": diff --git a/services/core/tests/integration/sandbox_specification_lifecycle_test.go b/services/core/tests/integration/sandbox_specification_lifecycle_test.go index f3a87dc6c..ff9903153 100644 --- a/services/core/tests/integration/sandbox_specification_lifecycle_test.go +++ b/services/core/tests/integration/sandbox_specification_lifecycle_test.go @@ -181,25 +181,17 @@ func TestSandboxSpecificationChangesPreserveEveryRetainedResource(t *testing.T) t.Fatal(err) } } - for _, field := range []string{"resources", "runtime"} { - changed := input - if field == "resources" { - changed.Resources.MemoryMiB *= 2 - } else { - runtime := *input.Runtime - runtime.SourceCommit = strings.Repeat("1", 40) - changed.Runtime = &runtime - } - changed.ExpectedGeneration = view.Generation - if _, _, err := changes.ClassifyChange(t.Context(), view.InstallationID, changed); err != nil { - t.Fatal(field, err) - } - next, err := changes.Update(SandboxResetTestContext(t.Context()), view.InstallationID, changed) - if err != nil || next.Generation != view.Generation+1 || next.OwnerEpoch != view.OwnerEpoch { - t.Fatal(field, next, err) - } - view = next + changed := input + changed.Resources.MemoryMiB *= 2 + changed.ExpectedGeneration = view.Generation + if _, _, err := changes.ClassifyChange(t.Context(), view.InstallationID, changed); err != nil { + t.Fatal(err) + } + next, err := changes.Update(SandboxResetTestContext(t.Context()), view.InstallationID, changed) + if err != nil || next.Generation != view.Generation+1 || next.OwnerEpoch != view.OwnerEpoch { + t.Fatal(next, err) } + view = next after, err := deployments.View(t.Context()) if err != nil || before.Resources != after.Resources { t.Fatal("online specification change altered ownership", err) @@ -226,11 +218,8 @@ func TestSandboxSpecificationChangesPreserveEveryRetainedResource(t *testing.T) } else if err := sessionService(t, s).DeleteSession(t.Context(), sessions.DeleteSessionCommand{TenantID: tenant, SessionID: session.ID}); err != nil { t.Fatal(err) } - changed := input - changed.Resources.MemoryMiB *= 2 - runtime := *input.Runtime - runtime.SourceCommit = strings.Repeat("2", 40) - changed.Runtime = &runtime + changed = input + changed.Resources.MemoryMiB *= 4 changed.ExpectedGeneration = view.Generation committed, err := changes.Update(SandboxResetTestContext(t.Context()), view.InstallationID, changed) if err != nil || committed.Generation != view.Generation+1 || committed.Reset != nil || committed.Resources != (deployment.Resources{}) || committed.Specification == nil || !reflect.DeepEqual(*committed.Specification, changed.DeploymentSpec) { diff --git a/services/core/tests/integration/sandbox_specification_test.go b/services/core/tests/integration/sandbox_specification_test.go index e69631350..fab731b79 100644 --- a/services/core/tests/integration/sandbox_specification_test.go +++ b/services/core/tests/integration/sandbox_specification_test.go @@ -1,18 +1,12 @@ package integration import ( - "strings" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) func SandboxDeploymentTestSpec(provider string) sandbox.DeploymentSpec { s := sandbox.DeploymentSpec{Resources: sandbox.Resources{CPUs: 2, MemoryMiB: 2048}} - if provider == "e2b" { - return s - } - s.Runtime = &sandbox.RuntimeRelease{SourceCommit: strings.Repeat("a", 40), ImageID: "sha256:" + strings.Repeat("b", 64), ImageManifestDigest: "sha256:" + strings.Repeat("c", 64), MicrosandboxRef: "oac-runtime@sha256:" + strings.Repeat("d", 64), RuntimeSHA256: strings.Repeat("e", 64), FirmwareSHA256: strings.Repeat("f", 64)} if provider == "microsandbox" { s.Resources.RootDiskMiB = 8192 s.Resources.EnvironmentDiskMiB = 8192 diff --git a/services/core/tools/microsandbox-provider/README.md b/services/core/tools/microsandbox-provider/README.md index e2d25e902..9c1cefe54 100644 --- a/services/core/tools/microsandbox-provider/README.md +++ b/services/core/tools/microsandbox-provider/README.md @@ -2,7 +2,7 @@ microsandbox runs each hosted Session in its own microVM on a Linux amd64 node with KVM, and it is the Sandbox Provider that supports idle suspension. Core forwards provider operations to the node over the [node protocol](../../../../contracts/agents-api/node-generation-protocol.md); the node's adapter ([`sandbox/microsandbox`](../../internal/sandbox/microsandbox)) runs this helper once per operation. The private helper wire version is 6; mismatches are rejected. The helper links the microsandbox Go SDK v0.7.8 with its FFI library, so Core and the node program stay CGO-free Go binaries. It implements the checkpoint operations of the provider-neutral `sandbox.SandboxProvider` with full snapshots. It has no daemon, lifecycle database, scheduler or network control plane. -[Sandbox Provider protocol](../../../../docs/sandbox-provider.md) owns the provider contract. [Sandbox deployment](../../../../contracts/agents-api/sandbox-deployment.md) owns the resources, Runtime release and suspension policy; the [nodes guide](../../../../docs/getting-started/nodes.md) owns node installation, host requirements, the node's directories and its network policy. +[Sandbox Provider protocol](../../../../docs/sandbox-provider.md) owns the provider contract. [Sandbox deployment](../../../../contracts/agents-api/sandbox-deployment.md) owns the resources and suspension policy; the [nodes guide](../../../../docs/getting-started/nodes.md) owns node installation, host requirements, the node's directories and its network policy. ## Installation checks @@ -12,7 +12,7 @@ Before every operation the helper checks that it was built with the exact offici The runtime home must be private (mode 0700), short, on local persistent storage and used by no other installation, profile or manual lifecycle tool. microsandbox uses Unix sockets there, so the node installer refuses a home whose path would exceed their limit. The home holds confidential VM disks, memory snapshots and SDK state; preserve it with the node identity and Core's database when recovering a host. Every managed lifecycle change goes through the provider. -The Runtime image is an immutable `repository@sha256:<64 lowercase hex>` reference that matches the saved Runtime release; bare image IDs and mutable tags are rejected. The node installer imports the distribution's image under that reference. To load an image by hand, `msb image load --tag repository@sha256:` must register the digest reference explicitly, with the manifest digest from `image inspect`, not the Docker image config ID. The image carries the daemon, Python 3, the native Harnesses and the shared Runtime helpers. The provider installs no registry credentials. Creation uses the SDK's `never` pull policy. +The Runtime image is an immutable `repository@sha256:<64 lowercase hex>` reference that the node's `provider.json` records as `native.image`; bare image IDs and mutable tags are rejected. The node installer imports the distribution's image under that reference. To load an image by hand, `msb image load --tag repository@sha256:` must register the digest reference explicitly, with the manifest digest from `image inspect`, not the Docker image config ID. The image carries the daemon, Python 3, the native Harnesses and the shared Runtime helpers. The provider installs no registry credentials. Creation uses the SDK's `never` pull policy. The checkpoint root is a separate adapter-owned directory, mode 0700, with a private `.oac-checkpoint-store` UUID marker. It is never derived from a workspace binding and must not be reachable through any guest mount. Archives contain confidential RAM and root-disk state. The marker and installation identity define the artifact domain; native release and binary hashes, Runtime image, kernel, CPU features and guest geometry define the conservative execution class. Filesystem paths, hostnames and local SDK homes do not define compatibility. Nodes can share this private root at different mount paths; the backing store must preserve durable writes, atomic rename and cross-host file locks. The installer's local default only advertises its own domain. diff --git a/services/web/distribution_test.go b/services/web/distribution_test.go index 51e0608df..3a540af8a 100644 --- a/services/web/distribution_test.go +++ b/services/web/distribution_test.go @@ -12,38 +12,29 @@ import ( "testing" ) -// activeRelease publishes an empty node payload release under root the way -// `oac init` does and returns the release directory. -func activeRelease(t *testing.T, root string, manifest map[string]any) string { +// publishedPayload writes an empty node payload manifest under root the way +// `oac init` does and returns root. +func publishedPayload(t *testing.T, root string, manifest map[string]any) string { t.Helper() - revision := strings.Repeat("a", 40) - release := filepath.Join(root, "releases", revision) - if err := os.MkdirAll(release, 0700); err != nil { - t.Fatal(err) - } - manifest["source_commit"] = revision + manifest["source_commit"] = strings.Repeat("a", 40) raw, _ := json.Marshal(manifest) - if os.WriteFile(filepath.Join(release, "manifest.json"), raw, 0600) != nil || - os.WriteFile(filepath.Join(root, "active.json"), []byte(`{"source_commit":"`+revision+`"}`), 0600) != nil { + if os.WriteFile(filepath.Join(root, "manifest.json"), raw, 0600) != nil { t.Fatal("cannot publish the node payload") } - return release + return root } func TestConsoleRequiresPublishedNodePayload(t *testing.T) { - payload := t.TempDir() - if err := os.WriteFile(filepath.Join(payload, "node-install.pyz"), []byte("bootstrap"), 0600); err != nil { - t.Fatal(err) - } + payload := publishedPayload(t, t.TempDir(), map[string]any{}) upstream, _ := url.Parse("http://127.0.0.1:1") if _, err := newConsole(config{origin: testOrigin, upstream: upstream, dist: t.TempDir(), coreKey: testCoreKey, nodePayloadDir: payload}); err == nil { - t.Fatal("console started from a node payload without active.json") + t.Fatal("console started from a node payload without node-install.pyz") } } func TestOfflineArtifactsAreManifestAllowlisted(t *testing.T) { dist, payload := t.TempDir(), t.TempDir() - release := activeRelease(t, payload, map[string]any{"artifacts": map[string]any{"native/bin/oac-node": map[string]string{"filename": "matched-node"}, "private/key": map[string]string{"filename": "private-key"}, "native/bin/oac-selfhost": map[string]string{"filename": "retired-launcher"}}}) + release := publishedPayload(t, payload, map[string]any{"artifacts": map[string]any{"native/bin/oac-node": map[string]string{"filename": "matched-node"}, "private/key": map[string]string{"filename": "private-key"}, "native/bin/oac-selfhost": map[string]string{"filename": "retired-launcher"}}}) for _, item := range []struct{ root, name, body string }{{dist, "index.html", "console"}, {release, "node-install.pyz", "bootstrap"}} { if err := os.WriteFile(filepath.Join(item.root, item.name), []byte(item.body), 0600); err != nil { t.Fatal(err) @@ -96,7 +87,7 @@ func TestConsoleReportsServableNodeProviders(t *testing.T) { for _, logical := range []string{"native/bin/oac-node", "images/runtime.tar.gz", "native/microsandbox/msb", "runtime/seccomp.json"} { artifacts[logical] = map[string]any{"filename": strings.ReplaceAll(logical, "/", "-"), "size": len("runtime-bytes")} } - release := activeRelease(t, payload, map[string]any{"artifacts": artifacts}) + release := publishedPayload(t, payload, map[string]any{"artifacts": artifacts}) write(filepath.Join(dist, "index.html"), "console") write(filepath.Join(release, "node-install.pyz"), "bootstrap") for logical := range artifacts { @@ -135,7 +126,7 @@ func TestConsoleReportsServableNodeProviders(t *testing.T) { } } -func TestRetainedPayloadsRemainHTTPReachableAfterPublication(t *testing.T) { +func TestNodePayloadServesOnlyPublishedFiles(t *testing.T) { root := t.TempDir() write := func(name, body string) { t.Helper() @@ -147,14 +138,11 @@ func TestRetainedPayloadsRemainHTTPReachableAfterPublication(t *testing.T) { t.Fatal(err) } } - a, b := strings.Repeat("a", 40), strings.Repeat("b", 40) - for _, revision := range []string{a, b} { - prefix := "releases/" + revision + "/" - write(prefix+"manifest.json", `{"source_commit":"`+revision+`","artifacts":{"native/bin/oac-node":{"filename":"node"},"private/key":{"filename":"key"}}}`) - write(prefix+"node-install.pyz", "installer-"+revision) - write(prefix+"artifacts/node", "binary-"+revision) - write(prefix+"artifacts/key", "secret-must-not-be-served") - } + write("manifest.json", `{"source_commit":"`+strings.Repeat("a", 40)+`","artifacts":{"native/bin/oac-node":{"filename":"node"},"private/key":{"filename":"key"}}}`) + write("node-install.pyz", "installer") + write("artifacts/node", "binary") + write("artifacts/key", "secret-must-not-be-served") + write("secrets/core.key", "secret-must-not-be-served") payload, err := os.OpenRoot(root) if err != nil { t.Fatal(err) @@ -167,33 +155,22 @@ func TestRetainedPayloadsRemainHTTPReachableAfterPublication(t *testing.T) { h.serveNodePayload(w, httptest.NewRequest("GET", "/node-install/"+path, nil)) return w.Code, w.Body.String() } - for _, revision := range []string{a, b} { - write("active.json", `{"source_commit":"`+revision+`"}`) - if code, body := request("node-install.pyz"); code != 200 || body != "installer-"+revision { - t.Fatal("active release not served", code, body) - } - // Add node commands verify the active release's installer without a Web restart. - config := httptest.NewRecorder() - h.serveConsoleConfiguration(config, httptest.NewRequest("GET", "/console/config", nil)) - digest := sha256.Sum256([]byte("installer-" + revision)) - if !strings.Contains(config.Body.String(), `"node_installer_sha256":"`+hex.EncodeToString(digest[:])+`"`) { - t.Fatal("installer digest does not follow the active release", config.Body.String()) - } - if code, body := request("releases/" + a + "/artifacts/node"); code != 200 || body != "binary-"+a { - t.Fatal("retained release unreachable", code, body) - } - for _, name := range []string{"active.json", "releases/" + a + "/artifacts/key", "releases/" + a + "/secrets/core.key", "releases/../../node-install.pyz"} { - if code, _ := request(name); code != 404 { - t.Fatal("non-public file exposed", name, code) - } - } + if code, body := request("node-install.pyz"); code != 200 || body != "installer" { + t.Fatal("installer not served", code, body) + } + // Add node commands verify the installer by this digest. + config := httptest.NewRecorder() + h.serveConsoleConfiguration(config, httptest.NewRequest("GET", "/console/config", nil)) + digest := sha256.Sum256([]byte("installer")) + if !strings.Contains(config.Body.String(), `"node_installer_sha256":"`+hex.EncodeToString(digest[:])+`"`) { + t.Fatal("installer digest does not follow the payload", config.Body.String()) } - write("releases/"+a+"/manifest.json", `{"source_commit":"`+b+`"}`) - if code, _ := request("releases/" + a + "/node-install.pyz"); code != 404 { - t.Fatal("mismatched release identity was served") + if code, body := request("artifacts/node"); code != 200 || body != "binary" { + t.Fatal("declared artifact unreachable", code, body) } - write("active.json", `{"source_commit":"../../"}`) - if code, _ := request("node-install.pyz"); code != 404 { - t.Fatal("invalid pointer did not fail closed") + for _, name := range []string{"artifacts/key", "secrets/core.key", "releases/" + strings.Repeat("a", 40) + "/node-install.pyz", "../node-install.pyz"} { + if code, _ := request(name); code != 404 { + t.Fatal("non-public file exposed", name, code) + } } } diff --git a/services/web/node_artifacts.go b/services/web/node_artifacts.go index c17869b87..2e96ecc00 100644 --- a/services/web/node_artifacts.go +++ b/services/web/node_artifacts.go @@ -22,9 +22,9 @@ type nodeManifest struct { Artifacts map[string]nodeArtifact `json:"artifacts"` } -func (h *console) readNodeManifest(prefix string) (nodeManifest, error) { +func (h *console) readNodeManifest() (nodeManifest, error) { var manifest nodeManifest - f, err := h.nodePayload.Open(prefix + "manifest.json") + f, err := h.nodePayload.Open("manifest.json") if err != nil { return manifest, err } @@ -33,12 +33,10 @@ func (h *console) readNodeManifest(prefix string) (nodeManifest, error) { if err != nil || len(raw) > 1024*1024 || json.Unmarshal(raw, &manifest) != nil { return manifest, errors.New("invalid node manifest") } - if prefix != "releases/"+manifest.SourceCommit+"/" { - return manifest, errors.New("node manifest release mismatch") - } return manifest, nil } +var payloadRevision = regexp.MustCompile(`^[0-9a-f]{40}$`) var nodeArtifactName = regexp.MustCompile(`^[A-Za-z0-9._-]+$`) var nodeArtifactDigest = regexp.MustCompile(`^[0-9a-f]{64}$`) diff --git a/services/web/node_artifacts_test.go b/services/web/node_artifacts_test.go index 8dc245484..6fdc01150 100644 --- a/services/web/node_artifacts_test.go +++ b/services/web/node_artifacts_test.go @@ -14,7 +14,6 @@ import ( func TestOnlineNodeArtifactsRedirectWithoutLocalPayload(t *testing.T) { root := t.TempDir() revision := strings.Repeat("a", 40) - prefix := "releases/" + revision + "/" write := func(name string, data []byte) { t.Helper() full := filepath.Join(root, name) @@ -31,8 +30,7 @@ func TestOnlineNodeArtifactsRedirectWithoutLocalPayload(t *testing.T) { manifest.Artifacts[logical] = nodeArtifact{Filename: "oac-" + revision + "-" + strings.ReplaceAll(logical, "/", "-"), Size: 9, SHA256: hex.EncodeToString(sum[:])} } raw, _ := json.Marshal(manifest) - write(prefix+"manifest.json", raw) - write("active.json", []byte(`{"source_commit":"`+revision+`"}`)) + write("manifest.json", raw) payload, err := os.OpenRoot(root) if err != nil { t.Fatal(err) @@ -45,31 +43,31 @@ func TestOnlineNodeArtifactsRedirectWithoutLocalPayload(t *testing.T) { name := manifest.Artifacts["images/runtime.tar.gz"].Filename for _, method := range []string{"GET", "HEAD"} { w := httptest.NewRecorder() - h.serveNodePayload(w, httptest.NewRequest(method, "/node-install/"+prefix+"artifacts/"+name, nil)) + h.serveNodePayload(w, httptest.NewRequest(method, "/node-install/artifacts/"+name, nil)) if w.Code != 307 || w.Header().Get("Location") != manifest.ArtifactBaseURL+"/"+name { t.Fatal(w.Code, w.Header()) } } - if _, err := os.Stat(filepath.Join(root, prefix, "artifacts")); !os.IsNotExist(err) { + if _, err := os.Stat(filepath.Join(root, "artifacts")); !os.IsNotExist(err) { t.Fatal("Web cached execution artifacts", err) } for _, name := range []string{"unknown", "../manifest.json", "private-key"} { w := httptest.NewRecorder() - h.serveNodePayload(w, httptest.NewRequest("GET", "/node-install/"+prefix+"artifacts/"+name, nil)) + h.serveNodePayload(w, httptest.NewRequest("GET", "/node-install/artifacts/"+name, nil)) if w.Code != 404 { t.Fatal(w.Code, name) } } - write(prefix+"artifacts/"+name, []byte("node-data")) + write("artifacts/"+name, []byte("node-data")) w := httptest.NewRecorder() - request := httptest.NewRequest("GET", "/node-install/"+prefix+"artifacts/"+name, nil) + request := httptest.NewRequest("GET", "/node-install/artifacts/"+name, nil) request.Header.Set("Range", "bytes=5-") h.serveNodePayload(w, request) if w.Code != 206 || w.Body.String() != "data" { t.Fatal("offline artifact did not take precedence", w.Code, w.Body.String()) } // A damaged local file must not be hidden by a remote availability fallback. - write(prefix+"artifacts/"+name, []byte("bad")) + write("artifacts/"+name, []byte("bad")) if len(h.nodeArtifacts()) != 0 { t.Fatal("damaged local Runtime was advertised") } diff --git a/services/web/node_installation.go b/services/web/node_installation.go index 4825af720..65dd5135b 100644 --- a/services/web/node_installation.go +++ b/services/web/node_installation.go @@ -8,7 +8,6 @@ import ( "io" "net/http" "os" - "regexp" "sort" "strings" @@ -33,64 +32,29 @@ var optionalPayloadFiles = func() map[string]bool { return files }() -var payloadRevision = regexp.MustCompile(`^[0-9a-f]{40}$`) - -// activePayloadPrefix reads the installer's atomic release pointer once per request. -func activePayloadPrefix(root *os.Root) (string, error) { - raw, err := root.ReadFile("active.json") - if err != nil || len(raw) > 256 { - return "", errors.New("invalid active node payload") - } - var pointer struct { - SourceCommit string `json:"source_commit"` - } - if json.Unmarshal(raw, &pointer) != nil || !payloadRevision.MatchString(pointer.SourceCommit) { - return "", errors.New("invalid active node payload") - } - return "releases/" + pointer.SourceCommit + "/", nil -} - -func (h *console) resolveNodePayload(name string) (string, bool) { - var prefix string - if strings.HasPrefix(name, "releases/") { - parts := strings.SplitN(name, "/", 3) - if len(parts) != 3 || !payloadRevision.MatchString(parts[1]) { - return "", false - } - prefix, name = "releases/"+parts[1]+"/", parts[2] - } else { - var err error - prefix, err = activePayloadPrefix(h.nodePayload) - if err != nil { - return "", false - } - } +func (h *console) resolveNodePayload(name string) bool { if !nodePayloadFiles[name] && (!strings.HasPrefix(name, "artifacts/") || strings.Contains(strings.TrimPrefix(name, "artifacts/"), "/")) { - return "", false + return false } - manifest, err := h.readNodeManifest(prefix) + manifest, err := h.readNodeManifest() if err != nil { - return "", false + return false } if nodePayloadFiles[name] { - return prefix + name, true + return true } for logical, entry := range manifest.Artifacts { if optionalPayloadFiles[logical] && entry.Filename != "" && "artifacts/"+entry.Filename == name { - return prefix + name, true + return true } } - return "", false + return false } // installerDigest returns the SHA-256 that Web's install commands verify // before running the named installer from the payload. func installerDigest(root *os.Root, name string) (string, error) { - prefix, err := activePayloadPrefix(root) - if err != nil { - return "", err - } - f, err := root.Open(prefix + name) + f, err := root.Open(name) if err != nil { return "", errors.New("installer " + name + " is missing from the node installation payload") } @@ -113,16 +77,15 @@ func (h *console) serveNodePayload(w http.ResponseWriter, r *http.Request) { return } name := strings.TrimPrefix(r.URL.Path, "/node-install/") - resolved, allowed := h.resolveNodePayload(name) - if !allowed { + if !h.resolveNodePayload(name) { http.NotFound(w, r) return } - f, err := h.nodePayload.Open(resolved) + f, err := h.nodePayload.Open(name) if err != nil { if errors.Is(err, os.ErrNotExist) { - if prefix, filename, ok := strings.Cut(resolved, "artifacts/"); ok { - if manifest, readErr := h.readNodeManifest(prefix); readErr == nil { + if filename, ok := strings.CutPrefix(name, "artifacts/"); ok { + if manifest, readErr := h.readNodeManifest(); readErr == nil { if target := manifest.artifactURL(filename); target != "" { http.Redirect(w, r, target, http.StatusTemporaryRedirect) return @@ -145,18 +108,13 @@ func (h *console) serveNodePayload(w http.ResponseWriter, r *http.Request) { // nodeArtifacts reports the providers whose node artifacts this console can serve: // each is present locally or has a pinned release download. Nodes verify every -// checksum themselves. It is read per -// request, so artifacts added by rerunning the installer show without a restart. +// checksum themselves. It is read per request. func (h *console) nodeArtifacts() []string { available := []string{} if h.nodePayload == nil { return available } - prefix, err := activePayloadPrefix(h.nodePayload) - if err != nil { - return available - } - manifest, err := h.readNodeManifest(prefix) + manifest, err := h.readNodeManifest() if err != nil { return available } @@ -165,7 +123,7 @@ func (h *console) nodeArtifacts() []string { for _, artifact := range artifacts { logical := artifact.Path entry, ok := manifest.Artifacts[logical] - info, err := h.nodePayload.Stat(prefix + "artifacts/" + entry.Filename) + info, err := h.nodePayload.Stat("artifacts/" + entry.Filename) local := err == nil && info.Mode().IsRegular() && info.Size() == entry.Size remote := errors.Is(err, os.ErrNotExist) && manifest.artifactURL(entry.Filename) != "" if !ok || entry.Filename == "" || strings.Contains(entry.Filename, "/") || (!local && !remote) { @@ -181,8 +139,8 @@ func (h *console) nodeArtifacts() []string { return available } -// serveConsoleConfiguration reads the active release on every request, so an -// upgrade that moves the pointer needs no Web restart. +// serveConsoleConfiguration reads the payload on every request, so artifacts +// added by rerunning the installer show without a Web restart. func (h *console) serveConsoleConfiguration(w http.ResponseWriter, _ *http.Request) { digest := "" if h.nodePayload != nil { diff --git a/services/web/node_installation_test.go b/services/web/node_installation_test.go index 6ee331c8f..277b36504 100644 --- a/services/web/node_installation_test.go +++ b/services/web/node_installation_test.go @@ -35,7 +35,7 @@ func TestPairedConsoleProxiesOnlyAdministration(t *testing.T) { defer upstream.Close() u, _ := url.Parse(upstream.URL) dist, payload := t.TempDir(), t.TempDir() - release := activeRelease(t, payload, map[string]any{}) + release := publishedPayload(t, payload, map[string]any{}) for _, file := range []struct{ path, value string }{{filepath.Join(dist, "index.html"), "console"}, {filepath.Join(release, "node-install.pyz"), "print('installer')"}, {filepath.Join(release, "self-hosted-install.pyz"), "print('self-hosted')"}, {filepath.Join(release, "caller.key"), "must-not-be-served"}} { if err := os.WriteFile(file.path, []byte(file.value), 0600); err != nil { t.Fatal(err)