From e297d24bc1cdff1fc04804522fd7eb6396a8b9c8 Mon Sep 17 00:00:00 2001 From: mintaka Date: Fri, 16 Oct 2026 22:28:51 -0400 Subject: [PATCH 01/10] docs(compass): design the Compass Server on Kubernetes (RIG-5559) Server image, a public server-manifests renderer, and Runner egress and CA values. Ledgers D1-D9 as DL-539 to DL-547. Co-authored-by: Matt Wilkinson --- .../compass/decisions/server/DL-539.md | 6 + .../compass/decisions/server/DL-540.md | 6 + .../compass/decisions/server/DL-541.md | 6 + .../compass/decisions/server/DL-542.md | 6 + .../compass/decisions/server/DL-543.md | 6 + .../compass/decisions/server/DL-544.md | 6 + .../compass/decisions/server/DL-545.md | 6 + .../compass/decisions/server/DL-546.md | 6 + .../compass/decisions/server/DL-547.md | 6 + docs/designs/compass/server-on-kubernetes.md | 626 ++++++++++++++++++ 10 files changed, 680 insertions(+) create mode 100644 docs/designs/compass/decisions/server/DL-539.md create mode 100644 docs/designs/compass/decisions/server/DL-540.md create mode 100644 docs/designs/compass/decisions/server/DL-541.md create mode 100644 docs/designs/compass/decisions/server/DL-542.md create mode 100644 docs/designs/compass/decisions/server/DL-543.md create mode 100644 docs/designs/compass/decisions/server/DL-544.md create mode 100644 docs/designs/compass/decisions/server/DL-545.md create mode 100644 docs/designs/compass/decisions/server/DL-546.md create mode 100644 docs/designs/compass/decisions/server/DL-547.md create mode 100644 docs/designs/compass/server-on-kubernetes.md diff --git a/docs/designs/compass/decisions/server/DL-539.md b/docs/designs/compass/decisions/server/DL-539.md new file mode 100644 index 0000000..bacfceb --- /dev/null +++ b/docs/designs/compass/decisions/server/DL-539.md @@ -0,0 +1,6 @@ +--- +id: DL-539 +decision: "Each stack child that runs as a container becomes its own pod: the Server is a one-replica Recreate Deployment, NATS a StatefulSet, Postgres a dev-only StatefulSet on k3s and an external managed database on EKS; the OTel collector is external and phase one deploys no LLM gateway." +status: "Active (Matt, 2026-10-17)" +record: ../../server-on-kubernetes.md#design +--- diff --git a/docs/designs/compass/decisions/server/DL-540.md b/docs/designs/compass/decisions/server/DL-540.md new file mode 100644 index 0000000..270b5b1 --- /dev/null +++ b/docs/designs/compass/decisions/server/DL-540.md @@ -0,0 +1,6 @@ +--- +id: DL-540 +decision: "The Server ships as `ghcr.io/rigelbuild/compass-server`, built like the Runner image: the `.#compass-server` nix closure (the binary, secretspec and the wrapper's bash) on the pinned distroless nonroot base, uid 65532, published by digest and gated by a `SERVER_IMAGE_CLOSURE_PATHS` set that covers every closure input." +status: "Active (Matt, 2026-10-17)" +record: ../../server-on-kubernetes.md#design +--- diff --git a/docs/designs/compass/decisions/server/DL-541.md b/docs/designs/compass/decisions/server/DL-541.md new file mode 100644 index 0000000..c5dc532 --- /dev/null +++ b/docs/designs/compass/decisions/server/DL-541.md @@ -0,0 +1,6 @@ +--- +id: DL-541 +decision: "Migrations stay in Server boot against a direct, session-mode database endpoint; there is no migration Job." +status: "Active (Matt, 2026-10-17)" +record: ../../server-on-kubernetes.md#design +--- diff --git a/docs/designs/compass/decisions/server/DL-542.md b/docs/designs/compass/decisions/server/DL-542.md new file mode 100644 index 0000000..b23bcb1 --- /dev/null +++ b/docs/designs/compass/decisions/server/DL-542.md @@ -0,0 +1,6 @@ +--- +id: DL-542 +decision: "The Server renderer names Secrets but holds no values; Server secrets are read through `dotenv://` from a read-only mounted Secret file, `XDG_STATE_HOME` and the state dir sit on an emptyDir, and a digest-pinned init container copies the admin token into a 0600 file." +status: "Active (Matt, 2026-10-17)" +record: ../../server-on-kubernetes.md#design +--- diff --git a/docs/designs/compass/decisions/server/DL-543.md b/docs/designs/compass/decisions/server/DL-543.md new file mode 100644 index 0000000..e6f91f4 --- /dev/null +++ b/docs/designs/compass/decisions/server/DL-543.md @@ -0,0 +1,6 @@ +--- +id: DL-543 +decision: "In-cluster NATS requires a user and password and a NetworkPolicy admits port 4222 only from the Server pod." +status: "Active (Matt, 2026-10-17)" +record: ../../server-on-kubernetes.md#design +--- diff --git a/docs/designs/compass/decisions/server/DL-544.md b/docs/designs/compass/decisions/server/DL-544.md new file mode 100644 index 0000000..6b21c6e --- /dev/null +++ b/docs/designs/compass/decisions/server/DL-544.md @@ -0,0 +1,6 @@ +--- +id: DL-544 +decision: "The Server terminates its own TLS with a long-lived certificate from the cluster's existing private-CA ClusterIssuer and restarts on renewal; `serverAddr` is the in-cluster Service and external access is TCP passthrough." +status: "Active (Matt, 2026-10-17)" +record: ../../server-on-kubernetes.md#design +--- diff --git a/docs/designs/compass/decisions/server/DL-545.md b/docs/designs/compass/decisions/server/DL-545.md new file mode 100644 index 0000000..bfe1292 --- /dev/null +++ b/docs/designs/compass/decisions/server/DL-545.md @@ -0,0 +1,6 @@ +--- +id: DL-545 +decision: "A k3s Runner-cluster entry keeps the default issuer and uses a `jwksFile` rendered from the cluster's public keys, so no Go change and no anonymous discovery is needed; the rendered Runner gets `serverCA` and an `egressAllow` list rendered as `--egress-allow`." +status: "Active (Matt, 2026-10-17)" +record: ../../server-on-kubernetes.md#design +--- diff --git a/docs/designs/compass/decisions/server/DL-546.md b/docs/designs/compass/decisions/server/DL-546.md new file mode 100644 index 0000000..224e3ff --- /dev/null +++ b/docs/designs/compass/decisions/server/DL-546.md @@ -0,0 +1,6 @@ +--- +id: DL-546 +decision: "Public compass holds `tools/server-manifests/`; the operator's infra repo holds per-cluster values, renders at a pinned compass commit, and Flux applies the output; a scheduled job there moves the pin and resolves both image digests from one commit." +status: "Active (Matt, 2026-10-17)" +record: ../../server-on-kubernetes.md#design +--- diff --git a/docs/designs/compass/decisions/server/DL-547.md b/docs/designs/compass/decisions/server/DL-547.md new file mode 100644 index 0000000..57b84d5 --- /dev/null +++ b/docs/designs/compass/decisions/server/DL-547.md @@ -0,0 +1,6 @@ +--- +id: DL-547 +decision: "Self-hosted k3s comes first and is done when Server and Runner run there, a session from an out-of-cluster CLI completes with agent egress to the provider and the forge, and dev points at the cluster on a fresh database; EKS follows with the same renderers, bare-metal Runner nodes and a spend cap per new meter." +status: "Active (Matt, 2026-10-17)" +record: ../../server-on-kubernetes.md#design +--- diff --git a/docs/designs/compass/server-on-kubernetes.md b/docs/designs/compass/server-on-kubernetes.md new file mode 100644 index 0000000..d20b4ca --- /dev/null +++ b/docs/designs/compass/server-on-kubernetes.md @@ -0,0 +1,626 @@ +# Compass Server on Kubernetes — self-hosted k3s first, then EKS + +> Freezes on merge; later changes supersede by citation, never rewrite. +> Unless a path says otherwise, every path is in +> [RigelBuild/compass](https://github.com/RigelBuild/compass) at `main`. +> Compass design records moved out of `docs/designs/`; this record cites them +> at commit [`53ab753`](https://github.com/RigelBuild/compass/tree/53ab753f089ef1293d66736f247d41d38063dc7f/docs/designs), +> where they last lived. + +**Named decisions this record introduces:** D1–D9, inline in +[Design](#design), ledgered as DL-539 to DL-547 in `decisions/server/`. + +## Open Questions + +Each question names a recommendation, and the design is written against it. +OQ-2 and OQ-3 are closed by the source and stay as rationale. + +**OQ-1 (non-load-bearing) — Bootstrap admin token from a Secret.** +`readSuppliedAdminToken` (`go/server/network_door.go`) rejects a symlink and +any group or other bit. A Secret volume file is a symlink, and with `fsGroup` +kubelet adds group bits, so a Go relaxation does not help either. + +1. An init container copies the token into a `0600` file on an emptyDir. + The init image is `busybox`, pinned by digest in the values, because the + distroless Server image has no `cp` and its wrapper's bash has no stable + path. +2. A Go change accepts a symlinked file with a `0400` target. +3. No supplied token: the Server mints one into its state dir on every + restart. + +**Recommendation:** 1. It needs no Go change, and it keeps the token stable +across restarts. + +**OQ-2 (closed) — Server secret provider.** `dotenv://` from a read-only +mounted Secret file. `SpecResolver.buildManifest` +(`go/internal/secrets/resolver.go`) declares the exact stored names +(`COMPASS_MASTER_KEY`, `SERVER_`). The flake check +`tools/flake-gate/compass-server-secretspec.nix` and the e2e fixture +(`go/e2e/fixture.go`) already boot through `dotenv://`. Unlike `env://`, the +file keeps the master key out of the Server's process env, and secretspec +re-reads it on each resolve, so a rotated Secret needs no restart. + +**OQ-3 (closed) — LLM gateway placement.** Phase one deploys no gateway. +Only `compass-stack` refuses to start without one (`go/internal/stack/stack.go`: +"gateway image is required; pass --gateway-image or --gateway-external"), and +D1 drops compass-stack on Kubernetes. `compass-server` has no gateway flag, +and no code under `go/` delivers a gateway URL to agents yet: that is T5 of +the [gateway record](https://github.com/RigelBuild/compass/blob/53ab753f089ef1293d66736f247d41d38063dc7f/docs/designs/server/compass-server-llm-gateway/design.md). +Agents keep today's shape, a raw provider key plus an egress allowlist (D7). +The gateway's Kubernetes placement, and its service token, belong to a later +record. + +**OQ-4 (non-load-bearing) — Server certificate.** DNS-01 needs only a public +zone, not a reachable host, so both options work on a self-hosted k3s cluster. + +1. A long-lived certificate (for example one year) from the cluster's + existing private-CA ClusterIssuer. The Runner needs a `serverCA` value + (T5), and clients need `--ca` or `ca_cert`. `serverAddr` stays + `https://compass-server..svc`. +2. A public DNS-01 certificate for the external name. This drops the CA + plumbing. The Runner then dials the external name, which must resolve + in-cluster (split-horizon DNS or a CoreDNS rewrite), and the `.svc` SAN is + lost. + +**Recommendation:** 1. It reuses an issuer that exists, and it keeps the +Runner on the in-cluster name. EKS may choose either. + +**OQ-5 (load-bearing for EKS only) — Managed Postgres and BYPASSRLS.** +`0001_init.sql` runs `CREATE ROLE compass_system NOLOGIN BYPASSRLS` under +`IF NOT EXISTS`, then always runs `ALTER ROLE compass_system BYPASSRLS`. +PostgreSQL lets only a superuser or a role that holds BYPASSRLS set that +attribute ([CREATE ROLE](https://www.postgresql.org/docs/current/sql-createrole.html)). +`WithSystemRole` (`go/internal/store/tenant_tx.go`) uses the role at runtime, +so it is load-bearing. The RDS master user is `NOSUPERUSER` +([rds_superuser](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Appendix.PostgreSQL.CommonDBATasks.Roles.rds_superuser.html)), +so roles created out of band do not help there. + +1. A managed provider whose owner role holds BYPASSRLS and CREATEROLE. Neon + grants BYPASSRLS to its owner role for projects created after + 2023-08-15. +2. An in-cluster Postgres operator on EKS, whose superuser runs the + migration unchanged. +3. RDS or Aurora, with a schema change that drops the BYPASSRLS role. This + is a Go and SQL record of its own. + +**Recommendation:** 1. It needs no code change. T7 confirms the owner role's +attributes before it wires the DSN. + +**OQ-6 (non-load-bearing) — Image lane reuse.** + +1. A new `tools/server-image/` imports the pure cores of + `tools/runner-image/`. +2. Generalize `tools/runner-image/` to build both images. + +**Recommendation:** 1. The Runner build carries a kernel and guest image that +the Server does not need. T3 (compass PR #2355) implements 1 and adds a +`--lane runner|server` flag to the shared publish and retag scripts. + +**OQ-7 (non-load-bearing) — Server image visibility.** The Runner package is +private (`tools/runner-image/retag.ts`). + +1. The Server package is public. +2. The Server package is private, and clusters need a pull secret. + +**Recommendation:** 1. The Server binary already ships in public release +assets. + +**OQ-8 (non-load-bearing) — Sessions across a Server restart.** A dropped +session stream ends `runner.Run` with "runner sessions loop" and the Runner +exits (`go/internal/runner/run.go`). The renderer has no liveness probe +because "a pid-1 restart tears down all node sessions" +(`tools/runner-manifests/render-core.ts`). Every Server deploy and every +certificate renewal therefore kills live sessions. + +1. Accept the loss in phase one. Deploys and renewals are rare and + operator-timed. +2. A separate record makes the Runner survive a Server restart: it re-dials, + re-enrolls with the hub, and keeps its microVMs. +3. Add in-process TLS reload to the door. This covers renewals only, not + deploys. + +**Recommendation:** 1 now, and file 2 as its own record. Option 3 is the +deferred follow-up F1 in [Plan](#plan). + +**OQ-9 (non-load-bearing) — Who runs Postgres on k3s.** + +1. The Server renderer keeps an optional Postgres StatefulSet, marked + dev-only, with a backup note. +2. The renderer never renders Postgres. The operator installs a Postgres + operator on k3s, and EKS uses a managed database. + +**Recommendation:** 1. It is the fastest path, and the operator's Postgres +operator is not installed on the k3s cluster today. An absent `postgres` value +already renders no Postgres objects, so moving to 2 later needs no renderer +change. The StatefulSet is not a supported product surface: it has no major +upgrade path and no point-in-time recovery. + +**OQ-10 (non-load-bearing) — Digest bumps.** A dev cluster tracks main, but +D8 pins both digests to one compass commit. + +1. Re-render by hand on each bump. +2. A scheduled job in the operator's infra repo resolves both digests at one + compass commit, re-renders, and opens a pull request. +3. Flux image automation. It bumps each image alone, which breaks the + one-commit pairing. + +**Recommendation:** 2, sized as a task in T6. Each merged bump restarts the +Server (OQ-8). + +**OQ-11 (non-load-bearing) — Dev state.** Moving the current dev database +needs a dump and restore and the same master key. `reconcileKeyState` +(`go/server/serve.go`) fails boot on a key mismatch. Today's dev Server reads +the key through secretspec's default chain. + +1. Fresh install. Operators re-enter secrets with `compass secret set` and + push agent configs again. +2. `pg_dump` the dev database, restore it, and seed the same master key. + +**Recommendation:** 1. Dev holds no history worth the key-handling risk, and +the restore path is untested. + +## Problem / Intent + +The Runner can run on Kubernetes: it has a digest-published image, a manifest +renderer, and projected-token identity. Nothing deploys a Compass Server +there, so the Runner's `serverAddr` has no target. This record adds a Server +image, a public renderer for the Server and its data services, the Runner +values that agents need, and the node and delivery steps. With them, Server +plus Runner run end to end on a self-hosted k3s cluster, and the dev environment moves +there. EKS follows. + +## Approach + +### Ground truth + +- **Stack children.** `spawnChain` (`go/internal/stack/stack.go`) starts + Postgres, the OTel collector, NATS, the LLM gateway, compass-server, then + compass-runner. The first four are podman containers. The Server and + Runner are host processes, and the Server receives the door as + `--listen-fd 3` (`serverSpec`, `go/internal/stack/spec.go`). +- **Boot order.** `Serve` (`go/server/serve.go`) binds listeners, then + `openFabricAndStore` opens NATS before the store. `store.Open` + (`go/internal/store/store.go`) pings once without retry. It then applies + the embedded migrations under the session-scoped advisory lock + `migrationLockKey`. +- **RLS roles.** `0001_init.sql` creates `compass_app` (NOLOGIN) and + `compass_system` (NOLOGIN BYPASSRLS), always re-asserts BYPASSRLS, and + grants both to the owner. The owner therefore needs CREATEROLE and + BYPASSRLS (OQ-5). +- **Door.** `loadNetworkTLS` (`go/server/network_door.go`) loads the keypair + once, with a TLS 1.3 floor and h2. It has no reload. The Server serves no + UI files. `ServeConfig.CORSAllowedOrigin` admits one browser origin. +- **Health.** `go/server/health.go` serves `GET /healthz` and grpc health on + every door. It reports NOT_SERVING once drain starts. `compass.runner` + reports SERVING only while a Runner is enrolled. +- **Secrets.** `resolveMasterKey` (`go/server/serve.go`) fails boot without a + 64-hex `COMPASS_MASTER_KEY`. The flake's `compass-server` is a + `wrapProgram` bash wrapper that puts secretspec on `PATH` (`flake.nix`). + `run` in `go/internal/secrets/resolver.go` discards secretspec's stderr, so + a bad provider surfaces only as `exit status 1` under + "resolve master key". secretspec writes an audit log under + `$XDG_STATE_HOME`, and a failed write does not block a resolve. +- **State dir.** `--state-dir` defaults to the socket's parent. + `ensurePrivateDir` (`go/server/socket.go`) creates missing dirs at `0700` + and leaves an existing dir alone. `warnIfSharedTokenDir` warns when the + token dir has group or other bits, as a fresh emptyDir root does. +- **Runner.** `renderRunnerManifests` (`tools/runner-manifests/render-core.ts`) + takes `serverAddr`, which must be absolute `https`, and emits env only, no + `args`. The renderer has no CA value, so the Runner trusts system roots + unless `COMPASS_RUNNER_CA` is set (`go/cmd/compass-runner/main.go`). The + hub holds one Runner (`go/internal/runnerhub/hub.go`). `Run` + (`go/internal/runner/run.go`) returns an error when the session stream + drops. +- **Agent egress.** The microVM always arms a default-deny firewall. + `--egress-allow` (`go/cmd/compass-runner/main.go`) is the only way to open + a host, and it has no env fallback. `parseEgress` treats an empty value as + "no host reachable". `isValidHost` (`go/internal/runtime/host.go`) admits + DNS names and bare IP literals, never `host:port`. Agents call providers + with a raw key from the provider seed + (`go/internal/runtime/secrets_materialize.go`). +- **Issuer keys.** `fetchHTTPS` (`go/internal/auth/workload_identity.go`) + sends no `Authorization` header, and k3s runs with anonymous auth off. A + `jwksFile` entry never fetches: its keys "change only on rollout". + `validateRunnerClusters` (`go/internal/auth/runner_clusters.go`) requires + issuers to be unique only among the configured entries, so the k3s default + issuer `https://kubernetes.default.svc.cluster.local` is valid. +- **Image lane.** `runner-image/Dockerfile` copies a nix closure onto a + digest-pinned distroless `static-debian12` nonroot base. + `publish-runner-image` and `release-runner-image` + (`.github/workflows/release.yml`) push by digest and re-tag `:vX.Y.Z` by + digest. `resolveAncestor` (`tools/runner-image/retag-core.ts`) finds the + newest `:git-` publish at or before a commit. +- **Node shape** ([spike findings](https://github.com/RigelBuild/compass/blob/53ab753f089ef1293d66736f247d41d38063dc7f/docs/designs/platform/compass-runner-containerization/spike-findings.md)). + `TestFullBoot` passed on a bare-metal k3s node. That node had kernel 6.18, + `/dev/kvm` `0666`, the generic device plugin and `hostUsers: false`. It has + no AppArmor. The Ubuntu VM run needed + `kernel.apparmor_restrict_unprivileged_userns=0`. + +### Design + +**Topology (D1).** Each stack child that runs as a container today becomes +its own pod. The Server container supervises nothing. + +| Child | k3s | EKS | +| --- | --- | --- | +| compass-server | Deployment, `replicas: 1`, `strategy: Recreate` | same | +| NATS | StatefulSet, 1 replica, PVC, `DefaultNatsImage`, the stack's R1 file JetStream config | same | +| Postgres | dev-only StatefulSet, PVC, `DefaultPostgresImage` (OQ-9) | external managed Postgres (OQ-5) | +| OTel collector | none: `OTEL_EXPORTER_OTLP_ENDPOINT` points at the operator's collector | same | +| LLM gateway | not deployed in phase one (OQ-3) | same | +| compass-runner | the existing DaemonSet, on one KVM node | same, on bare-metal nodes | + +The Server runs one replica because the hub holds one Runner and the +delivery-cutover record keeps a single Server. `Recreate` also keeps two +Servers from migrating at once. Each rollout drops live sessions (OQ-8). + +**Server image (D2).** `ghcr.io/rigelbuild/compass-server` follows the Runner +image's shape. `server-image/Dockerfile` uses the same pinned base and copies +the realised `.#compass-server` closure. The image runs as uid 65532, and the +staged symlink `/nix/store/.compass-server` is its entrypoint. No other +compass binary ships. The closure carries secretspec and the wrapper's bash, +plus glibc and tzdata; an image scan sees them. Publishing reuses the Runner +lane (OQ-6). `SERVER_IMAGE_CLOSURE_PATHS` gates it, and the set is listed in +T3. + +**Data and migrations (D3).** Migrations stay in Server boot; there is no +Job. + +- The DSN must name a direct, session-mode endpoint, because the migration + lock is session-scoped. +- One replica fits the default pool of 10. +- On k3s, the Postgres superuser is `compass`, its password comes from a + Secret, and it logs in with `scram-sha-256`. On the first migration that + superuser creates the RLS roles. The StatefulSet is dev-only (OQ-9); the + operator backs up its volume and the master key. +- On EKS, the owner role must hold CREATEROLE and BYPASSRLS (OQ-5). + Operators turn compute suspend off. A suspended database fails the single + ping, and the pod restarts until it answers. + +**Secrets (D4).** The renderer names Secrets but never holds a value. + +- `COMPASS_DATABASE_DSN`, `COMPASS_NATS_URL` and `COMPASS_S3_*` come from + Secrets as env. +- Server secrets come from one Secret key, `server.env`, mounted read-only + at `/etc/compass/secrets/`. The Server reads it through + `--secret-provider dotenv:///etc/compass/secrets/server.env` (OQ-2). The + file holds `COMPASS_MASTER_KEY` and the forge `SERVER_*` names. A wrong path + or a bad file fails boot with only `exit status 1`, because secretspec's + stderr is discarded; the README says to check the mount first. +- `XDG_STATE_HOME` is `/run/compass/xdg-state`, on the emptyDir, so + secretspec's audit log has a writable home on a read-only root. +- An init container copies the admin token into a `0600` file (OQ-1). +- The operator's infra repo creates every Secret with its own tooling. It + also backs up the master key, because losing it is unrecoverable + (`docs/self-host.md`). +- Agent provider keys are not Server secrets. An admin stores each one with + `compass secret set --kind provider --provider --delivery file`, + value on stdin. `--delivery file` matters: `Install` routes env delivery + before kind, so an env-delivered key never reaches the provider seed. + +**NATS access (D5).** The stack's NATS has no auth because it listens only on +loopback. In a cluster, a NetworkPolicy admits port 4222 only from the +Server pod. NATS also requires a user and password. The password reaches the +NATS config through a variable reference, and the credentials ride in +`COMPASS_NATS_URL`. The fabric's connect path already keeps URL credentials +out of its errors (`go/internal/fabric/fabric.go`). + +**Door, TLS and serverAddr (D6).** The Server terminates its own TLS. No +ingress may terminate it: the door refuses cleartext, and the Runner's bidi +stream needs h2 end to end. + +- **Certificate.** cert-manager issues a `kubernetes.io/tls` Secret from the + cluster's existing private-CA ClusterIssuer (OQ-4), with a long duration. + Its SANs cover the in-cluster name and the external name. +- **Renewal.** The door loads its keypair once. On renewal the operator's + reload annotation restarts the Server, which drops live sessions (OQ-8). + In-process reload is deferred (follow-up F1). +- **Services.** Service `compass-server` (ClusterIP) maps port 443 to + container port 50052. +- **serverAddr** is `https://compass-server..svc`. The Runner + trusts the CA through a new `serverCA` value (T5). +- **External access** is a LoadBalancer Service on 50052, passed through as + TCP, at an operator-owned DNS name. +- **Clients.** The CLI uses `--server-addr`, `--ca` and the admin token + (`go/cmd/compass/client.go`). The desktop app uses client mode with + `ca_cert` (`docs/self-host.md`). A browser origin comes from the + `corsAllowedOrigin` value. +- **Probes.** The startup and readiness probes are HTTPS `httpGet` on + `/healthz`. Readiness never depends on `compass.runner`, because the + Runner needs the Service before it can enroll. There is no liveness + probe. + +**In-cluster Runner (D7).** Three values make a rendered Runner useful. + +- **Identity.** The k3s entry keeps the cluster's default issuer and uses + `jwksFile`. The operator's infra repo reads the public keys with + `kubectl get --raw /openid/v1/jwks` at render time, and the Server renderer + writes them into the runner-clusters ConfigMap. This needs no Go change and + no anonymous discovery. A k3s signing-key rotation is a manual operator + step, and it then needs a re-render and a Server rollout + (`docs/self-host.md`). An EKS issuer serves discovery anonymously and uses + `jwksURI` discovery as today. `tokenFile` is deferred (follow-up F2). +- **CA.** `serverCA` mounts the private CA and sets `COMPASS_RUNNER_CA`. +- **Egress.** `egressAllow` renders the Runner container's + `args: ["--egress-allow="]`. The renderer applies the + `isValidHost` grammar, so `host:port` fails at render time, not at Runner + start. Phase one lists the provider API host, `github.com` and + `api.github.com`. + +**Delivery split (D8).** Public compass holds `tools/server-manifests/`, in +the shape of `tools/runner-manifests/`: values in, plain objects out, +digest-only images, no Secrets. The operator's infra repo holds the values +for each cluster. It renders at a pinned compass commit and commits the +output, and Flux applies it. It also owns Secrets, the Certificate, the +device plugin and node setup. Both image digests resolve from one compass +commit: each is that image's newest publish at or before the commit, so the +Server and Runner API versions match. A scheduled job in the operator's +infra repo moves the pin (OQ-10). + +**Rollout (D9).** k3s comes first, as the dev environment. It is done when +three things hold: + +- Server and Runner run there. +- A session started from an out-of-cluster CLI completes, with its agent + reaching the provider and the forge. +- Dev points at the cluster, on a fresh database (OQ-11). + +EKS follows with the same renderers and other values. Managed-plane detail is +out of scope; see `docs/concepts/self-host-and-managed.md` and the +[OSS-core boundary record](https://github.com/RigelBuild/compass/blob/53ab753f089ef1293d66736f247d41d38063dc7f/docs/designs/meta/oss-core-managed-boundary/design.md). + +## Alternatives considered + +- **One pod running `compass-stack up`.** Rejected. The supervisor drives + podman and host processes, which is the host-level bring-up the + distribution record keeps. +- **Ingress-terminated TLS, re-encrypted to the door.** Rejected. It adds an + h2 proxy hop and a second certificate. +- **A migration Job.** Rejected. No migrate-only entrypoint exists, and the + advisory lock already serializes migrations. +- **In-process TLS reload now.** Deferred (F1): every deploy drops sessions + the same way (OQ-8). +- **A unique k3s issuer plus `tokenFile`.** Deferred (F2): a live issuer + change needs a dual-issuer window, and it is a Go change. +- **Anonymous discovery via AuthenticationConfiguration.** Not taken: it needs + Kubernetes ≥1.34 and an API-server change. +- **A `COMPASS_EGRESS_ALLOW` env fallback.** Not taken: a Go PR that `args` + makes unnecessary. + +## Plan + +### Global Constraints + +- **Public repo.** No operator names, hosts, zones or private-product + detail. See `docs/concepts/self-host-and-managed.md`. +- **PRs.** T3, T4 and T5 are independent PRs off compass `main`; none + stacks on another. Each passes alone, red then green. T6 and T7 live in the + operator's infra repo. +- **Images.** Digest only. Renderers reject tags, as + `assertRunnerImageDigest` does. The init image is a value, digest-pinned + and checked the same way. +- **Secrets.** Renderers emit no Secret object and no secret value. +- **Version coupling.** Server and Runner digests come from one compass + commit. +- **Kubernetes.** ≥1.33, for user namespaces and for the identity record's + ≥1.32 floor. +- **Server pod.** uid 65532, `readOnlyRootFilesystem`, drop ALL, + `RuntimeDefault` seccomp. The socket, state dir and `XDG_STATE_HOME` live + on one emptyDir at `/run/compass`; the state dir is a child the Server + creates at `0700`. +- **Lanes.** TypeScript is Bun with `bun:test`, as in + `tools/runner-manifests/`. Go DB tests use `pgtest`. +- **Spend.** Every new usage-billed meter on EKS ships with an enforced cap + per layer, a budget alert that pages, and a row in the operator's spend-cap + register, in the same change. + +### Order + +```text +T3 server image ──┐ +T4 server renderer ┼──► T6 k3s wiring + bump job ──► T7 EKS values +T5 runner values ──┘ +Deferred, off the critical path: F1 door TLS reload, F2 tokenFile +``` + +T3 must merge and publish on `main` before T6 can deploy. T4 and T5 can +merge in any order. + +### T3 — Server image and publish lane (CI, nix) + +In flight as compass PR #2355. Its smoke runs the locally built OCI image: + +- `--version` prints `compass-server `. +- Starting with no DSN exits 1 and names the missing DSN. +- The image config user is `65532:65532`. +- With Postgres and NATS containers, a read-only root and + `--secret-provider dotenv:///secrets/key.env` from a read-only mount, boot + resolves the master key and WhoAmI answers. NATS is required, because the + Server opens the fabric before the store. This mirrors + `tools/flake-gate/compass-server-secretspec.nix`. +- Two builds give the same manifest digest. + +`SERVER_IMAGE_CLOSURE_PATHS` must cover every input of the closure: +`go list -deps ./cmd/compass-server` reaches `go/cmd`, `go/events`, `go/gen`, +`go/internal` and `go/server`, and the secretspec pin comes from +`devenv.lock`. `go/**` is a superset; a Runner-only Go change then publishes +a Server image too, which is the safe direction. + +```text +server-image/{Dockerfile,moon.yml,.dockerignore} // pinned base from runner-image/Dockerfile +tools/server-image/{build.ts,build-core.ts} // ENTRYPOINT_LINK = ".compass-server", IMAGE_USER = "65532:65532" +tools/runner-image/image-lane-core.ts // imageLaneConfig(lane: string | undefined): ImageLaneConfig | undefined +tools/runner-image/{publish.ts,retag.ts} // --lane runner|server +.moon/workspace.yml // compass-server-image: 'server-image' +.github/workflows/release.yml // jobs publish-server-image, release-server-image +SERVER_IMAGE_CLOSURE_PATHS: + server-image/** tools/server-image/** tools/runner-image/** go/** + tools/toolchain/secretspec-env.nix devenv.lock flake.nix flake.lock + version.txt .github/workflows/release.yml +image: ghcr.io/rigelbuild/compass-server@sha256:, :git- and :vX.Y.Z by digest +``` + +### T4 — Server manifests renderer (TypeScript) + +Tests use golden output plus invariants: + +- `replicas: 1` with `Recreate`. +- Digest-only images, the init image included. +- No `Secret` kind. +- The NetworkPolicy admits 4222 only from the Server label. +- Probes hit `/healthz`; no liveness probe. +- An absent `postgres` value renders no Postgres objects. +- The pod security context is as listed in Global Constraints. +- `XDG_STATE_HOME` and `--state-dir` sit on the emptyDir, and the state dir + is not the emptyDir root. +- A `runnerClusters` entry with `jwks` renders a ConfigMap key and a + matching `jwksFile`; an entry with both `jwks` and `jwksURI` is rejected. + +The README documents the render steps, the JWKS read, and the +`exit status 1` boot failure. + +```text +tools/server-manifests/render-core.ts + export type RunnerClusterValues = { + name: string; issuer: string; audience?: string; + namespace: string; serviceAccount: string; maxTokenLifetime?: string; + jwksURI?: string; jwks?: string; // jwks: JWKS JSON text; renders jwksFile + caFile?: string; + } + export type ServerDeployValues = { + namespace: string; serverImage: string; natsImage: string; initImage: string; + postgres?: { image: string; storage: string; secretName: string }; // dev-only + natsStorage: string; tlsSecretName: string; dsnSecretName: string; + natsSecretName: string; serverSecretsSecretName: string; // key server.env + adminTokenSecretName: string; s3SecretName?: string; + corsAllowedOrigin?: string; otlpEndpoint?: string; + external?: { port: number; loadBalancerClass?: string }; + runnerClusters: RunnerClusterValues[]; + } + export function parseServerDeployValues(raw: unknown): ServerDeployValues + export function renderServerManifests(v: ServerDeployValues): ReturnType + export function assertServerImageDigest(ref: string): void + // objects: ServiceAccount, Deployment, Service, Service(LoadBalancer)?, + // ConfigMap runner-clusters (clusters.yaml + .jwks.json), + // NATS StatefulSet+Service+ConfigMap, Postgres StatefulSet+Service?, NetworkPolicy x2 + // init container (initImage, uid 65532): cp the admin token to + // /run/compass/bootstrap/admin-token, chmod 0600 + // server args: --listen 0.0.0.0:50052 --tls-cert /etc/compass/tls/tls.crt + // --tls-key /etc/compass/tls/tls.key --state-dir /run/compass/state + // --socket /run/compass/state/server.sock + // --admin-token-file /run/compass/bootstrap/admin-token + // --runner-clusters /etc/compass/runner-clusters/clusters.yaml + // --secret-provider dotenv:///etc/compass/secrets/server.env + // env: XDG_STATE_HOME=/run/compass/xdg-state +tools/server-manifests/{render.ts,values.example.json,README.md,moon.yml} +.moon/workspace.yml // server-manifests +``` + +### T5 — Runner renderer `serverCA` and `egressAllow` (TypeScript) + +When `serverCA` is set, the renderer mounts the ConfigMap key and sets +`COMPASS_RUNNER_CA`. When `egressAllow` is non-empty, the container gets one +`--egress-allow=` arg with the hosts comma-joined. Invalid hosts fail +`parseRunnerDeployValues`. With both absent, the output is unchanged. Tests +cover each case and a `github.com:443` rejection. The README covers both +values and lists the hosts a session needs. + +```text +tools/runner-manifests/render-core.ts + RunnerDeployValues.serverCA?: { configMapName: string; key: string } + // mount /etc/compass/server-ca/; env COMPASS_RUNNER_CA + RunnerDeployValues.egressAllow?: string[] + // args: ["--egress-allow=" + hosts.join(",")]; grammar of go/internal/runtime/host.go isValidHost + export function assertEgressHost(host: string): void +``` + +### T6 — k3s wiring in the operator's infra repo (operator infra) + +Node, declared in the host's config as code: + +- Promote the spike's generic device-plugin slice to its permanent place. +- The `/dev/kvm` `0666` udev rule. +- The seccomp profile staged at the kubelet root. +- Confirm Kubernetes ≥1.33. No issuer change and no AppArmor sysctl: the + KVM node has no AppArmor. + +Cluster: + +- One Certificate against the existing private-CA ClusterIssuer, with a long + duration and the reload annotation on the Server Deployment. +- The Secrets from D4, including `server.env`. +- The k3s JWKS read into the values. +- Rendered output is committed. Flux Kustomizations apply data, then + Server, then Runners. +- A scheduled bump job (OQ-10). For one compass commit it resolves each + lane's digest with `resolveAncestor` from + `tools/runner-image/retag-core.ts`, re-renders both, and opens a pull + request. + +Dev move (OQ-11): a fresh install. The admin re-enters forge and provider +secrets with `compass secret set`, provider keys with `--kind provider +--delivery file`, and pushes agent configs again. + +Acceptance: + +- Flux reports Ready. +- `/healthz` returns 200. +- `compass.runner` reports SERVING. +- An out-of-cluster CLI session completes, and its agent reaches the + provider and pushes to the forge. +- Dev points at the cluster. +- The bump job opens one pull request for a new compass commit. + +```text +inputs: tools/server-manifests + tools/runner-manifests at one compass commit +Runner values: serverAddr https://compass-server..svc, serverCA set, + egressAllow [, github.com, api.github.com] +Server values: runnerClusters[0] = { issuer: https://kubernetes.default.svc.cluster.local, jwks: } +``` + +### T7 — EKS values (operator infra) + +First confirm OQ-5's owner-role attributes. Then wire: + +- The external Postgres direct endpoint, with compute suspend off. +- A passthrough NLB. +- The cluster's public issuer with `jwksURI` discovery. +- A bare-metal node group that exposes `/dev/kvm`, sized for the Runner's + `sessionCapacity`. Every Runner pod requests one `kvmResourceName` device. +- For each new meter (managed Postgres, the NLB, bare-metal nodes): an + enforced cap at each layer, a budget alert that pages, and a spend-cap + register row, in the same change. + +Acceptance matches T6, without the dev move. + +### Deferred follow-ups + +- **F1 — Door reloads its TLS keypair (Go).** Taken when OQ-8 option 2 is + ruled out or renewals become frequent. Tests: after a rewrite the next + handshake presents the new serial; a corrupt rewrite keeps the old serial; + existing connections survive. + +```text +go/server/network_door.go + func newCertReloader(certFile, keyFile string, log *slog.Logger) (*certReloader, error) + func (r *certReloader) GetCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) +``` + +- **F2 — `tokenFile` on a Runner cluster (Go).** Taken when a cluster needs + live key rotation without a rollout. Key fetches send + `Authorization: Bearer` with the file's contents, re-read on every fetch. + +```text +go/internal/auth/runner_clusters.go + RunnerCluster.TokenFile string // yaml:"tokenFile", optional +go/internal/auth/workload_identity.go + func fetchHTTPS(ctx context.Context, client *http.Client, u, bearer string) ([]byte, error) +``` + +## Tasks + +- [ ] T3 — Server image and publish lane (CI, nix), compass PR #2355 +- [ ] T4 — `tools/server-manifests` renderer (TypeScript) +- [ ] T5 — Runner renderer `serverCA` and `egressAllow` (TypeScript) +- [ ] T6 — k3s wiring, bump job and dev move (operator's infra repo) +- [ ] T7 — EKS values, bare-metal nodes and spend caps (operator's infra repo) From 564d77ae6395332db8052ee8e3e7f5a8c5f8bffa Mon Sep 17 00:00:00 2001 From: mintaka Date: Sat, 17 Oct 2026 04:10:38 -0400 Subject: [PATCH 02/10] docs(compass): wire the forge App, pin one Runner and guard the master key (RIG-5559) Review fixes: forge args and App secrets, one Runner pod per Server, and a boot-only master key. Co-authored-by: Matt Wilkinson --- .../compass/decisions/server/DL-539.md | 2 +- .../compass/decisions/server/DL-542.md | 2 +- .../compass/decisions/server/DL-546.md | 2 +- .../compass/decisions/server/DL-547.md | 2 +- docs/designs/compass/server-on-kubernetes.md | 170 ++++++++++++------ 5 files changed, 121 insertions(+), 57 deletions(-) diff --git a/docs/designs/compass/decisions/server/DL-539.md b/docs/designs/compass/decisions/server/DL-539.md index bacfceb..20bc263 100644 --- a/docs/designs/compass/decisions/server/DL-539.md +++ b/docs/designs/compass/decisions/server/DL-539.md @@ -1,6 +1,6 @@ --- id: DL-539 -decision: "Each stack child that runs as a container becomes its own pod: the Server is a one-replica Recreate Deployment, NATS a StatefulSet, Postgres a dev-only StatefulSet on k3s and an external managed database on EKS; the OTel collector is external and phase one deploys no LLM gateway." +decision: "Each stack child that runs as a container becomes its own pod: the Server is a one-replica Recreate Deployment with exactly one Runner pod per Server, NATS a StatefulSet, Postgres a dev-only StatefulSet on k3s and an external managed database on EKS; the OTel collector is external and phase one deploys no LLM gateway." status: "Active (Matt, 2026-10-17)" record: ../../server-on-kubernetes.md#design --- diff --git a/docs/designs/compass/decisions/server/DL-542.md b/docs/designs/compass/decisions/server/DL-542.md index b23bcb1..f41310b 100644 --- a/docs/designs/compass/decisions/server/DL-542.md +++ b/docs/designs/compass/decisions/server/DL-542.md @@ -1,6 +1,6 @@ --- id: DL-542 -decision: "The Server renderer names Secrets but holds no values; Server secrets are read through `dotenv://` from a read-only mounted Secret file, `XDG_STATE_HOME` and the state dir sit on an emptyDir, and a digest-pinned init container copies the admin token into a 0600 file." +decision: "The Server renderer names Secrets but holds no values; Server secrets are read through `dotenv://` from a read-only mounted Secret file whose master key never changes, forge App ids and secret names render as `--forge-*` args, `XDG_STATE_HOME` and the state dir sit on an emptyDir, and a digest-pinned init container copies the admin token into a 0600 file." status: "Active (Matt, 2026-10-17)" record: ../../server-on-kubernetes.md#design --- diff --git a/docs/designs/compass/decisions/server/DL-546.md b/docs/designs/compass/decisions/server/DL-546.md index 224e3ff..89a2279 100644 --- a/docs/designs/compass/decisions/server/DL-546.md +++ b/docs/designs/compass/decisions/server/DL-546.md @@ -1,6 +1,6 @@ --- id: DL-546 -decision: "Public compass holds `tools/server-manifests/`; the operator's infra repo holds per-cluster values, renders at a pinned compass commit, and Flux applies the output; a scheduled job there moves the pin and resolves both image digests from one commit." +decision: "Public compass holds `tools/server-manifests/`; the operator holds per-cluster values, renders at a pinned compass commit, and has Flux apply the output; the operator's scheduled job moves the pin, resolves both image digests from one commit, and fails when a lane has no publish for a closure change." status: "Active (Matt, 2026-10-17)" record: ../../server-on-kubernetes.md#design --- diff --git a/docs/designs/compass/decisions/server/DL-547.md b/docs/designs/compass/decisions/server/DL-547.md index 57b84d5..95b2e6b 100644 --- a/docs/designs/compass/decisions/server/DL-547.md +++ b/docs/designs/compass/decisions/server/DL-547.md @@ -1,6 +1,6 @@ --- id: DL-547 -decision: "Self-hosted k3s comes first and is done when Server and Runner run there, a session from an out-of-cluster CLI completes with agent egress to the provider and the forge, and dev points at the cluster on a fresh database; EKS follows with the same renderers, bare-metal Runner nodes and a spend cap per new meter." +decision: "Self-hosted k3s comes first and is done when Server and Runner run there, a session from an out-of-cluster CLI completes with agent egress to the provider and a brokered forge push, and dev points at the cluster on a fresh database; EKS follows, gated on the managed-Postgres ruling, with the same renderers, one bare-metal Runner node and a spend cap per new meter." status: "Active (Matt, 2026-10-17)" record: ../../server-on-kubernetes.md#design --- diff --git a/docs/designs/compass/server-on-kubernetes.md b/docs/designs/compass/server-on-kubernetes.md index d20b4ca..5b1df08 100644 --- a/docs/designs/compass/server-on-kubernetes.md +++ b/docs/designs/compass/server-on-kubernetes.md @@ -37,8 +37,12 @@ mounted Secret file. `SpecResolver.buildManifest` (`COMPASS_MASTER_KEY`, `SERVER_`). The flake check `tools/flake-gate/compass-server-secretspec.nix` and the e2e fixture (`go/e2e/fixture.go`) already boot through `dotenv://`. Unlike `env://`, the -file keeps the master key out of the Server's process env, and secretspec -re-reads it on each resolve, so a rotated Secret needs no restart. +file keeps the master key out of the Server's process env. secretspec +re-reads the file on each resolve, so rotated `SERVER_*` values apply without +a restart. `COMPASS_MASTER_KEY` is the exception: `resolveMasterKey` +(`go/server/serve.go`) reads it once at boot, and `reconcileKeyState` fails +boot when it differs from the stored key. It must never change in the +mounted Secret. **OQ-3 (closed) — LLM gateway placement.** Phase one deploys no gateway. Only `compass-stack` refuses to start without one (`go/internal/stack/stack.go`: @@ -83,8 +87,10 @@ so roles created out of band do not help there. 3. RDS or Aurora, with a schema change that drops the BYPASSRLS role. This is a Go and SQL record of its own. -**Recommendation:** 1. It needs no code change. T7 confirms the owner role's -attributes before it wires the DSN. +**Recommendation:** 1. It needs no code change. Whichever option is ruled, +the owner login role needs CREATEROLE and BYPASSRLS (or superuser) and must +own the target database, so it can create objects in `public`. T7 is gated +on this ruling. **OQ-6 (non-load-bearing) — Image lane reuse.** @@ -139,8 +145,8 @@ upgrade path and no point-in-time recovery. D8 pins both digests to one compass commit. 1. Re-render by hand on each bump. -2. A scheduled job in the operator's infra repo resolves both digests at one - compass commit, re-renders, and opens a pull request. +2. The operator's scheduled job resolves both digests at one compass commit, + re-renders, and opens a change for review. 3. Flux image automation. It bumps each image alone, which breaks the one-commit pairing. @@ -248,11 +254,15 @@ its own pod. The Server container supervises nothing. | Postgres | dev-only StatefulSet, PVC, `DefaultPostgresImage` (OQ-9) | external managed Postgres (OQ-5) | | OTel collector | none: `OTEL_EXPORTER_OTLP_ENDPOINT` points at the operator's collector | same | | LLM gateway | not deployed in phase one (OQ-3) | same | -| compass-runner | the existing DaemonSet, on one KVM node | same, on bare-metal nodes | +| compass-runner | the existing DaemonSet, on exactly one KVM node | same: one bare-metal node | The Server runs one replica because the hub holds one Runner and the delivery-cutover record keeps a single Server. `Recreate` also keeps two Servers from migrating at once. Each rollout drops live sessions (OQ-8). +Each cluster runs exactly one Runner pod per Server: `Hub.enroll` +(`go/internal/runnerhub/hub.go`) replaces the attached Runner and clears its +session bindings, so a second Runner would drop the first one's sessions. +Multi-Runner is a separate record. **Server image (D2).** `ghcr.io/rigelbuild/compass-server` follows the Runner image's shape. `server-image/Dockerfile` uses the same pinned base and copies @@ -273,9 +283,9 @@ Job. Secret, and it logs in with `scram-sha-256`. On the first migration that superuser creates the RLS roles. The StatefulSet is dev-only (OQ-9); the operator backs up its volume and the master key. -- On EKS, the owner role must hold CREATEROLE and BYPASSRLS (OQ-5). - Operators turn compute suspend off. A suspended database fails the single - ping, and the pod restarts until it answers. +- On EKS, the owner role must hold CREATEROLE and BYPASSRLS and own the + database (OQ-5). Operators turn compute suspend off. A suspended database + fails the single ping, and the pod restarts until it answers. **Secrets (D4).** The renderer names Secrets but never holds a value. @@ -284,20 +294,34 @@ Job. - Server secrets come from one Secret key, `server.env`, mounted read-only at `/etc/compass/secrets/`. The Server reads it through `--secret-provider dotenv:///etc/compass/secrets/server.env` (OQ-2). The - file holds `COMPASS_MASTER_KEY` and the forge `SERVER_*` names. A wrong path + file holds `COMPASS_MASTER_KEY` and the `SERVER_` values. A wrong path or a bad file fails boot with only `exit status 1`, because secretspec's stderr is discarded; the README says to check the mount first. +- `SERVER_*` values re-resolve on use. `COMPASS_MASTER_KEY` is read once at + boot and must never change: a different key fails boot in + `reconcileKeyState`. - `XDG_STATE_HOME` is `/run/compass/xdg-state`, on the emptyDir, so secretspec's audit log has a writable home on a read-only root. - An init container copies the admin token into a `0600` file (OQ-1). -- The operator's infra repo creates every Secret with its own tooling. It - also backs up the master key, because losing it is unrecoverable - (`docs/self-host.md`). +- The operator creates every Secret with its own tooling. It also backs up + the master key, because losing it is unrecoverable (`docs/self-host.md`). - Agent provider keys are not Server secrets. An admin stores each one with `compass secret set --kind provider --provider --delivery file`, value on stdin. `--delivery file` matters: `Install` routes env delivery before kind, so an env-delivered key never reaches the provider seed. +**Forge credential (D4).** Agents get forge tokens only from the Server's +App broker. `buildGitCredentialBroker` (`go/server/git_credential.go`) +returns nil unless the primary App id is set, and user `GH_TOKEN`-style +secrets are rejected and stripped (`IsReservedGitHubEnvName`, +`go/internal/secrets/secrets.go`). So the renderer takes a `forge` value and +renders it as `--forge-*` args: App ids, installation ids and secret +names, never values. The App private keys and the webhook secret are +`SERVER_` lines in `server.env`, PEM values double-quoted with `\n` +escapes as `dotenvForgeValue` (`go/e2e/fixture.go`) writes them. They are not +set with `compass secret set`. Agent push also needs the reviewer App, +because the forge-write path runs only when both Apps are configured. + **NATS access (D5).** The stack's NATS has no auth because it listens only on loopback. In a cluster, a NetworkPolicy admits port 4222 only from the Server pod. NATS also requires a user and password. The password reaches the @@ -333,7 +357,7 @@ stream needs h2 end to end. **In-cluster Runner (D7).** Three values make a rendered Runner useful. - **Identity.** The k3s entry keeps the cluster's default issuer and uses - `jwksFile`. The operator's infra repo reads the public keys with + `jwksFile`. The operator reads the public keys with `kubectl get --raw /openid/v1/jwks` at render time, and the Server renderer writes them into the runner-clusters ConfigMap. This needs no Go change and no anonymous discovery. A k3s signing-key rotation is a manual operator @@ -349,13 +373,15 @@ stream needs h2 end to end. **Delivery split (D8).** Public compass holds `tools/server-manifests/`, in the shape of `tools/runner-manifests/`: values in, plain objects out, -digest-only images, no Secrets. The operator's infra repo holds the values -for each cluster. It renders at a pinned compass commit and commits the -output, and Flux applies it. It also owns Secrets, the Certificate, the -device plugin and node setup. Both image digests resolve from one compass -commit: each is that image's newest publish at or before the commit, so the -Server and Runner API versions match. A scheduled job in the operator's -infra repo moves the pin (OQ-10). +digest-only images, no Secrets. The operator holds the values for each +cluster, renders at a pinned compass commit, commits the output, and has +Flux apply it. The operator also owns Secrets, the Certificate, the device +plugin and node config. Both image digests resolve from one compass commit: +each is that image's newest publish at or before the commit. That pairing +matches the Server and Runner API versions only while both closure sets +cover every Go input, `go/gen` included; T3 owns that guarantee. The +operator's scheduled job moves the pin (OQ-10), and it fails when a lane has +no publish for a closure change. **Rollout (D9).** k3s comes first, as the dev environment. It is done when three things hold: @@ -394,8 +420,9 @@ out of scope; see `docs/concepts/self-host-and-managed.md` and the - **Public repo.** No operator names, hosts, zones or private-product detail. See `docs/concepts/self-host-and-managed.md`. - **PRs.** T3, T4 and T5 are independent PRs off compass `main`; none - stacks on another. Each passes alone, red then green. T6 and T7 live in the - operator's infra repo. + stacks on another. Each passes alone, red then green. T6 and T7 are + operator responsibilities: values, Secrets, the Certificate, node config + and Flux inputs. - **Images.** Digest only. Renderers reject tags, as `assertRunnerImageDigest` does. The init image is a value, digest-pinned and checked the same way. @@ -417,9 +444,9 @@ out of scope; see `docs/concepts/self-host-and-managed.md` and the ### Order ```text -T3 server image ──┐ -T4 server renderer ┼──► T6 k3s wiring + bump job ──► T7 EKS values -T5 runner values ──┘ +#2357 runner closure ──► T3 server image ──┐ +T4 server renderer ────────────────────────┼──► T6 k3s operator ──► T7 EKS operator +T5 runner values ──────────────────────────┘ (gated on OQ-5) Deferred, off the critical path: F1 door TLS reload, F2 tokenFile ``` @@ -446,6 +473,12 @@ In flight as compass PR #2355. Its smoke runs the locally built OCI image: `devenv.lock`. `go/**` is a superset; a Runner-only Go change then publishes a Server image too, which is the safe direction. +T3 owns D8's version-pairing guarantee. It depends on compass PR #2357, +which widens `RUNNER_IMAGE_CLOSURE_PATHS` to `go/**`; without it a +`go/gen`-only change publishes a new Server and no new Runner. T3 adds a +`bun:test` that feeds `go/gen/compass/v1/compass.pb.go` to `closureChanges` +for both lanes and expects a hit in each. + ```text server-image/{Dockerfile,moon.yml,.dockerignore} // pinned base from runner-image/Dockerfile tools/server-image/{build.ts,build-core.ts} // ENTRYPOINT_LINK = ".compass-server", IMAGE_USER = "65532:65532" @@ -475,9 +508,17 @@ Tests use golden output plus invariants: is not the emptyDir root. - A `runnerClusters` entry with `jwks` renders a ConfigMap key and a matching `jwksFile`; an entry with both `jwks` and `jwksURI` is rejected. - -The README documents the render steps, the JWKS read, and the -`exit status 1` boot failure. +- A `forge` value renders `--forge-host`, `--forge-app-id`, + `--forge-installation-id`, `--forge-app-key-secret`, + `--forge-app-webhook-secret`, `--forge-reviewer-app-id`, + `--forge-reviewer-app-installation-id` and + `--forge-reviewer-app-key-secret`, plus `--forge-repos` when set. Ids + must be positive integers. An absent `forge` renders no `--forge-*` arg. + +The README documents the render steps, the JWKS read, the +`exit status 1` boot failure, and the `server.env` lines. It warns that +`COMPASS_MASTER_KEY` must never change in the mounted Secret: the Server +reads it once at boot, and a different key fails boot. ```text tools/server-manifests/render-core.ts @@ -495,8 +536,16 @@ tools/server-manifests/render-core.ts adminTokenSecretName: string; s3SecretName?: string; corsAllowedOrigin?: string; otlpEndpoint?: string; external?: { port: number; loadBalancerClass?: string }; + forge?: ForgeValues; runnerClusters: RunnerClusterValues[]; } + export type ForgeValues = { + host?: string; repos?: string[]; // repos: owner/name seed for --forge-repos + appId: number; installationId: number; + appKeySecret: string; appWebhookSecret: string; // SERVER_ in server.env + reviewerAppId: number; reviewerInstallationId: number; + reviewerAppKeySecret: string; + } export function parseServerDeployValues(raw: unknown): ServerDeployValues export function renderServerManifests(v: ServerDeployValues): ReturnType export function assertServerImageDigest(ref: string): void @@ -511,6 +560,7 @@ tools/server-manifests/render-core.ts // --admin-token-file /run/compass/bootstrap/admin-token // --runner-clusters /etc/compass/runner-clusters/clusters.yaml // --secret-provider dotenv:///etc/compass/secrets/server.env + // [--forge-* from forge; secret NAMEs only, never values] // env: XDG_STATE_HOME=/run/compass/xdg-state tools/server-manifests/{render.ts,values.example.json,README.md,moon.yml} .moon/workspace.yml // server-manifests @@ -534,32 +584,41 @@ tools/runner-manifests/render-core.ts export function assertEgressHost(host: string): void ``` -### T6 — k3s wiring in the operator's infra repo (operator infra) +### T6 — k3s operator responsibilities -Node, declared in the host's config as code: +Node config, declared as code: - Promote the spike's generic device-plugin slice to its permanent place. - The `/dev/kvm` `0666` udev rule. - The seccomp profile staged at the kubelet root. - Confirm Kubernetes ≥1.33. No issuer change and no AppArmor sysctl: the KVM node has no AppArmor. +- Exactly one KVM node carries the Runner (D1). -Cluster: +Certificates, Secrets and values: - One Certificate against the existing private-CA ClusterIssuer, with a long duration and the reload annotation on the Server Deployment. -- The Secrets from D4, including `server.env`. -- The k3s JWKS read into the values. +- The Secrets from D4. `server.env` holds `COMPASS_MASTER_KEY` and the + forge App keys and webhook secret as `SERVER_` lines. +- Server values: the k3s JWKS, the `forge` block (both Apps' ids and + secret names), and `initImage`. + +Flux inputs: + - Rendered output is committed. Flux Kustomizations apply data, then Server, then Runners. - A scheduled bump job (OQ-10). For one compass commit it resolves each lane's digest with `resolveAncestor` from - `tools/runner-image/retag-core.ts`, re-renders both, and opens a pull - request. + `tools/runner-image/retag-core.ts`. It runs `assertNoClosureChange` per lane + against that lane's closure set, so a lane with no publish for a closure + change fails the job instead of pairing a stale image. It then re-renders + both and opens a change for review. -Dev move (OQ-11): a fresh install. The admin re-enters forge and provider -secrets with `compass secret set`, provider keys with `--kind provider ---delivery file`, and pushes agent configs again. +Dev move (OQ-11): a fresh install. The admin re-enters provider keys with +`compass secret set --kind provider --delivery file`, grants each agent its +repos with `compass agent repo add `, and pushes agent +configs again. Acceptance: @@ -567,27 +626,32 @@ Acceptance: - `/healthz` returns 200. - `compass.runner` reports SERVING. - An out-of-cluster CLI session completes, and its agent reaches the - provider and pushes to the forge. + provider and pushes to the forge with a brokered App token. - Dev points at the cluster. -- The bump job opens one pull request for a new compass commit. +- The bump job opens one change for a new compass commit, and fails on a + closure change that has no publish. ```text inputs: tools/server-manifests + tools/runner-manifests at one compass commit Runner values: serverAddr https://compass-server..svc, serverCA set, egressAllow [, github.com, api.github.com] -Server values: runnerClusters[0] = { issuer: https://kubernetes.default.svc.cluster.local, jwks: } +Server values: runnerClusters[0] = { issuer: https://kubernetes.default.svc.cluster.local, jwks: }, + forge = { appId, installationId, appKeySecret, appWebhookSecret, + reviewerAppId, reviewerInstallationId, reviewerAppKeySecret } ``` -### T7 — EKS values (operator infra) +### T7 — EKS operator responsibilities -First confirm OQ-5's owner-role attributes. Then wire: +Gated on Matt's OQ-5 ruling. The database contract is then: a direct, +session-mode endpoint, compute suspend off, and an owner login role that +holds CREATEROLE and BYPASSRLS (or superuser) and owns the target database. +The operator checks those attributes before wiring the DSN. Then: -- The external Postgres direct endpoint, with compute suspend off. - A passthrough NLB. - The cluster's public issuer with `jwksURI` discovery. -- A bare-metal node group that exposes `/dev/kvm`, sized for the Runner's - `sessionCapacity`. Every Runner pod requests one `kvmResourceName` device. -- For each new meter (managed Postgres, the NLB, bare-metal nodes): an +- A bare-metal node group of exactly one node that exposes `/dev/kvm`, for + one Runner pod (D1). Size the instance by the Runner's `sessionCapacity`. +- For each new meter (managed Postgres, the NLB, the bare-metal node): an enforced cap at each layer, a budget alert that pages, and a spend-cap register row, in the same change. @@ -619,8 +683,8 @@ go/internal/auth/workload_identity.go ## Tasks -- [ ] T3 — Server image and publish lane (CI, nix), compass PR #2355 -- [ ] T4 — `tools/server-manifests` renderer (TypeScript) +- [ ] T3 — Server image and publish lane (CI, nix), compass PR #2355, after #2357 +- [ ] T4 — `tools/server-manifests` renderer with `forge` args (TypeScript) - [ ] T5 — Runner renderer `serverCA` and `egressAllow` (TypeScript) -- [ ] T6 — k3s wiring, bump job and dev move (operator's infra repo) -- [ ] T7 — EKS values, bare-metal nodes and spend caps (operator's infra repo) +- [ ] T6 — k3s operator: node config, Secrets, values, Flux inputs, bump job, dev move +- [ ] T7 — EKS operator, gated on OQ-5: database, one bare-metal node, spend caps From 0d3123ae5fcbb5227c9e804e1b6594799b73f0fd Mon Sep 17 00:00:00 2001 From: mintaka Date: Sat, 17 Oct 2026 04:35:53 -0400 Subject: [PATCH 03/10] docs(compass): scope Postgres role needs per option and make the reviewer App optional (RIG-5559) Review fixes: OQ-5 lists owner-role attributes per option, and brokered push needs only the primary App. Refs RIG-5559 Co-authored-by: Matt Wilkinson --- docs/designs/compass/server-on-kubernetes.md | 64 ++++++++++++-------- 1 file changed, 40 insertions(+), 24 deletions(-) diff --git a/docs/designs/compass/server-on-kubernetes.md b/docs/designs/compass/server-on-kubernetes.md index 5b1df08..3942c7b 100644 --- a/docs/designs/compass/server-on-kubernetes.md +++ b/docs/designs/compass/server-on-kubernetes.md @@ -87,10 +87,15 @@ so roles created out of band do not help there. 3. RDS or Aurora, with a schema change that drops the BYPASSRLS role. This is a Go and SQL record of its own. -**Recommendation:** 1. It needs no code change. Whichever option is ruled, -the owner login role needs CREATEROLE and BYPASSRLS (or superuser) and must -own the target database, so it can create objects in `public`. T7 is gated -on this ruling. +**Recommendation:** 1. It needs no code change. T7 is gated on this ruling. +The owner login role must own the target database, so it can create objects +in `public`. Its other attributes depend on the option: + +- Options 1 and 2: CREATEROLE plus BYPASSRLS, or superuser, because + `0001_init.sql` creates and re-asserts the BYPASSRLS role. +- Option 3: CREATEROLE only, for `compass_app` and the grant to the owner. + The schema change removes `compass_system`, so no role needs BYPASSRLS, + and the RDS master user qualifies. **OQ-6 (non-load-bearing) — Image lane reuse.** @@ -191,8 +196,8 @@ there. EKS follows. `migrationLockKey`. - **RLS roles.** `0001_init.sql` creates `compass_app` (NOLOGIN) and `compass_system` (NOLOGIN BYPASSRLS), always re-asserts BYPASSRLS, and - grants both to the owner. The owner therefore needs CREATEROLE and - BYPASSRLS (OQ-5). + grants both to the owner. Under today's schema the owner therefore needs + CREATEROLE and BYPASSRLS (OQ-5). - **Door.** `loadNetworkTLS` (`go/server/network_door.go`) loads the keypair once, with a TLS 1.3 floor and h2. It has no reload. The Server serves no UI files. `ServeConfig.CORSAllowedOrigin` admits one browser origin. @@ -283,8 +288,8 @@ Job. Secret, and it logs in with `scram-sha-256`. On the first migration that superuser creates the RLS roles. The StatefulSet is dev-only (OQ-9); the operator backs up its volume and the master key. -- On EKS, the owner role must hold CREATEROLE and BYPASSRLS and own the - database (OQ-5). Operators turn compute suspend off. A suspended database +- On EKS, the owner role must own the database and hold the attributes the + OQ-5 option requires. Operators turn compute suspend off. A suspended database fails the single ping, and the pod restarts until it answers. **Secrets (D4).** The renderer names Secrets but never holds a value. @@ -319,8 +324,12 @@ renders it as `--forge-*` args: App ids, installation ids and secret names, never values. The App private keys and the webhook secret are `SERVER_` lines in `server.env`, PEM values double-quoted with `\n` escapes as `dotenvForgeValue` (`go/e2e/fixture.go`) writes them. They are not -set with `compass secret set`. Agent push also needs the reviewer App, -because the forge-write path runs only when both Apps are configured. +set with `compass secret set`. Brokered git push needs only the primary App +and the agent's repo grants: `brokeredSecretResolver.ResolveFor` adds the +App token on the Runner's fetch, scoped to the repos that +`compass agent repo add` granted. The reviewer App is optional. It enables +only agent `ForgeCall` writes, which `wireForgeWriteCaller` +(`go/server/serve.go`) wires only when both Apps are configured. **NATS access (D5).** The stack's NATS has no auth because it listens only on loopback. In a cluster, a NetworkPolicy admits port 4222 only from the @@ -509,11 +518,12 @@ Tests use golden output plus invariants: - A `runnerClusters` entry with `jwks` renders a ConfigMap key and a matching `jwksFile`; an entry with both `jwks` and `jwksURI` is rejected. - A `forge` value renders `--forge-host`, `--forge-app-id`, - `--forge-installation-id`, `--forge-app-key-secret`, - `--forge-app-webhook-secret`, `--forge-reviewer-app-id`, + `--forge-installation-id`, `--forge-app-key-secret` and + `--forge-app-webhook-secret`, plus `--forge-repos` when set. Ids must be + positive integers. An absent `forge` renders no `--forge-*` arg. +- `forge.reviewer` renders `--forge-reviewer-app-id`, `--forge-reviewer-app-installation-id` and - `--forge-reviewer-app-key-secret`, plus `--forge-repos` when set. Ids - must be positive integers. An absent `forge` renders no `--forge-*` arg. + `--forge-reviewer-app-key-secret`. Without it, no reviewer arg renders. The README documents the render steps, the JWKS read, the `exit status 1` boot failure, and the `server.env` lines. It warns that @@ -543,8 +553,9 @@ tools/server-manifests/render-core.ts host?: string; repos?: string[]; // repos: owner/name seed for --forge-repos appId: number; installationId: number; appKeySecret: string; appWebhookSecret: string; // SERVER_ in server.env - reviewerAppId: number; reviewerInstallationId: number; - reviewerAppKeySecret: string; + reviewer?: { // optional: enables agent ForgeCall writes only + appId: number; installationId: number; appKeySecret: string; + }; } export function parseServerDeployValues(raw: unknown): ServerDeployValues export function renderServerManifests(v: ServerDeployValues): ReturnType @@ -600,9 +611,11 @@ Certificates, Secrets and values: - One Certificate against the existing private-CA ClusterIssuer, with a long duration and the reload annotation on the Server Deployment. - The Secrets from D4. `server.env` holds `COMPASS_MASTER_KEY` and the - forge App keys and webhook secret as `SERVER_` lines. -- Server values: the k3s JWKS, the `forge` block (both Apps' ids and - secret names), and `initImage`. + primary App key and webhook secret as `SERVER_` lines, plus the + reviewer App key when the reviewer is configured. +- Server values: the k3s JWKS, the `forge` block (the primary App's ids + and secret names, and `reviewer` only if agents need `ForgeCall` writes), + and `initImage`. Flux inputs: @@ -626,7 +639,10 @@ Acceptance: - `/healthz` returns 200. - `compass.runner` reports SERVING. - An out-of-cluster CLI session completes, and its agent reaches the - provider and pushes to the forge with a brokered App token. + provider and pushes to a granted repo with a brokered App token from the + primary App alone. +- With `reviewer` set, an agent `ForgeCall` write succeeds; without it, the + Server boots and push still works. - Dev points at the cluster. - The bump job opens one change for a new compass commit, and fails on a closure change that has no publish. @@ -637,15 +653,15 @@ Runner values: serverAddr https://compass-server..svc, serverCA set, egressAllow [, github.com, api.github.com] Server values: runnerClusters[0] = { issuer: https://kubernetes.default.svc.cluster.local, jwks: }, forge = { appId, installationId, appKeySecret, appWebhookSecret, - reviewerAppId, reviewerInstallationId, reviewerAppKeySecret } + reviewer?: { appId, installationId, appKeySecret } } ``` ### T7 — EKS operator responsibilities Gated on Matt's OQ-5 ruling. The database contract is then: a direct, -session-mode endpoint, compute suspend off, and an owner login role that -holds CREATEROLE and BYPASSRLS (or superuser) and owns the target database. -The operator checks those attributes before wiring the DSN. Then: +session-mode endpoint, compute suspend off, and an owner login role that owns +the target database and holds the attributes OQ-5 lists for the ruled +option. The operator checks those attributes before wiring the DSN. Then: - A passthrough NLB. - The cluster's public issuer with `jwksURI` discovery. From b21639254e56dd7cadeee83c506758dc5c2cb0ee Mon Sep 17 00:00:00 2001 From: mintaka Date: Sat, 17 Oct 2026 04:42:57 -0400 Subject: [PATCH 04/10] docs(compass): scope the brokered token to effective forge grants (RIG-5559) Review fix: the token covers the agent grants plus its owner grants. Refs RIG-5559 Co-authored-by: Matt Wilkinson --- docs/designs/compass/server-on-kubernetes.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/designs/compass/server-on-kubernetes.md b/docs/designs/compass/server-on-kubernetes.md index 3942c7b..0c0d893 100644 --- a/docs/designs/compass/server-on-kubernetes.md +++ b/docs/designs/compass/server-on-kubernetes.md @@ -326,8 +326,9 @@ names, never values. The App private keys and the webhook secret are escapes as `dotenvForgeValue` (`go/e2e/fixture.go`) writes them. They are not set with `compass secret set`. Brokered git push needs only the primary App and the agent's repo grants: `brokeredSecretResolver.ResolveFor` adds the -App token on the Runner's fetch, scoped to the repos that -`compass agent repo add` granted. The reviewer App is optional. It enables +App token on the Runner's fetch, scoped to the agent's effective grants +(`ListForgeScopeRepos`): its own grants from `compass agent repo add` plus +its owner's grants, including `--forge-scope-grants`. The reviewer App is optional. It enables only agent `ForgeCall` writes, which `wireForgeWriteCaller` (`go/server/serve.go`) wires only when both Apps are configured. From 6188329e1f4063d180bf9b7503b92cd2502e0c91 Mon Sep 17 00:00:00 2001 From: mintaka Date: Sat, 17 Oct 2026 10:53:07 -0400 Subject: [PATCH 05/10] docs(compass): narrow the Kubernetes record to the product changes (RIG-5559) The record now covers only what the OSS product needs to run on Kubernetes: the admin token read from a Secret volume, public images, user-provided Postgres, and paired orderable image tags. Co-authored-by: Matt Wilkinson --- .../compass/decisions/server/DL-539.md | 2 +- .../compass/decisions/server/DL-540.md | 2 +- .../compass/decisions/server/DL-541.md | 2 +- .../compass/decisions/server/DL-542.md | 2 +- .../compass/decisions/server/DL-543.md | 2 +- .../compass/decisions/server/DL-544.md | 2 +- .../compass/decisions/server/DL-545.md | 2 +- .../compass/decisions/server/DL-546.md | 2 +- .../compass/decisions/server/DL-547.md | 2 +- docs/designs/compass/server-on-kubernetes.md | 1108 ++++++++--------- 10 files changed, 504 insertions(+), 622 deletions(-) diff --git a/docs/designs/compass/decisions/server/DL-539.md b/docs/designs/compass/decisions/server/DL-539.md index 20bc263..2283295 100644 --- a/docs/designs/compass/decisions/server/DL-539.md +++ b/docs/designs/compass/decisions/server/DL-539.md @@ -1,6 +1,6 @@ --- id: DL-539 -decision: "Each stack child that runs as a container becomes its own pod: the Server is a one-replica Recreate Deployment with exactly one Runner pod per Server, NATS a StatefulSet, Postgres a dev-only StatefulSet on k3s and an external managed database on EKS; the OTel collector is external and phase one deploys no LLM gateway." +decision: "Each stack child that runs as a container becomes its own pod: the Server is a one-replica Recreate Deployment with exactly one Runner pod per Server, and NATS is a one-replica StatefulSet; Postgres and the OTel collector are operator-provided outside the renderer, and no LLM gateway is deployed." status: "Active (Matt, 2026-10-17)" record: ../../server-on-kubernetes.md#design --- diff --git a/docs/designs/compass/decisions/server/DL-540.md b/docs/designs/compass/decisions/server/DL-540.md index 270b5b1..336f86a 100644 --- a/docs/designs/compass/decisions/server/DL-540.md +++ b/docs/designs/compass/decisions/server/DL-540.md @@ -1,6 +1,6 @@ --- id: DL-540 -decision: "The Server ships as `ghcr.io/rigelbuild/compass-server`, built like the Runner image: the `.#compass-server` nix closure (the binary, secretspec and the wrapper's bash) on the pinned distroless nonroot base, uid 65532, published by digest and gated by a `SERVER_IMAGE_CLOSURE_PATHS` set that covers every closure input." +decision: "ghcr.io/rigelbuild/compass-server is built from the .#compass-server nix closure on a digest-pinned distroless static-debian12 base, runs as 65532:65532, is published by digest through the shared --lane runner|server publish and retag scripts, and is gated by SERVER_IMAGE_CLOSURE_PATHS." status: "Active (Matt, 2026-10-17)" record: ../../server-on-kubernetes.md#design --- diff --git a/docs/designs/compass/decisions/server/DL-541.md b/docs/designs/compass/decisions/server/DL-541.md index c5dc532..4b39c6c 100644 --- a/docs/designs/compass/decisions/server/DL-541.md +++ b/docs/designs/compass/decisions/server/DL-541.md @@ -1,6 +1,6 @@ --- id: DL-541 -decision: "Migrations stay in Server boot against a direct, session-mode database endpoint; there is no migration Job." +decision: "Migrations stay in Server boot with no migration Job, and the renderer never renders Postgres; the operator brings a Postgres reached through a direct session-mode endpoint whose login role owns the database and holds CREATEROLE plus BYPASSRLS, or superuser, and that answers at boot." status: "Active (Matt, 2026-10-17)" record: ../../server-on-kubernetes.md#design --- diff --git a/docs/designs/compass/decisions/server/DL-542.md b/docs/designs/compass/decisions/server/DL-542.md index f41310b..3799f7a 100644 --- a/docs/designs/compass/decisions/server/DL-542.md +++ b/docs/designs/compass/decisions/server/DL-542.md @@ -1,6 +1,6 @@ --- id: DL-542 -decision: "The Server renderer names Secrets but holds no values; Server secrets are read through `dotenv://` from a read-only mounted Secret file whose master key never changes, forge App ids and secret names render as `--forge-*` args, `XDG_STATE_HOME` and the state dir sit on an emptyDir, and a digest-pinned init container copies the admin token into a 0600 file." +decision: "The Server renderer names Secrets but holds no values; Server secrets are read through dotenv:// from a read-only mounted Secret file whose master key never changes, forge App ids and secret names render as --forge-* args, XDG_STATE_HOME and the state dir sit on an emptyDir, and the admin token Secret mounts directly at 0440 under pod fsGroup 65532 with no init container, which the Server accepts once it follows the volume symlink and allows group-read for a group it holds." status: "Active (Matt, 2026-10-17)" record: ../../server-on-kubernetes.md#design --- diff --git a/docs/designs/compass/decisions/server/DL-543.md b/docs/designs/compass/decisions/server/DL-543.md index e6f91f4..7059770 100644 --- a/docs/designs/compass/decisions/server/DL-543.md +++ b/docs/designs/compass/decisions/server/DL-543.md @@ -1,6 +1,6 @@ --- id: DL-543 -decision: "In-cluster NATS requires a user and password and a NetworkPolicy admits port 4222 only from the Server pod." +decision: "In-cluster NATS requires a user and password carried in COMPASS_NATS_URL, and a NetworkPolicy admits port 4222 only from the Server pod." status: "Active (Matt, 2026-10-17)" record: ../../server-on-kubernetes.md#design --- diff --git a/docs/designs/compass/decisions/server/DL-544.md b/docs/designs/compass/decisions/server/DL-544.md index 6b21c6e..2e4545c 100644 --- a/docs/designs/compass/decisions/server/DL-544.md +++ b/docs/designs/compass/decisions/server/DL-544.md @@ -1,6 +1,6 @@ --- id: DL-544 -decision: "The Server terminates its own TLS with a long-lived certificate from the cluster's existing private-CA ClusterIssuer and restarts on renewal; `serverAddr` is the in-cluster Service and external access is TCP passthrough." +decision: "The Server terminates its own TLS from an operator-supplied kubernetes.io/tls Secret and loads it once, so a renewed certificate takes effect on restart; no ingress terminates TLS, serverAddr is the in-cluster Service, external access is TCP passthrough, and a private CA reaches the Runner through serverCA and clients through --ca." status: "Active (Matt, 2026-10-17)" record: ../../server-on-kubernetes.md#design --- diff --git a/docs/designs/compass/decisions/server/DL-545.md b/docs/designs/compass/decisions/server/DL-545.md index bfe1292..9a2dd8d 100644 --- a/docs/designs/compass/decisions/server/DL-545.md +++ b/docs/designs/compass/decisions/server/DL-545.md @@ -1,6 +1,6 @@ --- id: DL-545 -decision: "A k3s Runner-cluster entry keeps the default issuer and uses a `jwksFile` rendered from the cluster's public keys, so no Go change and no anonymous discovery is needed; the rendered Runner gets `serverCA` and an `egressAllow` list rendered as `--egress-allow`." +decision: "A Runner-cluster entry whose issuer serves no anonymous discovery uses a jwksFile rendered from the cluster's public keys, so no Go change is needed; the rendered Runner gets serverCA and an egressAllow list rendered as --egress-allow, and its node exposes KVM as the kvmResourceName extended resource, stages the seccompProfilePath profile, and supports hostUsers: false." status: "Active (Matt, 2026-10-17)" record: ../../server-on-kubernetes.md#design --- diff --git a/docs/designs/compass/decisions/server/DL-546.md b/docs/designs/compass/decisions/server/DL-546.md index 89a2279..4585430 100644 --- a/docs/designs/compass/decisions/server/DL-546.md +++ b/docs/designs/compass/decisions/server/DL-546.md @@ -1,6 +1,6 @@ --- id: DL-546 -decision: "Public compass holds `tools/server-manifests/`; the operator holds per-cluster values, renders at a pinned compass commit, and has Flux apply the output; the operator's scheduled job moves the pin, resolves both image digests from one commit, and fails when a lane has no publish for a closure change." +decision: "Public compass holds tools/server-manifests/ and tools/runner-manifests/ while the operator holds values, Secrets, certificates, node config and apply; every main commit tags both the Server and Runner images :main--, where n is the first-parent count and each tag names that lane's newest publish at or before the commit, so a numerical image-automation policy on n bumps both images together with no operator bump job." status: "Active (Matt, 2026-10-17)" record: ../../server-on-kubernetes.md#design --- diff --git a/docs/designs/compass/decisions/server/DL-547.md b/docs/designs/compass/decisions/server/DL-547.md index 95b2e6b..fbbad4a 100644 --- a/docs/designs/compass/decisions/server/DL-547.md +++ b/docs/designs/compass/decisions/server/DL-547.md @@ -1,6 +1,6 @@ --- id: DL-547 -decision: "Self-hosted k3s comes first and is done when Server and Runner run there, a session from an out-of-cluster CLI completes with agent egress to the provider and a brokered forge push, and dev points at the cluster on a fresh database; EKS follows, gated on the managed-Postgres ruling, with the same renderers, one bare-metal Runner node and a spend cap per new meter." +decision: "Every Compass image (agent, runner, server, guest image, gateway) is public, renderers emit no imagePullSecrets, image tooling reads the registry anonymously and uses credentials only to write, and each publish job fails when its package is not public, because no GitHub API sets package visibility." status: "Active (Matt, 2026-10-17)" record: ../../server-on-kubernetes.md#design --- diff --git a/docs/designs/compass/server-on-kubernetes.md b/docs/designs/compass/server-on-kubernetes.md index 0c0d893..6710e2b 100644 --- a/docs/designs/compass/server-on-kubernetes.md +++ b/docs/designs/compass/server-on-kubernetes.md @@ -1,184 +1,66 @@ -# Compass Server on Kubernetes — self-hosted k3s first, then EKS +# Compass Server on Kubernetes > Freezes on merge; later changes supersede by citation, never rewrite. > Unless a path says otherwise, every path is in > [RigelBuild/compass](https://github.com/RigelBuild/compass) at `main`. -> Compass design records moved out of `docs/designs/`; this record cites them -> at commit [`53ab753`](https://github.com/RigelBuild/compass/tree/53ab753f089ef1293d66736f247d41d38063dc7f/docs/designs), -> where they last lived. **Named decisions this record introduces:** D1–D9, inline in [Design](#design), ledgered as DL-539 to DL-547 in `decisions/server/`. ## Open Questions -Each question names a recommendation, and the design is written against it. -OQ-2 and OQ-3 are closed by the source and stay as rationale. - -**OQ-1 (non-load-bearing) — Bootstrap admin token from a Secret.** -`readSuppliedAdminToken` (`go/server/network_door.go`) rejects a symlink and -any group or other bit. A Secret volume file is a symlink, and with `fsGroup` -kubelet adds group bits, so a Go relaxation does not help either. - -1. An init container copies the token into a `0600` file on an emptyDir. - The init image is `busybox`, pinned by digest in the values, because the - distroless Server image has no `cp` and its wrapper's bash has no stable - path. -2. A Go change accepts a symlinked file with a `0400` target. -3. No supplied token: the Server mints one into its state dir on every - restart. - -**Recommendation:** 1. It needs no Go change, and it keeps the token stable -across restarts. - -**OQ-2 (closed) — Server secret provider.** `dotenv://` from a read-only -mounted Secret file. `SpecResolver.buildManifest` -(`go/internal/secrets/resolver.go`) declares the exact stored names -(`COMPASS_MASTER_KEY`, `SERVER_`). The flake check -`tools/flake-gate/compass-server-secretspec.nix` and the e2e fixture -(`go/e2e/fixture.go`) already boot through `dotenv://`. Unlike `env://`, the -file keeps the master key out of the Server's process env. secretspec -re-reads the file on each resolve, so rotated `SERVER_*` values apply without -a restart. `COMPASS_MASTER_KEY` is the exception: `resolveMasterKey` -(`go/server/serve.go`) reads it once at boot, and `reconcileKeyState` fails -boot when it differs from the stored key. It must never change in the -mounted Secret. - -**OQ-3 (closed) — LLM gateway placement.** Phase one deploys no gateway. -Only `compass-stack` refuses to start without one (`go/internal/stack/stack.go`: -"gateway image is required; pass --gateway-image or --gateway-external"), and -D1 drops compass-stack on Kubernetes. `compass-server` has no gateway flag, -and no code under `go/` delivers a gateway URL to agents yet: that is T5 of -the [gateway record](https://github.com/RigelBuild/compass/blob/53ab753f089ef1293d66736f247d41d38063dc7f/docs/designs/server/compass-server-llm-gateway/design.md). -Agents keep today's shape, a raw provider key plus an egress allowlist (D7). -The gateway's Kubernetes placement, and its service token, belong to a later -record. - -**OQ-4 (non-load-bearing) — Server certificate.** DNS-01 needs only a public -zone, not a reachable host, so both options work on a self-hosted k3s cluster. - -1. A long-lived certificate (for example one year) from the cluster's - existing private-CA ClusterIssuer. The Runner needs a `serverCA` value - (T5), and clients need `--ca` or `ca_cert`. `serverAddr` stays - `https://compass-server..svc`. -2. A public DNS-01 certificate for the external name. This drops the CA - plumbing. The Runner then dials the external name, which must resolve - in-cluster (split-horizon DNS or a CoreDNS rewrite), and the `.svc` SAN is - lost. - -**Recommendation:** 1. It reuses an issuer that exists, and it keeps the -Runner on the in-cluster name. EKS may choose either. - -**OQ-5 (load-bearing for EKS only) — Managed Postgres and BYPASSRLS.** -`0001_init.sql` runs `CREATE ROLE compass_system NOLOGIN BYPASSRLS` under -`IF NOT EXISTS`, then always runs `ALTER ROLE compass_system BYPASSRLS`. -PostgreSQL lets only a superuser or a role that holds BYPASSRLS set that -attribute ([CREATE ROLE](https://www.postgresql.org/docs/current/sql-createrole.html)). -`WithSystemRole` (`go/internal/store/tenant_tx.go`) uses the role at runtime, -so it is load-bearing. The RDS master user is `NOSUPERUSER` -([rds_superuser](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Appendix.PostgreSQL.CommonDBATasks.Roles.rds_superuser.html)), -so roles created out of band do not help there. - -1. A managed provider whose owner role holds BYPASSRLS and CREATEROLE. Neon - grants BYPASSRLS to its owner role for projects created after - 2023-08-15. -2. An in-cluster Postgres operator on EKS, whose superuser runs the - migration unchanged. -3. RDS or Aurora, with a schema change that drops the BYPASSRLS role. This - is a Go and SQL record of its own. - -**Recommendation:** 1. It needs no code change. T7 is gated on this ruling. -The owner login role must own the target database, so it can create objects -in `public`. Its other attributes depend on the option: - -- Options 1 and 2: CREATEROLE plus BYPASSRLS, or superuser, because - `0001_init.sql` creates and re-asserts the BYPASSRLS role. -- Option 3: CREATEROLE only, for `compass_app` and the grant to the owner. - The schema change removes `compass_system`, so no role needs BYPASSRLS, - and the RDS master user qualifies. - -**OQ-6 (non-load-bearing) — Image lane reuse.** - -1. A new `tools/server-image/` imports the pure cores of - `tools/runner-image/`. -2. Generalize `tools/runner-image/` to build both images. - -**Recommendation:** 1. The Runner build carries a kernel and guest image that -the Server does not need. T3 (compass PR #2355) implements 1 and adds a -`--lane runner|server` flag to the shared publish and retag scripts. - -**OQ-7 (non-load-bearing) — Server image visibility.** The Runner package is -private (`tools/runner-image/retag.ts`). - -1. The Server package is public. -2. The Server package is private, and clusters need a pull secret. - -**Recommendation:** 1. The Server binary already ships in public release -assets. - -**OQ-8 (non-load-bearing) — Sessions across a Server restart.** A dropped -session stream ends `runner.Run` with "runner sessions loop" and the Runner -exits (`go/internal/runner/run.go`). The renderer has no liveness probe -because "a pid-1 restart tears down all node sessions" -(`tools/runner-manifests/render-core.ts`). Every Server deploy and every -certificate renewal therefore kills live sessions. - -1. Accept the loss in phase one. Deploys and renewals are rare and - operator-timed. -2. A separate record makes the Runner survive a Server restart: it re-dials, - re-enrolls with the hub, and keeps its microVMs. -3. Add in-process TLS reload to the door. This covers renewals only, not - deploys. - -**Recommendation:** 1 now, and file 2 as its own record. Option 3 is the -deferred follow-up F1 in [Plan](#plan). - -**OQ-9 (non-load-bearing) — Who runs Postgres on k3s.** - -1. The Server renderer keeps an optional Postgres StatefulSet, marked - dev-only, with a backup note. -2. The renderer never renders Postgres. The operator installs a Postgres - operator on k3s, and EKS uses a managed database. - -**Recommendation:** 1. It is the fastest path, and the operator's Postgres -operator is not installed on the k3s cluster today. An absent `postgres` value -already renders no Postgres objects, so moving to 2 later needs no renderer -change. The StatefulSet is not a supported product surface: it has no major -upgrade path and no point-in-time recovery. - -**OQ-10 (non-load-bearing) — Digest bumps.** A dev cluster tracks main, but -D8 pins both digests to one compass commit. - -1. Re-render by hand on each bump. -2. The operator's scheduled job resolves both digests at one compass commit, - re-renders, and opens a change for review. -3. Flux image automation. It bumps each image alone, which breaks the - one-commit pairing. - -**Recommendation:** 2, sized as a task in T6. Each merged bump restarts the -Server (OQ-8). - -**OQ-11 (non-load-bearing) — Dev state.** Moving the current dev database -needs a dump and restore and the same master key. `reconcileKeyState` -(`go/server/serve.go`) fails boot on a key mismatch. Today's dev Server reads -the key through secretspec's default chain. - -1. Fresh install. Operators re-enter secrets with `compass secret set` and - push agent configs again. -2. `pg_dump` the dev database, restore it, and seed the same master key. - -**Recommendation:** 1. Dev holds no history worth the key-handling risk, and -the restore path is untested. +**OQ-12 (non-load-bearing) — Multi-arch Server image.** P1 publishes +`linux/amd64` only. `tools/server-image/build.ts` exits with "server-image +targets linux/amd64, but this host is ${process.arch}", and `flake.nix` +lists only `x86_64-linux`. The Server has no KVM need, so an arm64 node could +run it. The Runner stays amd64. + +1. Publish a multi-arch index (`linux/amd64`, `linux/arm64`). This needs an + `aarch64-linux` closure, an arm64 publish job and an index push, as + `publishImageIndex` (`tools/agent-image-index/index.ts`) does for the + agent image. `manifestIdentity` (`tools/runner-image/retag-core.ts`) + rejects an index today ("an image index, or malformed"), so retag and the + P6 tagger must learn to verify one. +2. Keep amd64 only. Operators place the Server with the P2 `nodeSelector` + and `toleration` values. + +**Recommendation:** 1. Nothing binds the Server to amd64, and an operator can +then run it on arm64 nodes next to one amd64 KVM node. P1–P6 hold under +either answer: P2 adds the optional placement values in both cases, because +it has none today. Option 1 lands as a follow-up to P1. + +## Resolved decisions + +Question numbers match the review threads on this record. + +- **OQ-1 — Admin token from a Secret.** A Go change (P4). The supplied token + file may be a symlink, and it may be group-readable when the process holds + the file's group. There is no init container (D4). +- **OQ-2 — Server secret provider.** `dotenv://` from a read-only mounted + Secret file (D4). +- **OQ-3 — LLM gateway.** Not deployed by this record (D1). +- **OQ-5 — Postgres.** The operator brings a Postgres that meets the D3 + contract. +- **OQ-6 — Image lane reuse.** `tools/server-image/` reuses the pure cores of + `tools/runner-image/` behind `--lane runner|server` (P1). +- **OQ-7 — Image visibility.** Every Compass image is public (D9, P5). +- **OQ-8 — Sessions across a Server restart.** The Runner must survive a + Server restart. That is its own record. Until it lands, see + [Known limitation](#known-limitation). +- **OQ-9 — Who runs Postgres.** Never the renderer (D1, D3). +- **OQ-10 — Digest bumps.** Paired per-commit tags that image automation + follows (D8, P6). ## Problem / Intent -The Runner can run on Kubernetes: it has a digest-published image, a manifest +The Runner runs on Kubernetes: it has a digest-published image, a manifest renderer, and projected-token identity. Nothing deploys a Compass Server -there, so the Runner's `serverAddr` has no target. This record adds a Server -image, a public renderer for the Server and its data services, the Runner -values that agents need, and the node and delivery steps. With them, Server -plus Runner run end to end on a self-hosted k3s cluster, and the dev environment moves -there. EKS follows. +there, so the Runner's `serverAddr` has no target. This record lists the +product changes that let an operator run Server plus Runner on a cluster, and +the requirements the product places on that operator. How one operator runs +its own clusters is out of scope; see +[self-host and managed](https://github.com/RigelBuild/compass/blob/main/docs/concepts/self-host-and-managed.md) +and the [OSS-core boundary record](meta/oss-core-managed-boundary/design.md). ## Approach @@ -186,65 +68,63 @@ there. EKS follows. - **Stack children.** `spawnChain` (`go/internal/stack/stack.go`) starts Postgres, the OTel collector, NATS, the LLM gateway, compass-server, then - compass-runner. The first four are podman containers. The Server and - Runner are host processes, and the Server receives the door as - `--listen-fd 3` (`serverSpec`, `go/internal/stack/spec.go`). + compass-runner. The first four are podman containers. The Server gets the + door as `--listen-fd 3` (`serverSpec`, `go/internal/stack/spec.go`). - **Boot order.** `Serve` (`go/server/serve.go`) binds listeners, then - `openFabricAndStore` opens NATS before the store. `store.Open` - (`go/internal/store/store.go`) pings once without retry. It then applies - the embedded migrations under the session-scoped advisory lock - `migrationLockKey`. -- **RLS roles.** `0001_init.sql` creates `compass_app` (NOLOGIN) and - `compass_system` (NOLOGIN BYPASSRLS), always re-asserts BYPASSRLS, and - grants both to the owner. Under today's schema the owner therefore needs - CREATEROLE and BYPASSRLS (OQ-5). + `openFabricAndStore` opens NATS before the store. `Open` + (`go/internal/store/store.go`) pings once without retry ("store: ping"), + then applies the embedded migrations under the session-scoped advisory + lock `migrationLockKey`. +- **RLS roles.** `0001_init.sql` runs `CREATE ROLE compass_system NOLOGIN + BYPASSRLS`, always re-runs `ALTER ROLE compass_system BYPASSRLS`, and grants + `compass_app` and `compass_system` to `current_user`. PostgreSQL lets only + a superuser or a role holding BYPASSRLS set that attribute + ([CREATE ROLE](https://www.postgresql.org/docs/current/sql-createrole.html)). + `WithSystemRole` (`go/internal/store/tenant_tx.go`) runs `SET LOCAL ROLE + compass_system` at runtime. - **Door.** `loadNetworkTLS` (`go/server/network_door.go`) loads the keypair - once, with a TLS 1.3 floor and h2. It has no reload. The Server serves no - UI files. `ServeConfig.CORSAllowedOrigin` admits one browser origin. -- **Health.** `go/server/health.go` serves `GET /healthz` and grpc health on - every door. It reports NOT_SERVING once drain starts. `compass.runner` - reports SERVING only while a Runner is enrolled. + once, with a TLS 1.3 floor and h2. It has no reload. +- **Health.** `go/server/health.go` serves `GET /healthz` and grpc health. It + reports NOT_SERVING once drain starts. `compass.runner` is SERVING only + while a Runner is enrolled. - **Secrets.** `resolveMasterKey` (`go/server/serve.go`) fails boot without a - 64-hex `COMPASS_MASTER_KEY`. The flake's `compass-server` is a - `wrapProgram` bash wrapper that puts secretspec on `PATH` (`flake.nix`). - `run` in `go/internal/secrets/resolver.go` discards secretspec's stderr, so - a bad provider surfaces only as `exit status 1` under - "resolve master key". secretspec writes an audit log under - `$XDG_STATE_HOME`, and a failed write does not block a resolve. -- **State dir.** `--state-dir` defaults to the socket's parent. - `ensurePrivateDir` (`go/server/socket.go`) creates missing dirs at `0700` - and leaves an existing dir alone. `warnIfSharedTokenDir` warns when the - token dir has group or other bits, as a fresh emptyDir root does. + 64-hex `COMPASS_MASTER_KEY`, and `reconcileKeyState` fails boot when it + differs from the stored key. The resolver in + `go/internal/secrets/resolver.go` names the dotenv file when it cannot open + it ("secrets: dotenv provider file %s: %w"), and secretspec writes an audit + log under `$XDG_STATE_HOME`. +- **Admin token.** `readSuppliedAdminToken` (`go/server/network_door.go`) + `Lstat`s the path and fails on `info.Mode()&(os.ModeType|0o077) != 0`. A + Secret volume file is a symlink through `..data`. With `fsGroup`, kubelet's + `changeFilePermission` (`pkg/volume/volume_linux.go` in + kubernetes/kubernetes) runs `os.Lchown(filename, -1, int(*fsGroup))` and + adds `roMask = os.FileMode(0440)` on a read-only volume. The file fails + both checks today. - **Runner.** `renderRunnerManifests` (`tools/runner-manifests/render-core.ts`) - takes `serverAddr`, which must be absolute `https`, and emits env only, no - `args`. The renderer has no CA value, so the Runner trusts system roots - unless `COMPASS_RUNNER_CA` is set (`go/cmd/compass-runner/main.go`). The - hub holds one Runner (`go/internal/runnerhub/hub.go`). `Run` - (`go/internal/runner/run.go`) returns an error when the session stream - drops. -- **Agent egress.** The microVM always arms a default-deny firewall. + takes an absolute `https` `serverAddr`, and now `serverCA` and + `egressAllow`. The hub holds one Runner (`go/internal/runnerhub/hub.go`). +- **Agent egress.** The microVM arms a default-deny firewall. `--egress-allow` (`go/cmd/compass-runner/main.go`) is the only way to open - a host, and it has no env fallback. `parseEgress` treats an empty value as - "no host reachable". `isValidHost` (`go/internal/runtime/host.go`) admits - DNS names and bare IP literals, never `host:port`. Agents call providers - with a raw key from the provider seed - (`go/internal/runtime/secrets_materialize.go`). + a host. `isValidHost` (`go/internal/runtime/host.go`) admits DNS names and + IP literals, never `host:port`. - **Issuer keys.** `fetchHTTPS` (`go/internal/auth/workload_identity.go`) - sends no `Authorization` header, and k3s runs with anonymous auth off. A - `jwksFile` entry never fetches: its keys "change only on rollout". + sends no `Authorization` header. A `jwksFile` entry never fetches. `validateRunnerClusters` (`go/internal/auth/runner_clusters.go`) requires - issuers to be unique only among the configured entries, so the k3s default - issuer `https://kubernetes.default.svc.cluster.local` is valid. -- **Image lane.** `runner-image/Dockerfile` copies a nix closure onto a - digest-pinned distroless `static-debian12` nonroot base. - `publish-runner-image` and `release-runner-image` - (`.github/workflows/release.yml`) push by digest and re-tag `:vX.Y.Z` by - digest. `resolveAncestor` (`tools/runner-image/retag-core.ts`) finds the - newest `:git-` publish at or before a commit. -- **Node shape** ([spike findings](https://github.com/RigelBuild/compass/blob/53ab753f089ef1293d66736f247d41d38063dc7f/docs/designs/platform/compass-runner-containerization/spike-findings.md)). - `TestFullBoot` passed on a bare-metal k3s node. That node had kernel 6.18, - `/dev/kvm` `0666`, the generic device plugin and `hostUsers: false`. It has - no AppArmor. The Ubuntu VM run needed + issuers to be unique only among the configured entries. +- **Image lanes.** Each lane publishes `:git-` by digest only when its + closure set changed. `resolveAncestor` (`tools/runner-image/retag-core.ts`) + walks first-parent ancestors and returns the newest published one; + `assertNoClosureChange` fails when a skipped commit changed the closure. + `release-runner-image` and `release-server-image` + (`.github/workflows/release.yml`) use them to mint `:vX.Y.Z` by digest. +- **Tag order.** No published tag sorts by commit. Commit timestamps are not + unique: 29 of the last 100 main commits share their second with another + commit. Flux's `RegexFilter.Apply` (`internal/policy/filter.go` in + fluxcd/image-reflector-controller) stores `f.filtered[tag] = item`, so two + tags with the same extracted value collapse to one. +- **Node shape** ([spike findings](platform/compass-runner-containerization/spike-findings.md)). + `TestFullBoot` passed on a bare-metal node with `/dev/kvm` `0666`, a + device plugin and `hostUsers: false`. A host with AppArmor needed `kernel.apparmor_restrict_unprivileged_userns=0`. ### Design @@ -252,456 +132,458 @@ there. EKS follows. **Topology (D1).** Each stack child that runs as a container today becomes its own pod. The Server container supervises nothing. -| Child | k3s | EKS | -| --- | --- | --- | -| compass-server | Deployment, `replicas: 1`, `strategy: Recreate` | same | -| NATS | StatefulSet, 1 replica, PVC, `DefaultNatsImage`, the stack's R1 file JetStream config | same | -| Postgres | dev-only StatefulSet, PVC, `DefaultPostgresImage` (OQ-9) | external managed Postgres (OQ-5) | -| OTel collector | none: `OTEL_EXPORTER_OTLP_ENDPOINT` points at the operator's collector | same | -| LLM gateway | not deployed in phase one (OQ-3) | same | -| compass-runner | the existing DaemonSet, on exactly one KVM node | same: one bare-metal node | - -The Server runs one replica because the hub holds one Runner and the -delivery-cutover record keeps a single Server. `Recreate` also keeps two -Servers from migrating at once. Each rollout drops live sessions (OQ-8). -Each cluster runs exactly one Runner pod per Server: `Hub.enroll` -(`go/internal/runnerhub/hub.go`) replaces the attached Runner and clears its -session bindings, so a second Runner would drop the first one's sessions. -Multi-Runner is a separate record. +| Child | On Kubernetes | +| --- | --- | +| compass-server | Deployment, `replicas: 1`, `strategy: Recreate` | +| NATS | StatefulSet, 1 replica, PVC, `DefaultNatsImage`, the stack's R1 file JetStream config | +| Postgres | operator-provided, outside the renderer (D3) | +| OTel collector | operator-provided; `OTEL_EXPORTER_OTLP_ENDPOINT` points at it | +| LLM gateway | not deployed; see the [gateway record](server/compass-server-llm-gateway/design.md) | +| compass-runner | the existing DaemonSet, on exactly one KVM node | + +One Server replica, because the hub holds one Runner and the +[delivery-cutover record](infra/runtime/compass-managed-delivery-cutover/design.md) +keeps a single Server. `Recreate` also keeps two Servers from migrating at +once. Exactly one Runner pod per Server: `enroll` (`go/internal/runnerhub/hub.go`) +replaces the attached Runner and drives its bound accounts offline, so a +second Runner drops the first one's sessions. Only `compass-stack` needs a +gateway ("gateway image is required; pass --gateway-image or +--gateway-external", `go/internal/stack/stack.go`); `compass-server` has no +gateway flag. Agents keep a raw provider key plus an egress allowlist (D7). **Server image (D2).** `ghcr.io/rigelbuild/compass-server` follows the Runner -image's shape. `server-image/Dockerfile` uses the same pinned base and copies -the realised `.#compass-server` closure. The image runs as uid 65532, and the -staged symlink `/nix/store/.compass-server` is its entrypoint. No other -compass binary ships. The closure carries secretspec and the wrapper's bash, -plus glibc and tzdata; an image scan sees them. Publishing reuses the Runner -lane (OQ-6). `SERVER_IMAGE_CLOSURE_PATHS` gates it, and the set is listed in -T3. +image's shape. `server-image/Dockerfile` copies the realised +`.#compass-server` closure onto a digest-pinned distroless `static-debian12` +base. The image runs as `65532:65532`, with `/nix/store/.compass-server` as +its entrypoint. The closure carries secretspec and the wrapper's bash. +`SERVER_IMAGE_CLOSURE_PATHS` gates publishing. **Data and migrations (D3).** Migrations stay in Server boot; there is no -Job. - -- The DSN must name a direct, session-mode endpoint, because the migration - lock is session-scoped. +Job. The renderer never renders Postgres. The operator's Postgres must meet +this contract: + +- The DSN names a direct, session-mode endpoint. The migration lock is + session-scoped, so a transaction-mode pooler breaks it. +- The login role owns the target database, so it can create objects in + `public`. +- The login role holds CREATEROLE plus BYPASSRLS, or is superuser, because + `0001_init.sql` creates and re-asserts the BYPASSRLS role. A service whose + owner role cannot hold BYPASSRLS cannot run today's schema. +- The database answers at boot. A suspended database fails the single ping, + and the pod restarts until it answers. - One replica fits the default pool of 10. -- On k3s, the Postgres superuser is `compass`, its password comes from a - Secret, and it logs in with `scram-sha-256`. On the first migration that - superuser creates the RLS roles. The StatefulSet is dev-only (OQ-9); the - operator backs up its volume and the master key. -- On EKS, the owner role must own the database and hold the attributes the - OQ-5 option requires. Operators turn compute suspend off. A suspended database - fails the single ping, and the pod restarts until it answers. -**Secrets (D4).** The renderer names Secrets but never holds a value. +**Secrets (D4).** The renderer names Secrets but never holds a value. The +operator creates every Secret and backs up the master key; losing it is +unrecoverable (`docs/self-host.md`). - `COMPASS_DATABASE_DSN`, `COMPASS_NATS_URL` and `COMPASS_S3_*` come from Secrets as env. - Server secrets come from one Secret key, `server.env`, mounted read-only - at `/etc/compass/secrets/`. The Server reads it through - `--secret-provider dotenv:///etc/compass/secrets/server.env` (OQ-2). The - file holds `COMPASS_MASTER_KEY` and the `SERVER_` values. A wrong path - or a bad file fails boot with only `exit status 1`, because secretspec's - stderr is discarded; the README says to check the mount first. + and read through `--secret-provider dotenv:///etc/compass/secrets/server.env`. + The file holds `COMPASS_MASTER_KEY` and the `SERVER_` values. + `SpecResolver.buildManifest` (`go/internal/secrets/resolver.go`) declares + exactly those names. Unlike `env://`, the file keeps the master key out of + the process env. - `SERVER_*` values re-resolve on use. `COMPASS_MASTER_KEY` is read once at - boot and must never change: a different key fails boot in - `reconcileKeyState`. -- `XDG_STATE_HOME` is `/run/compass/xdg-state`, on the emptyDir, so - secretspec's audit log has a writable home on a read-only root. -- An init container copies the admin token into a `0600` file (OQ-1). -- The operator creates every Secret with its own tooling. It also backs up - the master key, because losing it is unrecoverable (`docs/self-host.md`). -- Agent provider keys are not Server secrets. An admin stores each one with - `compass secret set --kind provider --provider --delivery file`, - value on stdin. `--delivery file` matters: `Install` routes env delivery - before kind, so an env-delivered key never reaches the provider seed. - -**Forge credential (D4).** Agents get forge tokens only from the Server's -App broker. `buildGitCredentialBroker` (`go/server/git_credential.go`) -returns nil unless the primary App id is set, and user `GH_TOKEN`-style -secrets are rejected and stripped (`IsReservedGitHubEnvName`, -`go/internal/secrets/secrets.go`). So the renderer takes a `forge` value and -renders it as `--forge-*` args: App ids, installation ids and secret -names, never values. The App private keys and the webhook secret are -`SERVER_` lines in `server.env`, PEM values double-quoted with `\n` -escapes as `dotenvForgeValue` (`go/e2e/fixture.go`) writes them. They are not -set with `compass secret set`. Brokered git push needs only the primary App -and the agent's repo grants: `brokeredSecretResolver.ResolveFor` adds the -App token on the Runner's fetch, scoped to the agent's effective grants -(`ListForgeScopeRepos`): its own grants from `compass agent repo add` plus -its owner's grants, including `--forge-scope-grants`. The reviewer App is optional. It enables -only agent `ForgeCall` writes, which `wireForgeWriteCaller` -(`go/server/serve.go`) wires only when both Apps are configured. - -**NATS access (D5).** The stack's NATS has no auth because it listens only on -loopback. In a cluster, a NetworkPolicy admits port 4222 only from the -Server pod. NATS also requires a user and password. The password reaches the -NATS config through a variable reference, and the credentials ride in -`COMPASS_NATS_URL`. The fabric's connect path already keeps URL credentials -out of its errors (`go/internal/fabric/fabric.go`). + boot and must never change in the mounted Secret. +- `XDG_STATE_HOME` and `--state-dir` sit on one emptyDir under `/run/compass`. + The state dir is a child the Server creates at `0700` (`ensurePrivateDir`, + `go/server/socket.go`). +- The admin token Secret mounts directly, `defaultMode: 0440`, under pod + `fsGroup: 65532`. `--admin-token-file` names the projected file. P4 makes + the Server accept it. No init container and no init image. +- Agents get forge tokens only from the Server's App broker. + `buildGitCredentialBroker` (`go/server/git_credential.go`) returns nil + unless the primary App id is set, and `IsReservedGitHubEnvName` + (`go/internal/secrets/secrets.go`) strips user `GH_TOKEN`-style secrets. So + the renderer takes a `forge` value and renders `--forge-*` args: ids and + secret names, never values. The reviewer App is optional; + `wireForgeWriteCaller` (`go/server/serve.go`) wires agent `ForgeCall` + writes only when both Apps are set. +- An admin stores each agent provider key with `compass secret set + --kind provider --provider --delivery file`. + +**NATS access (D5).** The stack's NATS has no auth because it listens on +loopback. In a cluster, NATS requires a user and password, and a +NetworkPolicy admits port 4222 only from the Server pod. The password reaches +the NATS config through a variable reference, and the credentials ride in +`COMPASS_NATS_URL`. The fabric's connect path keeps URL credentials out of its +errors (`go/internal/fabric/fabric.go`). **Door, TLS and serverAddr (D6).** The Server terminates its own TLS. No ingress may terminate it: the door refuses cleartext, and the Runner's bidi stream needs h2 end to end. -- **Certificate.** cert-manager issues a `kubernetes.io/tls` Secret from the - cluster's existing private-CA ClusterIssuer (OQ-4), with a long duration. - Its SANs cover the in-cluster name and the external name. -- **Renewal.** The door loads its keypair once. On renewal the operator's - reload annotation restarts the Server, which drops live sessions (OQ-8). - In-process reload is deferred (follow-up F1). -- **Services.** Service `compass-server` (ClusterIP) maps port 443 to - container port 50052. -- **serverAddr** is `https://compass-server..svc`. The Runner - trusts the CA through a new `serverCA` value (T5). -- **External access** is a LoadBalancer Service on 50052, passed through as - TCP, at an operator-owned DNS name. +- **Certificate.** The operator supplies a `kubernetes.io/tls` Secret whose + SANs cover the in-cluster name and any external name. A private CA works; + the Runner's `serverCA` value and the client `--ca` carry it. +- **Renewal.** The door loads its keypair once, so a renewed certificate + takes effect on restart (see [Known limitation](#known-limitation)). +- **Services.** Service `compass-server` (ClusterIP) maps 443 to 50052. An + optional LoadBalancer Service on 50052 passes TCP through. +- **serverAddr** is `https://compass-server..svc`. - **Clients.** The CLI uses `--server-addr`, `--ca` and the admin token (`go/cmd/compass/client.go`). The desktop app uses client mode with - `ca_cert` (`docs/self-host.md`). A browser origin comes from the - `corsAllowedOrigin` value. -- **Probes.** The startup and readiness probes are HTTPS `httpGet` on - `/healthz`. Readiness never depends on `compass.runner`, because the - Runner needs the Service before it can enroll. There is no liveness - probe. - -**In-cluster Runner (D7).** Three values make a rendered Runner useful. - -- **Identity.** The k3s entry keeps the cluster's default issuer and uses - `jwksFile`. The operator reads the public keys with - `kubectl get --raw /openid/v1/jwks` at render time, and the Server renderer - writes them into the runner-clusters ConfigMap. This needs no Go change and - no anonymous discovery. A k3s signing-key rotation is a manual operator - step, and it then needs a re-render and a Server rollout - (`docs/self-host.md`). An EKS issuer serves discovery anonymously and uses - `jwksURI` discovery as today. `tokenFile` is deferred (follow-up F2). -- **CA.** `serverCA` mounts the private CA and sets `COMPASS_RUNNER_CA`. -- **Egress.** `egressAllow` renders the Runner container's - `args: ["--egress-allow="]`. The renderer applies the - `isValidHost` grammar, so `host:port` fails at render time, not at Runner - start. Phase one lists the provider API host, `github.com` and + `ca_cert` (`docs/self-host.md`). A browser origin comes from + `corsAllowedOrigin`. +- **Probes.** Startup and readiness are HTTPS `httpGet` on `/healthz`. + Readiness never depends on `compass.runner`, because the Runner needs the + Service before it can enroll. There is no liveness probe. + +**Runner values (D7).** + +- **Identity.** A cluster whose issuer serves no anonymous discovery uses a + `jwksFile` entry: the operator reads `/openid/v1/jwks` at render time and + passes it as a `jwks` value, and the Server renderer writes it into the + runner-clusters ConfigMap. This needs no Go change. A signing-key rotation + then needs a re-render and a Server rollout. An issuer with anonymous + discovery keeps `jwksURI`. +- **CA.** `serverCA` mounts the CA and sets `COMPASS_RUNNER_CA`. +- **Egress.** `egressAllow` renders one `--egress-allow=` arg. The + renderer applies the `isValidHost` grammar, so `host:port` fails at render + time. A session needs the provider API host, `github.com` and `api.github.com`. +- **Node.** The KVM node exposes `/dev/kvm` as the extended resource named by + `kvmResourceName`, stages the seccomp profile at `seccompProfilePath` + under the kubelet seccomp root, and supports `hostUsers: false`. + +**Delivery split and paired tags (D8).** Public compass holds +`tools/server-manifests/` and `tools/runner-manifests/`: values in, plain +objects out, digest-only images, no Secrets. The operator holds the values +for each cluster, the Secrets, the certificate, node config, and how the +output is applied. Every main commit gets `:main--` on both the +Server and Runner images (P6). `n` is the commit's first-parent count on +main. Each lane's tag names that lane's newest publish at or before the +commit, re-tagged by digest with no rebuild. Both tags of one `n` therefore +pair the images built from the same Go inputs, as long as both closure sets +cover all of `go/**` (compass PR #2357). Main requires linear history and +forbids force-pushes, so `n` rises by one per main commit and is unique. Image +automation follows the tags with a numerical policy on `n`: + +```yaml +filterTags: + pattern: '^main-(?P[1-9][0-9]*)-[0-9a-f]{12}$' + extract: '$n' +policy: + numerical: + order: asc +digestReflectionPolicy: IfNotPresent +``` -**Delivery split (D8).** Public compass holds `tools/server-manifests/`, in -the shape of `tools/runner-manifests/`: values in, plain objects out, -digest-only images, no Secrets. The operator holds the values for each -cluster, renders at a pinned compass commit, commits the output, and has -Flux apply it. The operator also owns Secrets, the Certificate, the device -plugin and node config. Both image digests resolve from one compass commit: -each is that image's newest publish at or before the commit. That pairing -matches the Server and Runner API versions only while both closure sets -cover every Go input, `go/gen` included; T3 owns that guarantee. The -operator's scheduled job moves the pin (OQ-10), and it fails when a lane has -no publish for a closure change. - -**Rollout (D9).** k3s comes first, as the dev environment. It is done when -three things hold: - -- Server and Runner run there. -- A session started from an out-of-cluster CLI completes, with its agent - reaching the provider and the forge. -- Dev points at the cluster, on a fresh database (OQ-11). - -EKS follows with the same renderers and other values. Managed-plane detail is -out of scope; see `docs/concepts/self-host-and-managed.md` and the -[OSS-core boundary record](https://github.com/RigelBuild/compass/blob/53ab753f089ef1293d66736f247d41d38063dc7f/docs/designs/meta/oss-core-managed-boundary/design.md). - -## Alternatives considered - -- **One pod running `compass-stack up`.** Rejected. The supervisor drives - podman and host processes, which is the host-level bring-up the - distribution record keeps. -- **Ingress-terminated TLS, re-encrypted to the door.** Rejected. It adds an - h2 proxy hop and a second certificate. -- **A migration Job.** Rejected. No migrate-only entrypoint exists, and the - advisory lock already serializes migrations. -- **In-process TLS reload now.** Deferred (F1): every deploy drops sessions - the same way (OQ-8). -- **A unique k3s issuer plus `tokenFile`.** Deferred (F2): a live issuer - change needs a dual-issuer window, and it is a Go change. -- **Anonymous discovery via AuthenticationConfiguration.** Not taken: it needs - Kubernetes ≥1.34 and an API-server change. -- **A `COMPASS_EGRESS_ALLOW` env fallback.** Not taken: a Go PR that `args` - makes unnecessary. +One automation over both policies bumps both images in one commit. No +operator bump job is needed. The tagger writes the Server tag, then the +Runner tag, seconds apart. A scan between the two writes sees a new Server +with the previous Runner for one interval; the next scan pairs them. + +**Public images (D9).** Every Compass image is public: +`compass-agent`, `compass-runner`, `compass-server`, `compass-guest-image` +and `compass-gateway` under `ghcr.io/rigelbuild`. On 2026-10-17 all five +answered `skopeo list-tags --no-creds`. Neither renderer emits +`imagePullSecrets`. Tooling must not assume registry auth for reads. GitHub's +REST packages API can list, get, delete and restore a package, but has no +call that sets visibility. A new package starts private, and making it public +is a one-way switch in the package settings: a manual step at its first +publish ([package visibility](https://docs.github.com/en/packages/learn-github-packages/configuring-a-packages-access-control-and-visibility)). + +### Known limitation + +A Server restart ends every live session. A dropped session stream ends +`Run` (`go/internal/runner/run.go`, "runner sessions loop: %w"), and the +Runner exits. The Runner has no liveness probe because "a pid-1 restart tears +down all node sessions" (`tools/runner-manifests/render-core.ts`). Each +Server deploy and each certificate renewal therefore drops sessions, and D8 +makes deploys frequent. A separate record makes the Runner survive a Server +restart: it re-dials, re-enrolls with the hub, and keeps its microVMs. + +### Alternatives considered + +- **One pod running `compass-stack up`.** Rejected: the supervisor drives + podman and host processes. +- **An init container that copies the admin token.** Rejected: it adds an + image value and a pod step for a check the Server can make itself (P4). +- **A renderer-owned Postgres StatefulSet.** Rejected: no upgrade path and no + point-in-time recovery, and it would read as a supported surface. +- **Ingress-terminated TLS.** Rejected: an extra h2 hop and a second + certificate. +- **A migration Job.** Rejected: no migrate-only entrypoint exists, and the + advisory lock serializes migrations. +- **An operator bump job, or per-image automation on `:git-`.** + Rejected: the first is a scheduled job per operator; the second has no + order and pairs nothing. +- **A timestamp tag.** Rejected: equal timestamps collapse in the policy. ## Plan ### Global Constraints -- **Public repo.** No operator names, hosts, zones or private-product - detail. See `docs/concepts/self-host-and-managed.md`. -- **PRs.** T3, T4 and T5 are independent PRs off compass `main`; none - stacks on another. Each passes alone, red then green. T6 and T7 are - operator responsibilities: values, Secrets, the Certificate, node config - and Flux inputs. -- **Images.** Digest only. Renderers reject tags, as - `assertRunnerImageDigest` does. The init image is a value, digest-pinned - and checked the same way. +- **Public repo.** No operator names, hosts, zones, cluster names or private + product detail. +- **PRs.** Each task is one compass PR, red then green. P2 merges after P4, + because its output boots only with P4. P6 stacks on P5: both edit + `tools/runner-image/` tests and `release.yml`. +- **Images.** Digest only. Renderers reject tags, as `assertRunnerImageDigest` + does. - **Secrets.** Renderers emit no Secret object and no secret value. -- **Version coupling.** Server and Runner digests come from one compass - commit. -- **Kubernetes.** ≥1.33, for user namespaces and for the identity record's - ≥1.32 floor. -- **Server pod.** uid 65532, `readOnlyRootFilesystem`, drop ALL, - `RuntimeDefault` seccomp. The socket, state dir and `XDG_STATE_HOME` live - on one emptyDir at `/run/compass`; the state dir is a child the Server - creates at `0700`. -- **Lanes.** TypeScript is Bun with `bun:test`, as in - `tools/runner-manifests/`. Go DB tests use `pgtest`. -- **Spend.** Every new usage-billed meter on EKS ships with an enforced cap - per layer, a budget alert that pages, and a row in the operator's spend-cap - register, in the same change. +- **Kubernetes.** ≥1.33, for user namespaces. +- **Server pod.** uid and gid 65532, `fsGroup: 65532`, + `readOnlyRootFilesystem`, drop ALL, `RuntimeDefault` seccomp, + `automountServiceAccountToken: false`. +- **Lanes.** TypeScript is Bun with `bun:test`. Go DB tests use the `pgtest` + build tag. ### Order ```text -#2357 runner closure ──► T3 server image ──┐ -T4 server renderer ────────────────────────┼──► T6 k3s operator ──► T7 EKS operator -T5 runner values ──────────────────────────┘ (gated on OQ-5) -Deferred, off the critical path: F1 door TLS reload, F2 tokenFile +P1 server image (#2355, merged) ─┐ +P3 runner values (#2360, merged) ┼──► operators render and deploy +P4 admin token (Go) ──► P2 server renderer (#2363) ┘ +P5 public images ──► P6 paired main tags +Deferred: F1 door TLS reload, F2 tokenFile ``` -T3 must merge and publish on `main` before T6 can deploy. T4 and T5 can -merge in any order. - -### T3 — Server image and publish lane (CI, nix) - -In flight as compass PR #2355. Its smoke runs the locally built OCI image: - -- `--version` prints `compass-server `. -- Starting with no DSN exits 1 and names the missing DSN. -- The image config user is `65532:65532`. -- With Postgres and NATS containers, a read-only root and - `--secret-provider dotenv:///secrets/key.env` from a read-only mount, boot - resolves the master key and WhoAmI answers. NATS is required, because the - Server opens the fabric before the store. This mirrors - `tools/flake-gate/compass-server-secretspec.nix`. -- Two builds give the same manifest digest. +### P1 — Server image and publish lane -`SERVER_IMAGE_CLOSURE_PATHS` must cover every input of the closure: -`go list -deps ./cmd/compass-server` reaches `go/cmd`, `go/events`, `go/gen`, -`go/internal` and `go/server`, and the secretspec pin comes from -`devenv.lock`. `go/**` is a superset; a Runner-only Go change then publishes -a Server image too, which is the safe direction. - -T3 owns D8's version-pairing guarantee. It depends on compass PR #2357, -which widens `RUNNER_IMAGE_CLOSURE_PATHS` to `go/**`; without it a -`go/gen`-only change publishes a new Server and no new Runner. T3 adds a -`bun:test` that feeds `go/gen/compass/v1/compass.pb.go` to `closureChanges` -for both lanes and expects a hit in each. +Merged as compass PR #2355, after #2357 widened `RUNNER_IMAGE_CLOSURE_PATHS` +to `go/**`. Its smoke checks `--version`, a missing-DSN exit 1, user +`65532:65532`, a read-only `dotenv://` boot with Postgres and NATS, and a +reproducible digest. ```text -server-image/{Dockerfile,moon.yml,.dockerignore} // pinned base from runner-image/Dockerfile -tools/server-image/{build.ts,build-core.ts} // ENTRYPOINT_LINK = ".compass-server", IMAGE_USER = "65532:65532" -tools/runner-image/image-lane-core.ts // imageLaneConfig(lane: string | undefined): ImageLaneConfig | undefined -tools/runner-image/{publish.ts,retag.ts} // --lane runner|server -.moon/workspace.yml // compass-server-image: 'server-image' -.github/workflows/release.yml // jobs publish-server-image, release-server-image -SERVER_IMAGE_CLOSURE_PATHS: - server-image/** tools/server-image/** tools/runner-image/** go/** - tools/toolchain/secretspec-env.nix devenv.lock flake.nix flake.lock - version.txt .github/workflows/release.yml -image: ghcr.io/rigelbuild/compass-server@sha256:, :git- and :vX.Y.Z by digest +server-image/{Dockerfile,moon.yml,.dockerignore} +tools/server-image/{build.ts,build-core.ts} // ENTRYPOINT_LINK = ".compass-server", IMAGE_USER = "65532:65532" +tools/runner-image/image-lane-core.ts // imageLaneConfig(lane: string | undefined): ImageLaneConfig | undefined +tools/runner-image/{publish.ts,retag.ts} // --lane runner|server +.github/workflows/release.yml // publish-server-image, release-server-image +SERVER_IMAGE_CLOSURE_PATHS: server-image/** tools/server-image/** tools/runner-image/** go/** + tools/toolchain/secretspec-env.nix devenv.lock flake.nix flake.lock version.txt + .github/workflows/release.yml ``` -### T4 — Server manifests renderer (TypeScript) - -Tests use golden output plus invariants: - -- `replicas: 1` with `Recreate`. -- Digest-only images, the init image included. -- No `Secret` kind. -- The NetworkPolicy admits 4222 only from the Server label. +### P2 — Server manifests renderer + +Open as compass PR #2363. Against its current head it must drop +`initImage`, the `init-admin-token` init container and the `postgres` value +with its StatefulSet, Service and NetworkPolicy. It must add `forge`, +`nodeSelector` and `toleration`. Tests use golden output plus invariants: + +- `replicas: 1` with `Recreate`; digest-only images. +- No `Secret` kind, no `initContainers`, no Postgres object. +- `parseServerDeployValues` rejects a `postgres` or `initImage` key with an + error naming the key, so an older values file fails instead of rendering + without them. +- The admin token volume has `defaultMode: 0o440`, and `--admin-token-file` + names the file in that volume. +- The NATS NetworkPolicy admits 4222 only from the Server label. - Probes hit `/healthz`; no liveness probe. -- An absent `postgres` value renders no Postgres objects. -- The pod security context is as listed in Global Constraints. - `XDG_STATE_HOME` and `--state-dir` sit on the emptyDir, and the state dir - is not the emptyDir root. -- A `runnerClusters` entry with `jwks` renders a ConfigMap key and a - matching `jwksFile`; an entry with both `jwks` and `jwksURI` is rejected. -- A `forge` value renders `--forge-host`, `--forge-app-id`, - `--forge-installation-id`, `--forge-app-key-secret` and - `--forge-app-webhook-secret`, plus `--forge-repos` when set. Ids must be - positive integers. An absent `forge` renders no `--forge-*` arg. -- `forge.reviewer` renders `--forge-reviewer-app-id`, - `--forge-reviewer-app-installation-id` and - `--forge-reviewer-app-key-secret`. Without it, no reviewer arg renders. - -The README documents the render steps, the JWKS read, the -`exit status 1` boot failure, and the `server.env` lines. It warns that -`COMPASS_MASTER_KEY` must never change in the mounted Secret: the Server -reads it once at boot, and a different key fails boot. - -```text -tools/server-manifests/render-core.ts - export type RunnerClusterValues = { - name: string; issuer: string; audience?: string; - namespace: string; serviceAccount: string; maxTokenLifetime?: string; - jwksURI?: string; jwks?: string; // jwks: JWKS JSON text; renders jwksFile - caFile?: string; - } - export type ServerDeployValues = { - namespace: string; serverImage: string; natsImage: string; initImage: string; - postgres?: { image: string; storage: string; secretName: string }; // dev-only - natsStorage: string; tlsSecretName: string; dsnSecretName: string; - natsSecretName: string; serverSecretsSecretName: string; // key server.env - adminTokenSecretName: string; s3SecretName?: string; - corsAllowedOrigin?: string; otlpEndpoint?: string; - external?: { port: number; loadBalancerClass?: string }; - forge?: ForgeValues; - runnerClusters: RunnerClusterValues[]; - } - export type ForgeValues = { - host?: string; repos?: string[]; // repos: owner/name seed for --forge-repos - appId: number; installationId: number; - appKeySecret: string; appWebhookSecret: string; // SERVER_ in server.env - reviewer?: { // optional: enables agent ForgeCall writes only - appId: number; installationId: number; appKeySecret: string; - }; - } - export function parseServerDeployValues(raw: unknown): ServerDeployValues - export function renderServerManifests(v: ServerDeployValues): ReturnType - export function assertServerImageDigest(ref: string): void - // objects: ServiceAccount, Deployment, Service, Service(LoadBalancer)?, - // ConfigMap runner-clusters (clusters.yaml + .jwks.json), - // NATS StatefulSet+Service+ConfigMap, Postgres StatefulSet+Service?, NetworkPolicy x2 - // init container (initImage, uid 65532): cp the admin token to - // /run/compass/bootstrap/admin-token, chmod 0600 - // server args: --listen 0.0.0.0:50052 --tls-cert /etc/compass/tls/tls.crt - // --tls-key /etc/compass/tls/tls.key --state-dir /run/compass/state - // --socket /run/compass/state/server.sock - // --admin-token-file /run/compass/bootstrap/admin-token - // --runner-clusters /etc/compass/runner-clusters/clusters.yaml - // --secret-provider dotenv:///etc/compass/secrets/server.env - // [--forge-* from forge; secret NAMEs only, never values] - // env: XDG_STATE_HOME=/run/compass/xdg-state -tools/server-manifests/{render.ts,values.example.json,README.md,moon.yml} -.moon/workspace.yml // server-manifests + is not its root. +- A `runnerClusters` entry with `jwks` renders a ConfigMap key and a matching + `jwksFile`; `jwks` plus `jwksURI` is rejected. +- `forge` renders `--forge-host`, `--forge-app-id`, `--forge-installation-id`, + `--forge-app-key-secret`, `--forge-app-webhook-secret`, and `--forge-repos` + when set. Ids are positive integers. `forge.reviewer` adds the three + `--forge-reviewer-*` args. An absent value renders no arg. +- `nodeSelector` and `toleration` render as in the Runner renderer, with + `operator: "Equal"`; absent values render none. + +The README covers the render steps, the JWKS read, the D3 Postgres contract, +the `server.env` lines, and the never-change rule for `COMPASS_MASTER_KEY`. + +```ts +// tools/server-manifests/render-core.ts +export type RunnerClusterValues = { + name: string; issuer: string; audience?: string; + namespace: string; serviceAccount: string; maxTokenLifetime?: string; + jwksURI?: string; jwks?: string; caFile?: string; +}; +export type ForgeValues = { + host?: string; repos?: string[]; + appId: number; installationId: number; + appKeySecret: string; appWebhookSecret: string; + reviewer?: { appId: number; installationId: number; appKeySecret: string }; +}; +export type ServerDeployValues = { + namespace: string; serverImage: string; natsImage: string; natsStorage: string; + tlsSecretName: string; dsnSecretName: string; natsSecretName: string; + serverSecretsSecretName: string; adminTokenSecretName: string; s3SecretName?: string; + corsAllowedOrigin?: string; otlpEndpoint?: string; + external?: { port: number; loadBalancerClass?: string }; + nodeSelector?: { key: string; value: string }; + toleration?: { key: string; value: string; effect: "NoSchedule" | "PreferNoSchedule" | "NoExecute" }; + forge?: ForgeValues; + runnerClusters: RunnerClusterValues[]; +}; +export function assertServerImageDigest(ref: string): void; +export function parseServerDeployValues(raw: unknown): ServerDeployValues; +export function renderServerManifests(v: ServerDeployValues): ReturnType; +// server args: --listen 0.0.0.0:50052 --tls-cert /etc/compass/tls/tls.crt +// --tls-key /etc/compass/tls/tls.key --state-dir /run/compass/state +// --socket /run/compass/state/server.sock +// --admin-token-file /var/run/compass/admin-token/token +// --runner-clusters /etc/compass/runner-clusters/clusters.yaml +// --secret-provider dotenv:///etc/compass/secrets/server.env [--forge-*] +// files: tools/server-manifests/{render.ts,values.example.json,README.md,moon.yml} ``` -### T5 — Runner renderer `serverCA` and `egressAllow` (TypeScript) +### P3 — Runner renderer `serverCA` and `egressAllow` -When `serverCA` is set, the renderer mounts the ConfigMap key and sets -`COMPASS_RUNNER_CA`. When `egressAllow` is non-empty, the container gets one -`--egress-allow=` arg with the hosts comma-joined. Invalid hosts fail -`parseRunnerDeployValues`. With both absent, the output is unchanged. Tests -cover each case and a `github.com:443` rejection. The README covers both -values and lists the hosts a session needs. +Merged as compass PRs #2358 and #2360. `serverCA` takes exactly one of +`configMapName` or `secretName`, plus `key`, mounts at +`/etc/compass/server-ca` and sets `COMPASS_RUNNER_CA`. `egressAllow` renders +one `--egress-allow=` arg and rejects hosts outside `isValidHost`. -```text -tools/runner-manifests/render-core.ts - RunnerDeployValues.serverCA?: { configMapName: string; key: string } - // mount /etc/compass/server-ca/; env COMPASS_RUNNER_CA - RunnerDeployValues.egressAllow?: string[] - // args: ["--egress-allow=" + hosts.join(",")]; grammar of go/internal/runtime/host.go isValidHost - export function assertEgressHost(host: string): void +```ts +// tools/runner-manifests/render-core.ts +RunnerDeployValues.egressAllow?: readonly string[]; +RunnerDeployValues.serverCA?: + | { configMapName: string; secretName?: undefined; key: string } + | { secretName: string; configMapName?: undefined; key: string }; ``` -### T6 — k3s operator responsibilities - -Node config, declared as code: - -- Promote the spike's generic device-plugin slice to its permanent place. -- The `/dev/kvm` `0666` udev rule. -- The seccomp profile staged at the kubelet root. -- Confirm Kubernetes ≥1.33. No issuer change and no AppArmor sysctl: the - KVM node has no AppArmor. -- Exactly one KVM node carries the Runner (D1). - -Certificates, Secrets and values: - -- One Certificate against the existing private-CA ClusterIssuer, with a long - duration and the reload annotation on the Server Deployment. -- The Secrets from D4. `server.env` holds `COMPASS_MASTER_KEY` and the - primary App key and webhook secret as `SERVER_` lines, plus the - reviewer App key when the reviewer is configured. -- Server values: the k3s JWKS, the `forge` block (the primary App's ids - and secret names, and `reviewer` only if agents need `ForgeCall` writes), - and `initImage`. - -Flux inputs: - -- Rendered output is committed. Flux Kustomizations apply data, then - Server, then Runners. -- A scheduled bump job (OQ-10). For one compass commit it resolves each - lane's digest with `resolveAncestor` from - `tools/runner-image/retag-core.ts`. It runs `assertNoClosureChange` per lane - against that lane's closure set, so a lane with no publish for a closure - change fails the job instead of pairing a stale image. It then re-renders - both and opens a change for review. - -Dev move (OQ-11): a fresh install. The admin re-enters provider keys with -`compass secret set --kind provider --delivery file`, grants each agent its -repos with `compass agent repo add `, and pushes agent -configs again. - -Acceptance: - -- Flux reports Ready. -- `/healthz` returns 200. -- `compass.runner` reports SERVING. -- An out-of-cluster CLI session completes, and its agent reaches the - provider and pushes to a granted repo with a brokered App token from the - primary App alone. -- With `reviewer` set, an agent `ForgeCall` write succeeds; without it, the - Server boots and push still works. -- Dev points at the cluster. -- The bump job opens one change for a new compass commit, and fails on a - closure change that has no publish. - -```text -inputs: tools/server-manifests + tools/runner-manifests at one compass commit -Runner values: serverAddr https://compass-server..svc, serverCA set, - egressAllow [, github.com, api.github.com] -Server values: runnerClusters[0] = { issuer: https://kubernetes.default.svc.cluster.local, jwks: }, - forge = { appId, installationId, appKeySecret, appWebhookSecret, - reviewer?: { appId, installationId, appKeySecret } } +### P4 — Admin token file from a Secret volume (Go) + +`readSuppliedAdminToken` opens the path once, which follows the symlink, and +checks the opened file's `Stat`. That removes the gap between check and read. +The file must be regular, with no setuid, setgid or sticky bit, no +group-write bit and no other bit. Group-read is allowed only when the file's +gid is the process's `os.Getegid()` or in `os.Getgroups()`. kubelet's 0440 +file under `fsGroup` passes; a 0640 file in a group the process does not +hold fails. `writeSuppliedAdminTokenFile` still copies the token to a 0600 +file under the state dir, and `reusableAdminToken` keeps its strict `Lstat` +check for that copy. Both are unchanged. + +Tests, red first: + +- `go/server/admin_token_mode_test.go` (`//go:build unix`), on the pure + check: 0600 passes; 0640 and 0440 pass with a held gid; 0640 fails with an + unheld gid; 0660, 0604, a directory and a setgid file fail. +- `TestIssueAndWriteAdminTokenRejectsMalformedSuppliedFiles` + (`go/server/admin_token_reuse_pgtest_test.go`) drops its "group readable" + and "symlink input" rejection cases and adds "group writable" (0660) and + "other readable" (0604). +- A new pgtest builds the Secret volume layout: `..data/token` at 0440 in + `os.Getegid()`, and `token -> ..data/token`. Boot registers the token, + `assertAdminTokenAuthenticates` passes, and the state-dir copy is 0600. + +The spec clause in `docs/specs/product/compass.md` ("SHALL name a private +regular file") and the `--admin-token-file` help in +`go/cmd/compass-server/main.go` ("Private file containing the supplied +bootstrap-admin token") change to the new rule. + +```go +// go/server/network_door.go (//go:build unix) +func readSuppliedAdminToken(path string) (string, error) // os.Open, f.Stat, io.ReadAll(f) +// checkSuppliedTokenMode is pure; gid comes from info.Sys().(*syscall.Stat_t).Gid. +func checkSuppliedTokenMode(mode os.FileMode, gid uint32, held []int) error ``` -### T7 — EKS operator responsibilities - -Gated on Matt's OQ-5 ruling. The database contract is then: a direct, -session-mode endpoint, compute suspend off, and an owner login role that owns -the target database and holds the attributes OQ-5 lists for the ruled -option. The operator checks those attributes before wiring the DSN. Then: - -- A passthrough NLB. -- The cluster's public issuer with `jwksURI` discovery. -- A bare-metal node group of exactly one node that exposes `/dev/kvm`, for - one Runner pod (D1). Size the instance by the Runner's `sessionCapacity`. -- For each new meter (managed Postgres, the NLB, the bare-metal node): an - enforced cap at each layer, a budget alert that pages, and a spend-cap - register row, in the same change. +### P5 — Public images and anonymous reads + +The five packages are public already. P5 makes the tooling match and keeps +it true. + +- `retag.ts` probes with `skopeo inspect --no-creds --raw`; only the copy + uses `REGISTRY_AUTH_FILE`. `classifyProbe` still reads 403 as ambiguous, + because a private package and a missing package give the same 403. +- Stale text goes: the `release-runner-image` step comment "The runner + package is private, so even the probes need auth", the `retag.ts` header + "so private packages work", and the `retag-core.test.ts` case name "a + private package without auth (403 Forbidden) is ambiguous" and its "on + this private package" comment. +- A new check runs as the last step of each of the five publish jobs. It + reads the just-published tag anonymously and fails with the package + settings path when the package is not public. A new package therefore + fails its first publish until someone sets it public: the one manual step, + because no API sets visibility. + +Tests: `assert-public-core.test.ts` maps a manifest to public, and a 403, a +"manifest unknown" and a timeout to failures that name the package. +`retag-core.test.ts` asserts the probe argv carries `--no-creds` and the copy +argv carries `--authfile`. + +```ts +// tools/runner-image/assert-public-core.ts +export function publicVerdict(result: ProbeResult, ref: string): { ok: true } | { ok: false; message: string }; +// tools/runner-image/assert-public.ts bun tools/runner-image/assert-public.ts --ref +// tools/runner-image/retag-core.ts +export function probeArgv(repo: string, sha12: string): string[]; // ["inspect","--no-creds","--raw","docker://:git-"] +``` -Acceptance matches T6, without the dev move. +### P6 — Paired main tags + +A new job `tag-main-images` in `.github/workflows/release.yml` runs on every +main push. It `needs: [publish-runner-image, publish-server-image]`, so it +runs after both lanes publish or skip, and never when either failed. It has +`permissions: {contents: read, packages: write}`, its own concurrency group +with `cancel-in-progress: false` and `queue: max`, and a checkout with +`fetch-depth: 0`; a shallow clone gives a wrong `n`. + +`tag-main.ts` computes `n` with `git rev-list --count --first-parent `. +For each lane it runs `resolveAncestor` over `git rev-list --first-parent +--max-count=50 ` and `assertNoClosureChange` over the diff from the +resolved commit, as `retag.ts` does. It resolves both lanes before any write. +Then it copies each digest to `:main--` with `--preserve-digests`, +Server first, and checks each with `assertCoherent`. A lane that published +this commit resolves at position 1. + +Tests, red first: + +- `retag-core.test.ts`: `mainTag(1638, <40-hex>)` returns + `main-1638-`; 0, a fraction and a short sha throw `EXIT.usage`. +- `resolveMainPair`: both lanes found at the head; a lane absent at the head + with no closure change resolves its ancestor; a lane absent with a closure + change throws, and the injected writer records no call. +- `tools/runner-image/closure-paths.test.ts`: `go/gen/compass/v1/compass.pb.go` + hits both closure sets through `closureChanges`; `tag-main-images` needs + both publish jobs and checks out with `fetch-depth: 0`. +- The tag matches the D8 pattern, and Flux's extract yields `n`. + +```ts +// tools/runner-image/retag-core.ts +export function mainTag(firstParentCount: number, commit: string): string; // "main--" +export async function resolveMainPair(input: { + ancestors: readonly string[]; + probe: Record Promise>; + changedSince: (commit: string) => Promise; + closurePaths: Record; +}): Promise>; +// tools/runner-image/tag-main.ts +// bun tools/runner-image/tag-main.ts --sha <40-hex> \ +// --server-repo ghcr.io/rigelbuild/compass-server --runner-repo ghcr.io/rigelbuild/compass-runner +// .github/workflows/release.yml: job tag-main-images +``` ### Deferred follow-ups -- **F1 — Door reloads its TLS keypair (Go).** Taken when OQ-8 option 2 is - ruled out or renewals become frequent. Tests: after a rewrite the next - handshake presents the new serial; a corrupt rewrite keeps the old serial; - existing connections survive. +- **F1 — Door reloads its TLS keypair (Go).** Taken if renewals must not + restart the Server. Tests: after a rewrite the next handshake presents the + new serial; a corrupt rewrite keeps the old one. -```text -go/server/network_door.go - func newCertReloader(certFile, keyFile string, log *slog.Logger) (*certReloader, error) - func (r *certReloader) GetCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) +```go +// go/server/network_door.go +func newCertReloader(certFile, keyFile string, log *slog.Logger) (*certReloader, error) +func (r *certReloader) GetCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) ``` - **F2 — `tokenFile` on a Runner cluster (Go).** Taken when a cluster needs - live key rotation without a rollout. Key fetches send - `Authorization: Bearer` with the file's contents, re-read on every fetch. - -```text -go/internal/auth/runner_clusters.go - RunnerCluster.TokenFile string // yaml:"tokenFile", optional -go/internal/auth/workload_identity.go - func fetchHTTPS(ctx context.Context, client *http.Client, u, bearer string) ([]byte, error) + key rotation without a rollout. Key fetches send `Authorization: Bearer` + with the file's contents, re-read on every fetch. + +```go +// go/internal/auth/runner_clusters.go +RunnerCluster.TokenFile string // yaml:"tokenFile", optional +// go/internal/auth/workload_identity.go +func fetchHTTPS(ctx context.Context, client *http.Client, u, bearer string) ([]byte, error) ``` ## Tasks -- [ ] T3 — Server image and publish lane (CI, nix), compass PR #2355, after #2357 -- [ ] T4 — `tools/server-manifests` renderer with `forge` args (TypeScript) -- [ ] T5 — Runner renderer `serverCA` and `egressAllow` (TypeScript) -- [ ] T6 — k3s operator: node config, Secrets, values, Flux inputs, bump job, dev move -- [ ] T7 — EKS operator, gated on OQ-5: database, one bare-metal node, spend caps +- [x] P1 — Server image and publish lane, compass PR #2355, after #2357 +- [ ] P2 — `tools/server-manifests` renderer: no Postgres, no init container, + `forge`, placement values (compass PR #2363), after P4 +- [x] P3 — Runner renderer `serverCA` and `egressAllow`, compass PRs #2358 + and #2360 +- [ ] P4 — Admin token file from a Secret volume (Go) +- [ ] P5 — Public images; tooling reads anonymously and checks visibility +- [ ] P6 — Paired `:main--` tags on every main commit, after P5 From 7b687d77b254122f6d6239097bf564f0693cd29d Mon Sep 17 00:00:00 2001 From: mintaka Date: Sat, 17 Oct 2026 18:25:15 -0400 Subject: [PATCH 06/10] docs(compass): fix the Kubernetes record review findings (RIG-5559) Postgres 16 floor, digest-only image updates, non-blocking admin token open, anonymous registry reads everywhere, and multi-arch images as D10 and P7. Co-authored-by: Matt Wilkinson --- .../compass/decisions/server/DL-541.md | 2 +- .../compass/decisions/server/DL-546.md | 2 +- .../compass/decisions/server/DL-547.md | 2 +- .../compass/decisions/server/DL-560.md | 6 + docs/designs/compass/server-on-kubernetes.md | 450 ++++++++++++------ 5 files changed, 302 insertions(+), 160 deletions(-) create mode 100644 docs/designs/compass/decisions/server/DL-560.md diff --git a/docs/designs/compass/decisions/server/DL-541.md b/docs/designs/compass/decisions/server/DL-541.md index 4b39c6c..c49fbb2 100644 --- a/docs/designs/compass/decisions/server/DL-541.md +++ b/docs/designs/compass/decisions/server/DL-541.md @@ -1,6 +1,6 @@ --- id: DL-541 -decision: "Migrations stay in Server boot with no migration Job, and the renderer never renders Postgres; the operator brings a Postgres reached through a direct session-mode endpoint whose login role owns the database and holds CREATEROLE plus BYPASSRLS, or superuser, and that answers at boot." +decision: "Migrations stay in Server boot with no migration Job, and the renderer never renders Postgres; the operator brings a Postgres reached through a direct session-mode endpoint whose login role owns the database and is either a CREATEROLE plus BYPASSRLS role on PostgreSQL 16 or later or a superuser on any version, and that answers at boot." status: "Active (Matt, 2026-10-17)" record: ../../server-on-kubernetes.md#design --- diff --git a/docs/designs/compass/decisions/server/DL-546.md b/docs/designs/compass/decisions/server/DL-546.md index 4585430..0e57d65 100644 --- a/docs/designs/compass/decisions/server/DL-546.md +++ b/docs/designs/compass/decisions/server/DL-546.md @@ -1,6 +1,6 @@ --- id: DL-546 -decision: "Public compass holds tools/server-manifests/ and tools/runner-manifests/ while the operator holds values, Secrets, certificates, node config and apply; every main commit tags both the Server and Runner images :main--, where n is the first-parent count and each tag names that lane's newest publish at or before the commit, so a numerical image-automation policy on n bumps both images together with no operator bump job." +decision: "Public compass holds tools/server-manifests/ and tools/runner-manifests/ while the operator holds values, Secrets, certificates, node config and apply; the head commit of every main push tags both the Server and Runner images :main--, where n is its first-parent count and each tag names that lane's newest publish at or before the commit, so a numerical image-automation policy on n bumps both images together with no operator bump job, and the automation writes only the digest, never the tag, into a pod spec." status: "Active (Matt, 2026-10-17)" record: ../../server-on-kubernetes.md#design --- diff --git a/docs/designs/compass/decisions/server/DL-547.md b/docs/designs/compass/decisions/server/DL-547.md index fbbad4a..74e9146 100644 --- a/docs/designs/compass/decisions/server/DL-547.md +++ b/docs/designs/compass/decisions/server/DL-547.md @@ -1,6 +1,6 @@ --- id: DL-547 -decision: "Every Compass image (agent, runner, server, guest image, gateway) is public, renderers emit no imagePullSecrets, image tooling reads the registry anonymously and uses credentials only to write, and each publish job fails when its package is not public, because no GitHub API sets package visibility." +decision: "Every Compass image (agent, runner, server, guest image, gateway) is public, renderers emit no imagePullSecrets, every registry read in the publish, index and release tooling is anonymous and credentials are used only for the write destination, and each publish job fails when its package is not public, because no GitHub API sets package visibility." status: "Active (Matt, 2026-10-17)" record: ../../server-on-kubernetes.md#design --- diff --git a/docs/designs/compass/decisions/server/DL-560.md b/docs/designs/compass/decisions/server/DL-560.md new file mode 100644 index 0000000..ae8065d --- /dev/null +++ b/docs/designs/compass/decisions/server/DL-560.md @@ -0,0 +1,6 @@ +--- +id: DL-560 +decision: "Every Compass image (agent, runner, server, guest image, gateway) is built in compass for linux/amd64 and linux/arm64 and published as an OCI index whose members are first pushed as :git--, and the retag and main-tag tooling verifies an index by its digest and member list." +status: "Active (Matt, 2026-10-17)" +record: ../../server-on-kubernetes.md#design +--- diff --git a/docs/designs/compass/server-on-kubernetes.md b/docs/designs/compass/server-on-kubernetes.md index 6710e2b..3146292 100644 --- a/docs/designs/compass/server-on-kubernetes.md +++ b/docs/designs/compass/server-on-kubernetes.md @@ -4,34 +4,14 @@ > Unless a path says otherwise, every path is in > [RigelBuild/compass](https://github.com/RigelBuild/compass) at `main`. -**Named decisions this record introduces:** D1–D9, inline in -[Design](#design), ledgered as DL-539 to DL-547 in `decisions/server/`. - -## Open Questions - -**OQ-12 (non-load-bearing) — Multi-arch Server image.** P1 publishes -`linux/amd64` only. `tools/server-image/build.ts` exits with "server-image -targets linux/amd64, but this host is ${process.arch}", and `flake.nix` -lists only `x86_64-linux`. The Server has no KVM need, so an arm64 node could -run it. The Runner stays amd64. - -1. Publish a multi-arch index (`linux/amd64`, `linux/arm64`). This needs an - `aarch64-linux` closure, an arm64 publish job and an index push, as - `publishImageIndex` (`tools/agent-image-index/index.ts`) does for the - agent image. `manifestIdentity` (`tools/runner-image/retag-core.ts`) - rejects an index today ("an image index, or malformed"), so retag and the - P6 tagger must learn to verify one. -2. Keep amd64 only. Operators place the Server with the P2 `nodeSelector` - and `toleration` values. - -**Recommendation:** 1. Nothing binds the Server to amd64, and an operator can -then run it on arm64 nodes next to one amd64 KVM node. P1–P6 hold under -either answer: P2 adds the optional placement values in both cases, because -it has none today. Option 1 lands as a follow-up to P1. +**Named decisions this record introduces:** D1–D10, inline in +[Design](#design), ledgered as DL-539 to DL-547 and DL-560 in +`decisions/server/`. ## Resolved decisions -Question numbers match the review threads on this record. +No question is open. Question numbers match the review threads on this +record. - **OQ-1 — Admin token from a Secret.** A Go change (P4). The supplied token file may be a symlink, and it may be group-readable when the process holds @@ -39,6 +19,9 @@ Question numbers match the review threads on this record. - **OQ-2 — Server secret provider.** `dotenv://` from a read-only mounted Secret file (D4). - **OQ-3 — LLM gateway.** Not deployed by this record (D1). +- **OQ-4 — Server certificate.** The operator supplies the certificate. The + product carries a private CA through the Runner's `serverCA` and the + client `--ca` (D6). - **OQ-5 — Postgres.** The operator brings a Postgres that meets the D3 contract. - **OQ-6 — Image lane reuse.** `tools/server-image/` reuses the pure cores of @@ -48,8 +31,12 @@ Question numbers match the review threads on this record. Server restart. That is its own record. Until it lands, see [Known limitation](#known-limitation). - **OQ-9 — Who runs Postgres.** Never the renderer (D1, D3). -- **OQ-10 — Digest bumps.** Paired per-commit tags that image automation +- **OQ-10 — Digest bumps.** Paired per-push tags that image automation follows (D8, P6). +- **OQ-11 — Retired.** It asked how one operator moves its own state, which + is not a product change. +- **OQ-12 — Multi-arch images.** Every Compass image is published for + `linux/amd64` and `linux/arm64` (D10, P7). ## Problem / Intent @@ -166,9 +153,14 @@ this contract: session-scoped, so a transaction-mode pooler breaks it. - The login role owns the target database, so it can create objects in `public`. -- The login role holds CREATEROLE plus BYPASSRLS, or is superuser, because - `0001_init.sql` creates and re-asserts the BYPASSRLS role. A service whose - owner role cannot hold BYPASSRLS cannot run today's schema. +- PostgreSQL ≥16 with a login role that holds CREATEROLE plus BYPASSRLS, or + a superuser on any version. `0001_init.sql` creates and re-asserts the + BYPASSRLS role. Before 16, "You must be a superuser to create a new role + having the BYPASSRLS attribute" + ([CREATE ROLE, 15](https://www.postgresql.org/docs/15/sql-createrole.html)); + from 16, "Only superuser roles or roles with BYPASSRLS can specify + BYPASSRLS" ([CREATE ROLE, 16](https://www.postgresql.org/docs/16/sql-createrole.html)). + A service whose owner role cannot hold BYPASSRLS cannot run today's schema. - The database answers at boot. A suspended database fails the single ping, and the pod restarts until it answers. - One replica fits the default pool of 10. @@ -252,14 +244,16 @@ stream needs h2 end to end. `tools/server-manifests/` and `tools/runner-manifests/`: values in, plain objects out, digest-only images, no Secrets. The operator holds the values for each cluster, the Secrets, the certificate, node config, and how the -output is applied. Every main commit gets `:main--` on both the -Server and Runner images (P6). `n` is the commit's first-parent count on -main. Each lane's tag names that lane's newest publish at or before the -commit, re-tagged by digest with no rebuild. Both tags of one `n` therefore -pair the images built from the same Go inputs, as long as both closure sets -cover all of `go/**` (compass PR #2357). Main requires linear history and -forbids force-pushes, so `n` rises by one per main commit and is unique. Image -automation follows the tags with a numerical policy on `n`: +output is applied. The head commit of every main push gets +`:main--` on both the Server and Runner images (P6). `n` is that +commit's first-parent count on main. Each lane's tag names that lane's newest +publish at or before the commit, re-tagged by digest with no rebuild. Both +tags of one `n` therefore pair the images built from the same Go inputs, as +long as both closure sets cover all of `go/**` (compass PR #2357). Main +requires linear history and forbids force-pushes, so `n` is unique and rises +with each push. A push of several commits tags only its head, so `n` has +gaps; a numerical policy does not care. Image automation follows the tags +with a numerical policy on `n`: ```yaml filterTags: @@ -271,7 +265,16 @@ policy: digestReflectionPolicy: IfNotPresent ``` -One automation over both policies bumps both images in one commit. No +Only the digest may change a pod spec. The tag changes on every push even +when a lane's digest does not, so a pod spec that carries the tag would roll +the Server and the Runner on every push and drop every session. The +renderers keep rejecting a tagged ref (`assertRunnerImageDigest`, +`assertServerImageDigest`). The operator's setter therefore writes only the +`:digest` field of the policy (`{"$imagepolicy": "::digest"}`), +for example on a kustomize `images[].digest` override of the rendered +output. Values files are JSON and carry no setter comment. + +One automation over both policies bumps both digests in one commit. No operator bump job is needed. The tagger writes the Server tag, then the Runner tag, seconds apart. A scan between the two writes sees a new Server with the previous Runner for one interval; the next scan pairs them. @@ -280,11 +283,39 @@ with the previous Runner for one interval; the next scan pairs them. `compass-agent`, `compass-runner`, `compass-server`, `compass-guest-image` and `compass-gateway` under `ghcr.io/rigelbuild`. On 2026-10-17 all five answered `skopeo list-tags --no-creds`. Neither renderer emits -`imagePullSecrets`. Tooling must not assume registry auth for reads. GitHub's -REST packages API can list, get, delete and restore a package, but has no -call that sets visibility. A new package starts private, and making it public -is a one-way switch in the package settings: a manual step at its first -publish ([package visibility](https://docs.github.com/en/packages/learn-github-packages/configuring-a-packages-access-control-and-visibility)). +`imagePullSecrets`. A command that only reads the registry (`skopeo +inspect`, `podman manifest add`) sends no credentials; a write (`skopeo +copy`, `podman manifest push`) uses them for the destination only (P5). +GitHub's REST packages API can list, get, delete and restore a +package, but has no call that sets visibility. A new package starts private, +and making it public is a one-way switch in the package settings: a manual +step at its first publish +([package visibility](https://docs.github.com/en/packages/learn-github-packages/configuring-a-packages-access-control-and-visibility)). + +**Multi-arch images (D10).** Every Compass image is published for +`linux/amd64` and `linux/arm64`, built in compass. `:git-`, +`:main--` and `:vX.Y.Z` name an OCI index of the two members, and +each member is pushed first as `:git--`. This is the agent +lane's shape today: `publish-image-amd64` and `publish-image-arm64` +(`.github/workflows/release.yml`) push the members, and +`publishImageIndex` (`tools/agent-image-index/index.ts`) composes and checks +the index against `EXPECTED_PLATFORMS = "linux/amd64,linux/arm64"`. The +other four lanes are single-arch now: + +- Runner: `IMAGE_PLATFORM = "linux/amd64"` and an x64-only host check + (`tools/runner-image/build.ts`). +- Server: "server-image targets linux/amd64, but this host is + ${process.arch}" (`tools/server-image/build.ts`). +- Gateway: `PLATFORM = "linux/amd64"`, and `layoutDigest` + (`tools/gateway-image/core.ts`) requires exactly one manifest. +- Guest image: one artifact manifest. `fetchGuestManifest` + (`go/internal/stack/materialize.go`) accepts only a single manifest. + +`flake.nix` lists `systems = [ "x86_64-linux" ]`. `manifestIdentity` +(`tools/runner-image/retag-core.ts`) refuses an index ("an image index, or +malformed"), so retag and the P6 tagger must learn to verify one. A KVM node +of either arch can then run the Runner, and the Server and NATS can run on +any node. ### Known limitation @@ -320,17 +351,24 @@ restart: it re-dials, re-enrolls with the hub, and keeps its microVMs. - **Public repo.** No operator names, hosts, zones, cluster names or private product detail. - **PRs.** Each task is one compass PR, red then green. P2 merges after P4, - because its output boots only with P4. P6 stacks on P5: both edit - `tools/runner-image/` tests and `release.yml`. + because its output boots only with P4. P5, P6 and P7 form one stack in + that order: all three edit `tools/runner-image/` and `release.yml`. - **Images.** Digest only. Renderers reject tags, as `assertRunnerImageDigest` - does. + does. Every image is an OCI index of `linux/amd64` and `linux/arm64` once + P7 lands. +- **Registry auth.** Reads send no credentials; writes use credentials for + the destination only. - **Secrets.** Renderers emit no Secret object and no secret value. - **Kubernetes.** ≥1.33, for user namespaces. +- **PostgreSQL.** ≥16 for a non-superuser owner role (D3). - **Server pod.** uid and gid 65532, `fsGroup: 65532`, `readOnlyRootFilesystem`, drop ALL, `RuntimeDefault` seccomp, `automountServiceAccountToken: false`. - **Lanes.** TypeScript is Bun with `bun:test`. Go DB tests use the `pgtest` build tag. +- **Spec impact.** P4 changes the requirement "The network door mints and + persists a bootstrap admin token" in `docs/specs/product/compass.md`. No + other task changes a spec. ### Order @@ -338,7 +376,7 @@ restart: it re-dials, re-enrolls with the hub, and keeps its microVMs. P1 server image (#2355, merged) ─┐ P3 runner values (#2360, merged) ┼──► operators render and deploy P4 admin token (Go) ──► P2 server renderer (#2363) ┘ -P5 public images ──► P6 paired main tags +P5 anonymous reads ──► P6 paired main tags ──► P7 multi-arch images Deferred: F1 door TLS reload, F2 tokenFile ``` @@ -347,18 +385,13 @@ Deferred: F1 door TLS reload, F2 tokenFile Merged as compass PR #2355, after #2357 widened `RUNNER_IMAGE_CLOSURE_PATHS` to `go/**`. Its smoke checks `--version`, a missing-DSN exit 1, user `65532:65532`, a read-only `dotenv://` boot with Postgres and NATS, and a -reproducible digest. - -```text -server-image/{Dockerfile,moon.yml,.dockerignore} -tools/server-image/{build.ts,build-core.ts} // ENTRYPOINT_LINK = ".compass-server", IMAGE_USER = "65532:65532" -tools/runner-image/image-lane-core.ts // imageLaneConfig(lane: string | undefined): ImageLaneConfig | undefined -tools/runner-image/{publish.ts,retag.ts} // --lane runner|server -.github/workflows/release.yml // publish-server-image, release-server-image -SERVER_IMAGE_CLOSURE_PATHS: server-image/** tools/server-image/** tools/runner-image/** go/** - tools/toolchain/secretspec-env.nix devenv.lock flake.nix flake.lock version.txt - .github/workflows/release.yml -``` +reproducible digest. Files: `server-image/{Dockerfile,moon.yml,.dockerignore}`, +`tools/server-image/{build.ts,build-core.ts}`, `--lane` in +`tools/runner-image/{publish.ts,retag.ts}`, and the `publish-server-image` and +`release-server-image` jobs. `SERVER_IMAGE_CLOSURE_PATHS` is +`server-image/**`, `tools/server-image/**`, `tools/runner-image/**`, `go/**`, +`tools/toolchain/secretspec-env.nix`, `devenv.lock`, `flake.nix`, +`flake.lock`, `version.txt` and `.github/workflows/release.yml`. ### P2 — Server manifests renderer @@ -367,7 +400,9 @@ Open as compass PR #2363. Against its current head it must drop with its StatefulSet, Service and NetworkPolicy. It must add `forge`, `nodeSelector` and `toleration`. Tests use golden output plus invariants: -- `replicas: 1` with `Recreate`; digest-only images. +- `replicas: 1` with `Recreate`. +- `serverImage` and `natsImage` accept only `repo@sha256:<64 hex>`. A + tagged ref, `repo:tag@sha256:…` included, fails parse with the field name. - No `Secret` kind, no `initContainers`, no Postgres object. - `parseServerDeployValues` rejects a `postgres` or `initImage` key with an error naming the key, so an older values file fails instead of rendering @@ -387,44 +422,11 @@ with its StatefulSet, Service and NetworkPolicy. It must add `forge`, - `nodeSelector` and `toleration` render as in the Runner renderer, with `operator: "Equal"`; absent values render none. -The README covers the render steps, the JWKS read, the D3 Postgres contract, -the `server.env` lines, and the never-change rule for `COMPASS_MASTER_KEY`. - -```ts -// tools/server-manifests/render-core.ts -export type RunnerClusterValues = { - name: string; issuer: string; audience?: string; - namespace: string; serviceAccount: string; maxTokenLifetime?: string; - jwksURI?: string; jwks?: string; caFile?: string; -}; -export type ForgeValues = { - host?: string; repos?: string[]; - appId: number; installationId: number; - appKeySecret: string; appWebhookSecret: string; - reviewer?: { appId: number; installationId: number; appKeySecret: string }; -}; -export type ServerDeployValues = { - namespace: string; serverImage: string; natsImage: string; natsStorage: string; - tlsSecretName: string; dsnSecretName: string; natsSecretName: string; - serverSecretsSecretName: string; adminTokenSecretName: string; s3SecretName?: string; - corsAllowedOrigin?: string; otlpEndpoint?: string; - external?: { port: number; loadBalancerClass?: string }; - nodeSelector?: { key: string; value: string }; - toleration?: { key: string; value: string; effect: "NoSchedule" | "PreferNoSchedule" | "NoExecute" }; - forge?: ForgeValues; - runnerClusters: RunnerClusterValues[]; -}; -export function assertServerImageDigest(ref: string): void; -export function parseServerDeployValues(raw: unknown): ServerDeployValues; -export function renderServerManifests(v: ServerDeployValues): ReturnType; -// server args: --listen 0.0.0.0:50052 --tls-cert /etc/compass/tls/tls.crt -// --tls-key /etc/compass/tls/tls.key --state-dir /run/compass/state -// --socket /run/compass/state/server.sock -// --admin-token-file /var/run/compass/admin-token/token -// --runner-clusters /etc/compass/runner-clusters/clusters.yaml -// --secret-provider dotenv:///etc/compass/secrets/server.env [--forge-*] -// files: tools/server-manifests/{render.ts,values.example.json,README.md,moon.yml} -``` +The README covers the render steps, the JWKS read, the `server.env` lines, +and the never-change rule for `COMPASS_MASTER_KEY`. It states the D3 +Postgres contract with its floor: PostgreSQL ≥16 for a CREATEROLE plus +BYPASSRLS owner, or a superuser on any version. It states that image +automation must write only the digest (D8). ### P3 — Runner renderer `serverCA` and `egressAllow` @@ -433,59 +435,64 @@ Merged as compass PRs #2358 and #2360. `serverCA` takes exactly one of `/etc/compass/server-ca` and sets `COMPASS_RUNNER_CA`. `egressAllow` renders one `--egress-allow=` arg and rejects hosts outside `isValidHost`. -```ts -// tools/runner-manifests/render-core.ts -RunnerDeployValues.egressAllow?: readonly string[]; -RunnerDeployValues.serverCA?: - | { configMapName: string; secretName?: undefined; key: string } - | { secretName: string; configMapName?: undefined; key: string }; -``` - ### P4 — Admin token file from a Secret volume (Go) -`readSuppliedAdminToken` opens the path once, which follows the symlink, and -checks the opened file's `Stat`. That removes the gap between check and read. -The file must be regular, with no setuid, setgid or sticky bit, no -group-write bit and no other bit. Group-read is allowed only when the file's -gid is the process's `os.Getegid()` or in `os.Getgroups()`. kubelet's 0440 -file under `fsGroup` passes; a 0640 file in a group the process does not -hold fails. `writeSuppliedAdminTokenFile` still copies the token to a 0600 -file under the state dir, and `reusableAdminToken` keeps its strict `Lstat` -check for that copy. Both are unchanged. +`readSuppliedAdminToken` opens the path once with +`O_RDONLY|O_NONBLOCK|O_CLOEXEC`, which follows the symlink, and checks the +open file's `Stat` before it reads. `O_NONBLOCK` keeps the open of a FIFO +from blocking boot; today `Lstat` fails a FIFO before any open. A non-regular +file fails before any read. Checking the open file removes the gap between +check and read. The file must be regular, with no setuid, setgid or sticky +bit, no group-write bit and no other bit. Group-read is allowed only when +the file's gid is the process's `os.Getegid()` or in `os.Getgroups()`. +kubelet's 0440 file under `fsGroup` passes; a 0640 file in a group the +process does not hold fails. `writeSuppliedAdminTokenFile` still copies the +token to a 0600 file under the state dir, and `reusableAdminToken` keeps its +strict `Lstat` check for that copy. Both are unchanged. Tests, red first: - `go/server/admin_token_mode_test.go` (`//go:build unix`), on the pure check: 0600 passes; 0640 and 0440 pass with a held gid; 0640 fails with an - unheld gid; 0660, 0604, a directory and a setgid file fail. + unheld gid; 0660, 0604, a directory, a FIFO and a setgid file fail. - `TestIssueAndWriteAdminTokenRejectsMalformedSuppliedFiles` (`go/server/admin_token_reuse_pgtest_test.go`) drops its "group readable" - and "symlink input" rejection cases and adds "group writable" (0660) and - "other readable" (0604). + and "symlink input" rejection cases. It adds "group writable" (0660), + "other readable" (0604), and "fifo": a `syscall.Mkfifo` at 0600 with no + writer, plus a symlink to it. Both fail with "must be a private regular + file" within a 5-second test deadline. A plain `os.Open` would block there. - A new pgtest builds the Secret volume layout: `..data/token` at 0440 in `os.Getegid()`, and `token -> ..data/token`. Boot registers the token, `assertAdminTokenAuthenticates` passes, and the state-dir copy is 0600. -The spec clause in `docs/specs/product/compass.md` ("SHALL name a private -regular file") and the `--admin-token-file` help in -`go/cmd/compass-server/main.go` ("Private file containing the supplied -bootstrap-admin token") change to the new rule. - -```go -// go/server/network_door.go (//go:build unix) -func readSuppliedAdminToken(path string) (string, error) // os.Open, f.Stat, io.ReadAll(f) -// checkSuppliedTokenMode is pure; gid comes from info.Sys().(*syscall.Stat_t).Gid. -func checkSuppliedTokenMode(mode os.FileMode, gid uint32, held []int) error -``` - -### P5 — Public images and anonymous reads - -The five packages are public already. P5 makes the tooling match and keeps -it true. - -- `retag.ts` probes with `skopeo inspect --no-creds --raw`; only the copy - uses `REGISTRY_AUTH_FILE`. `classifyProbe` still reads 403 as ambiguous, - because a private package and a missing package give the same 403. +Spec impact: the requirement "The network door mints and persists a +bootstrap admin token" (`docs/specs/product/compass.md`) says the file +"SHALL name a private regular file". It changes to: a regular file, reached +directly or through symlinks, with no other bits and no group-write bit, and +group-read only for a group the Server process holds. The +`--admin-token-file` help in `go/cmd/compass-server/main.go` ("Private file +containing the supplied bootstrap-admin token") changes to match. + +### P5 — Anonymous registry reads + +The five packages are public already. P5 makes every registry read in +compass anonymous, so D9 holds, and keeps the packages public. + +- `retag.ts` probes and verifies with `skopeo inspect --no-creds --raw`. Only + the copy uses `--dest-authfile`. `classifyProbe` still reads 403 as + ambiguous, because a private package and a missing package give the same + 403. +- The `release-image` job (`.github/workflows/release.yml`) has three + `skopeo inspect --authfile "$REGISTRY_AUTH_FILE"` reads: the ancestor probe + and the two digest checks after the copy. All three use `--no-creds`. Its + `skopeo copy --multi-arch all` uses `--dest-authfile`. +- `inspectCommand` (`tools/agent-image-index/index.ts`) builds every `skopeo + inspect` there with `--authfile`; it uses `--no-creds`. `podman manifest + add` drops `--authfile`; `podman manifest push` keeps it. +- `agent-image/publish.sh` reads the remote tag twice with `inspect --raw + --authfile`. Both use `--no-creds`. +- The guest and gateway `publish.ts` pass `auth` to their `inspect` reads. + The reads drop it; the push keeps it. - Stale text goes: the `release-runner-image` step comment "The runner package is private, so even the probes need auth", the `retag.ts` header "so private packages work", and the `retag-core.test.ts` case name "a @@ -497,24 +504,24 @@ it true. fails its first publish until someone sets it public: the one manual step, because no API sets visibility. -Tests: `assert-public-core.test.ts` maps a manifest to public, and a 403, a -"manifest unknown" and a timeout to failures that name the package. -`retag-core.test.ts` asserts the probe argv carries `--no-creds` and the copy -argv carries `--authfile`. +Tests, red first: -```ts -// tools/runner-image/assert-public-core.ts -export function publicVerdict(result: ProbeResult, ref: string): { ok: true } | { ok: false; message: string }; -// tools/runner-image/assert-public.ts bun tools/runner-image/assert-public.ts --ref -// tools/runner-image/retag-core.ts -export function probeArgv(repo: string, sha12: string): string[]; // ["inspect","--no-creds","--raw","docker://:git-"] -``` +- `assert-public-core.test.ts` maps a manifest to public, and a 403, a + "manifest unknown" and a timeout to failures that name the package. +- `retag-core.test.ts`: `probeArgv` carries `--no-creds` and no `--authfile`. +- `tools/agent-image-index/index.test.ts`: the recorded commands carry + `--no-creds` on every `skopeo inspect`, and `--authfile` only on `podman + manifest push`. +- A new `tools/runner-image/registry-auth.test.ts` reads `release.yml`, + `agent-image/publish.sh` and the guest and gateway `publish.ts`. It fails + on any `inspect` that names `--authfile` or `auth`. ### P6 — Paired main tags A new job `tag-main-images` in `.github/workflows/release.yml` runs on every -main push. It `needs: [publish-runner-image, publish-server-image]`, so it -runs after both lanes publish or skip, and never when either failed. It has +main push and tags the push's head commit. It +`needs: [publish-runner-image, publish-server-image]`, so it runs after both +lanes publish or skip, and never when either failed. It has `permissions: {contents: read, packages: write}`, its own concurrency group with `cancel-in-progress: false` and `queue: max`, and a checkout with `fetch-depth: 0`; a shallow clone gives a wrong `n`. @@ -539,8 +546,109 @@ Tests, red first: both publish jobs and checks out with `fetch-depth: 0`. - The tag matches the D8 pattern, and Flux's extract yields `n`. +### P7 — Multi-arch images + +Every lane publishes `linux/amd64` and `linux/arm64` members and one OCI +index, as the agent lane does (D10). + +- **Closures.** `flake.nix` `systems` adds `aarch64-linux`, so + `.#compass-server`, `.#compass-runner` and the guest assets evaluate on + both. `guest-image/default.nix` already selects its agent member per + system (`agentPin.platformFor`). +- **Build guards.** `tools/runner-image/build.ts` and + `tools/server-image/build.ts` take the platform from the host + (`x64` → `linux/amd64`, `arm64` → `linux/arm64`) and fail on any other + host. `PLATFORM` in `tools/gateway-image/core.ts` becomes a parameter. The + `IMAGE_PLATFORM` comment in `tools/runner-image/build.ts` ("the cluster + nodes are amd64") goes. +- **Jobs.** Each of `publish-runner-image`, `publish-server-image`, + `publish-guest-image` and `publish-gateway-image` becomes a matrix over + `ubuntu-latest` and `ubuntu-24.04-arm`, as `publish-image-amd64` and + `publish-image-arm64` are. Each leg pushes `:git--` by digest. + The change gate runs once, in a `-gate` job that both legs need. +- **Index.** A new `publish--index` job per lane needs both legs and + composes `:git-`. `publishImageIndex` moves from the agent-only + `IMAGE_REF` to a `repo` input, so all five lanes share it and its + `EXPECTED_PLATFORMS` check. +- **Guest artifact.** The guest index carries two artifact manifests with + `platform` set. `fetchGuestManifest` (`go/internal/stack/materialize.go`) + accepts an index and picks the member for `runtime.GOARCH`; a digest that + names one manifest still works. +- **Retag and the P6 tagger.** `manifestIdentity` accepts an index and + returns the index digest plus each member's platform and digest. + `assertCoherent` compares the index digest and the member list. Every copy + uses `--multi-arch all --preserve-digests`. `tag-main-images` needs the + index jobs, not the legs. + +Tests, red first: + +- `retag-core.test.ts`: `manifestIdentity` on a two-member index returns + both members; an index missing `linux/arm64` throws `EXIT.badManifest`; + `assertCoherent` fails on a member digest change. +- `tools/agent-image-index/index.test.ts`: `publishImageIndex` with a + `compass-server` repo composes the index from the `-amd64` and `-arm64` + tags. +- `go/internal/stack/materialize_test.go`: an index resolves to the + `runtime.GOARCH` member; an index with no such member fails and names the + arch. +- `tools/ci-matrix/index.test.ts`: every publish lane has an arm64 leg and an + index job. +- `tools/runner-image/build-core.test.ts` and + `tools/server-image/build-core.test.ts`: `hostPlatform("arm64")` returns + `linux/arm64`, and `hostPlatform("ia32")` throws. + +### Interfaces + ```ts -// tools/runner-image/retag-core.ts +// tools/server-manifests/render-core.ts (P2) +export type RunnerClusterValues = { + name: string; issuer: string; audience?: string; + namespace: string; serviceAccount: string; maxTokenLifetime?: string; + jwksURI?: string; jwks?: string; caFile?: string; +}; +export type ForgeValues = { + host?: string; repos?: string[]; + appId: number; installationId: number; + appKeySecret: string; appWebhookSecret: string; + reviewer?: { appId: number; installationId: number; appKeySecret: string }; +}; +export type ServerDeployValues = { + namespace: string; serverImage: string; natsImage: string; natsStorage: string; + tlsSecretName: string; dsnSecretName: string; natsSecretName: string; + serverSecretsSecretName: string; adminTokenSecretName: string; s3SecretName?: string; + corsAllowedOrigin?: string; otlpEndpoint?: string; + external?: { port: number; loadBalancerClass?: string }; + nodeSelector?: { key: string; value: string }; + toleration?: { key: string; value: string; effect: "NoSchedule" | "PreferNoSchedule" | "NoExecute" }; + forge?: ForgeValues; + runnerClusters: RunnerClusterValues[]; +}; +export function assertServerImageDigest(ref: string): void; +export function parseServerDeployValues(raw: unknown): ServerDeployValues; +export function renderServerManifests(v: ServerDeployValues): ReturnType; +// server args: --listen 0.0.0.0:50052 --tls-cert /etc/compass/tls/tls.crt +// --tls-key /etc/compass/tls/tls.key --state-dir /run/compass/state +// --socket /run/compass/state/server.sock +// --admin-token-file /var/run/compass/admin-token/token +// --runner-clusters /etc/compass/runner-clusters/clusters.yaml +// --secret-provider dotenv:///etc/compass/secrets/server.env [--forge-*] +// files: tools/server-manifests/{render.ts,values.example.json,README.md,moon.yml} + +// tools/runner-manifests/render-core.ts (P3, merged) +RunnerDeployValues.egressAllow?: readonly string[]; +RunnerDeployValues.serverCA?: + | { configMapName: string; secretName?: undefined; key: string } + | { secretName: string; configMapName?: undefined; key: string }; + +// tools/runner-image/assert-public-core.ts (P5) +export function publicVerdict(result: ProbeResult, ref: string): { ok: true } | { ok: false; message: string }; +// tools/runner-image/assert-public.ts: bun tools/runner-image/assert-public.ts --ref +// tools/runner-image/retag-core.ts (P5) +export function probeArgv(repo: string, sha12: string): string[]; // ["inspect","--no-creds","--raw","docker://:git-"] +// tools/agent-image-index/index.ts (P5): authFile leaves the read path +function inspectCommand(tag: string, ...args: string[]): string[]; // ["skopeo","inspect",...args,"--no-creds",tag] + +// tools/runner-image/retag-core.ts (P6) export function mainTag(firstParentCount: number, commit: string): string; // "main--" export async function resolveMainPair(input: { ancestors: readonly string[]; @@ -548,10 +656,34 @@ export async function resolveMainPair(input: { changedSince: (commit: string) => Promise; closurePaths: Record; }): Promise>; -// tools/runner-image/tag-main.ts +// tools/runner-image/tag-main.ts (P6) // bun tools/runner-image/tag-main.ts --sha <40-hex> \ // --server-repo ghcr.io/rigelbuild/compass-server --runner-repo ghcr.io/rigelbuild/compass-runner // .github/workflows/release.yml: job tag-main-images + +// tools/runner-image/retag-core.ts (P7) +export type Platform = "linux/amd64" | "linux/arm64"; +export type ManifestIdentity = + | { kind: "manifest"; manifestDigest: string; configDigest: string } + | { kind: "index"; manifestDigest: string; members: { platform: Platform; digest: string }[] }; +export function manifestIdentity(raw: string): ManifestIdentity; +export function assertCoherent(source: ManifestIdentity, release: ManifestIdentity): void; +// tools/runner-image/build-core.ts and tools/server-image/build-core.ts (P7) +export function hostPlatform(arch: NodeJS.Architecture): Platform; +// tools/agent-image-index/index.ts (P7) +export type PublishInput = { repo: string; sha: string; authFile: string; runnerTemp: string }; +export async function publishImageIndex(input: PublishInput, run: CommandRunner): Promise; +// tools/gateway-image/core.ts (P7): the PLATFORM const becomes a platform parameter +``` + +```go +// go/server/network_door.go (P4, //go:build unix) +func readSuppliedAdminToken(path string) (string, error) // os.OpenFile(path, O_RDONLY|O_NONBLOCK|O_CLOEXEC, 0), f.Stat, io.ReadAll(f) +// checkSuppliedTokenMode is pure; gid comes from info.Sys().(*syscall.Stat_t).Gid. +func checkSuppliedTokenMode(mode os.FileMode, gid uint32, held []int) error + +// go/internal/stack/materialize.go (P7) +func selectGuestMember(raw []byte, arch string) (digest string, err error) // index → member digest for runtime.GOARCH ``` ### Deferred follow-ups @@ -581,9 +713,13 @@ func fetchHTTPS(ctx context.Context, client *http.Client, u, bearer string) ([]b - [x] P1 — Server image and publish lane, compass PR #2355, after #2357 - [ ] P2 — `tools/server-manifests` renderer: no Postgres, no init container, - `forge`, placement values (compass PR #2363), after P4 + `forge`, placement values, digest-only images (compass PR #2363), after P4 - [x] P3 — Runner renderer `serverCA` and `egressAllow`, compass PRs #2358 and #2360 -- [ ] P4 — Admin token file from a Secret volume (Go) -- [ ] P5 — Public images; tooling reads anonymously and checks visibility -- [ ] P6 — Paired `:main--` tags on every main commit, after P5 +- [ ] P4 — Admin token file from a Secret volume (Go), with the spec change +- [ ] P5 — Anonymous registry reads in every lane; publish fails on a private + package +- [ ] P6 — Paired `:main--` tags on the head of every main push, + after P5 +- [ ] P7 — Multi-arch (`linux/amd64`, `linux/arm64`) images in every lane, + index-aware retag and tagger, after P6 From 887262250024f0f398b65a01eae19f154b94e25c Mon Sep 17 00:00:00 2001 From: mintaka Date: Sat, 17 Oct 2026 21:10:39 -0400 Subject: [PATCH 07/10] docs(compass): cover the arm64 guest boot path and explicit anonymous reads (RIG-5559) Co-authored-by: Matt Wilkinson --- .../compass/decisions/server/DL-560.md | 2 +- docs/designs/compass/server-on-kubernetes.md | 301 ++++++++++++++---- 2 files changed, 242 insertions(+), 61 deletions(-) diff --git a/docs/designs/compass/decisions/server/DL-560.md b/docs/designs/compass/decisions/server/DL-560.md index ae8065d..201636f 100644 --- a/docs/designs/compass/decisions/server/DL-560.md +++ b/docs/designs/compass/decisions/server/DL-560.md @@ -1,6 +1,6 @@ --- id: DL-560 -decision: "Every Compass image (agent, runner, server, guest image, gateway) is built in compass for linux/amd64 and linux/arm64 and published as an OCI index whose members are first pushed as :git--, and the retag and main-tag tooling verifies an index by its digest and member list." +decision: "Every Compass image (agent, runner, server, guest image, gateway) is built in compass for linux/amd64 and linux/arm64 and published as an OCI index whose members are first pushed as :git--, the retag and main-tag tooling verifies an index by its digest and member list, and the guest boot path chooses its kernel file (bzImage or Image) and serial console (ttyS0 or ttyAMA0) per arch." status: "Active (Matt, 2026-10-17)" record: ../../server-on-kubernetes.md#design --- diff --git a/docs/designs/compass/server-on-kubernetes.md b/docs/designs/compass/server-on-kubernetes.md index 3146292..b0453e1 100644 --- a/docs/designs/compass/server-on-kubernetes.md +++ b/docs/designs/compass/server-on-kubernetes.md @@ -8,10 +8,27 @@ [Design](#design), ledgered as DL-539 to DL-547 and DL-560 in `decisions/server/`. +## Open Questions + +- **OQ-13 — arm64 microVM boot proof.** P7 builds the arm64 guest and + unit-tests its kernel name and console, but no GitHub-hosted arm64 runner + exposes `/dev/kvm`, so no CI job can boot it. No P7 code change depends + on the answer; only the arm64 Runner support claim does. + 1. No pre-merge arm64 boot. The Runner's startup boot canary + (`runMicroVMPreflight`, `go/cmd/compass-runner/main.go`) fails closed + on a node that cannot boot the guest. The Runner README says amd64 + only until `compass-go:test-microvm` passes on an arm64 KVM host with + GICv3. + 2. A self-hosted arm64 bare-metal runner for the `microvm` job. It proves + every PR, but GitHub says self-hosted runners + ["should almost never be used for public repositories"](https://docs.github.com/en/actions/reference/security/secure-use#hardening-for-self-hosted-runners), + because a fork PR can run code on the host. + + **Recommendation:** option 1. P7 is written against it. + ## Resolved decisions -No question is open. Question numbers match the review threads on this -record. +Question numbers match the review threads on this record. - **OQ-1 — Admin token from a Secret.** A Go change (P4). The supplied token file may be a symlink, and it may be group-readable when the process holds @@ -313,9 +330,39 @@ other four lanes are single-arch now: `flake.nix` lists `systems = [ "x86_64-linux" ]`. `manifestIdentity` (`tools/runner-image/retag-core.ts`) refuses an index ("an image index, or -malformed"), so retag and the P6 tagger must learn to verify one. A KVM node -of either arch can then run the Runner, and the Server and NATS can run on -any node. +malformed"), so retag and the P6 tagger must learn to verify one. + +The guest boot path is also amd64-only, in three ways. The +[guest agent pin record](infra/runtime/compass-guest-agent-pin-multiarch/design.md) +named them and left them to a follow-up (its T5). P7 is that follow-up. + +- **Kernel file.** At the pinned nixpkgs, `linuxPackages.kernel.target` is + `bzImage` on `x86_64-linux` and `Image` on `aarch64-linux`. `bzImage` is + hard-coded in `kernelImagePath` (`tools/runner-image/build-core.ts`), the + kernel path in `tools/guest-image/publish.ts`, `guestImageEnv` + (`tools/microvm-boot-test/run-core.ts`), the `microvm` job in + `.github/workflows/ci.yml`, and the kernel `annotationName` in + `guestAssets` (`go/internal/stack/materialize.go`). That name must equal + `ASSET_FILENAMES.kernel` (`tools/guest-image/publish-core.ts`), because + `validateGuestLayer` checks the layer annotation. +- **Boot modules.** `moduleConfigCheck` (`guest-image/default.nix`) fails + unless eight symbols are `=m`. The aarch64 kernel config has + `CONFIG_VIRTIO_PCI`, `CONFIG_VIRTIO_BLK`, `CONFIG_VIRTIO_NET` and + `CONFIG_PACKET` as `=y`, so the arm64 initrd build fails. A built-in module + is harmless: kmod `modprobe` exits 0 for it, and `makeModulesClosure` + records it as a `builtin` dependency. +- **Console.** `vmmArgs` (`go/internal/runtime/microvm/launch.go`) always + appends `console=ttyS0`. The arm64 serial device is `ttyAMA0`, as + cloud-hypervisor's AArch64 example shows, so guestd's stderr would miss + the captured console. + +P7 fixes all three, with a unit test per arch for each. No CI job can boot +an arm64 guest: GitHub's hosted arm64 runners expose no `/dev/kvm` +([runner-images #14062](https://github.com/actions/runner-images/issues/14062)). +So P7 adds an arm64 build gate, not a boot gate. When P7 lands, the Server +and NATS run on a node of either arch. The Runner image ships both members, +but the Runner is supported on amd64 only until an arm64 microVM boot +passes (OQ-13). ### Known limitation @@ -478,43 +525,90 @@ containing the supplied bootstrap-admin token") changes to match. The five packages are public already. P5 makes every registry read in compass anonymous, so D9 holds, and keeps the packages public. -- `retag.ts` probes and verifies with `skopeo inspect --no-creds --raw`. Only - the copy uses `--dest-authfile`. `classifyProbe` still reads 403 as - ambiguous, because a private package and a missing package give the same - 403. -- The `release-image` job (`.github/workflows/release.yml`) has three - `skopeo inspect --authfile "$REGISTRY_AUTH_FILE"` reads: the ancestor probe - and the two digest checks after the copy. All three use `--no-creds`. Its - `skopeo copy --multi-arch all` uses `--dest-authfile`. -- `inspectCommand` (`tools/agent-image-index/index.ts`) builds every `skopeo - inspect` there with `--authfile`; it uses `--no-creds`. `podman manifest - add` drops `--authfile`; `podman manifest push` keeps it. -- `agent-image/publish.sh` reads the remote tag twice with `inspect --raw - --authfile`. Both use `--no-creds`. -- The guest and gateway `publish.ts` pass `auth` to their `inspect` reads. - The reads drop it; the push keeps it. -- Stale text goes: the `release-runner-image` step comment "The runner - package is private, so even the probes need auth", the `retag.ts` header - "so private packages work", and the `retag-core.test.ts` case name "a - private package without auth (403 Forbidden) is ambiguous" and its "on - this private package" comment. -- A new check runs as the last step of each of the five publish jobs. It - reads the just-published tag anonymously and fails with the package - settings path when the package is not public. A new package therefore - fails its first publish until someone sets it public: the one manual step, - because no API sets visibility. - -Tests, red first: - +Dropping `--authfile` is not enough. Every publish job writes +`REGISTRY_AUTH_FILE=$RUNNER_TEMP/ghcr-auth.json` to `$GITHUB_ENV` +(`.github/workflows/release.yml`). skopeo's `sharedImageFlags` defaults +`--authfile` to `os.Getenv("REGISTRY_AUTH_FILE")`. podman's +`GetDefaultAuthFile` returns the same variable first, and `podman manifest +add` has no `--no-creds`. So each read names its anonymity: + +- Every `skopeo inspect` of a registry ref passes `--no-creds`. skopeo then + sets an empty `DockerAuthConfig`, which `GetCredentials` + (`containers/image`, `pkg/docker/config`) returns before any auth file. +- Every `skopeo copy` passes `--src-no-creds` and `--dest-authfile `, + never a bare `--authfile`. The rule holds for a local source too, where + the source flag is a no-op. +- `podman manifest add` passes `--authfile /anonymous-auth.json`, + a file holding `{}`. A named auth file turns off every fallback path in + `getAuthFilePaths` (`if !userSpecifiedPath`). `podman manifest push` keeps + the login file. + +The sites: + +- `tools/runner-image/`: `retag.ts` builds its probe, copy and verify argv + with new `inspectArgv` and `copyArgv` in `retag-core.ts`. Its copy today + passes no auth flag and leans on the environment; it now requires + `REGISTRY_AUTH_FILE` and fails `EXIT.usage` without it. `tagArgs` + (`publish-core.ts`) moves from `--authfile` to the copy pair. +- `tools/agent-image-index/index.ts`: `inspectCommand` drops `authFile` and + passes `--no-creds`; `composeLocalIndex` adds members with the empty auth + file. +- `tools/guest-image/publish.ts` and `tools/gateway-image/publish.ts`: the + `auth` spread leaves both `inspect` reads, which pass `--no-creds`. The + copy passes `--src-no-creds`, plus `--dest-authfile` when + `REGISTRY_AUTH_FILE` is set. +- `tools/release-notes/index.ts`: `gatherImage` adds `--no-creds`. + `tools/guest-image/pin-agent-image.ts` passes it already. +- The `release-image` job: three `skopeo inspect --authfile + "$REGISTRY_AUTH_FILE"` reads (the ancestor probe and two digest checks) + and one `skopeo copy --multi-arch all`. +- `agent-image/publish.sh`: two remote `inspect --raw --authfile` reads and + one `copy "nix:$SPEC" "$dest" --authfile`. The pinned fork skopeo + (`tools/toolchain/skopeo-nix2container-env.nix`, 1.24.1) has `--no-creds`, + `--src-no-creds` and `--dest-authfile`. + +`classifyProbe` still reads 403 as ambiguous, because a private package and +a missing package give the same 403. + +Stale text goes: the `release-runner-image` step comment "The runner package +is private, so even the probes need auth", the `retag.ts` header "so private +packages work", and the `retag-core.test.ts` case name "a private package +without auth (403 Forbidden) is ambiguous" and its "on this private package" +comment. + +A new check runs as the last step of each of the five publish jobs. It +reads the just-published tag anonymously and fails with the package settings +path when the package is not public. A new package therefore fails its first +publish until someone sets it public: the one manual step, because no API +sets visibility. + +Tests, red first. Each asserts the flag on every read, not the absence of +auth: + +- `retag-core.test.ts`: `inspectArgv` carries `--no-creds`; `copyArgv` + carries `--src-no-creds` and `--dest-authfile` and no `--authfile`. +- `publish-core.test.ts`: `tagArgs` carries the copy pair and no + `--authfile`. +- `tools/agent-image-index/index.test.ts`: every recorded `skopeo inspect` + carries `--no-creds`; every `podman manifest add` carries the empty auth + file, whose content is `{}`; only `podman manifest push` carries the + login file. +- `tools/guest-image/publish.test.ts`: the case "threads --authfile to + every registry call when the login pinned one" now requires `--no-creds` + on each logged `inspect` and `--src-no-creds --dest-authfile + /tmp/creds.json` on the `copy`. +- `tools/gateway-image/publish.test.ts`: the case "passes the auth file to + probe, copy, and re-read" expects `inspect --no-creds --raw`, then `copy + --src-no-creds --dest-authfile /tmp/registry-auth.json`, then `inspect + --no-creds --raw`. +- A new `tools/runner-image/registry-auth.test.ts` joins continuation lines + in `release.yml` and `agent-image/publish.sh`. Every skopeo `inspect` of a + non-`nix:` ref must carry `--no-creds`. Every `copy` must carry + `--src-no-creds` and `--dest-authfile`. Only `login` may carry + `--authfile`. It checks the `gatherImage` template the same way. Today's + sources fail all three rules. - `assert-public-core.test.ts` maps a manifest to public, and a 403, a "manifest unknown" and a timeout to failures that name the package. -- `retag-core.test.ts`: `probeArgv` carries `--no-creds` and no `--authfile`. -- `tools/agent-image-index/index.test.ts`: the recorded commands carry - `--no-creds` on every `skopeo inspect`, and `--authfile` only on `podman - manifest push`. -- A new `tools/runner-image/registry-auth.test.ts` reads `release.yml`, - `agent-image/publish.sh` and the guest and gateway `publish.ts`. It fails - on any `inspect` that names `--authfile` or `auth`. ### P6 — Paired main tags @@ -555,12 +649,13 @@ index, as the agent lane does (D10). `.#compass-server`, `.#compass-runner` and the guest assets evaluate on both. `guest-image/default.nix` already selects its agent member per system (`agentPin.platformFor`). -- **Build guards.** `tools/runner-image/build.ts` and - `tools/server-image/build.ts` take the platform from the host - (`x64` → `linux/amd64`, `arm64` → `linux/arm64`) and fail on any other - host. `PLATFORM` in `tools/gateway-image/core.ts` becomes a parameter. The - `IMAGE_PLATFORM` comment in `tools/runner-image/build.ts` ("the cluster - nodes are amd64") goes. +- **Build guards.** `hostPlatform` (`tools/runner-image/build-core.ts`) + maps the host (`x64` → `linux/amd64`, `arm64` → `linux/arm64`) and throws + on any other host. `tools/runner-image/build.ts` and + `tools/server-image/build.ts` call it; the server lane already imports + this module. `PLATFORM` in `tools/gateway-image/core.ts` becomes a + parameter. The `IMAGE_PLATFORM` comment in `tools/runner-image/build.ts` + ("the cluster nodes are amd64") goes. - **Jobs.** Each of `publish-runner-image`, `publish-server-image`, `publish-guest-image` and `publish-gateway-image` becomes a matrix over `ubuntu-latest` and `ubuntu-24.04-arm`, as `publish-image-amd64` and @@ -579,6 +674,49 @@ index, as the agent lane does (D10). `assertCoherent` compares the index digest and the member list. Every copy uses `--multi-arch all --preserve-digests`. `tag-main-images` needs the index jobs, not the legs. +- **Kernel file.** `kernelFileName` (`tools/runner-image/build-core.ts`) + maps `linux/amd64` to `bzImage` and `linux/arm64` to `Image`. It is the + only TypeScript table. `kernelImagePath`, `buildArgs`, the kernel path in + `tools/guest-image/publish.ts`, `assetFilenames` (it replaces + `ASSET_FILENAMES` in `tools/guest-image/publish-core.ts`) and + `guestImageEnv` take the platform and call it. Each entry point passes + `hostPlatform(process.arch)`. `tools/guest-image/moon.yml` and + `tools/microvm-boot-test/moon.yml` add `/tools/runner-image/*-core.ts` to + their inputs, as `tools/server-image/moon.yml` does. In Go, a new + `guestKernelName(goarch)` (`go/internal/stack/materialize.go`) returns + `Image` for `arm64` and `bzImage` otherwise, and the kernel entry of + `guestAssets` takes its `annotationName` from + `guestKernelName(runtime.GOARCH)`. An amd64 artifact then fails + `validateGuestLayer` on an arm64 host. `pinSmoke` (`guest-image/default.nix`), + which the `compass-guest-image:eval-arm64` task evaluates, also asserts + that `compass-guest-kernel.target` is `bzImage` for `x86_64-linux` and + `Image` for `aarch64-linux`, so a nixpkgs move that renames the file fails + a gate. The `microvm` job in `ci.yml` stays on amd64 and keeps + `$kernel/bzImage`. +- **Boot modules.** `moduleConfigCheck` (`guest-image/default.nix`) accepts + `=m` or `=y`. It fails only when a symbol is absent or `is not set`. It + becomes a function of the config file, so a fixture test can run it. +- **Console.** A new `serialConsole(goarch)` + (`go/internal/runtime/microvm/launch.go`) returns `ttyS0` for `amd64` and + `ttyAMA0` for `arm64`, and errors on any other arch. `launch` calls it + with `runtime.GOARCH` before it starts a child and passes the result to + `vmmArgs`, which appends `console=`. +- **arm64 build gate.** A new `guest-arm64` job in `.github/workflows/ci.yml` + runs on `ubuntu-24.04-arm` when `microvm_affected` is true, as + `agent-fod-arm64` does, and the rollup checks it the same way. It runs + `bun tools/microvm-boot-test/run.ts --build-only`. That mode builds the + guest and the VMM stack, fails when the kernel file is missing, and skips + `go test`. It cannot boot: hosted arm64 runners have no `/dev/kvm`. +- **arm64 Runner support (OQ-13).** `tools/runner-manifests/README.md` + gains the line "Runner nodes must be amd64." The PR that removes it + carries a passing `moon run compass-go:test-microvm` log from an arm64 + KVM host with GICv3. Until then, the Runner's boot canary fails closed on + a node that cannot boot the guest. +- **Docs.** `docs/self-host-guest-image.md` names `Image` beside `bzImage` + in its file table and adds an `org.compass.guest.layer.Image` row. + Comments that name only `ttyS0` or `bzImage` name both: + `go/internal/runtime/microvm/config.go`, `go/internal/guestd/guestd.go`, + `go/cmd/compass-guestd/main.go` and `guest-image/default.nix`. Tests, red first: @@ -590,12 +728,29 @@ Tests, red first: tags. - `go/internal/stack/materialize_test.go`: an index resolves to the `runtime.GOARCH` member; an index with no such member fails and names the - arch. -- `tools/ci-matrix/index.test.ts`: every publish lane has an arm64 leg and an - index job. -- `tools/runner-image/build-core.test.ts` and - `tools/server-image/build-core.test.ts`: `hostPlatform("arm64")` returns - `linux/arm64`, and `hostPlatform("ia32")` throws. + arch. `guestKernelName("arm64")` is `Image` and + `guestKernelName("amd64")` is `bzImage`. +- `tools/runner-image/closure-paths.test.ts`, which already parses + `release.yml`: each of the four publish jobs has an `ubuntu-latest` and an + `ubuntu-24.04-arm` leg, and each `publish--index` job needs both. +- `tools/runner-image/build-core.test.ts`: `hostPlatform("arm64")` returns + `linux/arm64`, and `hostPlatform("ia32")` throws. For `linux/arm64`, + `kernelImagePath` and `buildArgs(…).GUEST_KERNEL` end in `/Image`; for + `linux/amd64` they keep `/bzImage`. +- `tools/guest-image/publish-core.test.ts`: `layoutPlan` for `linux/arm64` + writes `org.compass.guest.layer.Image` and no `bzImage` key. +- `tools/microvm-boot-test/run-core.test.ts`: `guestImageEnv` for + `linux/arm64` sets `COMPASS_TEST_GUEST_KERNEL` to `/Image`. +- `go/internal/runtime/microvm/launch_cmdline_test.go`: `serialConsole` + returns `ttyS0` for `amd64` and `ttyAMA0` for `arm64`, and errors for + `riscv64`. `vmmArgs` given `ttyAMA0` carries `console=ttyAMA0` and no + `ttyS0`. +- A new `guest-image/module-check-tests.nix`, built by the + `compass-guest-image:test` task: a config with all eight symbols `=m` or + `=y` passes; one symbol `is not set`, or absent, fails with + "BUILD-BREAK". +- `compass-guest-image:eval-arm64`: the kernel-target assertion passes at + the pin, and fails when its expected map swaps the two names. ### Interfaces @@ -643,10 +798,15 @@ RunnerDeployValues.serverCA?: // tools/runner-image/assert-public-core.ts (P5) export function publicVerdict(result: ProbeResult, ref: string): { ok: true } | { ok: false; message: string }; // tools/runner-image/assert-public.ts: bun tools/runner-image/assert-public.ts --ref -// tools/runner-image/retag-core.ts (P5) -export function probeArgv(repo: string, sha12: string): string[]; // ["inspect","--no-creds","--raw","docker://:git-"] +// tools/runner-image/retag-core.ts (P5); retag.ts builds every registry argv with these +export function inspectArgv(ref: string): string[]; // ["inspect","--no-creds","--raw","docker://"] +export function copyArgv(sourceRef: string, destRef: string, destAuthFile: string): string[]; +// ["copy","--preserve-digests","--src-no-creds","--dest-authfile",,"docker://","docker://"] +// tools/runner-image/publish-core.ts (P5): signature unchanged, argv moves to the copy pair +export function tagArgs(repo: string, digest: string, sha12: string, authFile: string): string[]; // tools/agent-image-index/index.ts (P5): authFile leaves the read path function inspectCommand(tag: string, ...args: string[]): string[]; // ["skopeo","inspect",...args,"--no-creds",tag] +// composeLocalIndex writes "{}" to /anonymous-auth.json and passes it to every `podman manifest add` // tools/runner-image/retag-core.ts (P6) export function mainTag(firstParentCount: number, commit: string): string; // "main--" @@ -661,15 +821,30 @@ export async function resolveMainPair(input: { // --server-repo ghcr.io/rigelbuild/compass-server --runner-repo ghcr.io/rigelbuild/compass-runner // .github/workflows/release.yml: job tag-main-images -// tools/runner-image/retag-core.ts (P7) -export type Platform = "linux/amd64" | "linux/arm64"; +// tools/runner-image/retag-core.ts (P7); Platform comes from build-core.ts export type ManifestIdentity = | { kind: "manifest"; manifestDigest: string; configDigest: string } | { kind: "index"; manifestDigest: string; members: { platform: Platform; digest: string }[] }; export function manifestIdentity(raw: string): ManifestIdentity; export function assertCoherent(source: ManifestIdentity, release: ManifestIdentity): void; -// tools/runner-image/build-core.ts and tools/server-image/build-core.ts (P7) -export function hostPlatform(arch: NodeJS.Architecture): Platform; +// tools/runner-image/build-core.ts (P7); tools/server-image/build.ts imports hostPlatform +export type Platform = "linux/amd64" | "linux/arm64"; +export function hostPlatform(arch: NodeJS.Architecture): Platform; // x64 → linux/amd64, arm64 → linux/arm64, else throws +export function kernelFileName(platform: Platform): "bzImage" | "Image"; +export function kernelImagePath(kernelDir: string, platform: Platform): string; // / +export function buildArgs(outputs: RunnerImageOutputs, platform: Platform): Record; +// buildctlArgs(contextDir, outputs, platform, output) passes its platform to buildArgs +// tools/guest-image/publish-core.ts (P7): replaces ASSET_FILENAMES +export function assetFilenames(platform: Platform): Readonly>; +export function layoutPlan( + assets: Readonly>, + provenance: { revision: string; agentImageDigest: string }, + platform: Platform, +): LayoutPlan; +// tools/microvm-boot-test/run-core.ts (P7) +export function guestImageEnv(outPaths: readonly string[], platform: Platform): Record; +// tools/microvm-boot-test/run.ts (P7): --build-only realises the guest and VMM attrs, +// checks / exists, and exits 0 without `go test` // tools/agent-image-index/index.ts (P7) export type PublishInput = { repo: string; sha: string; authFile: string; runnerTemp: string }; export async function publishImageIndex(input: PublishInput, run: CommandRunner): Promise; @@ -682,8 +857,13 @@ func readSuppliedAdminToken(path string) (string, error) // os.OpenFile(path, O_ // checkSuppliedTokenMode is pure; gid comes from info.Sys().(*syscall.Stat_t).Gid. func checkSuppliedTokenMode(mode os.FileMode, gid uint32, held []int) error -// go/internal/stack/materialize.go (P7) +// go/internal/stack/materialize.go (P7, //go:build unix) func selectGuestMember(raw []byte, arch string) (digest string, err error) // index → member digest for runtime.GOARCH +func guestKernelName(goarch string) string // "Image" for arm64, else "bzImage"; guestAssets' kernel annotationName + +// go/internal/runtime/microvm/launch.go (P7, //go:build unix) +func serialConsole(goarch string) (string, error) // amd64 → "ttyS0", arm64 → "ttyAMA0", else error +func vmmArgs(cfg BootConfig, consolePath, console string, opts launchOptions) []string // appends "console=" ``` ### Deferred follow-ups @@ -722,4 +902,5 @@ func fetchHTTPS(ctx context.Context, client *http.Client, u, bearer string) ([]b - [ ] P6 — Paired `:main--` tags on the head of every main push, after P5 - [ ] P7 — Multi-arch (`linux/amd64`, `linux/arm64`) images in every lane, - index-aware retag and tagger, after P6 + index-aware retag and tagger, per-arch guest kernel file, boot modules and + console, arm64 guest build gate, after P6 From 3f8c10b475629b69de644824cf0ce1e11636257a Mon Sep 17 00:00:00 2001 From: mintaka Date: Sat, 17 Oct 2026 21:41:10 -0400 Subject: [PATCH 08/10] docs(compass): keep the pre-push destination probe authenticated (RIG-5559) Co-authored-by: Matt Wilkinson --- .../compass/decisions/server/DL-547.md | 2 +- docs/designs/compass/server-on-kubernetes.md | 96 +++++++++++++------ 2 files changed, 68 insertions(+), 30 deletions(-) diff --git a/docs/designs/compass/decisions/server/DL-547.md b/docs/designs/compass/decisions/server/DL-547.md index 74e9146..a069388 100644 --- a/docs/designs/compass/decisions/server/DL-547.md +++ b/docs/designs/compass/decisions/server/DL-547.md @@ -1,6 +1,6 @@ --- id: DL-547 -decision: "Every Compass image (agent, runner, server, guest image, gateway) is public, renderers emit no imagePullSecrets, every registry read in the publish, index and release tooling is anonymous and credentials are used only for the write destination, and each publish job fails when its package is not public, because no GitHub API sets package visibility." +decision: "Every Compass image (agent, runner, server, guest image, gateway) is public, renderers emit no imagePullSecrets, every registry read in the publish, index and release tooling is anonymous and credentials are used only for the write destination, except the pre-push probe of the tag a push is about to write, which is part of the write and uses the login file because GHCR answers an anonymous probe of a package that does not exist yet with 403, and each publish job fails when its package is not public, because no GitHub API sets package visibility." status: "Active (Matt, 2026-10-17)" record: ../../server-on-kubernetes.md#design --- diff --git a/docs/designs/compass/server-on-kubernetes.md b/docs/designs/compass/server-on-kubernetes.md index b0453e1..df4f47f 100644 --- a/docs/designs/compass/server-on-kubernetes.md +++ b/docs/designs/compass/server-on-kubernetes.md @@ -303,6 +303,10 @@ answered `skopeo list-tags --no-creds`. Neither renderer emits `imagePullSecrets`. A command that only reads the registry (`skopeo inspect`, `podman manifest add`) sends no credentials; a write (`skopeo copy`, `podman manifest push`) uses them for the destination only (P5). +One read is part of a write: the probe of the exact tag a push is about to +write uses the login file. GHCR answers an anonymous request for a package +that does not exist yet with 403, and every probe classifier aborts on +403, so an anonymous probe would stop the push that creates the package. GitHub's REST packages API can list, get, delete and restore a package, but has no call that sets visibility. A new package starts private, and making it public is a one-way switch in the package settings: a manual @@ -404,7 +408,8 @@ restart: it re-dials, re-enrolls with the hub, and keeps its microVMs. does. Every image is an OCI index of `linux/amd64` and `linux/arm64` once P7 lands. - **Registry auth.** Reads send no credentials; writes use credentials for - the destination only. + the destination only. The probe of the tag a push is about to write is + part of the write and uses the login file (D9). - **Secrets.** Renderers emit no Secret object and no secret value. - **Kubernetes.** ≥1.33, for user namespaces. - **PostgreSQL.** ≥16 for a non-superuser owner role (D3). @@ -523,7 +528,8 @@ containing the supplied bootstrap-admin token") changes to match. ### P5 — Anonymous registry reads The five packages are public already. P5 makes every registry read in -compass anonymous, so D9 holds, and keeps the packages public. +compass anonymous except the destination probe, so D9 holds, and keeps the +packages public. Dropping `--authfile` is not enough. Every publish job writes `REGISTRY_AUTH_FILE=$RUNNER_TEMP/ghcr-auth.json` to `$GITHUB_ENV` @@ -535,6 +541,18 @@ add` has no `--no-creds`. So each read names its anonymity: - Every `skopeo inspect` of a registry ref passes `--no-creds`. skopeo then sets an empty `DockerAuthConfig`, which `GetCredentials` (`containers/image`, `pkg/docker/config`) returns before any auth file. +- The exception is the destination probe: the `inspect` of the exact tag a + push is about to write, whose answer decides publish, skip or abort. It is + part of the write and uses the same credentials as the push. On + 2026-10-17, an anonymous probe of a package that does not exist got + "Requesting bearer token: received unexpected HTTP status: 403 Forbidden"; + a missing tag on an existing public package got "manifest unknown". The + pre-push classifiers free the tag only on a not-found answer: + `tagDisposition` (`tools/guest-image/publish-core.ts`, + `tools/gateway-image/core.ts`), `guard_immutable` + (`agent-image/publish.sh`) and `isManifestUnknown` + (`tools/agent-image-index/index.ts`). So an anonymous probe would abort + the push that creates a new package. - Every `skopeo copy` passes `--src-no-creds` and `--dest-authfile `, never a bare `--authfile`. The rule holds for a local source too, where the source flag is a no-op. @@ -551,21 +569,25 @@ The sites: `REGISTRY_AUTH_FILE` and fails `EXIT.usage` without it. `tagArgs` (`publish-core.ts`) moves from `--authfile` to the copy pair. - `tools/agent-image-index/index.ts`: `inspectCommand` drops `authFile` and - passes `--no-creds`; `composeLocalIndex` adds members with the empty auth - file. + passes `--no-creds`. `guardImmutableTag` probes `:git-` with a new + `destinationProbeCommand` that carries the login file. `composeLocalIndex` + adds members with the empty auth file. - `tools/guest-image/publish.ts` and `tools/gateway-image/publish.ts`: the - `auth` spread leaves both `inspect` reads, which pass `--no-creds`. The - copy passes `--src-no-creds`, plus `--dest-authfile` when + pre-push probe keeps its `auth` spread, the login file when + `REGISTRY_AUTH_FILE` is set. The post-push re-read passes `--no-creds`. + The copy passes `--src-no-creds`, plus `--dest-authfile` when `REGISTRY_AUTH_FILE` is set. - `tools/release-notes/index.ts`: `gatherImage` adds `--no-creds`. `tools/guest-image/pin-agent-image.ts` passes it already. - The `release-image` job: three `skopeo inspect --authfile "$REGISTRY_AUTH_FILE"` reads (the ancestor probe and two digest checks) - and one `skopeo copy --multi-arch all`. -- `agent-image/publish.sh`: two remote `inspect --raw --authfile` reads and - one `copy "nix:$SPEC" "$dest" --authfile`. The pinned fork skopeo - (`tools/toolchain/skopeo-nix2container-env.nix`, 1.24.1) has `--no-creds`, - `--src-no-creds` and `--dest-authfile`. + and one `skopeo copy --multi-arch all`. All three are source or + verification reads, so all pass `--no-creds`. +- `agent-image/publish.sh`: the `guard_immutable` probe of `$dest` keeps + `--authfile "$REGISTRY_AUTH_FILE"`. The post-copy re-read passes + `--no-creds`, and the `copy "nix:$SPEC" "$dest"` passes the copy pair. + The pinned fork skopeo (`tools/toolchain/skopeo-nix2container-env.nix`, + 1.24.1) has `--no-creds`, `--src-no-creds` and `--dest-authfile`. `classifyProbe` still reads 403 as ambiguous, because a private package and a missing package give the same 403. @@ -580,33 +602,42 @@ A new check runs as the last step of each of the five publish jobs. It reads the just-published tag anonymously and fails with the package settings path when the package is not public. A new package therefore fails its first publish until someone sets it public: the one manual step, because no API -sets visibility. +sets visibility. On a new package, the anonymous post-push re-read fails +first, with 403. The re-run after the package is set public finds the tag +through the destination probe, skips the push, and passes the check. -Tests, red first. Each asserts the flag on every read, not the absence of -auth: +Tests, red first. Each asserts the flag on every read, and the login file on +the destination probe: - `retag-core.test.ts`: `inspectArgv` carries `--no-creds`; `copyArgv` carries `--src-no-creds` and `--dest-authfile` and no `--authfile`. - `publish-core.test.ts`: `tagArgs` carries the copy pair and no `--authfile`. -- `tools/agent-image-index/index.test.ts`: every recorded `skopeo inspect` - carries `--no-creds`; every `podman manifest add` carries the empty auth - file, whose content is `{}`; only `podman manifest push` carries the - login file. +- `tools/agent-image-index/index.test.ts`: the `guardImmutableTag` probe of + `:git-` carries `--authfile ` and no `--no-creds`; every other + recorded `skopeo inspect` carries `--no-creds`; every `podman manifest add` + carries the empty auth file, whose content is `{}`; only `podman manifest + push` and the destination probe carry the login file. - `tools/guest-image/publish.test.ts`: the case "threads --authfile to - every registry call when the login pinned one" now requires `--no-creds` - on each logged `inspect` and `--src-no-creds --dest-authfile - /tmp/creds.json` on the `copy`. + every registry call when the login pinned one" now requires `--authfile + /tmp/creds.json` on the pre-push `inspect`, `--src-no-creds --dest-authfile + /tmp/creds.json` on the `copy`, and `--no-creds` on the re-read. A new case + stubs skopeo to answer 403 to an `inspect` without `--authfile` and + "manifest unknown" to one with it, and requires the `copy` to run. - `tools/gateway-image/publish.test.ts`: the case "passes the auth file to - probe, copy, and re-read" expects `inspect --no-creds --raw`, then `copy - --src-no-creds --dest-authfile /tmp/registry-auth.json`, then `inspect - --no-creds --raw`. + probe, copy, and re-read" expects `inspect --authfile + /tmp/registry-auth.json --raw`, then `copy --src-no-creds --dest-authfile + /tmp/registry-auth.json`, then `inspect --no-creds --raw`. - A new `tools/runner-image/registry-auth.test.ts` joins continuation lines in `release.yml` and `agent-image/publish.sh`. Every skopeo `inspect` of a non-`nix:` ref must carry `--no-creds`. Every `copy` must carry `--src-no-creds` and `--dest-authfile`. Only `login` may carry - `--authfile`. It checks the `gatherImage` template the same way. Today's - sources fail all three rules. + `--authfile`. It checks the `gatherImage` template the same way. One case + is allowed: the body of `guard_immutable` in `agent-image/publish.sh` + (from `guard_immutable() {` to the next line that is exactly `}`) holds + exactly one `inspect`, and it must carry `--authfile + "$REGISTRY_AUTH_FILE"` and no `--no-creds`. Today's sources fail the + first three rules. - `assert-public-core.test.ts` maps a manifest to public, and a 403, a "manifest unknown" and a timeout to failures that name the package. @@ -804,9 +835,16 @@ export function copyArgv(sourceRef: string, destRef: string, destAuthFile: strin // ["copy","--preserve-digests","--src-no-creds","--dest-authfile",,"docker://","docker://"] // tools/runner-image/publish-core.ts (P5): signature unchanged, argv moves to the copy pair export function tagArgs(repo: string, digest: string, sha12: string, authFile: string): string[]; -// tools/agent-image-index/index.ts (P5): authFile leaves the read path +// tools/agent-image-index/index.ts (P5): authFile leaves every read except the destination probe function inspectCommand(tag: string, ...args: string[]): string[]; // ["skopeo","inspect",...args,"--no-creds",tag] +function destinationProbeCommand(authFile: string, tag: string): string[]; // ["skopeo","inspect","--raw","--authfile",authFile,tag] +// guardImmutableTag probes :git- with destinationProbeCommand; nothing else calls it // composeLocalIndex writes "{}" to /anonymous-auth.json and passes it to every `podman manifest add` +// tools/guest-image/publish.ts, tools/gateway-image/publish.ts (P5): +// pre-push probe ["inspect",...auth,"--raw","docker://"], auth = ["--authfile",REGISTRY_AUTH_FILE] or [] +// copy ["copy","--src-no-creds",...destAuth,...,"oci:","docker://"], destAuth = ["--dest-authfile",REGISTRY_AUTH_FILE] or [] +// re-read ["inspect","--no-creds","--raw","docker://"] +// agent-image/publish.sh (P5): guard_immutable keeps `inspect --raw --authfile "$REGISTRY_AUTH_FILE" "$dest"` // tools/runner-image/retag-core.ts (P6) export function mainTag(firstParentCount: number, commit: string): string; // "main--" @@ -897,8 +935,8 @@ func fetchHTTPS(ctx context.Context, client *http.Client, u, bearer string) ([]b - [x] P3 — Runner renderer `serverCA` and `egressAllow`, compass PRs #2358 and #2360 - [ ] P4 — Admin token file from a Secret volume (Go), with the spec change -- [ ] P5 — Anonymous registry reads in every lane; publish fails on a private - package +- [ ] P5 — Anonymous registry reads in every lane, except the authenticated + destination probe; publish fails on a private package - [ ] P6 — Paired `:main--` tags on the head of every main push, after P5 - [ ] P7 — Multi-arch (`linux/amd64`, `linux/arm64`) images in every lane, From 7a78ca04eb299d620815344fddb23e39e7c6b6ab Mon Sep 17 00:00:00 2001 From: mintaka Date: Sun, 18 Oct 2026 00:39:27 -0400 Subject: [PATCH 09/10] docs(compass): point the Runner volume root at a quota subdirectory (RIG-5559) Adds P8. P2 is now a follow-up to the merged first renderer. Co-authored-by: Matt Wilkinson --- .../compass/decisions/server/DL-545.md | 2 +- docs/designs/compass/server-on-kubernetes.md | 74 ++++++++++++++++++- 2 files changed, 71 insertions(+), 5 deletions(-) diff --git a/docs/designs/compass/decisions/server/DL-545.md b/docs/designs/compass/decisions/server/DL-545.md index 9a2dd8d..761a161 100644 --- a/docs/designs/compass/decisions/server/DL-545.md +++ b/docs/designs/compass/decisions/server/DL-545.md @@ -1,6 +1,6 @@ --- id: DL-545 -decision: "A Runner-cluster entry whose issuer serves no anonymous discovery uses a jwksFile rendered from the cluster's public keys, so no Go change is needed; the rendered Runner gets serverCA and an egressAllow list rendered as --egress-allow, and its node exposes KVM as the kvmResourceName extended resource, stages the seccompProfilePath profile, and supports hostUsers: false." +decision: "A Runner-cluster entry whose issuer serves no anonymous discovery uses a jwksFile rendered from the cluster's public keys, so no Go change is needed; the rendered Runner gets serverCA and an egressAllow list rendered as --egress-allow, and its node exposes KVM as the kvmResourceName extended resource, stages the seccompProfilePath profile, and supports hostUsers: false; the Runner's microVM volume root is the volumes subdirectory of the session-volume hostPath mount, and the operator puts the project quota on that subdirectory, never on the hostPath itself, so the quota check has an unprojected reference above it." status: "Active (Matt, 2026-10-17)" record: ../../server-on-kubernetes.md#design --- diff --git a/docs/designs/compass/server-on-kubernetes.md b/docs/designs/compass/server-on-kubernetes.md index df4f47f..f326b3a 100644 --- a/docs/designs/compass/server-on-kubernetes.md +++ b/docs/designs/compass/server-on-kubernetes.md @@ -256,6 +256,12 @@ stream needs h2 end to end. - **Node.** The KVM node exposes `/dev/kvm` as the extended resource named by `kvmResourceName`, stages the seccomp profile at `seccompProfilePath` under the kubelet seccomp root, and supports `hostUsers: false`. +- **Session volumes.** `hostPaths.sessionVolumeRoot` is a directory with no + project id. The operator creates `volumes/` inside it and puts the project + id, `FS_XFLAG_PROJINHERIT` and the limits on `volumes/`. The Runner's + volume root is that subdirectory (P8). The quota check needs an + unprojected reference above the volume root, and in the pod the only one + is the top of the hostPath mount. **Delivery split and paired tags (D8).** Public compass holds `tools/server-manifests/` and `tools/runner-manifests/`: values in, plain @@ -403,7 +409,8 @@ restart: it re-dials, re-enrolls with the hub, and keeps its microVMs. product detail. - **PRs.** Each task is one compass PR, red then green. P2 merges after P4, because its output boots only with P4. P5, P6 and P7 form one stack in - that order: all three edit `tools/runner-image/` and `release.yml`. + that order: all three edit `tools/runner-image/` and `release.yml`. P8 + edits only `tools/runner-manifests/` and is independent of the rest. - **Images.** Digest only. Renderers reject tags, as `assertRunnerImageDigest` does. Every image is an OCI index of `linux/amd64` and `linux/arm64` once P7 lands. @@ -426,7 +433,8 @@ restart: it re-dials, re-enrolls with the hub, and keeps its microVMs. ```text P1 server image (#2355, merged) ─┐ -P3 runner values (#2360, merged) ┼──► operators render and deploy +P3 runner values (#2360, merged) ┤ +P8 runner volume root ───────────┼──► operators render and deploy P4 admin token (Go) ──► P2 server renderer (#2363) ┘ P5 anonymous reads ──► P6 paired main tags ──► P7 multi-arch images Deferred: F1 door TLS reload, F2 tokenFile @@ -447,7 +455,7 @@ reproducible digest. Files: `server-image/{Dockerfile,moon.yml,.dockerignore}`, ### P2 — Server manifests renderer -Open as compass PR #2363. Against its current head it must drop +The first renderer merged as compass PR #2363. A follow-up PR against main must drop `initImage`, the `init-admin-token` init container and the `postgres` value with its StatefulSet, Service and NetworkPolicy. It must add `forge`, `nodeSelector` and `toleration`. Tests use golden output plus invariants: @@ -783,6 +791,56 @@ Tests, red first: - `compass-guest-image:eval-arm64`: the kernel-target assertion passes at the pin, and fails when its expected map swaps the two names. +### P8 — Runner volume root below the session mount + +`renderRunnerManifests` (`tools/runner-manifests/render-core.ts`) mounts +`hostPaths.sessionVolumeRoot` at `SESSION_MOUNT_PATH` +(`/var/lib/compass/sessions`) and sets `COMPASS_MICROVM_VOLUME_ROOT` to +`SESSION_MOUNT_PATH` too, with `COMPASS_MICROVM_QUOTA_REQUIRED=true`. +`mountRoot` (`go/internal/runtime/microvm_quota_linux.go`) walks up until +`st_dev` changes. In the pod, that boundary is the top of the hostPath bind +mount. So for the volume root, `mountRoot` returns `distinct=false`, and +`readVolumeQuota` fails startup: "point --microvm-volume-root (or +$COMPASS_MICROVM_VOLUME_ROOT) at a SUBDIRECTORY of the quota'd filesystem +rather than at its mount point". Every rendered Runner fails its preflight. + +The fix is a subdirectory. A new const `SESSION_VOLUME_ROOT` is +`${SESSION_MOUNT_PATH}/volumes`, and `COMPASS_MICROVM_VOLUME_ROOT` takes it. +The mount and the hostPath (`type: "Directory"`) are unchanged. + +The quota also moves down. The kernel projects the quota into `statfs` only +for an inode that has the project-inherit flag: `xfs_fs_statfs` +(`fs/xfs/xfs_super.c`) checks `XFS_DIFLAG_PROJINHERIT`, and `ext4_statfs` +(`fs/ext4/super.c`) checks `EXT4_INODE_PROJINHERIT`. If the hostPath itself +carried the project, its root would read projected totals, and the check +would compare the project with itself. So the operator puts the project on +`/volumes` and leaves `sessionVolumeRoot` without one. +The renderer cannot create or check the subdirectory. A missing one fails +closed: `readVolumeQuota` returns `statfs "": …`, as +`TestReadVolumeQuotaAbsentPath` requires. + +The alternative keeps the volume root at the mount and makes the hostPath +the quota'd directory. It does not work: the volume root is still the top of +the mount, so `distinct` stays false. Any layout that passes needs an +unprojected directory at the top of the mount and the project below it. + +The README's "Other node prerequisites" bullet for the session tree says to +put the project quota on `/volumes`, owned by uid and gid +65532, and not on `sessionVolumeRoot`. + +Tests, red first, in `tools/runner-manifests/render-core.test.ts`: + +- "uses runtime settings consumed by compass-runner": + `COMPASS_MICROVM_VOLUME_ROOT` is `/var/lib/compass/sessions/volumes`. +- New: the volume root is a strict subdirectory of the `session-volumes` + `mountPath`. The test reads both from the rendered DaemonSet, so it fails + on today's output, where they are equal. +- "mounts only the session tree, runtime tree, and projected token" keeps + `/var/lib/compass/sessions` as the mount path. + +The Runner renderer has no golden file. `tools/server-manifests/golden.yaml` +holds no Runner object, so it does not change. + ### Interfaces ```ts @@ -826,6 +884,11 @@ RunnerDeployValues.serverCA?: | { configMapName: string; secretName?: undefined; key: string } | { secretName: string; configMapName?: undefined; key: string }; +// tools/runner-manifests/render-core.ts (P8) +const SESSION_MOUNT_PATH = "/var/lib/compass/sessions"; // unchanged: session-volumes mountPath +const SESSION_VOLUME_ROOT = `${SESSION_MOUNT_PATH}/volumes`; // new: COMPASS_MICROVM_VOLUME_ROOT value +// RunnerDeployValues.hostPaths is unchanged; the operator creates /volumes with the project quota + // tools/runner-image/assert-public-core.ts (P5) export function publicVerdict(result: ProbeResult, ref: string): { ok: true } | { ok: false; message: string }; // tools/runner-image/assert-public.ts: bun tools/runner-image/assert-public.ts --ref @@ -931,7 +994,7 @@ func fetchHTTPS(ctx context.Context, client *http.Client, u, bearer string) ([]b - [x] P1 — Server image and publish lane, compass PR #2355, after #2357 - [ ] P2 — `tools/server-manifests` renderer: no Postgres, no init container, - `forge`, placement values, digest-only images (compass PR #2363), after P4 + `forge`, placement values, digest-only images (follow-up to compass PR #2363), after P4 - [x] P3 — Runner renderer `serverCA` and `egressAllow`, compass PRs #2358 and #2360 - [ ] P4 — Admin token file from a Secret volume (Go), with the spec change @@ -942,3 +1005,6 @@ func fetchHTTPS(ctx context.Context, client *http.Client, u, bearer string) ([]b - [ ] P7 — Multi-arch (`linux/amd64`, `linux/arm64`) images in every lane, index-aware retag and tagger, per-arch guest kernel file, boot modules and console, arm64 guest build gate, after P6 +- [ ] P8 — Runner renderer sets `COMPASS_MICROVM_VOLUME_ROOT` to + `/var/lib/compass/sessions/volumes`, with the README quota layout; + independent of P2 From d0abefcdbc48ee5d3b9c0aeb9f7be8ee3e7b5352 Mon Sep 17 00:00:00 2001 From: mintaka Date: Sun, 18 Oct 2026 11:00:25 -0400 Subject: [PATCH 10/10] docs(compass): resolve OQ-13 and add the Runner TLS server name task (RIG-5559) Records the arm64 build-gate ruling as DL-577 and adds P9, a Runner --server-name that sets tls.Config.ServerName. Co-authored-by: Matt Wilkinson --- .../compass/decisions/server/DL-545.md | 2 +- .../compass/decisions/server/DL-577.md | 6 + docs/designs/compass/server-on-kubernetes.md | 136 +++++++++++++++--- 3 files changed, 120 insertions(+), 24 deletions(-) create mode 100644 docs/designs/compass/decisions/server/DL-577.md diff --git a/docs/designs/compass/decisions/server/DL-545.md b/docs/designs/compass/decisions/server/DL-545.md index 761a161..236a95a 100644 --- a/docs/designs/compass/decisions/server/DL-545.md +++ b/docs/designs/compass/decisions/server/DL-545.md @@ -1,6 +1,6 @@ --- id: DL-545 -decision: "A Runner-cluster entry whose issuer serves no anonymous discovery uses a jwksFile rendered from the cluster's public keys, so no Go change is needed; the rendered Runner gets serverCA and an egressAllow list rendered as --egress-allow, and its node exposes KVM as the kvmResourceName extended resource, stages the seccompProfilePath profile, and supports hostUsers: false; the Runner's microVM volume root is the volumes subdirectory of the session-volume hostPath mount, and the operator puts the project quota on that subdirectory, never on the hostPath itself, so the quota check has an unprojected reference above it." +decision: "A Runner-cluster entry whose issuer serves no anonymous discovery uses a jwksFile rendered from the cluster's public keys, so no Go change is needed; the rendered Runner gets serverCA, an optional serverName that sets the Runner's TLS server name through --server-name so a certificate naming only the public host verifies against the in-cluster serverAddr, and an egressAllow list rendered as --egress-allow, and its node exposes KVM as the kvmResourceName extended resource, stages the seccompProfilePath profile, and supports hostUsers: false; the Runner's microVM volume root is the volumes subdirectory of the session-volume hostPath mount, and the operator puts the project quota on that subdirectory, never on the hostPath itself, so the quota check has an unprojected reference above it." status: "Active (Matt, 2026-10-17)" record: ../../server-on-kubernetes.md#design --- diff --git a/docs/designs/compass/decisions/server/DL-577.md b/docs/designs/compass/decisions/server/DL-577.md new file mode 100644 index 0000000..af51748 --- /dev/null +++ b/docs/designs/compass/decisions/server/DL-577.md @@ -0,0 +1,6 @@ +--- +id: DL-577 +decision: "The arm64 microVM guest gets no pre-merge boot: the P7 arm64 build gate is the only pre-merge proof, the Runner's startup boot canary fails closed on a node that cannot boot the guest, Runner nodes stay amd64 until an arm64 KVM boot passes, and no self-hosted arm64 CI runner is added." +status: "Active (Matt, 2026-10-18)" +record: ../../server-on-kubernetes.md#resolved-decisions +--- diff --git a/docs/designs/compass/server-on-kubernetes.md b/docs/designs/compass/server-on-kubernetes.md index f326b3a..361fe1d 100644 --- a/docs/designs/compass/server-on-kubernetes.md +++ b/docs/designs/compass/server-on-kubernetes.md @@ -6,25 +6,8 @@ **Named decisions this record introduces:** D1–D10, inline in [Design](#design), ledgered as DL-539 to DL-547 and DL-560 in -`decisions/server/`. - -## Open Questions - -- **OQ-13 — arm64 microVM boot proof.** P7 builds the arm64 guest and - unit-tests its kernel name and console, but no GitHub-hosted arm64 runner - exposes `/dev/kvm`, so no CI job can boot it. No P7 code change depends - on the answer; only the arm64 Runner support claim does. - 1. No pre-merge arm64 boot. The Runner's startup boot canary - (`runMicroVMPreflight`, `go/cmd/compass-runner/main.go`) fails closed - on a node that cannot boot the guest. The Runner README says amd64 - only until `compass-go:test-microvm` passes on an arm64 KVM host with - GICv3. - 2. A self-hosted arm64 bare-metal runner for the `microvm` job. It proves - every PR, but GitHub says self-hosted runners - ["should almost never be used for public repositories"](https://docs.github.com/en/actions/reference/security/secure-use#hardening-for-self-hosted-runners), - because a fork PR can run code on the host. - - **Recommendation:** option 1. P7 is written against it. +`decisions/server/`. The OQ-13 ruling is DL-577. DL-545 (D7) also covers +`serverName` (P9). ## Resolved decisions @@ -54,6 +37,15 @@ Question numbers match the review threads on this record. is not a product change. - **OQ-12 — Multi-arch images.** Every Compass image is published for `linux/amd64` and `linux/arm64` (D10, P7). +- **OQ-13 — arm64 microVM boot proof.** No pre-merge arm64 boot; the P7 + arm64 build gate is the only pre-merge proof, and P7 is unchanged. No + GitHub-hosted arm64 runner exposes `/dev/kvm`. The Runner's startup boot + canary (`runMicroVMPreflight`, `go/cmd/compass-runner/main.go`) fails + closed on a node that cannot boot the guest, and the Runner README says + amd64 only until an arm64 boot passes (P7). A self-hosted arm64 runner was + rejected: GitHub says self-hosted runners + ["should almost never be used for public repositories"](https://docs.github.com/en/actions/reference/security/secure-use#hardening-for-self-hosted-runners) + (DL-577). ## Problem / Intent @@ -107,6 +99,11 @@ and the [OSS-core boundary record](meta/oss-core-managed-boundary/design.md). - **Runner.** `renderRunnerManifests` (`tools/runner-manifests/render-core.ts`) takes an absolute `https` `serverAddr`, and now `serverCA` and `egressAllow`. The hub holds one Runner (`go/internal/runnerhub/hub.go`). +- **Runner TLS name.** `NewCATrustClient` (`go/internal/runner/tls.go`) sets + only `RootCAs` and `MinVersion`, and without `--ca` the Runner uses + `http.DefaultClient`. Go's `addTLS` (`net/http/transport.go`) fills an + empty `ServerName` with the dialed host, so the Runner always checks the + certificate against the `--server` host. - **Agent egress.** The microVM arms a default-deny firewall. `--egress-allow` (`go/cmd/compass-runner/main.go`) is the only way to open a host. `isValidHost` (`go/internal/runtime/host.go`) admits DNS names and @@ -225,8 +222,10 @@ ingress may terminate it: the door refuses cleartext, and the Runner's bidi stream needs h2 end to end. - **Certificate.** The operator supplies a `kubernetes.io/tls` Secret whose - SANs cover the in-cluster name and any external name. A private CA works; - the Runner's `serverCA` value and the client `--ca` carry it. + SANs cover the in-cluster name and any external name. A certificate that + names only the public host also works when the Runner sets `serverName` + (P9). A private CA works; the Runner's `serverCA` value and the client + `--ca` carry it. - **Renewal.** The door loads its keypair once, so a renewed certificate takes effect on restart (see [Known limitation](#known-limitation)). - **Services.** Service `compass-server` (ClusterIP) maps 443 to 50052. An @@ -249,6 +248,8 @@ stream needs h2 end to end. then needs a re-render and a Server rollout. An issuer with anonymous discovery keeps `jwksURI`. - **CA.** `serverCA` mounts the CA and sets `COMPASS_RUNNER_CA`. +- **Server name.** `serverName` sets `COMPASS_SERVER_NAME`, the name the + Runner checks on the certificate (P9). - **Egress.** `egressAllow` renders one `--egress-allow=` arg. The renderer applies the `isValidHost` grammar, so `host:port` fails at render time. A session needs the provider API host, `github.com` and @@ -410,7 +411,9 @@ restart: it re-dials, re-enrolls with the hub, and keeps its microVMs. - **PRs.** Each task is one compass PR, red then green. P2 merges after P4, because its output boots only with P4. P5, P6 and P7 form one stack in that order: all three edit `tools/runner-image/` and `release.yml`. P8 - edits only `tools/runner-manifests/` and is independent of the rest. + edits only `tools/runner-manifests/`. P9 edits the Runner's TLS client, + its flags and `tools/runner-manifests/`, but no line that P8 edits. P8 and + P9 are independent of the rest. - **Images.** Digest only. Renderers reject tags, as `assertRunnerImageDigest` does. Every image is an OCI index of `linux/amd64` and `linux/arm64` once P7 lands. @@ -426,7 +429,8 @@ restart: it re-dials, re-enrolls with the hub, and keeps its microVMs. - **Lanes.** TypeScript is Bun with `bun:test`. Go DB tests use the `pgtest` build tag. - **Spec impact.** P4 changes the requirement "The network door mints and - persists a bootstrap admin token" in `docs/specs/product/compass.md`. No + persists a bootstrap admin token" in `docs/specs/product/compass.md`. P9 + adds a `compass-runner --server-name` requirement to the same file. No other task changes a spec. ### Order @@ -435,6 +439,7 @@ restart: it re-dials, re-enrolls with the hub, and keeps its microVMs. P1 server image (#2355, merged) ─┐ P3 runner values (#2360, merged) ┤ P8 runner volume root ───────────┼──► operators render and deploy +P9 runner TLS server name ───────┤ P4 admin token (Go) ──► P2 server renderer (#2363) ┘ P5 anonymous reads ──► P6 paired main tags ──► P7 multi-arch images Deferred: F1 door TLS reload, F2 tokenFile @@ -841,6 +846,76 @@ Tests, red first, in `tools/runner-manifests/render-core.test.ts`: The Runner renderer has no golden file. `tools/server-manifests/golden.yaml` holds no Runner object, so it does not change. +### P9 — Runner TLS server name + +An operator's certificate may name only the public host, while +`serverAddr` is the in-cluster Service (D6). The Runner then checks the +certificate against the Service name and fails with `x509.HostnameError` +(see [Ground truth](#ground-truth), Runner TLS name). The Server needs no +change: the door holds one keypair (`loadNetworkTLS`, +`go/server/network_door.go`), and Go's `getCertificate` +(`crypto/tls/common.go`) returns a lone certificate whatever the SNI. + +`compass-runner` gains `--server-name`, default `$COMPASS_SERVER_NAME`. It +sets `tls.Config.ServerName` in `go/internal/runner/tls.go`: the name the +Runner checks on the certificate and sends as SNI. It changes no trust +root, so the certificate must still chain to `--ca` or the host roots. A new +`runner.NewServerClient(caPath, serverName)` builds the client. An empty +`caPath` leaves `RootCAs` nil, which means the host roots, so the setting +also works without `--ca`. A non-empty `caPath` is read with the +`NewCATrustClient` errors. The client keeps TLS 1.3 minimum and HTTP/1.1 +plus HTTP/2. Like the `--ca` client today, it reads no proxy variables. +`NewCATrustClient` keeps its contract and its CLI and e2e callers; both +functions share one transport builder. + +In `go/cmd/compass-runner/main.go`, a new `runnerHTTPClient` replaces the +`--ca` block. It resolves both values with `orEnv`. When both are empty, it +returns `http.DefaultClient`, as `httpClientFor` (`go/cmd/compass/client.go`) +does. `Dial` (`go/internal/runner/runner.go`) already uses that client for a +nil one, so with no server name the behaviour is unchanged. + +The Runner renderer gains an optional `serverName`. It uses the +`isValidEgressHost` grammar, so a port or a URL fails at render time. When +set, the container gets `COMPASS_SERVER_NAME`, placed after +`COMPASS_RUNNER_CA`. When unset, the output is unchanged. In +`tools/runner-manifests/README.md`, "Two optional values:" becomes three. +The new `serverName` bullet says: set it when `serverAddr` is the in-cluster +Service and the certificate names only a public host. + +P9 is independent of P2 and P4. It shares three files in +`tools/runner-manifests/` with P8, but no lines: P8 edits the +`COMPASS_MICROVM_VOLUME_ROOT` entry, the runtime-settings test and the +README's node prerequisites. + +Spec impact: under `## Runner enrollment CLI` in +`docs/specs/product/compass.md`, a new requirement follows the `--ca` +requirement: "`compass-runner --server-name` sets the name checked on the +Server's certificate". The Runner SHALL check the certificate against that +name instead of the `--server` host, and SHALL trust the same roots. When it +is unset, the Runner SHALL check the `--server` host. Its scenario is the +first test below. + +Tests, red first: + +- `go/internal/runner/tls_test.go`, new `TestServerNameOverride`: a + `certgen` certificate for `compass.example.test` only, served by + `startTLSRunnerServer` at `127.0.0.1`, with that certificate as `caPath`. + `NewServerClient(caPath, "compass.example.test")` enrolls. + `NewServerClient(caPath, "")` fails at the handshake with + `x509.HostnameError`. +- `NewServerClient("", "compass.example.test")` has a nil `RootCAs`, that + `ServerName` and `MinVersion` TLS 1.3. A missing or non-PEM `caPath` fails + with the `NewCATrustClient` messages. +- `go/cmd/compass-runner/main_test.go`, new + `TestServerNameFlagAndEnvironmentFallback`, shaped like + `TestAgentBatchingFlagAndEnvironmentFallback`: both values unset returns + `http.DefaultClient`; `$COMPASS_SERVER_NAME` alone gives a client with + that `ServerName`; the flag overrides the environment. +- `tools/runner-manifests/render-core.test.ts`, new + `describe("serverName")` after `describe("serverCA")`: unset renders no + `COMPASS_SERVER_NAME`; set renders it; `""`, `compass.example.test:443` + and `https://compass.example.test` throw "serverName". + ### Interfaces ```ts @@ -889,6 +964,9 @@ const SESSION_MOUNT_PATH = "/var/lib/compass/sessions"; // unchanged: session-vo const SESSION_VOLUME_ROOT = `${SESSION_MOUNT_PATH}/volumes`; // new: COMPASS_MICROVM_VOLUME_ROOT value // RunnerDeployValues.hostPaths is unchanged; the operator creates /volumes with the project quota +// tools/runner-manifests/render-core.ts (P9) +RunnerDeployValues.serverName?: string; // isValidEgressHost grammar; set → env COMPASS_SERVER_NAME after COMPASS_RUNNER_CA + // tools/runner-image/assert-public-core.ts (P5) export function publicVerdict(result: ProbeResult, ref: string): { ok: true } | { ok: false; message: string }; // tools/runner-image/assert-public.ts: bun tools/runner-image/assert-public.ts --ref @@ -965,6 +1043,15 @@ func guestKernelName(goarch string) string // "Image" for arm64, else "bzImage"; // go/internal/runtime/microvm/launch.go (P7, //go:build unix) func serialConsole(goarch string) (string, error) // amd64 → "ttyS0", arm64 → "ttyAMA0", else error func vmmArgs(cfg BootConfig, consolePath, console string, opts launchOptions) []string // appends "console=" + +// go/internal/runner/tls.go (P9, //go:build unix) +// caPath "" → RootCAs nil (host roots), else the --ca pool with NewCATrustClient's errors; +// serverName "" → Go checks the dialed host, else tls.Config.ServerName; TLS 1.3, HTTP/1.1 and HTTP/2 +func NewServerClient(caPath, serverName string) (*http.Client, error) + +// go/cmd/compass-runner/main.go (P9, //go:build unix): flag --server-name, default $COMPASS_SERVER_NAME +// orEnv on both values; both empty → http.DefaultClient, else runner.NewServerClient +func runnerHTTPClient(caFlag, serverNameFlag string) (connecthttp.HTTPClient, error) ``` ### Deferred follow-ups @@ -1008,3 +1095,6 @@ func fetchHTTPS(ctx context.Context, client *http.Client, u, bearer string) ([]b - [ ] P8 — Runner renderer sets `COMPASS_MICROVM_VOLUME_ROOT` to `/var/lib/compass/sessions/volumes`, with the README quota layout; independent of P2 +- [ ] P9 — Runner `--server-name` (`$COMPASS_SERVER_NAME`) sets + `tls.Config.ServerName`, with or without `--ca`; renderer value + `serverName`, README and spec requirement; independent of P2 and P4