Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@ anchor: class Compute
created: 2026-10-15T11:45:12Z
norm: '1'
sig: 3b062341dbce15d0
body_hash: 87ac2186f431e5a9
raw_hash: f37fce530dddb402
vouched: 2026-10-16T16:31:35Z
body_hash: f01b9e5fb5980efb
raw_hash: 6e6c9ba386a99925
vouched: 2026-10-17T00:39:18Z
lines: 49-325
---

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@ anchor: class Compute
created: 2026-10-15T12:04:18Z
norm: '1'
sig: 3b062341dbce15d0
body_hash: 87ac2186f431e5a9
raw_hash: f37fce530dddb402
vouched: 2026-10-16T16:31:35Z
body_hash: f01b9e5fb5980efb
raw_hash: 6e6c9ba386a99925
vouched: 2026-10-17T00:39:18Z
lines: 51-331
---

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@ anchor: class Compute
created: 2026-10-15T12:19:54Z
norm: '1'
sig: 3b062341dbce15d0
body_hash: 87ac2186f431e5a9
raw_hash: f37fce530dddb402
vouched: 2026-10-16T16:31:35Z
body_hash: f01b9e5fb5980efb
raw_hash: 6e6c9ba386a99925
vouched: 2026-10-17T00:39:18Z
lines: 51-353
---

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
---
id: 01M53GWM3DTFPKRXZ527SPHVVY
anchor: '@file'
created: 2026-10-16T23:31:22Z
norm: '1'
sig: 05a0174fe5ad7d26
body_hash: ac18c9b485431b2d
raw_hash: 5191ea554b4043b4
lines: 1-419
---

a container restart loses the runner, so running means the sandbox reports running and the runner answers /health
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
---
id: 01M53MS0DWRV2FSVSYXB1ECNPG
anchor: class Compute
created: 2026-10-17T00:39:18Z
norm: '1'
sig: 3b062341dbce15d0
body_hash: f01b9e5fb5980efb
raw_hash: 6e6c9ba386a99925
lines: 58-408
---

start replaces a running sandbox only when its project is gone, so work newer than the checkpoint is never destroyed
19 changes: 16 additions & 3 deletions bun.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 2 additions & 6 deletions services/compute/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,12 +1,8 @@
FROM oven/bun:1.4.0-debian
# The Sandbox SDK's Cloudflare adapter talks to this image's control plane on port 3000.
FROM docker.io/cloudflare/sandbox:0.12.9

RUN apt-get update && apt-get install -y --no-install-recommends git curl ca-certificates tar gzip \
&& rm -rf /var/lib/apt/lists/*
RUN curl -fsSL https://lane.lukeed.com | LANE_VERSION=v0.1.0 LANE_INSTALL=/usr/local/bin sh

WORKDIR /app
COPY dist/runner.js ./runner.js
RUN mkdir -p /workspace

EXPOSE 8080
CMD ["bun", "--no-env-file", "runner.js"]
111 changes: 82 additions & 29 deletions services/compute/index.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
import type { Sandbox as CloudflareSandbox } from "@cloudflare/sandbox";
import { cloudflare } from "@sandbox-sdk/cloudflare";
import { create, type Sandbox } from "@sandbox-sdk/core";
import { DurableObject } from "cloudflare:workers";

import { validateName } from "@ace/channel";
Expand All @@ -8,10 +11,12 @@ import { authorized, refused } from "@ace/directory/scope";
import channels from "../channel/src/index";
import type { Env as ChannelEnv } from "../channel/src/object";

export { Sandbox } from "@cloudflare/sandbox";
export { HostedChannel } from "../channel/src/object";

type Env = ChannelEnv & {
COMPUTE: DurableObjectNamespace<Compute>;
Sandbox: DurableObjectNamespace<CloudflareSandbox>;
GITHUB_TOKEN?: string;
};
const SIZES = ["standard-1", "standard-2", "standard-3", "standard-4"] as const;
Expand All @@ -35,7 +40,7 @@ type Manifest = {
};
const CHUNK = 64 * 1024;
const MAX = 16 * 1024 * 1024;
const INACTIVITY = 6 * 60 * 60 * 1000;
const INACTIVITY = "6h";

function size(value: unknown): Size {
if (!SIZES.includes(value as Size)) throw new Error(`Size must be ${SIZES.join(", ")}`);
Expand All @@ -52,14 +57,40 @@ async function digest(bytes: Uint8Array): Promise<string> {
/** Pilot checkpoints contain filesystem state only. Channel history remains in pi. */
export class Compute extends DurableObject<Env> {
#queue: Promise<unknown> = Promise.resolve();
#sandbox: Promise<Sandbox> | undefined;

constructor(ctx: DurableObjectState, env: Env) {
super(ctx, env);
if (ctx.container?.running) {
void ctx.blockConcurrencyWhile(() => ctx.container!.setInactivityTimeout(INACTIVITY));
/** A container that restarted has lost its runner, so only a live runner counts as running. */
async #running(): Promise<boolean> {
return await (await this.#box()).status() === "running" && await this.#healthy();
}

async #healthy(): Promise<boolean> {
try {
await this.#call("/health");
return true;
} catch {
return false;
}
}

/** The workspace is a Sandbox SDK sandbox; only this adapter choice names a provider. */
#box(): Promise<Sandbox> {
this.#sandbox ||= create({
adapter: cloudflare({
binding: this.env.Sandbox,
id: this.#settings().id,
options: { sleepAfter: INACTIVITY },
}),
});
return this.#sandbox;
}

async #destroy() {
const box = await this.#box();
this.#sandbox = undefined;
await box.stop();
}

#settings(): Settings {
const settings = this.ctx.storage.kv.get<Settings>("settings");
if (!settings) throw new Error("Initialize this pilot first");
Expand Down Expand Up @@ -87,24 +118,36 @@ export class Compute extends DurableObject<Env> {
this.ctx.storage.kv.put("frozen", action === "freeze");
}

/**
* Bodies travel as files and the request runs inside the sandbox, so the runner needs no
* exposed port on any provider.
*/
async #call(path: string, body?: unknown): Promise<Response> {
if (!this.ctx.container?.running) throw new Error("The sandbox is stopped");
const response = await this.ctx.container.getTcpPort(8080).fetch(`http://container${path}`, {
method: body === undefined ? "GET" : "POST",
body: body === undefined
? undefined
: body instanceof Uint8Array
? body
: JSON.stringify(body),
});
const box = await this.#box();
let post = "";
if (body !== undefined) {
await box.files.write(
"/tmp/ace-request",
body instanceof Uint8Array ? body : JSON.stringify(body),
);
post = "-X POST --data-binary @/tmp/ace-request";
}
const result = await box.process.shell(
`curl -sS -o /tmp/ace-reply -w '%{http_code}' ${post} http://localhost:8080${path}; rm -f /tmp/ace-request`,
// Marked so the runner's quiescence check ignores this request's own processes.
{ timeout: 300_000, env: { ACE_CONTROL: "1" } },
);
const status = Number(result.stdout);
if (!status) throw new Error(`${path}: ${result.stderr}`);
const response = new Response(await box.files.read("/tmp/ace-reply"), { status });
if (!response.ok) throw new Error(`${path}: ${await response.text()}`);
return response;
}

async #boot(selected: Size) {
const container = this.ctx.container!;
container.start({ image: container.images.base, instance: selected, enableInternet: true });
await container.setInactivityTimeout(INACTIVITY);
/** The container's instance type is fixed in wrangler.jsonc; the SDK has no portable size. */
async #boot(_selected: Size) {
const box = await this.#box();
await box.process.spawnShell("bun --no-env-file runner.js", { cwd: "/app" });
for (let attempt = 0; attempt < 120; attempt++) {
try {
await this.#call("/health");
Expand Down Expand Up @@ -186,13 +229,22 @@ export class Compute extends DurableObject<Env> {

async #start(selected: Size) {
const settings = this.#settings();
if (this.ctx.container!.running) {
if (this.ctx.storage.kv.get("frozen") && !this.ctx.storage.kv.get("ready")) {
await this.ctx.container!.destroy();
} else {
const incomplete = this.ctx.storage.kv.get("frozen") && !this.ctx.storage.kv.get("ready");
const box = await this.#box();
if (!incomplete && await box.status() === "running") {
if (await this.#healthy()) {
throw new Error("Sandbox is already running; use resize to change its size");
}
// A restarted container comes back without the project; a runner that merely exited
// leaves work newer than the checkpoint, so it restarts in place.
if ((await box.process.shell("test -d /workspace/project")).ok) {
await this.#channel("freeze");
await this.#boot(selected);
return this.#connect();
}
}
// Clears an incomplete start or a container that restarted without its runner.
await this.#destroy();
const manifest = this.ctx.storage.kv.get<Manifest>("checkpoint");
if (settings.started && !manifest) {
throw new Error(
Expand All @@ -207,7 +259,7 @@ export class Compute extends DurableObject<Env> {
await this.#connect(false);
} catch (error) {
// No channel input can reach this incomplete replacement while it is frozen.
await this.ctx.container!.destroy();
await this.#destroy();
throw error;
}
this.ctx.storage.kv.put("settings", { ...settings, size: selected, started: true });
Expand All @@ -220,7 +272,7 @@ export class Compute extends DurableObject<Env> {
* previous attempt having done it, so a retry after a partial failure finishes the job.
*/
async #delete(id: string) {
if (this.ctx.container!.running) await this.ctx.container!.destroy();
await this.#destroy();
const channel = this.env.CHANNEL.getByName(id);
// A frozen channel refuses deletion; a thaw refused because it is active or gone is harmless.
await channel.fetch(`https://channel/channels/${id}/transfer`, {
Expand All @@ -241,12 +293,13 @@ export class Compute extends DurableObject<Env> {
}
const action = named || "status";
if (action === "status" && request.method === "GET") {
const running = await this.#running();
return Response.json({
running: this.ctx.container!.running,
running,
settings: this.ctx.storage.kv.get("settings"),
checkpoint: this.ctx.storage.kv.get("checkpoint"),
frozen: !!this.ctx.storage.kv.get("frozen"),
runner: this.ctx.container!.running ? await (await this.#call("/status")).json() : null,
runner: running ? await (await this.#call("/status")).json() : null,
});
}
if (request.method !== "POST") return new Response("Use POST", { status: 405 });
Expand Down Expand Up @@ -301,7 +354,7 @@ export class Compute extends DurableObject<Env> {
}
if (
["exec", "checkpoint", "stop", "resize", "resume"].includes(action)
&& !this.ctx.container!.running
&& !(await this.#running())
) {
throw new Error("The sandbox is stopped; use start to restore it");
}
Expand Down Expand Up @@ -331,14 +384,14 @@ export class Compute extends DurableObject<Env> {
}
case "stop": {
const checkpoint = await this.#save();
await this.ctx.container!.destroy();
await this.#destroy();
await this.#channel("thaw");
return Response.json({ stopped: true, checkpoint });
}
case "resize": {
const selected = size(body.size);
const checkpoint = await this.#save();
await this.ctx.container!.destroy();
await this.#destroy();
await this.#start(selected);
return Response.json({ size: selected, checkpoint });
}
Expand Down
5 changes: 4 additions & 1 deletion services/compute/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,11 @@
"dependencies": {
"@ace/channel": "workspace:*",
"@ace/directory": "workspace:*",
"@cloudflare/sandbox": "0.12.9",
"@earendil-works/chord": "catalog:pi",
"@earendil-works/pi-durable": "catalog:pi"
"@earendil-works/pi-durable": "catalog:pi",
"@sandbox-sdk/cloudflare": "file:./vendor/sandbox-sdk/cloudflare",
"@sandbox-sdk/core": "file:./vendor/sandbox-sdk/core"
},
"devDependencies": {
"@cloudflare/workers-types": "catalog:cloudflare",
Expand Down
9 changes: 8 additions & 1 deletion services/compute/quiescence.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
import { readdir, readFile } from "node:fs/promises";

/** Set by the controller on the commands it runs to reach the runner. */
export const CONTROL = "ACE_CONTROL=1";

let baseline: Map<number, string> | undefined;

// A dedicated container records its platform processes before accepting work. Start times
Expand Down Expand Up @@ -44,7 +47,11 @@ async function processes(): Promise<Map<number, string>> {
if (fields.length < 20 || !/^\d+$/.test(fields[19])) {
throw new Error(`Cannot inspect process ${pid} before checkpointing`);
}
if (fields[0] !== "Z") found.set(pid, fields[19]);
if (fields[0] === "Z") return;
// The controller reaches the runner through processes it marks, which are not workspace work.
const environ = await readFile(`/proc/${pid}/environ`, "utf8").catch(() => "");
if (environ.split("\0").includes(CONTROL)) return;
found.set(pid, fields[19]);
}),
);
return found;
Expand Down
Loading