diff --git a/src/routes/(0)index.mdx b/src/routes/(0)index.mdx index 8b70e8489..8b3bc515a 100644 --- a/src/routes/(0)index.mdx +++ b/src/routes/(0)index.mdx @@ -33,6 +33,8 @@ Developers moving from Solid 1 or SolidStart have their own guides under [Migrat If you know what you need and want the signature, go to the [Reference](/reference). It is organized by import specifier, so `solid-js`, `@solidjs/web`, and `@solidjs/router` each have their own section. +If you write Solid with a coding assistant, [Solid for AI tools](/guides/ai-tools) lists the cheatsheet, skills, and diagnostics that ship with the packages and gives you a rules block to paste into its instructions file. + ## A reading order The Learn pages build on each other. diff --git a/src/routes/(3)building-apps/(3)server-functions/(6)streaming-ai-responses.mdx b/src/routes/(3)building-apps/(3)server-functions/(6)streaming-ai-responses.mdx new file mode 100644 index 000000000..be97180e3 --- /dev/null +++ b/src/routes/(3)building-apps/(3)server-functions/(6)streaming-ai-responses.mdx @@ -0,0 +1,270 @@ +--- +title: "Streaming AI responses" +version: "2.0" +description: "Forward a model's token stream through a server function into a store one chunk at a time, stop it from the browser or by asking a new question, and render the Markdown without re-rendering what has already landed." +--- + +The storefront gets a support assistant: a shopper types a question and the answer appears word by word as the model produces it. +Halfway through, the shopper asks a different question, or presses Stop, or the model call fails and they want to try again. +Written by hand, that is a `fetch` with a body reader, a flag for "streaming", an `AbortController` kept somewhere, and a text state that is replaced on every chunk. + +In Solid the model call is a server function that returns an async generator, the browser side is a projection that writes one store commit per chunk, and cancellation is what the reactive graph does with a computation it no longer needs. +This guide builds that assistant and ends with the cancellation contract, because deciding who may stop a generation is the part that does not fall out of the code. + +## Stream from the server + +A `"use server"` function can be an async generator; the transport sends each yielded value over the open response as it is produced, as [Reads, streams, and live data](/building-apps/server-functions/reads-and-live-data#return-a-stream) describes. +Read the request's abort signal and hand it to the model call: + +```ts +// src/data/assistant.ts +import { getRequestEvent } from "@solidjs/web"; + +export async function* answer(question: string) { + "use server"; + const signal = getRequestEvent()?.request.signal; + const stream = await model.stream({ prompt: question, signal }); + for await (const chunk of stream) { + yield chunk.text; + } +} +``` + +`model.stream` is a stand-in for the provider SDK; provider SDKs take an `AbortSignal` for a streaming call under a name such as `signal` or `abortSignal`. +The parts that are Solid's are exact: [`getRequestEvent()`](/reference/solid-web/request-response/get-request-event) returns the request event on the server, and `request` is the standard `Request`, whose `signal` aborts when the client disconnects before the response has ended. + +Call `answer("Do you ship to Portugal?")` from the browser and the first words arrive before the model has finished the sentence. +Each `yield` is one chunk on the wire; the function's `for await` loop is the only buffering there is. + +:::caution[The signal is the host's to keep live] +`request.signal` fires on a dropped connection only when the server integration aborts it when the socket closes. +The Vite plugin's dev server does; a hand-written Node server must abort the controller on the response's `close` event, or the signal never fires and the model keeps generating for a reader that is gone. +::: + +## Collect the chunks in a store + +In the browser, [`createProjection`](/reference/solid-js/stores/create-projection) with an async generator turns the stream into a store. +Each `yield` commits the writes made to the draft since the previous one, so the component reads `reply.text` and sees it grow: + +```tsx +// src/pages/Assistant.tsx +import { Errored, Show, createProjection, createSignal } from "solid-js"; +import { invoke } from "@solidjs/web/server-functions"; +import { answer } from "../data/assistant"; + +type Reply = { text: string; done: boolean }; + +function Assistant() { + const [question, setQuestion] = createSignal(""); + let current: AbortController | undefined; + + const reply = createProjection( + async function* (draft: Reply) { + const asked = question(); + draft.text = ""; + draft.done = !asked; + yield; + if (!asked) return; + + const controller = (current = new AbortController()); + try { + const chunks = await invoke( + answer, + { signal: controller.signal }, + asked + ); + for await (const chunk of chunks) { + draft.text += chunk; + yield; + } + } catch (error) { + if (!controller.signal.aborted) throw error; + } + draft.done = true; + }, + { text: "", done: true }, + { seedLoadingValue: true } + ); + + return ( + <> +
{ + event.preventDefault(); + const data = new FormData(event.currentTarget); + setQuestion(String(data.get("question") ?? "")); + }} + > + + + +
+ ( +

+ {String(error())} +

+ )} + > +

{reply.text}

+ + Writing… + +
+ + ); +} +``` + +Submit a question and the paragraph fills in chunk by chunk while "Writing…" shows; when the generator finishes, `done` flips and the indicator goes away. +Submit a second question while the first is still streaming and the text clears and starts again with the new answer; nothing from the first answer arrives afterwards. + +What happens on that second submit is the whole cancellation story on the client side. +The projection's function read `question()`, so the new value re-runs it. +A re-run supersedes the previous flight: the runtime calls `return()` on the previous generator, which exits its `for await` loop, which in turn closes the server function's iterable and aborts the request. +Draft writes the old generator makes after that point are ignored, which is why a late chunk from the first answer cannot land in the second. + +The `seedLoadingValue` option makes the seed, `{ text: "", done: true }`, the store's first committed value, so the first render shows an empty reply and no [`Loading`](/concepts/boundaries) boundary is needed above the paragraph. + +### Two ways to stop + +The second submit stops the first answer by superseding the read. +That is the shape to prefer: there is no controller to keep, and anything that re-runs the projection, including a `refresh(reply)`, cancels the same way. + +The Stop button is the explicit form. +[`invoke()`](/reference/solid-web/server-functions/invoke) calls a server function reference with options that belong to one call, and `signal` is one of them: aborting it rejects the call and cancels the request. +The `catch` turns that rejection into a normal end when the abort was ours, so the text so far stays on screen and `done` flips; any other failure is rethrown. +Without the `catch`, Stop would be an error the boundary shows, which is not what the shopper asked for. + +The controller lives in a plain variable rather than a signal on purpose: assigning it is not a reactive write, so the projection's function may do it, and nothing renders from it. + +### Errors and retry + +A rejected model call, a network failure after the first chunk, or an exception in the generator ends the projection's flight with an error, and the nearest [`Errored`](/reference/solid-js/components-jsx/errored) boundary renders its fallback. +The fallback's `reset` re-runs the sources that failed, so Retry asks the same question again with no state to clear by hand. + +## Render Markdown without re-rendering the answer + +A model answers in Markdown, and parsing the whole text on every chunk gets slower as the answer grows and rebuilds DOM the reader is looking at. +Split the answer into completed blocks and one trailing block: a completed block never changes, so it is parsed once and keyed; only the trailing block is re-rendered per chunk. + +```tsx +// src/pages/Assistant.tsx +import { For, createProjection, createSignal } from "solid-js"; +import { answer } from "../data/assistant"; + +type Block = { id: number; source: string }; +type Reply = { blocks: Block[]; tail: string; done: boolean }; + +function Assistant() { + const [question, setQuestion] = createSignal(""); + let nextId = 0; + + const reply = createProjection( + async function* (draft: Reply) { + const asked = question(); + draft.blocks = []; + draft.tail = ""; + draft.done = !asked; + yield; + if (!asked) return; + + for await (const chunk of await answer(asked)) { + draft.tail += chunk; + const parts = draft.tail.split("\n\n"); + draft.tail = parts.pop() ?? ""; + for (const source of parts) draft.blocks.push({ id: nextId++, source }); + yield; + } + if (draft.tail) draft.blocks.push({ id: nextId++, source: draft.tail }); + draft.tail = ""; + draft.done = true; + }, + { blocks: [], tail: "", done: true }, + { seedLoadingValue: true } + ); + + return ( +
+ block.id}> + {(block) =>
} + +
+
+ ); +} +``` + +`renderMarkdown` is a stand-in for the Markdown library of your choice; sanitize its output before using `innerHTML` on text a model produced. +A blank line closes a block, which is a good enough boundary for paragraphs, headings, and lists; a parser that reports where a block ends can replace the `split`. + +Ask a long question and watch the DOM: each completed paragraph is inserted once and never touched again, and only the last `div` changes as chunks arrive. +`For` keyed by `id` keeps a block's element when the array is replaced, for example when the finished answer is written back from a saved conversation, and `push` on the draft adds a row without rebuilding the others. + +## The cancellation contract + +Three parties can stop a generation, and the code above connects them in one direction. +State the contract in the server function's comment, because a reader of either side cannot see the other: + +1. The browser aborts: a superseding read, a Stop button, or a closed tab ends the iteration or the request. +2. The server's `request.signal` aborts, and the runtime calls `return()` on the generator, so its `finally` blocks run. +3. The generator forwarded that signal to the model call, so the provider stops generating. + +Step 3 is the one the author decides. +A generator that does not pass the signal on still ends, but only at its next `yield`, after the model has produced one more chunk for nobody; a model call that runs to completion for a reader who left is the cost of forgetting it. + +Forwarding is right for a chat answer, where the shopper is the only consumer and an abandoned answer has no value. +It is wrong for a durable job: a report the shopper asked to be generated into their account should finish whether or not they stay on the page, and reloading the page should show its progress rather than start it again. +Do not forward the signal to such a job. +Start it with an action, store its output on the server as it is produced, expose progress as a [live source](/building-apps/server-functions/reads-and-live-data#declare-a-live-source) the page reads, and expose Stop as a second action that the job checks between steps. +In that shape the request's abort means only "this reader left", which is the correct meaning; the job's own lifetime belongs to the server. + +:::note[What a superseded read cannot do] +Cancellation from the reactive graph stops the stream the component was reading. +It cannot stop work the server function started and did not tie to the signal, such as a job enqueued before the first `yield`. +If the function has side effects beyond the stream, give them the server-owned shape above. +::: + +## Common problems + +### The old answer's chunks appear after a new question + +The generator reads the question after its first `await`, so the re-run tracked nothing and the first flight was never superseded. +Read every reactive input before the first `await` or `yield`, as `asked` is read above. + +### Stop shows the error fallback + +The abort's rejection reached the boundary. +Catch it in the generator and rethrow only when `controller.signal.aborted` is false. + +### The model keeps running after the tab closes + +`request.signal` never fired. +Check that the server integration aborts the request on a closed socket, and that the generator passes the signal to the model call rather than only reading it. + +### The fallback flashes before the first chunk + +The projection was read under a `Loading` boundary without `seedLoadingValue`, so the seed was not observable until the first `yield` landed, and the boundary showed its fallback for that window. +Pass `{ seedLoadingValue: true }` when the seed is a valid first render, or keep the boundary and accept the fallback. + +## Recap + +- Return an async generator from a `"use server"` function and pass `getRequestEvent()?.request.signal` to the model call. +- Consume the stream in `createProjection(async function* (draft) { ... })`; each `yield` commits the draft writes made since the last one. +- Read every reactive input before the first `await` so a change supersedes the flight, which closes the previous generator and aborts its request. +- Use `invoke(fn, { signal }, ...args)` for an explicit Stop, and catch the abort so the text so far stays on screen. +- Put completed Markdown blocks in a `For` keyed by `id` and re-render only the trailing block. +- Forward the signal when the reader is the only consumer; for a durable job, do not forward it, and expose Stop as an action. + +## Next steps + +- [Reads, streams, and live data](/building-apps/server-functions/reads-and-live-data): the transport behind the stream, and `live()` for a value that replaces itself over time. +- [Metadata and transport](/building-apps/server-functions/metadata-and-transport#configure-one-call): the other per-call options `invoke()` accepts, and what it refuses. +- [Mutations](/concepts/mutations): actions and optimistic stores, for the server-owned job shape. +- [Chat state for the AI SDK](/guides/ai-sdk-chat): the same ideas through the AI SDK's chat class and a store-backed adapter. diff --git a/src/routes/(3)building-apps/(3)server-functions/(7)durable-generation.mdx b/src/routes/(3)building-apps/(3)server-functions/(7)durable-generation.mdx new file mode 100644 index 000000000..81353df7f --- /dev/null +++ b/src/routes/(3)building-apps/(3)server-functions/(7)durable-generation.mdx @@ -0,0 +1,437 @@ +--- +title: "Durable generation" +version: "2.0" +description: "Run an AI reply as a job that outlives the request that started it, store the reply as it grows, and deliver it through a live source so a closed tab loses nothing and two tabs read the same reply." +--- + +The support assistant from [Streaming AI responses](/building-apps/server-functions/streaming-ai-responses) answers while the shopper watches, and stops when they leave. +That is the right contract for a quick question. +It is the wrong one for a summary of a forty-message order dispute: the shopper closes the tab to find the order number, comes back, and the summary starts over, because the request that was writing it ended with the tab. + +The difference is who owns the generation. +In the streaming guide the request owns it, and the reactive graph cancels it. +On this page a job on the server owns it: an action starts the job and returns, the job writes the reply into storage as the model produces it, and every page that shows the reply reads what is stored through a live source. +Pick the request-scoped shape when the reader is the only consumer and an abandoned answer has no value; pick this shape when the reply must finish without its reader, be found again on reload, or be read from more than one tab. + +Two deliveries are shown. +The client-owned one sends the reply as data through [`live()`](/reference/solid-web/server-functions/live) and renders it in the browser. +The server-owned one renders the whole thread on the server as a live server component, which is the shape of the `chat` example in the Solid repository; server components are an experimental preview, and that section says so. +The job, the storage, and the actions are the same in both. + +## The job + +Three modules carry the shape: a store the job writes into, a registry of running jobs that no request owns, and the actions that start and stop one. + +The store keeps each reply and tells watchers when it changes: + +```ts +// src/server/replies.ts +export type Reply = { + id: string; + text: string; + status: "generating" | "complete" | "stopped" | "failed"; + error?: string; +}; + +const replies = new Map(); +const waiters = new Map void>>(); + +export function write(id: string, patch: Partial) { + const current = replies.get(id)!; + replies.set(id, { ...current, ...patch, version: current.version + 1 }); + const waiting = waiters.get(id); + waiters.delete(id); + waiting?.forEach((wake) => wake()); +} + +function changed(id: string, signal?: AbortSignal) { + return new Promise((wake) => { + (waiters.get(id) ?? waiters.set(id, new Set()).get(id)!).add(wake); + signal?.addEventListener("abort", () => wake(), { once: true }); + }); +} + +// The reply as it stands now, then again after every write. +export async function* watch(id: string, signal?: AbortSignal) { + let seen = -1; + while (!signal?.aborted) { + const current = replies.get(id); + if (current && current.version !== seen) { + seen = current.version; + yield current as Reply; + continue; + } + await changed(id, signal); + } +} +``` + +`watch` checks the version after every yield before it waits, so a write that lands while the consumer holds the previous value is not missed. +The `chat` example keeps the same shape in `src/server/db.ts`, written through to disk so a reply survives a restart. + +The job reads from the model and writes into the store. +It belongs to the module, not to a request: + +```ts +// src/server/generations.ts +import * as replies from "./replies"; + +const running = new Map(); + +export function start(replyId: string, history: Message[]) { + const abort = new AbortController(); + running.set(replyId, abort); + void (async () => { + let text = ""; + try { + for await (const chunk of model.stream({ + history, + signal: abort.signal, + })) { + text += chunk.text; + replies.write(replyId, { text }); + } + replies.write(replyId, { status: "complete" }); + } catch (error) { + if (abort.signal.aborted) replies.write(replyId, { status: "stopped" }); + else + replies.write(replyId, { + status: "failed", + error: "The model stopped answering", + }); + } finally { + running.delete(replyId); + } + })(); +} + +export function stop(replyId: string) { + running.get(replyId)?.abort(); +} +``` + +`model.stream` is a stand-in for the provider SDK, as in the streaming guide. +The signal it receives is the job's own, not a request's: Stop aborts it, and nothing else does. +A reply that fails keeps the text that had arrived, with the reason beside it; a reply that was stopped keeps its text and says so. +Write at most every few tens of milliseconds when the model is fast, as the example does, since every write wakes every watcher. + +The actions start and stop a job and return at once: + +```ts +// src/data/replies.ts +import { action } from "@solidjs/router"; +import * as generations from "../server/generations"; +import * as replies from "../server/replies"; +import * as threads from "../server/threads"; + +export const sendMessage = action( + async (threadId: string, messageId: string, text: string) => { + "use server"; + const { replyId, history } = await threads.append( + threadId, + messageId, + text + ); + generations.start(replyId, history); + }, + "sendMessage" +); + +export const stopReply = action(async (replyId: string) => { + "use server"; + generations.stop(replyId); +}, "stopReply"); +``` + +`sendMessage` stores the shopper's message and an empty reply with status `generating`, starts the job, and returns. +The response closes while the job is still writing; the server-function runtime does not dispose work a function started, so the job runs to its end whether or not anyone is watching. +`threads.append` is the thread half of the store, which keeps the message list and the history the model reads; the `chat` example's `db.send` is one. +Both actions are Solid Router [actions](/routing/solid-router/data#mutate-with-actions), so they work as a form before JavaScript has loaded and from code through `useAction`. + +:::caution[An orphan is a fact, not a hang] +A server restart ends every running job and leaves the replies they were writing at `generating`. +Mark them `failed` with a reason when the store loads, as the `chat` example does, so a reader sees a message with a Retry rather than a reply that never finishes. +::: + +## Client-owned: the browser renders the reply + +The live source yields the reply so far, whole, on every write: + +```ts +// src/data/replies.ts +import { getRequestEvent } from "@solidjs/web"; +import { GET, live } from "@solidjs/web/server-functions"; +import * as replies from "../server/replies"; + +export const watchReply = live( + GET(async function* (replyId: string) { + "use server"; + const signal = getRequestEvent()?.request.signal; + for await (const reply of replies.watch(replyId, signal)) yield reply; + }) +); +``` + +The request's signal ends the watcher when the reader leaves; the job does not see it. +The first yield on every connection is the current reply, which is the contract `live()` reconnects on, as [Declare a live source](/building-apps/server-functions/reads-and-live-data#declare-a-live-source) describes. + +In the browser, [`createProjection`](/reference/solid-js/stores/create-projection) with an async generator turns the stream into a store: + +```tsx +// src/pages/Reply.tsx +import { Show, createProjection } from "solid-js"; +import { stopReply, watchReply } from "../data/replies"; + +type ReplyView = { text: string; status: string; error?: string }; + +function Reply(props: { replyId: string }) { + const reply = createProjection( + async function* (draft: ReplyView) { + for await (const next of watchReply(props.replyId)) { + draft.text = next.text; + draft.status = next.status; + draft.error = next.error; + yield; + } + }, + { text: "", status: "generating" }, + { seedLoadingValue: true } + ); + + return ( +
+

{reply.text}

+ +
+ +
+
+ +

{reply.error}

+
+
+ ); +} +``` + +Send a message, close the tab while the reply is being written, and open the page again. +The component calls `watchReply` again, the first yield is the reply as far as it had got, and the paragraph keeps growing from there. +Open the page in two tabs and both show the same reply; press Stop in one and the status changes in both, because Stop aborted the job and the job wrote `stopped` into the store every watcher reads. + +The projection reads `props.replyId` before its first `await`, so a different reply re-runs it, which ends the previous iteration and its connection. +A projection rather than a memo keeps readers of `reply.status` out of every text update, and gives the Markdown block split from [Render Markdown without re-rendering the answer](/building-apps/server-functions/streaming-ai-responses#render-markdown-without-re-rendering-the-answer) a draft to write into. +`refresh(reply)` re-runs the projection, which opens a new connection; it does not touch the job. + +Every call site that iterates a live source holds its own connection. +When the thread list and the thread both need the same reply, hoist the read, or declare it with Solid Router's [`liveQuery`](/reference/solid-router/data#livequery), which keeps one connection per key for every consumer; the spelling is `liveQuery(watchReply, "reply")`, and a function already declared with `live(GET())` passes through unchanged. + +## Hold the bubble until the stream confirms + +The shopper's own message should be on screen the moment they press Enter, and should give way to the server's row when the thread shows it. +The thread is read through a live source, so the confirmation arrives on the stream rather than in the action's response. +That is what [`until`](/reference/solid-js/lifecycle-actions/until) waits for: + +```tsx +// src/components/Composer.tsx +import { For, action, createOptimisticStore, until } from "solid-js"; +import { useAction } from "@solidjs/router"; +import { sendMessage } from "../data/replies"; + +type Pending = { id: string; text: string }; + +function Composer(props: { threadId: string; seen: string[] }) { + const send = useAction(sendMessage); + const [outbox, setOutbox] = createOptimisticStore([]); + + const submit = action(function* (text: string) { + const id = crypto.randomUUID(); + setOutbox((list) => { + list.push({ id, text }); + }); + yield send(props.threadId, id, text); + yield until(() => props.seen.includes(id), { timeout: 15_000 }); + }); + + return ( + <> + !props.seen.includes(m.id))}> + {(message) =>

{message.text}

} +
+
{ + event.preventDefault(); + const data = new FormData(event.currentTarget); + void submit(String(data.get("text") ?? "")); + }} + > +