diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0a09bcc..c11b600 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,13 +26,13 @@ jobs: run: dotnet tool restore - name: Restore - run: dotnet restore ManagedCode.LlmTck.slnx + run: dotnet restore ManagedCode.LlmTestServer.slnx - name: Format - run: dotnet format ManagedCode.LlmTck.slnx --verify-no-changes --no-restore + run: dotnet format ManagedCode.LlmTestServer.slnx --verify-no-changes --no-restore - name: Build - run: dotnet build ManagedCode.LlmTck.slnx --configuration Release --no-restore + run: dotnet build ManagedCode.LlmTestServer.slnx --configuration Release --no-restore - name: Test run: bash scripts/coverage.sh @@ -43,9 +43,6 @@ jobs: - name: Verify installed NuGet consumer run: python3 scripts/verify-package.py - - name: Verify release recovery - run: python3 scripts/test-release.py - - name: Upload verification evidence if: always() uses: actions/upload-artifact@v7.0.1 @@ -60,3 +57,31 @@ jobs: with: name: packages path: artifacts/packages/*.nupkg + + container-smoke: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v7.0.0 + + - name: Setup .NET + uses: actions/setup-dotnet@v5.4.0 + with: + dotnet-version: 10.0.x + + - name: Publish SDK container to local daemon without registry push + run: >- + dotnet publish samples/ManagedCode.LlmTestServer.Service/ManagedCode.LlmTestServer.Service.csproj + --configuration Release --runtime linux-x64 --self-contained false + /t:PublishContainer -p:ContainerImageTag=ci -p:ContainerRuntimeIdentifier=linux-x64 + -p:UseAppHost=false + + - name: Verify actual container HTTP behavior + run: python3 scripts/verify-container.py --image managedcode/llm-test-server:ci + + - name: Upload container verification evidence + if: always() + uses: actions/upload-artifact@v7.0.1 + with: + name: container-verification + path: artifacts/container-verification diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml new file mode 100644 index 0000000..4124beb --- /dev/null +++ b/.github/workflows/pages.yml @@ -0,0 +1,77 @@ +name: Website + +on: + pull_request: + branches: [main] + paths: ['site/**', '.github/workflows/pages.yml'] + push: + branches: [main] + paths: ['site/**', '.github/workflows/pages.yml'] + workflow_run: + workflows: [Release] + types: [completed] + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: pages-${{ github.ref }} + cancel-in-progress: false + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7.0.0 + with: + ref: ${{ github.event.workflow_run.head_sha || github.sha }} + - uses: actions/setup-node@v7.1.0 + with: + node-version: 24 + cache: npm + cache-dependency-path: site/package-lock.json + - name: Install site tooling + run: npm ci --prefix site + - name: Validate HTML, CSS and JavaScript + run: npm run lint --prefix site + - name: Build static preview + run: npm run build --prefix site + + deploy: + if: github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.head_branch == 'main' + needs: validate + runs-on: ubuntu-latest + permissions: + contents: read + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - uses: actions/checkout@v7.0.0 + with: + ref: ${{ github.event.workflow_run.head_sha || github.sha }} + # Select GitHub Actions as the repository Pages source before the first deployment. + - name: Require the verified release before publishing the landing page + env: + GH_TOKEN: ${{ github.token }} + run: | + version=$(python3 -c 'import xml.etree.ElementTree as e; print(e.parse("Directory.Build.props").findtext(".//Version"))') + gh release view "v$version" --json assets --jq '.assets[].name' | grep -qx 'nuget-published.txt' + - name: Configure GitHub Pages + id: pages + uses: actions/configure-pages@v6.0.0 + - name: Build for canonical Pages URL + env: + PAGES_BASE_URL: ${{ steps.pages.outputs.base_url }} + run: python3 site/build.py --base-url "$PAGES_BASE_URL" + - name: Upload static website + uses: actions/upload-pages-artifact@v5.0.0 + with: + path: site/dist + - name: Deploy website + id: deployment + uses: actions/deploy-pages@v5.0.1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 98b4dab..7f3839d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,6 +21,7 @@ concurrency: jobs: pack: + if: github.ref == 'refs/heads/main' runs-on: ubuntu-latest env: NUGET_API_KEY: ${{ secrets.NUGET_API_KEY }} @@ -53,15 +54,15 @@ jobs: - name: Restore if: ${{ steps.release.outputs.should_release == 'true' }} - run: dotnet restore ManagedCode.LlmTck.slnx + run: dotnet restore ManagedCode.LlmTestServer.slnx - name: Format if: ${{ steps.release.outputs.should_release == 'true' }} - run: dotnet format ManagedCode.LlmTck.slnx --verify-no-changes --no-restore + run: dotnet format ManagedCode.LlmTestServer.slnx --verify-no-changes --no-restore - name: Build if: ${{ steps.release.outputs.should_release == 'true' }} - run: dotnet build ManagedCode.LlmTck.slnx --configuration Release --no-restore + run: dotnet build ManagedCode.LlmTestServer.slnx --configuration Release --no-restore - name: Test if: ${{ steps.release.outputs.should_release == 'true' }} @@ -75,10 +76,6 @@ jobs: if: ${{ steps.release.outputs.should_release == 'true' }} run: python3 scripts/verify-package.py - - name: Verify release recovery - if: ${{ steps.release.outputs.should_release == 'true' }} - run: python3 scripts/test-release.py - - name: Upload verification evidence if: always() uses: actions/upload-artifact@v7.0.1 @@ -88,21 +85,68 @@ jobs: artifacts/coverage artifacts/package-consumer - - name: Publish container image + - name: Publish and verify standalone SDK container locally + if: ${{ steps.release.outputs.should_release == 'true' }} + env: + IMAGE_TAG: ${{ steps.release.outputs.version }} + run: | + dotnet publish samples/ManagedCode.LlmTestServer.Service/ManagedCode.LlmTestServer.Service.csproj \ + --configuration Release --runtime linux-x64 --self-contained false \ + /t:PublishContainer -p:ContainerRuntimeIdentifier=linux-x64 \ + -p:ContainerImageTag="$IMAGE_TAG" -p:UseAppHost=false + python3 scripts/verify-container.py --image "managedcode/llm-test-server:$IMAGE_TAG" + + - name: Publish versioned multi-platform SDK container if: ${{ steps.release.outputs.should_release == 'true' }} env: IMAGE_TAG: ${{ steps.release.outputs.version }} + GHCR_TOKEN: ${{ github.token }} run: | - echo "${{ github.token }}" | docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin - docker build \ - -f samples/ManagedCode.LlmTck.Service/Dockerfile \ - -t "ghcr.io/managedcode/llm-tck:$IMAGE_TAG" \ - . - docker push "ghcr.io/managedcode/llm-tck:$IMAGE_TAG" - - - name: Publish packages and finalize release + printf '%s' "$GHCR_TOKEN" | docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin + dotnet publish samples/ManagedCode.LlmTestServer.Service/ManagedCode.LlmTestServer.Service.csproj \ + --configuration Release --self-contained false /t:PublishContainer \ + -p:ContainerRegistry=ghcr.io -p:ContainerImageTag="$IMAGE_TAG" \ + -p:UseAppHost=false + + - name: Verify anonymous public pull and actual container behavior + if: ${{ steps.release.outputs.should_release == 'true' }} + env: + IMAGE_TAG: ${{ steps.release.outputs.version }} + run: | + python3 scripts/verify-container.py \ + --image "ghcr.io/managedcode/llm-test-server:$IMAGE_TAG" --anonymous-pull \ + --require-platform linux/amd64 --require-platform linux/arm64 || { + echo "::error::Public container delivery failed. In the managedcode organization package settings for llm-test-server, set package visibility to Public, then rerun this release for the same version. packages:write and public repository visibility alone do not make a GHCR image public." + exit 1 + } + + - name: Upload container verification evidence + if: always() + uses: actions/upload-artifact@v7.0.1 + with: + name: release-container-verification + path: artifacts/container-verification + + - name: Publish packages and create release if: ${{ steps.release.outputs.should_release == 'true' }} env: GH_TOKEN: ${{ github.token }} TAG_NAME: ${{ steps.release.outputs.tag }} run: bash scripts/release.sh publish + + - name: Verify actual GitHub and NuGet publication + if: ${{ steps.release.outputs.should_release == 'true' }} + env: + GH_TOKEN: ${{ github.token }} + run: python3 scripts/verify-release.py + + - name: Verify public installed NuGet consumer + if: ${{ steps.release.outputs.should_release == 'true' }} + run: python3 scripts/verify-package.py --public-feed + + - name: Mark verified delivery complete + if: ${{ steps.release.outputs.should_release == 'true' }} + env: + GH_TOKEN: ${{ github.token }} + TAG_NAME: ${{ steps.release.outputs.tag }} + run: bash scripts/release.sh finalize diff --git a/.gitignore b/.gitignore index 3c05a70..34f649d 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,7 @@ coverage.cobertura.xml *.suo *.swp *.tmp + +# Launch production materials remain local until owner approval. +/marketing/ +.playwright-cli diff --git a/AGENTS.md b/AGENTS.md index 413f5e0..5d6c1ce 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,8 +1,8 @@ -# LLM TCK Contributor Guide +# LLM Test Server Contributor Guide ## Scope -This repository builds `ManagedCode.LlmTck`, a deterministic Technology Compatibility Kit for LLM APIs. Treat the runtime, protocol adapters, clients, Aspire integration, docs, and samples as separate package surfaces. +This repository builds `ManagedCode.LlmTestServer`, a deterministic LLM API mock server for LLM APIs. Treat the runtime, protocol adapters, clients, Aspire integration, docs, and samples as separate package surfaces. Follows [MCAF](https://mcaf.managed-code.com/). @@ -60,16 +60,16 @@ Rule format: ## Commands -- Restore: `dotnet restore ManagedCode.LlmTck.slnx` -- Build: `dotnet build ManagedCode.LlmTck.slnx --configuration Release --no-restore` -- Test: `dotnet test --project tests/ManagedCode.LlmTck.Tests/ManagedCode.LlmTck.Tests.csproj --configuration Release --no-build --verbosity normal` +- Restore: `dotnet restore ManagedCode.LlmTestServer.slnx` +- Build: `dotnet build ManagedCode.LlmTestServer.slnx --configuration Release --no-restore` +- Test: `dotnet test --project tests/ManagedCode.LlmTestServer.Tests/ManagedCode.LlmTestServer.Tests.csproj --configuration Release --no-build --verbosity normal` - Pack: `for project in src/*/*.csproj; do dotnet pack "$project" --configuration Release --no-build --output artifacts/packages; done` - Release: pushing to `main` must start the release workflow; it reads the package version from `Directory.Build.props`, creates the matching `vX.Y.Z` tag and GitHub Release, attaches package artifacts, and publishes NuGet packages only when that version is not already released. -- Format check: `dotnet format ManagedCode.LlmTck.slnx --verify-no-changes` +- Format check: `dotnet format ManagedCode.LlmTestServer.slnx --verify-no-changes` - Tool restore: `dotnet tool restore` - Coverage gate: production code line coverage must stay at or above 90%; measure with coverlet/reportgenerator when coverage expectations change. - Coverage: `bash scripts/coverage.sh` (maps deterministic CI source paths before collection and enforces 90% total production line coverage). -- Quality gate: `dotnet format ManagedCode.LlmTck.slnx --verify-no-changes --no-restore && dotnet build ManagedCode.LlmTck.slnx --configuration Release --no-restore && dotnet test --project tests/ManagedCode.LlmTck.Tests/ManagedCode.LlmTck.Tests.csproj --configuration Release --no-build --verbosity normal && git diff --check` +- Quality gate: `dotnet format ManagedCode.LlmTestServer.slnx --verify-no-changes --no-restore && dotnet build ManagedCode.LlmTestServer.slnx --configuration Release --no-restore && dotnet test --project tests/ManagedCode.LlmTestServer.Tests/ManagedCode.LlmTestServer.Tests.csproj --configuration Release --no-build --verbosity normal && git diff --check` `global.json` opts `dotnet test` into `Microsoft.Testing.Platform`; do not add VSTest-specific logger arguments to normal test commands. @@ -77,13 +77,33 @@ Rule format: ### Likes +- In marketing avatar videos, keep the presenter addressing the camera in frontal shots without side profiles or sustained off-camera gaze. Preserve natural blinking, small eye movements and relaxed facial/head motion; camera contact must feel conversational rather than frozen or tense. +- Match the avatar video's background, lighting and wardrobe colours to the actual website and product screenshots so all launch assets form one coherent visual style when edited together. +- Ship one complete v1.0.0 launch batch on a separate branch; the owner performs the merge. Do not merge, publish intermediate versions, or trigger releases before the complete product and launch preparation are ready. +- Publish the product landing page through GitHub Pages using the dashboard visual language and shared styles/components where practical; substantiate product sections with real server/browser screenshots and demo recordings. +- Keep README.md product-first: what LLM Test Server is, why to use it, what to do with it, and where to go next, in plain language. Move deep technical detail into docs/ and link to it. Every code sample must use real public APIs and be compile-checked against the current packages before it ships; never invent APIs or pad the README with technical noise. +- Lead README, landing and launch material with visually interesting captures (token flow, insights charts), not plain list or table screenshots. +- Keep the landing page polished, clear and fully responsive: showcase every major capability (provider matrix, request inspector, token analytics, tools/conversations, streaming, faults, Aspire/Docker setup) with crafted visual elements, not plain text blocks. Review it at desktop, tablet and mobile widths after every change. +- Make the dashboard visually rich with donut/pie charts across many dimensions of recorded traffic (providers, models, outcomes, operations, latency, tokens, cache, tool usage), each with a value-and-share legend; stable entity colors across filters. +- Use one distinctive, symbol-only product mark (no letter monogram) consistently for the launch thumbnail, favicons, dashboard header and OG/social images. It must not resemble the owner's other brands: no brackets around a square or token (that is the Prostir.build mark), and check prostir.build and managed-code.com marks before proposing one. +- The product website lives at https://llmtest.managed-code.com; use that canonical URL in site metadata, README, launch listings and social posts. +- Position the product as .NET-first but usable from any stack: the prebuilt Docker image plus the management API work for teams outside .NET, so launch copy and video must say so. +- Keep launch marketing drafts, generated images, videos, screenshots and production files under marketing/ and leave that directory uncommitted until the owner authorizes publication. +- Deliver both NuGet packages and a publicly pullable prebuilt Docker image, configurable through the management API. +- Build and publish container images through the .NET SDK container targets (`dotnet publish /t:PublishContainer`); do not use Dockerfile/docker build as the distribution build mechanism. + - Keep checked-in UI screenshots synchronized with the current rendered product; after visible dashboard changes, refresh and verify the affected images in a real browser before commit and push. - When token usage is requested, count response tokens with `tiktoken` and include the measured usage in replies; if `tiktoken` is unavailable, report that blocker instead of guessing token counts. - Token usage workflow changes need executable code and tests, not only contributor-guide documentation, so callers can verify usage accounting behavior directly. ### Dislikes -- Do not introduce real-LLM toggles, external project environment variable names, or WA.Storied-specific constants into this repository; LlmTck tests and samples must remain deterministic, repo-owned, and free of cross-project naming leaks. +- Treat ManagedCode.LlmTestServer as a completely new brand: remove all old project/package/type/configuration names, legacy aliases, and backward-compatibility paths. Do not provide a migration layer or preserve old branded diagnostic codes, fixture IDs, package identities, or release/deprecation machinery. + +- Do not ship template-looking or AI-generic UI: no colored left-border cards or chat bubbles, no unbalanced card compositions with lopsided visuals, no gradient-glow icon tiles, no walls of numbers where a diagram would explain more. Study high-quality references (for example Dribbble analytics dashboards) before visual work, keep every card's visual balanced and centered on one idea, and keep copy short and clear. +- Do not mention competing products in README, documentation, source, tests, or other repository artifacts; describe our own contracts and capabilities directly. +- Keep only integration tests that start the server and verify real HTTP/provider SDK behavior. Do not add or retain unit tests, fake clients, source/document scans, reflection-based test guards, or assertions about implementation fragments. Exercise every supported API and parameter, streaming mode, error boundary, and complete tool/conversation flow through the running server. +- Do not introduce real-LLM toggles, external project environment variable names, or WA.Storied-specific constants into this repository; LlmTestServer tests and samples must remain deterministic, repo-owned, and free of cross-project naming leaks. ## Code Quality Skills @@ -98,39 +118,45 @@ Rule format: ## Design Rules +- Identify TypeSafe as the provider and Jev as its model family in user-facing documentation; name System One as the native API/protocol and make its package, route and typed client discoverable instead of hiding TypeSafe behind the protocol name. + +- Tool fixtures must explicitly simulate distinct errors, timeouts, internal failures or connection crashes, literal null, empty/no result, and content-filter refusals. Keep tool-result failures distinct from provider/transport failures, expose the observed outcome in diagnostics, and verify recovery plus conversation/cache state through real server integrations; expected faults must not be rejected as invalid mock configuration. + - Before claiming that all review findings are fixed, recheck every finding against final behavior and tests; report removed capability claims and temporary mitigations separately from implemented functionality. - Prefer official .NET packages and abstractions before writing custom protocol code. -- Tests that fake `IChatClient` should use the dotnet/extensions pattern: a tiny fake class with delegate callbacks, not Moq or NSubstitute. -- Keep provider wire formats in provider packages such as `ManagedCode.LlmTck.OpenAI`; keep deterministic scenario behavior in `ManagedCode.LlmTck`. -- Keep the provider package matrix explicit. Do not ship the TCK as only OpenAI-compatible; maintain package surfaces for OpenAI, Azure OpenAI, Microsoft Foundry, Anthropic, Gemini, Groq, Mistral, Ollama, Cohere, Amazon Bedrock, OpenRouter, DeepSeek, and Perplexity. +- Provider compatibility tests must use the official provider SDK clients against the running server wherever an official client exists. Raw HTTP supplements malformed-request, exact wire-format and unsupported-boundary checks; it is not sufficient proof for supported success, streaming, tool or conversation flows. Do not substitute fake `IChatClient` implementations or direct runtime/builder calls for integration behavior. +- Keep provider wire formats in provider packages such as `ManagedCode.LlmTestServer.OpenAI`; keep deterministic scenario behavior in `ManagedCode.LlmTestServer`. +- Keep the provider package matrix explicit. Do not ship the Test Server as only OpenAI-compatible; maintain package surfaces for OpenAI, Azure OpenAI, Microsoft Foundry, Anthropic, Gemini, Groq, Mistral, Ollama, Cohere, Amazon Bedrock, OpenRouter, DeepSeek, and Perplexity. - Azure OpenAI and Microsoft Foundry compatibility must be proven with official Azure SDK clients, not only raw HTTP requests. - Azure SDK clients must select their own API version. Do not pin a dated version in consumers or SDK compatibility tests to satisfy the emulator; deployment routes must enforce their documented supported Azure versions and reject unknown versions, including syntactically valid future dates. Do not weaken server validation to make client tests pass. -- Built-in defaults, samples, and scenario examples must use official provider model IDs such as `gpt-4.1-mini`, `text-embedding-3-small`, `gpt-image-1`, and `gpt-4o-mini-tts`; do not use synthetic `llm-tck-*` model IDs for normal fixtures. +- Source current built-in model IDs and capability defaults from the original official provider SDK model definitions and current official model documentation. Prefer SDK model identifiers in SDK tests; do not invent models or treat stale hand-maintained string constants as an authoritative catalogue. Keep custom fixture models explicitly configurable. - Keep custom `AddModel(...)` support, but expose typed convenience APIs for common official fixture models so callers do not have to hand-type model IDs for standard chat, embedding, image, and audio setup. - Reasoning-capable models must carry deterministic reasoning-token usage through runtime summaries and provider response usage details; do not flatten reasoning usage into ordinary visible output tokens only. - Prompt-cache accounting is provider-specific: expose cache read/write usage for every provider surface that has a documented cache contract, and report no cache usage for providers or operations without one. - Compatibility tag values in C# code must come from named constants everywhere they are assigned or asserted, so `CompatibilityTags` cannot drift through inline string literals. -- Provider `ApiContract` operation id values and behavior-evidence operation ids in C# must come from `LlmTckProviderOperationIds`; do not type raw operation-id strings such as `videos.content.retrieve`. -- The client package must expose a universal pre-test configuration API so tests can spawn a client, reset or configure the hosted TCK, load models, auth, datasets, scenarios, scripted errors, embeddings, images, and audio fixtures without hand-authoring raw DTOs. +- Provider `ApiContract` operation id values and behavior-evidence operation ids in C# must come from `LlmTestServerProviderOperationIds`; do not type raw operation-id strings such as `videos.content.retrieve`. +- The client package must expose a universal pre-test configuration API so tests can spawn a client, reset or configure the hosted Test Server, load models, auth, datasets, scenarios, scripted errors, embeddings, images, and audio fixtures without hand-authoring raw DTOs. - The browser admin panel must be served from `/`, and runtime control APIs must use `/admin-api/...` paths; do not add hidden or aliased admin/control routes for reset, configure, models, assertions, or operator UI. -- The browser admin panel must be implemented as Blazor server-side components in `ManagedCode.LlmTck.Hosting`; do not serve it from a static HTML string endpoint, and use component state plus `StateHasChanged()` for refresh behavior instead of client-side polling fetch code. +- The browser admin panel must be implemented as Blazor server-side components in `ManagedCode.LlmTestServer.Hosting`; do not serve it from a static HTML string endpoint, and use component state plus `StateHasChanged()` for refresh behavior instead of client-side polling fetch code. - The browser admin panel is a full-viewport debugging surface, not a narrow landing page; show models/fixtures, actions, calls/requests, responses, provider/control API endpoints, assertions, and runtime events as separate grouped, inspectable areas so users can inspect each part without digging through one collapsed feed. - The browser admin dashboard must be request-centric: use a scannable table with one entry per provider request, show provider and model at a glance, and let operators inspect the full request, response, multi-turn conversation, cache accounting, token usage, timing, status, and related metadata without reconstructing a call from separate feeds; keep retention explicitly bounded and visibly mark the exceptional payload that exceeds that diagnostic budget. +- Show request and explicitly scoped conversation analytics from recorded traffic as visual token composition, not text tables and footnotes: split input tokens by system prompt, user, assistant, tool calls, tool results and tool schemas with the runtime tokenizer so the parts reconcile exactly with reported input, then show cache read/write/uncached and output/reasoning through flow (Sankey) and donut charts with value-and-share legends, plus per-request context growth for conversations. Keep reasoning/cache subsets out of double-counted totals, distinguish missing usage from zero, and disclose retention limits briefly instead of in walls of caveat text. - Keep the request inbox visually primary and easy to scan; do not compress it beside a narrow, endlessly tall inspector. Constrain long conversations and payloads to independent scroll regions, collapse oversized system/tool content by default while keeping it expandable, and validate the resulting hierarchy with realistic long multi-turn data at desktop, tablet, and mobile sizes. - Keep the dashboard's useful horizontal master-detail structure, but use an Apple-inspired content-first visual language rather than Material or Fluent styling: quiet system typography, generous clean surfaces, restrained separators and status color, sidebar depth only where it clarifies hierarchy, and no grid of badges or boxed metrics. Keep the filter/navigation rail visible at ordinary laptop widths and the selected response visible in the first reading-pane viewport; on narrow mobile screens use an explicit list-to-detail transition with a Back action. -- Render JSON payloads as properly indented code with an SF Mono-style font and preserve readable Unicode characters instead of exposing serializer escape sequences such as `\u0027`. +- Every chart mark, bar, segment and dot in the dashboard needs a styled hover tooltip with the label, value and share; do not rely on slow native title tooltips. +- Render JSON payloads as properly indented, syntax-highlighted code (keys, strings, numbers, literals) with an SF Mono-style font, and preserve readable Unicode characters instead of exposing serializer escape sequences such as `\u0027`. - Use Microsoft Fluent UI Blazor primitives for dashboard overlays and dismissible controls, but theme them to the dashboard's Apple-inspired visual language instead of adopting stock Fluent styling. Popovers must close on outside click, Escape, and completed actions. Blend Prostir.build's restrained glass, lime selection rail, and technical polish with managed-code.com's warm canvas, ink typography, and tiny coral-to-magenta accents without copying either site's branding or marketing-page layout. - The browser admin panel must auto-refresh by default every 15 seconds, but do not add visible countdown UI unless the user explicitly asks for it. -- Provider APIs must be explicitly namespaced by provider, such as `/openai`, `/anthropic`, `/gemini`, or `/azure-openai`; do not expose generic root `/v1/*` provider routes that make the TCK look like only the OpenAI API. +- Provider APIs must be explicitly namespaced by provider, such as `/openai`, `/anthropic`, `/gemini`, or `/azure-openai`; do not expose generic root `/v1/*` provider routes that make the Test Server look like only the OpenAI API. - Aspire examples must show endpoint retrieval from the Aspire resource (`GetEndpoint("http")` or `CreateHttpClient(...)`) and API key wiring so users see the complete integration path. -- Aspire sample services must use `ManagedCode.LlmTck.Hosting` service-defaults methods such as `AddServiceDefaults()` and `MapDefaultEndpoints()` for health/liveness wiring; do not create a separate ServiceDefaults sample project, and do not hand-roll `app.MapGet("/health", ...)` or placeholder root status endpoints in sample `Program.cs` files. -- Aspire integration must expose a package-owned `builder.AddLlmTck()` entry point so consumers can install the Aspire NuGet package and add the TCK without caller-supplied project paths or `Projects.*` metadata types. -- `builder.AddLlmTck()` must be .NET/Aspire-first and must not require Docker or a container runtime by default; container-backed hosting may exist only as an explicit opt-in API. +- Aspire sample services must use `ManagedCode.LlmTestServer.Hosting` service-defaults methods such as `AddServiceDefaults()` and `MapDefaultEndpoints()` for health/liveness wiring; do not create a separate ServiceDefaults sample project, and do not hand-roll `app.MapGet("/health", ...)` or placeholder root status endpoints in sample `Program.cs` files. +- Aspire integration must expose a package-owned `builder.AddLlmTestServer()` entry point so consumers can install the Aspire NuGet package and add the Test Server without caller-supplied project paths or `Projects.*` metadata types. +- `builder.AddLlmTestServer()` must be .NET/Aspire-first and must not require Docker or a container runtime by default; container-backed hosting may exist only as an explicit opt-in API. - Aspire integration must start an Aspire AppHost in tests before a change is considered covered, with the AppHost model built directly in test code or a test fixture. - Aspire test coverage should build the AppHost directly inside the test code or fixture; do not add a standalone `Obhost`/test AppHost file just to host Aspire for tests. - Aspire tests and samples must resolve service/provider URLs from the Aspire resource endpoint allocated by the AppHost; do not hard-code `127.0.0.1`, `localhost`, ports, or invented provider endpoints in configuration. -- Do not add an Aspire provider-endpoint setter to `ManagedCode.LlmTck.Aspire`; consumers must use the endpoint exposed by the `LlmTckResource`. +- Do not add an Aspire provider-endpoint setter to `ManagedCode.LlmTestServer.Aspire`; consumers must use the endpoint exposed by the `LlmTestServerResource`. - Do not add Aspire compatibility environment flags, sample-service config echoes, or fluent compatibility methods unless they drive real route/runtime behavior; prove compatibility through actual provider clients instead. - Provider API compatibility must be doc-backed: every claimed provider method, route, streaming mode, modality, and API-version constraint must be represented in a checked-in contract with official documentation links and covered by tests before the provider package claims support. - Provider API verification must cover the full claimed provider matrix: every implemented `ApiContract` operation for every provider needs behavior-test evidence, and fixes that touch provider compatibility must rerun the provider contract/evidence tests plus the full quality gate before closeout. @@ -139,26 +165,26 @@ Rule format: ## Ownership Map -- `src/ManagedCode.LlmTck`: provider-neutral runtime, scenarios, assertions, and model catalog. -- `src/ManagedCode.LlmTck.SystemOne`: Jev/Kev native SystemOne decision contracts and compatibility profile. -- `src/ManagedCode.LlmTck.Cloudflare`: Workers AI Clef/flash native decision envelopes, validation and compatibility profile. -- `src/ManagedCode.LlmTck.OpenAI`: OpenAI-compatible request/response shapes and mapping. -- `src/ManagedCode.LlmTck.AzureOpenAI`: Azure OpenAI compatibility profile and future Azure OpenAI wire contracts. -- `src/ManagedCode.LlmTck.Foundry`: Microsoft Foundry compatibility profile and future Foundry wire contracts. -- `src/ManagedCode.LlmTck.Anthropic`: Anthropic Messages compatibility profile and future Anthropic wire contracts. -- `src/ManagedCode.LlmTck.Gemini`: Gemini compatibility profile and future Gemini wire contracts. -- `src/ManagedCode.LlmTck.Groq`: Groq compatibility profile and future Groq wire contracts. -- `src/ManagedCode.LlmTck.Mistral`: Mistral compatibility profile and future Mistral wire contracts. -- `src/ManagedCode.LlmTck.Ollama`: Ollama compatibility profile and future Ollama wire contracts. -- `src/ManagedCode.LlmTck.Cohere`: Cohere compatibility profile and future Cohere wire contracts. -- `src/ManagedCode.LlmTck.Bedrock`: Amazon Bedrock compatibility profile and future Bedrock wire contracts. -- `src/ManagedCode.LlmTck.OpenRouter`: OpenRouter compatibility profile and future OpenRouter wire contracts. -- `src/ManagedCode.LlmTck.DeepSeek`: DeepSeek compatibility profile and future DeepSeek wire contracts. -- `src/ManagedCode.LlmTck.Perplexity`: Perplexity compatibility profile and future Perplexity wire contracts. -- `src/ManagedCode.LlmTck.Hosting`: ASP.NET Core provider and control endpoints. -- `src/ManagedCode.LlmTck.Client`: control client and `Microsoft.Extensions.AI` clients. -- `src/ManagedCode.LlmTck.Aspire`: Aspire AppHost extension methods. +- `src/ManagedCode.LlmTestServer`: provider-neutral runtime, scenarios, assertions, and model catalog. +- `src/ManagedCode.LlmTestServer.SystemOne`: Jev/Kev native SystemOne decision contracts and compatibility profile. +- `src/ManagedCode.LlmTestServer.Cloudflare`: Workers AI Clef/flash native decision envelopes, validation and compatibility profile. +- `src/ManagedCode.LlmTestServer.OpenAI`: OpenAI-compatible request/response shapes and mapping. +- `src/ManagedCode.LlmTestServer.AzureOpenAI`: Azure OpenAI compatibility profile and future Azure OpenAI wire contracts. +- `src/ManagedCode.LlmTestServer.Foundry`: Microsoft Foundry compatibility profile and future Foundry wire contracts. +- `src/ManagedCode.LlmTestServer.Anthropic`: Anthropic Messages compatibility profile and future Anthropic wire contracts. +- `src/ManagedCode.LlmTestServer.Gemini`: Gemini compatibility profile and future Gemini wire contracts. +- `src/ManagedCode.LlmTestServer.Groq`: Groq compatibility profile and future Groq wire contracts. +- `src/ManagedCode.LlmTestServer.Mistral`: Mistral compatibility profile and future Mistral wire contracts. +- `src/ManagedCode.LlmTestServer.Ollama`: Ollama compatibility profile and future Ollama wire contracts. +- `src/ManagedCode.LlmTestServer.Cohere`: Cohere compatibility profile and future Cohere wire contracts. +- `src/ManagedCode.LlmTestServer.Bedrock`: Amazon Bedrock compatibility profile and future Bedrock wire contracts. +- `src/ManagedCode.LlmTestServer.OpenRouter`: OpenRouter compatibility profile and future OpenRouter wire contracts. +- `src/ManagedCode.LlmTestServer.DeepSeek`: DeepSeek compatibility profile and future DeepSeek wire contracts. +- `src/ManagedCode.LlmTestServer.Perplexity`: Perplexity compatibility profile and future Perplexity wire contracts. +- `src/ManagedCode.LlmTestServer.Hosting`: ASP.NET Core provider and control endpoints. +- `src/ManagedCode.LlmTestServer.Client`: control client and `Microsoft.Extensions.AI` clients. +- `src/ManagedCode.LlmTestServer.Aspire`: Aspire AppHost extension methods. - `samples`: runnable AppHost and service sample. -- `tests`: TUnit unit, endpoint, client, and Aspire integration coverage. +- `tests`: TUnit server API, client SDK, and Aspire integration coverage. - `docs`: architecture, feature specs, ADRs, and test inventory. - `site`: static public landing page. diff --git a/Directory.Build.props b/Directory.Build.props index 8e6f0a5..69e8433 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -18,13 +18,13 @@ ManagedCode ManagedCode - ManagedCode.LlmTck + ManagedCode.LlmTestServer GitHub - https://github.com/managedcode/LlmTck - https://github.com/managedcode/LlmTck + https://github.com/managedcode/LlmTestServer + https://github.com/managedcode/LlmTestServer MIT README.md - llm;tck;testing;aspire;openai-compatible;azure-openai;anthropic;gemini;groq;mistral;foundry;ai;dotnet;jev;typesafe;system-one;decision-models;multimodal + llm;test-server;testing;aspire;openai-compatible;azure-openai;anthropic;gemini;groq;mistral;foundry;ai;dotnet;jev;typesafe;system-one;decision-models;multimodal true true true diff --git a/Directory.Packages.props b/Directory.Packages.props index 6de6c2e..fed354e 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -4,6 +4,8 @@ + + diff --git a/ManagedCode.LlmTck.slnx b/ManagedCode.LlmTck.slnx deleted file mode 100644 index 71cbd02..0000000 --- a/ManagedCode.LlmTck.slnx +++ /dev/null @@ -1,30 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ManagedCode.LlmTestServer.slnx b/ManagedCode.LlmTestServer.slnx new file mode 100644 index 0000000..6f76531 --- /dev/null +++ b/ManagedCode.LlmTestServer.slnx @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/README.md b/README.md index d1861e0..21a9d0f 100644 --- a/README.md +++ b/README.md @@ -1,1052 +1,275 @@ -# LLM Technology Compatibility Kit - -[![CI](https://github.com/managedcode/LlmTck/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/managedcode/LlmTck/actions/workflows/ci.yml) -[![Release](https://github.com/managedcode/LlmTck/actions/workflows/release.yml/badge.svg?branch=main)](https://github.com/managedcode/LlmTck/actions/workflows/release.yml) -[![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTck.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck) -[![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTck.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck) -[![License](https://img.shields.io/github/license/managedcode/LlmTck.svg?style=flat-square)](LICENSE) - -`ManagedCode.LlmTck` is a deterministic Technology Compatibility Kit for LLM APIs. -It gives tests a local provider-compatible server that can be hosted by Aspire, -scripted with explicit scenarios, and called through regular HTTP, official SDKs, -or `Microsoft.Extensions.AI`. - -The hosted surface is provider-namespaced. OpenAI-compatible routes live under -`/openai`, Azure OpenAI under `/azure-openai`, Microsoft Foundry under -`/microsoft-foundry`, Anthropic Messages under `/anthropic`, and every other -provider family uses its own namespace. Provider packages carry doc-backed API -contracts, and `ManagedCode.LlmTck.Hosting` only claims routes that are mapped by -`MapLlmTck()` and covered by tests. - -Use it when a test needs to prove what an application sends to an LLM provider, -script deterministic provider responses, exercise retry/error/model-routing paths, -or assert that no unexpected LLM calls occurred. When a bearer token is configured, -both provider endpoints and `/admin-api/*` control endpoints require it. - -## Jev / TypeSafe System One - -Jev is supported through its native System One API, including Jev 1.14 image assets -and bounding boxes. The emulator accepts JSON base64 and multipart PNG/JPEG uploads, -structured instructions and criteria, mixed `noul`/`choice`/`score`/`box` batches, -native confidence, usage and provider-resolved model versions. Image and box requests -require `TypeSafe-Danger-Zone-Images: true`; long-context requests can explicitly send -`TypeSafe-Danger-Zone-Long-Context: true`. It uses deterministic fixtures and never -calls a production model or estimates image tokens. - -- Native TCK route: `POST /systemone/v1/systemone`. -- Contracts: `ManagedCode.LlmTck.SystemOne`. -- Typed client: `LlmTckClient.CreateSystemOneClient()` from `ManagedCode.LlmTck.Client`. -- Register `jev-1.14.0` or `jev-preview` with `AddDecisionModel(id, LlmTckDecisionProvider.TypeSafe)`. -- [Jev 1.14 guide and limits](docs/Features/Jev114.md), [native decision fixtures](#native-decision-api-fixtures), - [official images API](https://docs.typesafe.ai/images). - -## Request Dashboard - -The Blazor dashboard at `/` is an operator-focused request inbox. It keeps provider, -model, status, latency, token/cache usage, multi-turn context, sanitized wire payloads, -stream chunks, and the deterministic runtime result together for each request. - -![LLM TCK provider request dashboard showing a selected multi-turn OpenAI request](https://raw.githubusercontent.com/managedcode/LlmTck/main/docs/images/admin-dashboard-overview.jpg) - -| Conversation timeline | Runtime diagnostics | -| --- | --- | -| ![Chronological multi-turn request and provider response](https://raw.githubusercontent.com/managedcode/LlmTck/main/docs/images/admin-dashboard-conversation.jpg) | ![Fluent UI runtime diagnostics dialog with fixtures, provider APIs, assertions, and events](https://raw.githubusercontent.com/managedcode/LlmTck/main/docs/images/admin-dashboard-runtime.jpg) | - -## Packages - -Package links open the latest stable release on NuGet; badges update automatically. - -| Package | Latest version | Downloads | Description | -| --- | --- | --- | --- | -| [ManagedCode.LlmTck](https://www.nuget.org/packages/ManagedCode.LlmTck) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTck.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTck.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck) | Provider-neutral scenario runtime and assertion state. | -| [ManagedCode.LlmTck.OpenAI](https://www.nuget.org/packages/ManagedCode.LlmTck.OpenAI) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTck.OpenAI.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.OpenAI) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTck.OpenAI.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.OpenAI) | OpenAI-compatible wire contracts. | -| [ManagedCode.LlmTck.AzureOpenAI](https://www.nuget.org/packages/ManagedCode.LlmTck.AzureOpenAI) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTck.AzureOpenAI.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.AzureOpenAI) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTck.AzureOpenAI.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.AzureOpenAI) | Azure OpenAI compatibility profile and future wire contracts. | -| [ManagedCode.LlmTck.Foundry](https://www.nuget.org/packages/ManagedCode.LlmTck.Foundry) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTck.Foundry.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.Foundry) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTck.Foundry.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.Foundry) | Microsoft Foundry compatibility profile and future wire contracts. | -| [ManagedCode.LlmTck.Anthropic](https://www.nuget.org/packages/ManagedCode.LlmTck.Anthropic) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTck.Anthropic.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.Anthropic) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTck.Anthropic.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.Anthropic) | Anthropic Messages compatibility profile and wire contracts. | -| [ManagedCode.LlmTck.Gemini](https://www.nuget.org/packages/ManagedCode.LlmTck.Gemini) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTck.Gemini.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.Gemini) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTck.Gemini.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.Gemini) | Gemini compatibility profile and future wire contracts. | -| [ManagedCode.LlmTck.Groq](https://www.nuget.org/packages/ManagedCode.LlmTck.Groq) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTck.Groq.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.Groq) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTck.Groq.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.Groq) | Groq compatibility profile and future wire contracts. | -| [ManagedCode.LlmTck.Mistral](https://www.nuget.org/packages/ManagedCode.LlmTck.Mistral) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTck.Mistral.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.Mistral) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTck.Mistral.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.Mistral) | Mistral compatibility profile and future wire contracts. | -| [ManagedCode.LlmTck.Ollama](https://www.nuget.org/packages/ManagedCode.LlmTck.Ollama) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTck.Ollama.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.Ollama) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTck.Ollama.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.Ollama) | Ollama compatibility profile and future wire contracts. | -| [ManagedCode.LlmTck.Cohere](https://www.nuget.org/packages/ManagedCode.LlmTck.Cohere) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTck.Cohere.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.Cohere) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTck.Cohere.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.Cohere) | Cohere compatibility profile and future wire contracts. | -| [ManagedCode.LlmTck.Bedrock](https://www.nuget.org/packages/ManagedCode.LlmTck.Bedrock) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTck.Bedrock.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.Bedrock) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTck.Bedrock.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.Bedrock) | Amazon Bedrock compatibility profile and future wire contracts. | -| [ManagedCode.LlmTck.OpenRouter](https://www.nuget.org/packages/ManagedCode.LlmTck.OpenRouter) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTck.OpenRouter.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.OpenRouter) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTck.OpenRouter.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.OpenRouter) | OpenRouter compatibility profile and future wire contracts. | -| [ManagedCode.LlmTck.DeepSeek](https://www.nuget.org/packages/ManagedCode.LlmTck.DeepSeek) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTck.DeepSeek.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.DeepSeek) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTck.DeepSeek.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.DeepSeek) | DeepSeek compatibility profile and future wire contracts. | -| [ManagedCode.LlmTck.Perplexity](https://www.nuget.org/packages/ManagedCode.LlmTck.Perplexity) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTck.Perplexity.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.Perplexity) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTck.Perplexity.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.Perplexity) | Perplexity compatibility profile and future wire contracts. | -| [ManagedCode.LlmTck.SystemOne](https://www.nuget.org/packages/ManagedCode.LlmTck.SystemOne) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTck.SystemOne.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.SystemOne) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTck.SystemOne.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.SystemOne) | Native Jev/Kev SystemOne decision contracts. | -| [ManagedCode.LlmTck.Cloudflare](https://www.nuget.org/packages/ManagedCode.LlmTck.Cloudflare) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTck.Cloudflare.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.Cloudflare) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTck.Cloudflare.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.Cloudflare) | Cloudflare Workers AI Clef/flash decision envelopes and validation. | -| [ManagedCode.LlmTck.Hosting](https://www.nuget.org/packages/ManagedCode.LlmTck.Hosting) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTck.Hosting.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.Hosting) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTck.Hosting.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.Hosting) | ASP.NET Core endpoint mapping. | -| [ManagedCode.LlmTck.Client](https://www.nuget.org/packages/ManagedCode.LlmTck.Client) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTck.Client.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.Client) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTck.Client.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.Client) | Control client plus `IChatClient`, `IEmbeddingGenerator>`, and `IImageGenerator` implementations. | -| [ManagedCode.LlmTck.Aspire](https://www.nuget.org/packages/ManagedCode.LlmTck.Aspire) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTck.Aspire.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.Aspire) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTck.Aspire.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTck.Aspire) | Aspire AppHost extension methods. | - -## Current Functionality - -This is the currently implemented behavior, not future intent: - -| Surface | What works now | -| --- | --- | -| Hosting | `builder.Services.AddLlmTck(...)` registers one deterministic runtime; `app.MapLlmTck()` maps the Blazor server-side browser panel at `/`, JSON control endpoints under `/admin-api`, and provider-namespaced HTTP routes. | -| Runtime configuration | Configure at host startup, through `LlmTckClient.ConfigureAsync(...)`, or through `POST /admin-api/configure`. Configuration replaces the current runtime state and resets assertions, scenario positions, fault counters, and prompt-cache entries. | -| Reset | `ResetAsync()` and `POST /admin-api/reset` keep the current models/scenarios/fixtures but clear assertion events, scenario response positions, rate-limit counters, and prompt-cache entries. | -| Models | Default models use official fixture IDs for chat, embeddings, images, audio, and video. Typed helpers register broader OpenAI model families, and `AddModel(...)` supports custom deployment or provider IDs. Model kind is enforced on every route. | -| Chat scenarios | Contains matching, exact prompt matching, queued responses, streaming chunks, scripted errors, delayed responses, cancellation-safe queues, and scenario-specific bearer tokens are implemented. | -| Modalities | Chat, embeddings, image generation/edit/variation, audio speech/transcription/translation, OpenAI/Azure video, Gemini long-running video/file output, and Bedrock text/embedding/image invoke shapes return deterministic fixtures. | -| Provider matrix | OpenAI, Azure OpenAI, Microsoft Foundry, Anthropic, Gemini, Groq, Mistral, Ollama, Cohere, Amazon Bedrock, OpenRouter, DeepSeek, and Perplexity are exposed under explicit provider namespaces. Each hosted operation is represented in a provider `ApiContract` and behavior evidence tests. | -| SDK clients | Official OpenAI, Azure OpenAI, Azure AI Inference / Foundry, Amazon Bedrock, and `Microsoft.Extensions.AI` clients can call the hosted provider routes. The client package also creates control, chat, embedding, image, and audio clients with the same bearer token. | -| Auth | A global bearer token protects provider and `/admin-api/*` endpoints. Anthropic `x-api-key` and Azure `api-key` headers are accepted as provider-native ways to pass the same token. Scenarios can add their own bearer-token requirement. | -| Fault simulation | Provider-neutral rate-limit and content-filter simulation flow through every provider family and return provider-shaped errors. Scripted per-scenario failures remain available for exact test paths. | -| Assertions | `/admin-api/assertions` returns counters plus every runtime event with request preview, response/error text, model id, scenario id, event kind, and deterministic usage. | -| Usage accounting | Input, output, reasoning, total, cached-input, and cache-write token values are computed deterministically and projected into provider-specific response envelopes when that provider surface documents usage fields. | -| Browser panel | `/` shows models/fixtures, actions, control and provider endpoint groups, grouped runtime events, captured requests, captured responses, counters, and per-event usage. It auto-refreshes every 15 seconds using Blazor component state and `StateHasChanged()`. | -| Aspire | `builder.AddLlmTck()` starts the packaged .NET service without Docker by default; `WithApiKey(...)` wires auth, `GetHttpEndpoint()` and provider-specific endpoint helpers expose URLs, and `AddLlmTckContainer()` is an explicit container opt-in. | - -## Provider Packages - -Provider packages keep vendor-specific protocol and compatibility metadata out of the provider-neutral runtime. The package surface exists now so applications can select a provider family explicitly; provider-specific DTOs and endpoint mappings are added inside each package as support grows. - -| Package | Provider ID | Protocol family | Primary hosted route | -| --- | --- | --- | --- | -| `ManagedCode.LlmTck.OpenAI` | `openai` | OpenAI | `/openai/v1` | -| `ManagedCode.LlmTck.AzureOpenAI` | `azure-openai` | Azure OpenAI | `/azure-openai/openai/deployments/{deployment}/chat/completions` | -| `ManagedCode.LlmTck.Foundry` | `microsoft-foundry` | Microsoft Foundry | `/microsoft-foundry/models/chat/completions` | -| `ManagedCode.LlmTck.Anthropic` | `anthropic` | Anthropic Messages | `/anthropic/v1/messages` | -| `ManagedCode.LlmTck.Gemini` | `gemini` | Gemini | `/gemini/v1beta/models/{model}:generateContent` | -| `ManagedCode.LlmTck.Groq` | `groq` | Groq | `/groq/openai/v1/chat/completions` | -| `ManagedCode.LlmTck.Mistral` | `mistral` | Mistral | `/mistral/v1/chat/completions` | -| `ManagedCode.LlmTck.Ollama` | `ollama` | Ollama | `/ollama/api/chat` | -| `ManagedCode.LlmTck.Cohere` | `cohere` | Cohere | `/cohere/v2/chat` | -| `ManagedCode.LlmTck.Bedrock` | `bedrock` | Amazon Bedrock | `/bedrock/model/{modelId}/converse` | -| `ManagedCode.LlmTck.OpenRouter` | `openrouter` | OpenRouter | `/openrouter/api/v1/chat/completions` | -| `ManagedCode.LlmTck.DeepSeek` | `deepseek` | DeepSeek | `/deepseek/v1/chat/completions` | -| `ManagedCode.LlmTck.Perplexity` | `perplexity` | Perplexity | `/perplexity/v1/sonar` | -| `ManagedCode.LlmTck.SystemOne` | `systemone` | Jev / TypeSafe and Kev native decisions | `/systemone/v1/systemone` | -| `ManagedCode.LlmTck.Cloudflare` | `cloudflare` | Clef / Clef Flash native decisions | `/cloudflare/client/v4/accounts/{accountId}/ai/run/@cf/cloudflare/clef` | - -Each provider profile also carries a doc-backed `ApiContract` with official documentation links, retrieval date, API version or header requirements, documented operations, streaming support, and whether `ManagedCode.LlmTck.Hosting` currently maps the operation. - -Use a provider profile when code needs a stable package-level declaration without starting a host: - -```csharp -using ManagedCode.LlmTck.AzureOpenAI; - -var profile = AzureOpenAiCompatibility.Profile; -Console.WriteLine(profile.Id); // azure-openai -``` - -## OpenAI SDK - -The official OpenAI .NET SDK expects its `Endpoint` to be the API root. Use -`/openai/v1` for this TCK, or `llmTck.GetOpenAiEndpoint()` from the Aspire -package when wiring an AppHost resource. - -```csharp -using OpenAI; -using OpenAI.Chat; -using System.ClientModel; - -var chat = new ChatClient( - "gpt-4.1-mini", - new ApiKeyCredential("test-key"), - new OpenAIClientOptions - { - Endpoint = new Uri("http://localhost:5000/openai/v1"), - }); - -await foreach (var update in chat.CompleteChatStreamingAsync( - [new UserChatMessage("What is the invoice total?")])) -{ - Console.Write(update.ContentUpdate.Count > 0 ? update.ContentUpdate[0].Text : string.Empty); -} -``` - -## Azure OpenAI And Foundry SDKs - -The OpenAI v1 API is available at `/azure-openai/openai/v1/` and `/microsoft-foundry/openai/v1/`, with `chat/completions`, `responses` and `embeddings`. Point the official OpenAI client endpoint at the corresponding base URL and supply the configured API key. These v1 routes accept bearer or `api-key` authentication and need no dated `api-version` query parameter. Responses supports ordinary JSON and documented SSE events. See the [September 2026 API review](docs/Features/ProviderApiReview20260907.md) for the verified SDK versions and provider contracts. - -Azure OpenAI deployment routes use the deployment name as the TCK model id. API keys sent through the official SDK's `api-key` header are accepted anywhere a bearer token would be accepted. - -```csharp -using Azure.AI.OpenAI; -using OpenAI.Chat; -using OpenAI.Embeddings; -using System.ClientModel; +# LLM Test Server -var azure = new AzureOpenAIClient( - new Uri("http://localhost:5000/azure-openai/"), - new ApiKeyCredential("test-key")); +[![CI](https://github.com/managedcode/LlmTestServer/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/managedcode/LlmTestServer/actions/workflows/ci.yml) +[![Release](https://github.com/managedcode/LlmTestServer/actions/workflows/release.yml/badge.svg?branch=main)](https://github.com/managedcode/LlmTestServer/actions/workflows/release.yml) +[![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTestServer.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer) +[![License](https://img.shields.io/github/license/managedcode/LlmTestServer.svg?style=flat-square)](LICENSE) -ChatClient chat = azure.GetChatClient("gpt-4.1-mini"); -EmbeddingClient embeddings = azure.GetEmbeddingClient("text-embedding-3-small"); +**A local, deterministic LLM provider for .NET tests, with a dashboard that shows where every token goes.** -var answer = await chat.CompleteChatAsync( -[ - new UserChatMessage("What is the invoice total?"), -]); +[![Token flow of a five-request conversation: system prompt, tool schemas, user, assistant, tool calls and tool results flowing into cache read, cache write, uncached input, response and reasoning](docs/images/showcase-token-flow.jpg)](docs/images/showcase-token-flow.jpg) -var vector = await embeddings.GenerateEmbeddingAsync("invoice"); -``` - -Microsoft Foundry / Azure AI Inference clients call the `/microsoft-foundry/chat/completions` and `/microsoft-foundry/embeddings` routes. The inference routes require `api-version=2024-05-01-preview`, which the official SDK supplies. Set `Model` to the model id configured in LLM TCK. - -```csharp -using Azure; -using Azure.AI.Inference; - -var endpoint = new Uri("http://localhost:5000/microsoft-foundry/"); -var credential = new AzureKeyCredential("test-key"); - -var chat = new ChatCompletionsClient(endpoint, credential); -var embeddings = new EmbeddingsClient(endpoint, credential); - -var completion = await chat.CompleteAsync( - new ChatCompletionsOptions( - [ - new ChatRequestUserMessage("What is the invoice total?"), - ]) - { - Model = "gpt-4.1-mini", - }); - -var embedding = await embeddings.EmbedAsync( - new EmbeddingsOptions(["invoice"]) - { - Model = "text-embedding-3-small", - }); -``` - -## Configuration Model - -LLM TCK is configured with one `LlmTckConfiguration`. - -You can apply that configuration in two places: +[What it is](#what-it-is) · [Token usage and cache hits](#token-usage-and-prompt-cache-hits) · [Quick start](#quick-start) · [Capabilities](#what-you-can-do) · [Providers](#providers) · [TypeSafe / Jev](#typesafe-and-jev) · [Docs](#learn-more) -- at host startup through `builder.Services.AddLlmTck(options => ...)` -- at test runtime through `LlmTckClient.ConfigureAsync(...)` or `POST /admin-api/configure` +## What it is -Runtime configuration is a replacement, not a merge. Applying a new configuration resets scenario positions and assertion events. The runtime snapshots the configuration, so later mutations to your builder/list objects do not change the running provider. +LLM Test Server is a small ASP.NET Core server that speaks the HTTP APIs of 13 LLM providers, each under its own +namespace: `/openai/v1`, `/anthropic/v1`, `/gemini/v1beta`, `/azure-openai`, and so on. Your application keeps its +official SDK or `Microsoft.Extensions.AI` client and only changes the base URL. Every answer, tool call, stream chunk +and failure comes from a fixture your test configures, so the same test produces the same result on every run — with +no external model calls and no production credentials. Run it from an Aspire AppHost, inside your own ASP.NET Core host, +or as a standalone container. -The default configuration advertises official OpenAI fixture model IDs: +Native decision APIs are included too: **TypeSafe Jev**, Kev through System One, and Cloudflare Clef. +Use the typed client from `ManagedCode.LlmTestServer.Client` with the contracts in +`ManagedCode.LlmTestServer.SystemOne` — see the [TypeSafe/Jev example](docs/Reference/Clients.md#typesafe-jev-through-system-one). -| Model ID | Kind | -| --- | --- | -| `gpt-4.1-mini` | Chat | -| `text-embedding-3-small` | Embedding | -| `gpt-image-1` | Image | -| `gpt-4o-mini-tts` | Audio | -| `sora-2` | Video | - -Model IDs and model kinds are enforced. A chat request to an embedding model, or an embedding request to an unknown model, returns `404` with `llm_tck_unknown_model`. +## Why use it -Use typed helpers for broader OpenAI model registration when tests need more than the defaults: +- **Deterministic tests for LLM features.** Script answers, tool calls, multi-turn conversations, streams, rate limits + and content filters instead of mocking SDK internals. +- **Your real integration code runs.** Requests travel through the same provider SDKs and wire formats your + application uses in production. +- **Proof of what your application sent.** Every request is recorded with its messages, payloads, usage and outcome; + tests can query, export and verify that journal. +- **Token and cache visibility.** See how much of each prompt is system text, tool schemas or tool results, and how + much of it the prompt cache served. +- **Safe in parallel.** Isolated sessions give each test its own configuration, queues, cache and journal on one + shared server. -```csharp -options.AddKnownOpenAiModels(reasoningTokens: 128); - -// Or register only one modality family. -options.AddCurrentOpenAiChatModels(reasoningTokens: 128); -options.AddOpenAiEmbeddingModels(); -options.AddOpenAiImageModels(); -options.AddOpenAiAudioModels(); -options.AddOpenAiVideoModels(); -``` +## Token usage and prompt-cache hits -Individual helpers are also available, including `AddGpt55(...)`, `AddGpt54Mini(...)`, `AddGpt5Nano(...)`, `AddGpt41()`, `AddGpt4OMini()`, `AddTextEmbedding3Large()`, `AddGptImage2()`, `AddGptImage15()`, `AddGptImage1Mini()`, `AddTts1()`, `AddTts1Hd()`, and `AddSora2Pro()`. Keep using `AddModel(...)` for custom deployment names or provider-specific IDs. +Every chat request gets deterministic token usage from the server's tokenizer, and the provider response carries it in +that provider's own usage fields. Three things are tracked for each request: -### Fault Simulation +- **What the input is made of.** Input tokens are split by role — system prompt, tool schemas, user, assistant text, + tool-call arguments and tool results. The parts add up exactly to the reported input and are recorded as + `InputComposition` on the request's usage. +- **How the prompt cache served it.** A repeated prompt prefix reports cache reads; the first request that makes a + prefix cacheable reports cache writes. Thresholds follow each provider (for example 1,024 tokens in 128-token steps on + OpenAI-compatible routes, 2,048 on Gemini, and explicit cache markers on Anthropic and Bedrock). +- **What the model produced.** Output tokens, with reasoning tokens for reasoning-capable fixture models reported + separately. -Fault simulation is configured on the provider-neutral runtime and applies through every provider namespace. Use it when a test needs retry, moderation, or provider-error handling without scripting each provider route separately. +Read it from a test through the request journal (`GET /admin-api/requests`): ```csharp -builder.Services.AddLlmTck(options => +var journal = await control.GetRequestsAsync(); +foreach (var request in journal.Requests) { - // Allow two accepted runtime requests, then return 429 too_many_requests. - options.SimulateRateLimitAfter(2); - - // Return 400 content_filter whenever a request text contains either term. - options.SimulateContentFilter("blocked phrase", "unsafe fixture"); -}); + if (request.Usage is not { InputComposition: { } parts } usage) continue; + Console.WriteLine($"{request.ModelId}: {usage.InputTokens} input = {parts.SystemTokens} system + " + + $"{parts.ToolDefinitionTokens} tool schemas + {parts.UserTokens} user + {parts.AssistantTokens} assistant + " + + $"{parts.ToolCallTokens} tool calls + {parts.ToolResultTokens} tool results + {parts.OtherTokens} other; " + + $"{usage.CachedInputTokens} read from cache, {usage.CacheCreationInputTokens} written"); +} ``` -`SimulateRateLimitAfter(0)` makes the first valid runtime request return `429` with `too_many_requests`. `SimulateContentFilter(...)` checks chat messages, embedding inputs, image prompts, audio speech input, audio transcription/translation prompts, and video prompts. `ConfigureAsync(...)` and `ResetAsync()` both reset the rate-limit counter so tests stay deterministic. +Or open the dashboard: the **Tokens** tab draws the flow above for one request or a whole labeled conversation, with +context-mix and prompt-cache donuts and per-request context growth. **Traffic insights** (`/?view=insights`) charts +all recorded traffic by provider, model, outcome, latency, cache and context mix; clicking a slice filters the inbox. -## ASP.NET Core Startup +[![Traffic insights: donuts, capsule bars and columns across providers, models, outcomes, cache and latency](docs/images/showcase-insights.jpg)](docs/images/showcase-insights.jpg) -The simplest host config is one chat scenario: +Details: [dashboard analytics](docs/Features/DashboardAnalytics.md) · +[token usage and prompt-cache accounting](docs/Reference/TokenUsageAndPromptCache.md). -```csharp -using ManagedCode.LlmTck.Hosting; -using ManagedCode.LlmTck.Models; +## Quick start -var builder = WebApplication.CreateBuilder(args); +> **Status:** the v1.0.0 NuGet packages and container image are not published yet. Until they are, build and pack +> the repository (version `0.1.18`) and use the local feed — see [installation](docs/Reference/Installation.md). -builder.Services.AddLlmTck(options => -{ - options.AddChatScenario( - "blue-whale", - scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) - .WhenUserContains("largest animal") - .Responds("blue whale", "blue ", "whale")); -}); - -var app = builder.Build(); -app.MapLlmTck(); -app.Run(); -``` - -A fuller config can advertise your application's real model names while still serving deterministic fixtures: - -```csharp -using ManagedCode.LlmTck.Hosting; -using ManagedCode.LlmTck.Models; +Install the libraries in your existing AppHost or integration-test project from the locally packed feed: -var builder = WebApplication.CreateBuilder(args); - -builder.Services.AddLlmTck(options => -{ - options.RequireBearerToken("test-key"); - - options.AddDefaultOpenAiModels(); - - options.WithDefaultEmbeddingVector(0.125f, 0.25f, 0.5f, 1.0f); - options.WithDefaultAudio(File.ReadAllBytes("fixtures/ok.wav"), "audio/wav"); - - options.AddChatScenario( - "invoice-total", - scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) - .WhenUserContains("invoice total") - .Responds( - "{\"total\":42.50}", - "{\"total\":", - "42.50", - "}")); - - options.AddChatScenario( - "provider-rate-limit", - scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) - .WhenUserContains("rate limit") - .Fails(429, "rate_limit_exceeded", "Scripted rate limit from LLM TCK.")); - - options.AddChatScenario( - "slow-response", - scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) - .WhenUserContains("slow") - .Responds("done") - .DelaysBy(250)); -}); - -var app = builder.Build(); -app.MapLlmTck(); -app.Run(); +```sh +dotnet add package ManagedCode.LlmTestServer.Aspire --version 0.1.18 --no-restore +dotnet add package ManagedCode.LlmTestServer.Client --version 0.1.18 --no-restore +dotnet restore --source /absolute/path/to/LlmTestServer/artifacts/packages \ + --source https://api.nuget.org/v3/index.json ``` -## Aspire AppHost +### In an Aspire AppHost -Install the Aspire integration package in the AppHost: - -```bash -dotnet add package ManagedCode.LlmTck.Aspire -``` - -Then add the package-owned TCK resource directly: +`AddLlmTestServer()` starts the packaged .NET service; Docker is not required. ```csharp -using ManagedCode.LlmTck.Aspire; +using ManagedCode.LlmTestServer.Aspire; var builder = DistributedApplication.CreateBuilder(args); +var llm = builder.AddLlmTestServer().WithApiKey("test-key"); -var llmTck = builder - .AddLlmTck() - .WithApiKey("test-key"); - -builder - .AddProject("consumer") - .WithReference(llmTck) - .WithEnvironment("LLM_PROVIDER_ENDPOINT", llmTck.GetHttpEndpoint()) - .WithEnvironment("OPENAI_ENDPOINT", llmTck.GetOpenAiEndpoint()) - .WaitFor(llmTck); +builder.AddProject("api") + .WithReference(llm) + .WithEnvironment("OPENAI_ENDPOINT", llm.GetOpenAiEndpoint()) + .WithEnvironment("OPENAI_API_KEY", "test-key") + .WaitFor(llm); builder.Build().Run(); ``` -`AddLlmTck()` creates a `LlmTckResource` backed by the packaged .NET LLM TCK service executable and exposes its `http` endpoint. It does not require a consumer service project reference, generated `Projects.*` metadata type, project path, Docker, or a container runtime. Consumer resources should reference the TCK resource, wait for it, and use `llmTck.GetHttpEndpoint()` when they need the provider-compatible base URL. `.WithApiKey("test-key")` sets `LlmTck:RequiredBearerToken` so both provider endpoints and `/admin-api/*` control endpoints require the same bearer token. - -Use `AddLlmTckContainer()` only when you explicitly want a container-backed resource, for example for a deployment or container-runtime smoke test. The container mode uses a `ghcr.io/managedcode/llm-tck` image tagged with the same release version as the installed `ManagedCode.LlmTck.Aspire` package; it is not the default local Aspire path. - -## Control Panel And Token Usage - -Open the TCK resource endpoint from the Aspire dashboard to inspect the running configuration at `/`. The browser admin panel is rendered by Blazor server-side rendering from `ManagedCode.LlmTck.Hosting`. If the resource was configured with `.WithApiKey("test-key")` or `RequireBearerToken("test-key")`, enter the same token in the control panel before refreshing. - -The panel reads `/admin-api/models` and `/admin-api/assertions`. It shows advertised models, actions, control endpoint groups, provider endpoint groups, grouped runtime events, captured requests, captured responses, assertion counters, and deterministic token usage. Auto-refresh is enabled by default and runs every 15 seconds on the server-side Blazor component; there is no client-side polling script and no visible countdown. - -Token usage is counted with the repo-owned tiktoken-compatible counter and reported both as summary totals and per runtime event with `inputTokens`, `cachedInputTokens`, `cacheCreationInputTokens`, `outputTokens`, `reasoningTokens`, and `totalTokens`. Provider response envelopes also receive the same deterministic usage values: OpenAI-compatible chat and Responses usage including reasoning-token and prompt-cache details when configured, Anthropic sync messages and streaming `message_start`/`message_delta`, Gemini `usageMetadata` including long-running video operation results, Cohere chat usage, Ollama prompt/eval counts, and Bedrock Converse usage. - -For reasoning-capable fixture models, configure a deterministic reasoning-token count on the model. LLM TCK includes those tokens in `outputTokens` and exposes the split in `reasoningTokens`; OpenAI-compatible responses also include `completion_tokens_details.reasoning_tokens` or `output_tokens_details.reasoning_tokens`. - -```csharp -options.AddGpt5Nano(reasoningTokens: 128); -``` - -## Prompt Cache Accounting - -Prompt cache accounting is deterministic runtime state for testing provider usage handling. LLM TCK does not call a real cache service and does not hide prompt text; it calculates repeated prompt-prefix usage from the requests that reach the runtime. - -Cache entries are created only for successful chat or Responses requests on provider surfaces with a cache policy. A failed, unmatched, unauthorized, cancelled, unknown-model, or scripted-error request does not create a cache entry. `ConfigureAsync(...)` and `ResetAsync()` both clear prompt-cache entries so each test can start from a known cache state. - -The cache key includes provider cache policy, model id, an optional provider cache key or session id, the rounded cacheable token count, and the cumulative message prefix. The first successful request with a cacheable prefix reports cache-write tokens and zero cache-read tokens. Ollama exposes cache reads only and does not report cache-write tokens. A later successful request with the same cache key reports cache-read tokens and zero cache-write tokens for the already cached prefix. - -Provider cache thresholds follow the current runtime policy: - -| Policy | Provider routes | Minimum cacheable tokens | Increment | -| --- | --- | ---: | ---: | -| OpenAI-compatible | OpenAI, Groq, Azure OpenAI, Microsoft Foundry, Perplexity, Responses API routes that use the OpenAI-compatible shape | 1024 | 128 | -| OpenRouter | OpenRouter chat and Responses routes; `session_id` or `x-session-id` can become the cache key when no explicit prompt cache key is sent | 1024 | 128 | -| DeepSeek | DeepSeek chat routes | 1024 | 128 | -| Anthropic | Anthropic Messages requests that include `cache_control` markers | 1024 | 128 | -| Mistral | Mistral chat routes | 64 | 64 | -| Gemini | Gemini generateContent routes | 2048 | 128 | -| Ollama | Ollama chat routes; deterministic cache reads only | 1 | 1 | -| Bedrock | Bedrock Converse requests that include `cachePoint` or `cache_control` markers | 1024 | 128 | - -Cache accounting is a breakdown of input usage, not an extra billable token bucket. `totalTokens` remains `inputTokens + outputTokens`; `cachedInputTokens` and `cacheCreationInputTokens` explain how much of the input was read from, or written to, the deterministic prompt cache. - -Provider envelopes expose the same runtime usage through provider-native field names: - -| Provider surface | Cache fields returned | -| --- | --- | -| OpenAI, Groq, Azure OpenAI, Microsoft Foundry, Mistral, Perplexity chat | `usage.prompt_tokens_details.cached_tokens` | -| OpenAI, Groq, Azure OpenAI, Microsoft Foundry, Perplexity Responses | `usage.input_tokens_details.cached_tokens` | -| OpenRouter chat and Responses | `cached_tokens` plus `cache_write_tokens` in the matching prompt/input token details object | -| DeepSeek chat | `usage.prompt_cache_hit_tokens` and `usage.prompt_cache_miss_tokens` | -| Anthropic Messages | `usage.cache_creation_input_tokens` and `usage.cache_read_input_tokens` when cache usage exists | -| Gemini generateContent | `usageMetadata.cachedContentTokenCount` | -| Ollama chat | `prompt_eval_cached_count`, reported on the final NDJSON chunk for streaming | -| Bedrock Converse and ConverseStream metadata | `usage.cacheReadInputTokens` and `usage.cacheWriteInputTokens` | - -![LLM TCK request dashboard showing per-request token and cache usage](docs/images/admin-dashboard-token-usage.png) - -![LLM TCK mobile request detail showing token, cache, conversation, and response data](docs/images/admin-dashboard-mobile-request-detail.png) - -## Chat Scenarios - -### Contains Match - -`WhenUserContains(...)` adds a user-message contains matcher. All configured match messages must be present somewhere in the actual request. - -```csharp -options.AddChatScenario( - "support-refund", - scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) - .WhenUserContains("refund") - .Responds("I can help with a refund.")); -``` - -### Exact Match - -Use exact matching when a test must prove the complete prompt contract: - -```csharp -using ManagedCode.LlmTck.Scenarios; - -options.AddChatScenario( - "exact-system-contract", - scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) - .WithExactMatch( - new LlmTckMessage { Role = "system", Content = "Return JSON only." }, - new LlmTckMessage { Role = "user", Content = "Give me the invoice total." }) - .Responds("{\"total\":42.50}")); -``` - -### Queued Responses - -Each matched request consumes the next response. This makes multi-turn flows deterministic: - -```csharp -options.AddChatScenario( - "two-turn-plan", - scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) - .WhenUserContains("make a plan") - .Responds("First draft") - .Responds("Revised draft")); -``` - -After the queue is exhausted, the provider returns `409` with `llm_tck_scenario_exhausted`. - -### Streaming Chunks - -The first `Responds` argument is the non-streaming response. The remaining arguments are the streaming chunks: - -```csharp -options.AddChatScenario( - "streaming-answer", - scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) - .WhenUserContains("stream this") - .Responds("blue whale", "blue ", "whale")); -``` - -For `stream: true`, LLM TCK emits SSE `data:` frames and a final `data: [DONE]` marker. All chunks in one response share the same response id. +### In a test -### Scripted Errors - -Use `Fails(...)` to test client error handling without waiting for a real provider to fail: - -```csharp -options.AddChatScenario( - "content-policy-error", - scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) - .WhenUserContains("blocked fixture") - .Fails(400, "content_filter", "The scripted request was rejected.")); -``` - -### Delays And Cancellation - -Use `DelaysBy(...)` to test timeout and cancellation behavior: +Add `ManagedCode.LlmTestServer.Aspire`, `ManagedCode.LlmTestServer.Client` and `Aspire.Hosting.Testing`, then +configure a response and call it: ```csharp -options.AddChatScenario( - "timeout-path", - scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) - .WhenUserContains("slow path") - .Responds("eventual answer") - .DelaysBy(1_500)); -``` - -If the caller cancels during the delay, the queued response is not consumed. - -### Scenario-Specific Token - -A scenario can require its own bearer token: - -```csharp -options.AddChatScenario( - "tenant-a-only", - scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) - .RequireBearerToken("tenant-a-key") - .WhenUserContains("tenant secret") - .Responds("tenant-a response")); -``` - -Use either a global token or per-scenario tokens. If both are configured for a scenario, the same request must satisfy both checks, so different global and scenario tokens intentionally make that scenario unreachable. - -## Modalities - -### Embeddings - -Configure a fixed embedding vector. Every input value receives the same deterministic vector: - -```csharp -options.AddTextEmbedding3Small(); -options.WithDefaultEmbeddingVector(0.01f, 0.02f, 0.03f, 0.04f); -``` - -```csharp -using ManagedCode.LlmTck.Client; +using Aspire.Hosting.Testing; +using ManagedCode.LlmTestServer.Aspire; +using ManagedCode.LlmTestServer.Client; using Microsoft.Extensions.AI; +using ChatMessage = Microsoft.Extensions.AI.ChatMessage; -using var httpClient = new HttpClient { BaseAddress = new Uri("http://localhost:5000") }; -using IEmbeddingGenerator> embeddings = - new LlmTckEmbeddingGenerator(httpClient, "text-embedding-3-small"); +var builder = DistributedApplicationTestingBuilder.Create([]); +builder.AddLlmTestServer("llm").WithApiKey("test-key"); +await using var app = await builder.BuildAsync(); +await app.StartAsync(); +await app.ResourceNotifications.WaitForResourceHealthyAsync("llm"); -var result = await embeddings.GenerateAsync(["alpha", "beta"]); +var control = new LlmTestServerClient(app.CreateHttpClient("llm", "http"), "test-key"); +await control.ConfigureAsync(config => config.RequireBearerToken("test-key").UseGpt41Mini() + .UseChatScenario("greeting", scenario => scenario.WhenUserContains("hello").Responds("Hello from your test!"))); +using var chat = control.CreateChatClient(); +var reply = await chat.GetResponseAsync([new ChatMessage(ChatRole.User, "hello")]); // "Hello from your test!" ``` -### Images - -Image generation returns a deterministic base64 PNG by default: +Your application can use its official SDK against the same server. In the Aspire test above, use its allocated +endpoint and the same fixture key — for example with OpenAI: ```csharp -options.AddGptImage1(); -``` - -```csharp -using ManagedCode.LlmTck.Client; -using Microsoft.Extensions.AI; - -using var httpClient = new HttpClient { BaseAddress = new Uri("http://localhost:5000") }; -using IImageGenerator images = new LlmTckImageGenerator(httpClient, "gpt-image-1"); - -var image = await images.GenerateAsync( - new ImageGenerationRequest { Prompt = "compatibility marker" }); -``` - -### Audio - -Audio speech returns deterministic bytes with a matching media type. The default fixture is a minimal WAV payload: - -```csharp -options.AddGpt4OMiniTts(); -options.WithDefaultAudio(File.ReadAllBytes("fixtures/speech.wav"), "audio/wav"); -``` - -```csharp -var audio = await httpClient.PostAsJsonAsync( - "/openai/v1/audio/speech", - new { model = "gpt-4o-mini-tts", input = "hello", voice = "alloy" }); - -audio.EnsureSuccessStatusCode(); -var bytes = await audio.Content.ReadAsByteArrayAsync(); -``` - -## Runtime Reconfiguration - -Use runtime reconfiguration when each test needs a different provider script. - -### Configure Before A Test - -Use `ManagedCode.LlmTck.Client` as the universal test control surface. A test can start or discover the hosted TCK, create one `LlmTckClient`, load all required models, datasets, scripted responses, errors, embeddings, images, and audio fixtures, and then call the provider through regular clients. - -```csharp -using ManagedCode.LlmTck.Client; -using Microsoft.Extensions.AI; - -using var httpClient = new HttpClient { BaseAddress = new Uri("http://localhost:5000") }; -var tck = LlmTckClient.Create(httpClient, bearerToken: "test-key"); - -await tck.ConfigureAsync( - config => config - .RequireBearerToken("test-key") - .UseChatModel("test-chat") - .UseEmbeddingModel("test-embedding") - .UseImageModel("test-image") - .UseAudioModel("test-audio") - .UseEmbeddingVector(0.9f, 0.8f) - .UseImageDataUri("data:image/png;base64,Zm9v") - .UseAudio([1, 2, 3, 4], "audio/test") - .UseDataset( - "checkout-flow", - dataset => dataset - .AddChatScenario( - "invoice-total", - scenario => scenario - .ForModel("test-chat") - .WhenUserContains("invoice total") - .Responds("{\"total\":42.50}", "{\"total\":", "42.50", "}")) - .AddChatScenario( - "provider-rate-limit", - scenario => scenario - .ForModel("test-chat") - .WhenUserContains("rate limit") - .Fails(429, "rate_limit_exceeded", "Scripted rate limit."))), - resetFirst: true); - -using IChatClient chat = tck.CreateChatClient("test-chat"); -using IEmbeddingGenerator> embeddings = - tck.CreateEmbeddingGenerator("test-embedding"); -using IImageGenerator images = tck.CreateImageGenerator("test-image"); - -var response = await chat.GetResponseAsync( -[ - new ChatMessage(ChatRole.User, "What is the invoice total?"), -]); - -var vector = await embeddings.GenerateAsync(["invoice"]); -var image = await images.GenerateAsync(new ImageGenerationRequest { Prompt = "fixture" }); -var audio = await tck.GenerateAudioAsync("test-audio", "speak this"); -var assertions = await tck.GetAssertionsAsync(); -``` - -`resetFirst: true` clears previous assertion events and scenario positions before the new configuration is applied. Named datasets are active after configuration; they group scenarios so a test can load a whole conversation set without flattening everything by hand. - -### Configure With `LlmTckClient` - -```csharp -using ManagedCode.LlmTck.Client; -using ManagedCode.LlmTck.Configuration; -using ManagedCode.LlmTck.Models; - -using var httpClient = new HttpClient { BaseAddress = new Uri("http://localhost:5000") }; -httpClient.DefaultRequestHeaders.Authorization = - new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", "test-key"); - -var control = new LlmTckClient(httpClient); - -var configuration = new LlmTckConfigurationBuilder() - .RequireBearerToken("test-key") - .AddModel("docs-chat", LlmTckModelKind.Chat) - .AddModel("docs-embedding", LlmTckModelKind.Embedding) - .AddChatScenario( - "docs-blue-whale", - scenario => scenario - .ForModel("docs-chat") - .WhenUserContains("largest animal") - .Responds("blue whale", "blue ", "whale")) - .WithDefaultEmbeddingVector(0.125f, 0.25f) - .Build(); - -await control.ConfigureAsync(configuration); -``` - -If the current runtime already has a bearer token, the configure request must use that current token. If the new configuration sets a different token, future provider and control requests must use the new token. - -### Configure With Raw JSON - -`POST /admin-api/configure` accepts readable enum values such as `"chat"` and `"contains"`: - -```bash -curl -X POST http://localhost:5000/admin-api/configure \ - -H "content-type: application/json" \ - -H "authorization: Bearer test-key" \ - -d '{ - "requiredBearerToken": "test-key", - "models": [ - { "id": "docs-chat", "kind": "chat" }, - { "id": "docs-embedding", "kind": "embedding" }, - { "id": "docs-image", "kind": "image" }, - { "id": "docs-audio", "kind": "audio" } - ], - "chatScenarios": [ - { - "id": "docs-blue-whale", - "modelId": "docs-chat", - "match": { - "mode": "contains", - "messages": [ - { "role": "user", "content": "largest animal" } - ] - }, - "responses": [ - { - "content": "blue whale", - "streamChunks": [ "blue ", "whale" ] - } - ] - } - ], - "defaultEmbeddingVector": [ 0.125, 0.25 ], - "defaultAudioMediaType": "audio/wav" - }' -``` +using System.ClientModel; +using OpenAI; +using OpenAI.Chat; -Then call the configured model: - -```bash -curl -X POST http://localhost:5000/openai/v1/chat/completions \ - -H "content-type: application/json" \ - -H "authorization: Bearer test-key" \ - -d '{ - "model": "docs-chat", - "messages": [ - { "role": "user", "content": "What is the largest animal?" } - ] - }' +await control.ResetAsync(); // Reuse the configured fixture for this separate SDK example. +using var providerHttp = app.CreateHttpClient("llm", "http"); +var openAiChat = new ChatClient("gpt-4.1-mini", new ApiKeyCredential("test-key"), + new OpenAIClientOptions { Endpoint = new Uri(providerHttp.BaseAddress!, "openai/v1") }); +ChatCompletion completion = await openAiChat.CompleteChatAsync([new UserChatMessage("hello")]); +Console.WriteLine($"{completion.Usage.InputTokenCount} input tokens, " + + $"{completion.Usage.InputTokenDetails.CachedTokenCount} from cache"); ``` -## Microsoft.Extensions.AI - -Use the client package when you want tests to call the fake provider through normal `Microsoft.Extensions.AI` abstractions: - -```csharp -using ManagedCode.LlmTck.Client; -using Microsoft.Extensions.AI; - -using var httpClient = new HttpClient -{ - BaseAddress = new Uri("http://localhost:5000"), -}; -httpClient.DefaultRequestHeaders.Authorization = - new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", "test-key"); - -using IChatClient chatClient = new LlmTckChatClient(httpClient); +### As a standalone container -var response = await chatClient.GetResponseAsync( -[ - new ChatMessage(ChatRole.User, "What color is the largest animal?"), -]); +Build the image with the .NET SDK (use `linux-x64` in both RID arguments on amd64), run it, then configure it over +HTTP — any language can drive it: -Console.WriteLine(response.Text); +```sh +dotnet publish samples/ManagedCode.LlmTestServer.Service/ManagedCode.LlmTestServer.Service.csproj \ + --configuration Release --runtime linux-arm64 --self-contained false \ + /t:PublishContainer -p:ContainerRuntimeIdentifiers=linux-arm64 -p:ContainerImageTag=local -p:UseAppHost=false +docker run --rm -p 8080:8080 -e LlmTestServer__RequiredBearerToken=local-test-key managedcode/llm-test-server:local ``` -When a bearer token is required, prefer creating the modality clients from the control client so the same token is applied consistently: - -```csharp -var tck = LlmTckClient.Create(httpClient, bearerToken: "test-key"); - -using IChatClient chatClient = tck.CreateChatClient("docs-chat"); -using IEmbeddingGenerator> embeddings = - tck.CreateEmbeddingGenerator("docs-embedding"); -using IImageGenerator images = tck.CreateImageGenerator("docs-image"); +```sh +curl --fail-with-body http://localhost:8080/admin-api/configure \ + -H 'Authorization: Bearer local-test-key' -H 'Content-Type: application/json' \ + --data '{"requiredBearerToken":"local-test-key","models":[{"id":"gpt-4.1-mini","kind":"chat"}],"chatScenarios":[{"id":"hello","modelId":"gpt-4.1-mini","responses":[{"content":"Hello from your fixture"}]}]}' +curl --fail-with-body http://localhost:8080/openai/v1/chat/completions \ + -H 'Authorization: Bearer local-test-key' -H 'Content-Type: application/json' \ + --data '{"model":"gpt-4.1-mini","messages":[{"role":"user","content":"Hello"}]}' ``` -Streaming uses the same client: - -```csharp -await foreach (var update in chatClient.GetStreamingResponseAsync( - [new ChatMessage(ChatRole.User, "stream this")])) -{ - Console.Write(update.Text); -} -``` - -## Assertions And Reset - -The assertion summary is intentionally simple: it tells the test whether requests matched configured scenarios, missed, failed auth, used unknown models, exhausted a response queue, or returned scripted errors. - -```csharp -var control = new LlmTckClient(httpClient); -var assertions = await control.GetAssertionsAsync(); - -if (assertions.Unmatched > 0 || assertions.ModelNotFound > 0) -{ - throw new InvalidOperationException("The application called an unexpected LLM path."); -} -``` - -Reset clears assertion events and scenario response positions without replacing the current configuration: - -```csharp -await control.ResetAsync(); -``` +Open `http://localhost:8080/` for the dashboard. When a key is configured it protects provider routes and +`/admin-api/*`. More: [Docker distribution](docs/Features/DockerDistribution.md), +[embedding in ASP.NET Core](docs/Reference/Installation.md#aspnet-core-startup). -## Hosted Endpoint Families +## What you can do -Provider endpoints are explicitly namespaced. The table below shows the route -families and representative operations; the full method/path contract lives in -each provider profile's `ApiContract` and is guarded by -`ProviderApiContractTests`. See -[`docs/Features/ProviderApiContractCoverage.md`](docs/Features/ProviderApiContractCoverage.md) -for the current coverage rules and behavior-test inventory. - -| Provider | Namespace | Representative hosted operations | +| Capability | What it covers | Guide | | --- | --- | --- | -| OpenAI | `/openai` | `/openai/v1/models`, `/openai/v1/decisions`, `/openai/v1/chat/completions`, `/openai/v1/responses`, `/openai/v1/embeddings`, `/openai/v1/images/*`, `/openai/v1/audio/*`, `/openai/v1/videos*` | -| Azure OpenAI | `/azure-openai` | `/azure-openai/openai/v1/{chat/completions,responses,embeddings}`, deployment chat/embeddings/images/audio, and `/azure-openai/openai/v1/video/generations/*` | -| Microsoft Foundry | `/microsoft-foundry` | `/microsoft-foundry/openai/v1/{chat/completions,responses,embeddings}`, `/microsoft-foundry/chat/completions`, `/microsoft-foundry/embeddings`, and `/microsoft-foundry/models/*` aliases | -| Anthropic | `/anthropic` | `/anthropic/v1/messages` | -| Gemini | `/gemini` | `/gemini/v1beta/models/{model}:generateContent`, streaming content, embeddings, long-running video operations, and generated files | -| Groq | `/groq` | `/groq/openai/v1/models`, chat completions, Responses, speech, transcription, and translation routes | -| Mistral | `/mistral` | `/mistral/v1/chat/completions`, `/mistral/v1/embeddings` | -| Ollama | `/ollama` | `/ollama/api/chat`, `/ollama/api/embed` | -| Cohere | `/cohere` | `/cohere/v2/chat`, `/cohere/v2/embed` | -| Amazon Bedrock | `/bedrock` | `/bedrock/model/{modelId}/converse`, converse stream, invoke, and invoke-with-response-stream | -| OpenRouter | `/openrouter` | `/openrouter/api/v1/models`, chat completions, Responses | -| DeepSeek | `/deepseek` | `/deepseek/models`, `/deepseek/v1/chat/completions` | -| Perplexity | `/perplexity` | `/perplexity/v1/sonar` | -| SystemOne (Jev/Kev) | `/systemone` | `/systemone/v1/systemone` | -| Cloudflare (Clef/flash) | `/cloudflare` | `/cloudflare/client/v4/accounts/{accountId}/ai/run/@cf/cloudflare/clef`, `clef-flash` | - -The Blazor server-side rendered browser control panel lives at `/`. JSON control endpoints live under -`/admin-api`: `GET /admin-api/models`, `GET /admin-api/assertions`, -`POST /admin-api/configure`, and `POST /admin-api/reset`. - -Audio speech returns a deterministic WAV fixture by default. When -`requiredBearerToken` is configured, all provider and control endpoints require -`Authorization: Bearer `. - -## Common Test Shapes - -### Replace Provider Calls In Integration Tests - -1. Start the LLM TCK service in Aspire or an ASP.NET Core test host. -2. Configure the expected scenarios for the test. -3. Point the application under test at the LLM TCK base URL. -4. Run the application flow. -5. Assert `GetAssertionsAsync()` has no unmatched, unknown-model, auth-failed, or exhausted events. - -### Test Retry Logic - -```csharp -options.AddChatScenario( - "retry-then-success", - scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) - .WhenUserContains("retry") - .Fails(429, "rate_limit_exceeded", "Try again.") - .Responds("success after retry")); -``` - -### Test Prompt Contract Drift - -```csharp -options.AddChatScenario( - "exact-contract", - scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) - .WithExactMatch( - new LlmTckMessage { Role = "system", Content = "Return JSON only." }, - new LlmTckMessage { Role = "user", Content = "Summarize invoice INV-42." }) - .Responds("{\"summary\":\"paid\"}")); -``` - -If the application changes the prompt, the request becomes unmatched and the test can fail on the assertion summary. - -### Test Wrong Model Wiring - -Configure only the models your application is allowed to call: - -```csharp -options.AddModel("approved-chat-model", LlmTckModelKind.Chat); -``` - -If the app calls `old-chat-model` or calls `approved-chat-model` through the embeddings endpoint, LLM TCK returns `llm_tck_unknown_model`. - -## Build And Test - -```bash -dotnet restore ManagedCode.LlmTck.slnx -dotnet build ManagedCode.LlmTck.slnx --configuration Release --no-restore -dotnet test --project tests/ManagedCode.LlmTck.Tests/ManagedCode.LlmTck.Tests.csproj --configuration Release --no-build --verbosity normal -for project in src/*/*.csproj; do dotnet pack "$project" --configuration Release --no-build --output artifacts/packages; done -``` - -`global.json` opts `dotnet test` into `Microsoft.Testing.Platform` for .NET 10. - -## Compatibility boundaries and verification - -Tool calls and structured output use explicit deterministic fixtures: `CallsTool(...)`, `CallsTools(...)` and `RespondsJson(...)`. Provider adapters preserve tool history and streaming events; `LlmTckChatClient` supports the standard Microsoft.Extensions.AI function-invocation loop. Fixtures must satisfy tool selection, argument schemas and response schemas; mismatches return 409 without consuming the response. See [supported contracts and tests](docs/Features/ToolAndStructuredFixtures.md). - -OpenAI chat streaming supports `stream_options.include_usage`; the control library's `IChatClient` carries token usage and terminal updates, including reasoning and cache details. Authentication failures and scripted errors before generation do not accrue output/reasoning tokens. - -Video create operations retain completed fixture jobs. Retrieve/content/delete require a previously created ID, and reset or reconfiguration clears jobs. OpenAI fixtures return `status=completed` and `progress=100`; IDs remain unique within a run. Dataset-local scenario IDs are isolated, including concurrent cancellation rollback. - -Prompt-cache storage is bounded to 4096 prefixes by default. Use `WithPromptCacheCapacity(capacity)` to change the limit; zero disables caching. Eviction is deterministic FIFO, and reset/configure clears the cache. - -Legacy Azure deployment APIs accept the dated `api-version` selected automatically by the Azure SDK. Use the default `AzureOpenAIClientOptions` constructor, or omit options entirely; do not pin a version for the emulator. These routes require exactly one supported deployment API version: the documented `2024-10-21` GA or `2025-04-01-preview` contract. Missing, malformed, duplicate, and unknown versions are rejected, including valid-looking future dates. Azure/Foundry `/openai/v1` endpoints require no dated query parameter. - -After restore/build, run `bash scripts/coverage.sh` for the production 90% coverage gate. After packing, run `python3 scripts/verify-package.py` for an isolated NuGet consumer, test-owned Aspire host and Chromium dashboard test; this requires Python 3 and PowerShell (`pwsh`). CI installs Chromium dependencies and saves coverage/browser evidence. `python3 scripts/test-release.py` verifies partial-release recovery locally without external publication. Release delivery is marked complete only after NuGet push and GitHub artifact attachment succeed. - -## Native decision API fixtures - -Native typed clients are available through `LlmTckClient.CreateSystemOneClient()` -and `LlmTckClient.CreateCloudflareDecisionClient(accountId)`. Clef accepts embedded -PNG/JPEG/WebP data URLs or base64 objects; Jev accepts PNG/JPEG reference assets as -JSON or multipart, with explicit image/long-context opt-ins. See the -[official-contract audit and supported limits](docs/Features/DecisionVisionAudit.md). - - -Jev, Kev, Cloudflare Clef/Clef Flash and OpenAI Decisions use explicit decision fixtures. The TCK hosts their native JSON shapes under provider namespaces and does not contact production providers. Declare the provider profile explicitly; model names never select an API implicitly. - -| Provider | TCK POST route | Production POST endpoint | Model helper | Official contract | -| --- | --- | --- | --- | --- | -| TypeSafe Jev | `/systemone/v1/systemone` | `https://api.typesafe.ai/v1/systemone` | `AddJevLatest()` | [OpenAPI](https://api.typesafe.ai/openapi.json), [primitives](https://docs.typesafe.ai/primitives/choice) | -| Kev | `/systemone/v1/systemone` | `{kev-server}/v1/systemone` | `AddKevLatest()` | [native request models](https://github.com/jaredpalmer/kev/blob/main/kev/api.py), [server/auth](https://github.com/jaredpalmer/kev/blob/main/kev/serve.py) | -| Cloudflare Clef | `/cloudflare/client/v4/accounts/{accountId}/ai/run/@cf/cloudflare/clef` | `https://api.cloudflare.com/client/v4/accounts/{accountId}/ai/run/@cf/cloudflare/clef` | `AddClef()` | [Clef API](https://developers.cloudflare.com/workers-ai/models/clef/), [REST envelope/auth](https://developers.cloudflare.com/workers-ai/get-started/rest-api/) | -| Cloudflare Clef Flash | `/cloudflare/client/v4/accounts/{accountId}/ai/run/@cf/cloudflare/clef-flash` | `https://api.cloudflare.com/client/v4/accounts/{accountId}/ai/run/@cf/cloudflare/clef-flash` | `AddClefFlash()` | [Clef Flash API](https://developers.cloudflare.com/workers-ai/models/clef-flash/) | -| OpenAI Decisions | `/openai/v1/decisions` | `https://api.openai.com/v1/decisions` | `AddOpenAiDecisionModel("gpt-6-luna")` | [create API](https://developers.openai.com/api/reference/resources/decisions/methods/create/) | - -Production APIs use `Authorization: Bearer `; Kev checks `KEV_API_KEY` only when configured and otherwise permits unauthenticated calls. For the TCK, `RequireBearerToken` controls the expected test token and protects both provider routes and `/admin-api/*`. Tests do not need production credentials. Resolve the hosted base URL from the Aspire resource (`GetEndpoint("http")` / `CreateHttpClient(...)`), then append the TCK route. - -### SystemOne: Jev and Kev - -Jev requires `model`, string/object/array `state`, and a nonempty question map. Kev accepts any JSON `state`, including numbers, booleans and null; instructions and criterion descriptions also accept arbitrary JSON. Kev defaults an omitted `model` to `kev-latest`; the TCK enables that default only when `AddKevLatest()` explicitly registers the Kev profile. Binary questions use `type: "noul"`; optional `criteria` maps the native `true` and `false` descriptions. Choices use an opaque ID-to-description map. Scores use an ordered criteria array. - -```json -{ - "model": "kev-latest", - "state": { "message": "Please help quickly" }, - "questions": { - "urgent": { "type": "noul", "instructions": "Is urgent?", "criteria": { "true": "needs prompt help", "false": "can wait" } }, - "route": { "type": "choice", "instructions": "Choose the queue", "criteria": { "fast": "urgent", "regular": "routine" } }, - "severity": { "type": "score", "instructions": "Assess severity", "criteria": ["low", "medium", "high"] } - } -} -``` - -Use `model: "jev-latest"` for Jev. Native responses contain keyed `answers`: `noul`, or `choice`/`score` with `confidence` and `probabilities`; score `legend` keys are zero-based numeric strings. `usage` contains `input_tokens` and `output_tokens`. Scores are native ordinal values and can exceed one. - -```csharp -using ManagedCode.LlmTck.Decisions; - -options.AddKevLatest().AddDecisionScenario(new LlmTckDecisionScenario -{ - Id = "priority", ModelId = "kev-latest", - Answers = new() - { - ["urgent"] = new() { Kind = LlmTckDecisionKind.Predicate, Probability = 0.95 }, - ["route"] = new() { Kind = LlmTckDecisionKind.Choice, Choice = "fast", Confidence = 0.9, - Probabilities = new() { ["fast"] = 0.95, ["regular"] = 0.05 } }, - ["severity"] = new() { Kind = LlmTckDecisionKind.Score, Score = 1.75, Confidence = 0.625, - Probabilities = new() { ["0"] = 0.05, ["1"] = 0.15, ["2"] = 0.8 } }, - }, - Usage = new() { InputTokens = 100, OutputTokens = 10, TotalTokens = 110 }, -}); -``` - -Kev score legends contain rendered strings, while Jev retains string/object/array criteria. Kev responses include `latency_ms`; `LlmTckDecisionScenario.Metadata` supplies deterministic latency and optional `Truncated`, `StateTokens`, `StateTokensUsed` fields. The route echoes or generates `x-typesafe-request-id`, including on errors. Invalid SystemOne request shapes return HTTP 422 with a native `detail` validation array; runtime/admission faults retain `detail` strings. Kev bearer failures include `WWW-Authenticate: Bearer`. This compatibility surface covers `/v1/systemone`; Kev demo `/permute` and `/separate` routes are outside its scope. - -### Cloudflare Workers AI: Clef and Clef Flash +| Chat scenarios | Contains or exact prompt matching, queued replies, streaming chunks, delays, scripted errors, per-scenario keys | [Configuration](docs/Reference/Configuration.md) | +| Tool calls and structured output | Function calls validated against your schemas, JSON responses, well-known tools, tool failures | [Tools](docs/Features/ToolAndStructuredFixtures.md) · [Well-known tools](docs/Features/WellKnownFunctionTools.md) · [Tool faults](docs/Features/ToolFaultFixtures.md) | +| Multi-turn conversations | Typed step graphs with branches, tool-result checks and terminal states | [Conversation DSL](docs/Features/ConversationDSL.md) | +| Streaming | Provider-native SSE and NDJSON streams with usage where the provider reports it, plus delivery faults | [Stream delivery faults](docs/Features/StreamDeliveryFaults.md) | +| Failure paths | 429 rate limits and content filters on every provider family | [Fault simulation](docs/Reference/Configuration.md#fault-simulation) | +| More than text | Embeddings, images, speech, transcription, translation and video jobs | [Modalities](docs/Reference/Configuration.md#modalities) | +| Native provider tools | OpenAI web search, file search and code; Anthropic web search; Responses continuation | [OpenAI web search](docs/Features/OpenAiNativeWebSearch.md) · [File/code](docs/Features/OpenAiNativeFileCode.md) · [Anthropic](docs/Features/AnthropicNativeWebSearch.md) · [Continuation](docs/Features/ResponsesContinuation.md) | +| Parallel tests | Isolated sessions with their own configuration, queues, cache and journal | [Test sessions](docs/Features/TestSessions.md) | +| Verification | Query and export the request journal, verify counts, order and tool arguments, fix IDs and clocks | [Verification and determinism](docs/Features/RequestVerificationAndDeterminism.md) | +| Native decision APIs | TypeSafe Jev, Kev (System One), Cloudflare Clef, OpenAI Decisions | [Decision contracts](docs/Features/DecisionModels.md) · [Jev 1.14](docs/Features/Jev114.md) | +| Official SDKs | OpenAI, Azure OpenAI, Foundry, Anthropic, Google Gen AI, Bedrock and `Microsoft.Extensions.AI` | [Clients and SDKs](docs/Reference/Clients.md) · [SDK compatibility](docs/Features/ProviderSdkCompatibility.md) | + +Everything is a fixture you choose: nothing is hidden behind retries or randomness, and the server never calls a real +model. + +## Dashboard tour + +The dashboard at `/` is a request inbox: one row per provider call, with the full request, response, conversation, +payloads, stream chunks, tokens and runtime result one click away. + +| Request overview | Conversation | Runtime diagnostics | +| --- | --- | --- | +| [![Request overview with conversation strip, tool-call card and highlighted arguments](docs/images/showcase-overview.jpg)](docs/images/showcase-overview.jpg) | [![Conversation with per-message token bars](docs/images/showcase-conversation.jpg)](docs/images/showcase-conversation.jpg) | [![Runtime diagnostics with assertion and fixture charts](docs/images/admin-dashboard-runtime.jpg)](docs/images/admin-dashboard-runtime.jpg) | +| What was sent, what came back, and what it cost in tokens. | Each turn with its role and token share. | Fixtures, provider routes, assertions and the event timeline. | -Send the same SystemOne question shape to the account/model route with body `model: "clef"` or `"clef-flash"` matching the route. Every question requires instructions. Binary `criteria.true` and `criteria.false` accept arbitrary JSON, including numbers, booleans and null; `criteria` itself must remain an object. State and instructions retain their documented string/object/array types, choice descriptions accept string/object/array/null, and score descriptions accept string/object/array. Responses wrap the native answer object in the Workers AI envelope: +## Providers -```json -{ - "result": { - "model": "clef", - "answers": { "urgent": { "type": "noul", "noul": 0.95 } }, - "usage": { "input_tokens": 100, "output_tokens": 0 } - }, - "success": true, - "errors": [], - "messages": [] -} -``` +Each provider family lives under its own namespace and keeps its native request and response shapes. -Configure `AddClef()` / `AddClefFlash()` and fixtures with the matching short model ID; native surrounding model whitespace is accepted. Runtime/auth failures remain non-success HTTP responses and use `success: false` with native numeric `errors[].code`. Explicit numeric scenario error codes are preserved, including 5004/400 (invalid input), 3007 or 3008/408 (timeout), 3036 or 3040/429 (capacity/rate limit). See [native errors](https://developers.cloudflare.com/workers-ai/platform/errors/). For native error cases, configure the explicit numeric `LlmTckScenarioError.Code`. Unmapped TCK-only errors such as fixture mismatch use the HTTP status as a synthetic code; that fallback is not a claimed Cloudflare native error code. +| Provider | Namespace | Provider | Namespace | +| --- | --- | --- | --- | +| OpenAI | `/openai` | Mistral | `/mistral` | +| Azure OpenAI | `/azure-openai` | Ollama | `/ollama` | +| Microsoft Foundry | `/microsoft-foundry` | Cohere | `/cohere` | +| Anthropic | `/anthropic` | Amazon Bedrock | `/bedrock` | +| Gemini | `/gemini` | OpenRouter | `/openrouter` | +| Groq | `/groq` | DeepSeek | `/deepseek` | +| Perplexity | `/perplexity` | TypeSafe Jev / Kev (System One) | `/systemone` | +| Cloudflare Clef / Clef Flash | `/cloudflare` | | | + +Operation coverage differs by provider and grows over time. A provider being listed does not mean every API of that +provider is implemented — check the [operation coverage](docs/Features/ProviderApiContractCoverage.md) and +[routes reference](docs/Reference/ProvidersAndRoutes.md) for the exact contracts. + +## TypeSafe and Jev + +**TypeSafe is the provider; Jev is its model family. System One is the native API.** +Use `ManagedCode.LlmTestServer.SystemOne` for its request and response contracts, and +`ManagedCode.LlmTestServer.Client` for `control.CreateSystemOneClient()`. +The native server route is `POST /systemone/v1/systemone`. + +Configure deterministic answers for predicates, choices, scores and Jev 1.14 bounding boxes. +JSON image assets and multipart uploads are supported with explicit image opt-in. +Kev uses the same protocol package with its own compatibility profile. + +[Configure and call Jev from a test](docs/Reference/Clients.md#typesafe-jev-through-system-one) · +[Jev 1.14 images and limits](docs/Features/Jev114.md) · +[Native decision fixtures](docs/Features/DecisionModels.md) -Optional `images` is an array of inline data URLs or `{ "content_type": "image/png", "base64": "..." }` objects. PNG/JPEG/WebP are supported; remote URLs are rejected. Limits: four images, 4 MiB decoded file bytes and 16 megapixels per image, 8 MiB combined decoded bytes, and 13 MiB for the complete request body. Image dimensions and file type are read as metadata; compressed pixel streams are not decoded, so full raster validity is outside this deterministic mock validation. The TCK retains supplied images in the matched input and trace. See [input schema](https://developers.cloudflare.com/workers-ai/models/clef/schema-input.json), [output schema](https://developers.cloudflare.com/workers-ai/models/clef/schema-output.json). +## Packages -### OpenAI Decisions +| Package | Use it for | +| --- | --- | +| `ManagedCode.LlmTestServer.Aspire` | Add the server to an AppHost or Aspire test with `AddLlmTestServer()`. | +| `ManagedCode.LlmTestServer.Client` | Configure, reset, query and verify from tests; `IChatClient`, embedding and image clients. | +| `ManagedCode.LlmTestServer.Hosting` | Embed the server and dashboard in your own ASP.NET Core host. | +| `ManagedCode.LlmTestServer` | The provider-neutral runtime: models, scenarios, usage and assertions. | +| `ManagedCode.LlmTestServer.` | Wire contracts for each provider family listed above. | +| `ManagedCode.LlmTestServer.SystemOne` | Native TypeSafe Jev and Kev contracts for the System One API. | +| `ManagedCode.LlmTestServer.Cloudflare` | Native Cloudflare Clef and Clef Flash decision contracts. | -OpenAI uses `input` and an ordered question array. Predicate questions return one probability; boolean choices can carry separate true/false descriptions. Choice values are typed strings or booleans. Score `levels` use ordered string labels and optional descriptions; responses retain the zero-based level index and original label. +The full list with NuGet links is in [providers, packages and routes](docs/Reference/ProvidersAndRoutes.md#packages). -```json -{ - "model": "gpt-6-luna", - "input": "Please help quickly", - "questions": [ - { "type": "predicate", "name": "urgent", "instructions": "Is urgent?" }, - { "type": "choice", "name": "route", "instructions": "Choose the queue", "choices": [ - { "value": "fast", "description": "urgent" }, { "value": false, "description": "defer" } - ] }, - { "type": "score", "name": "severity", "instructions": "Assess severity", "levels": [ - { "label": "low", "description": "low" }, { "label": "medium", "description": "medium" }, { "label": "high", "description": "high" } - ] } - ] -} -``` +## Learn more -OpenAI fixture question IDs use `OpenAiDecisionFixtureIds.ForQuestionIndex(index)` for the zero-based position, regardless of optional native names. Choice fixture IDs use `ForChoiceValue(string/bool)` canonical JSON literals so string `"true"` differs from boolean `true`. Responses preserve names (including `null`), duplicate names and answer order. +- [Installation and hosting](docs/Reference/Installation.md) — local feed, container, ASP.NET Core, Aspire, building from source +- [Configuration and scenarios](docs/Reference/Configuration.md) — models, scenarios, modalities, faults, assertions +- [Clients and SDKs](docs/Reference/Clients.md) — control client, official SDKs, `Microsoft.Extensions.AI`, TypeSafe client +- [Token usage and prompt cache](docs/Reference/TokenUsageAndPromptCache.md) and [dashboard analytics](docs/Features/DashboardAnalytics.md) +- [Providers, packages and routes](docs/Reference/ProvidersAndRoutes.md) and [operation coverage](docs/Features/ProviderApiContractCoverage.md) +- [Architecture](docs/Architecture.md), [test inventory](docs/Testing/AcceptanceCriteriaTestInventory.md) and [delivery plan](docs/Features/LlmTestServerDeliveryPlan.md) -```csharp -using ManagedCode.LlmTck.Decisions; -using ManagedCode.LlmTck.OpenAI; +## Build from source -var fast = OpenAiDecisionFixtureIds.ForChoiceValue("fast"); -var defer = OpenAiDecisionFixtureIds.ForChoiceValue(false); -options.AddOpenAiDecisionModel("gpt-6-luna").AddDecisionScenario(new LlmTckDecisionScenario -{ - Id = "priority-openai", ModelId = "gpt-6-luna", - Answers = new() - { - [OpenAiDecisionFixtureIds.ForQuestionIndex(0)] = new() { Probability = 0.95 }, - [OpenAiDecisionFixtureIds.ForQuestionIndex(1)] = new() { Kind = LlmTckDecisionKind.Choice, Choice = fast, - Confidence = 0.9, Probabilities = new() { [fast] = 0.9, [defer] = 0.1 } }, - [OpenAiDecisionFixtureIds.ForQuestionIndex(2)] = new() { Kind = LlmTckDecisionKind.Score, Score = 1.75, - Confidence = 0.8, Probabilities = new() { ["0"] = 0.05, ["1"] = 0.15, ["2"] = 0.8 } }, - }, - Usage = new() { InputTokens = 100, CachedInputTokens = 4, OutputTokens = 0, TotalTokens = 100 }, -}); +```sh +dotnet restore ManagedCode.LlmTestServer.slnx +dotnet build ManagedCode.LlmTestServer.slnx --configuration Release --no-restore +dotnet test --project tests/ManagedCode.LlmTestServer.Tests/ManagedCode.LlmTestServer.Tests.csproj --configuration Release --no-build ``` -`input` can also be an array of user messages: string content or `input_text` / `input_image` parts. Images use embedded base64 data URLs; external URLs, file IDs, audio, tools and non-user roles are rejected. Usage retains input/output/total tokens, cached/cache-write input details and reasoning output details, including zero defaults. Set `Refused = true` on an answer fixture to emit a native `{"type":"refusal","name":...}` answer in a successful HTTP response. - -### Limits, fixture matching and failures - -| Profile | Choice criteria | Score criteria | Other native constraints | -| --- | --- | --- | --- | -| TypeSafe Jev | Up to 255 | 1..10 | Optional instructions and binary criteria; state is string/object/array, including JSON descriptions and nullable optional instructions. | -| Kev | 1..255 | 1..255 | Any JSON state/instructions/descriptions, empty opaque IDs; probabilities/native scores rounded to four decimals. | -| Cloudflare Clef/Flash | 2..255 | 2..10 | 1..64 questions; ASCII letter/digit/underscore/dot/hyphen IDs, max 100 characters; required nonempty instructions; four-decimal probabilities/native scores. | -| OpenAI | 2..255 distinct typed values | Ordered labels; no undocumented count cap | Optional names/descriptions, empty strings and repeated score labels preserved; max 128 embedded images; model/names/instructions/labels/descriptions max 1,048,576 characters; no invented choice-value length cap. | +Packing, coverage and package verification are described in [installation](docs/Reference/Installation.md#build-and-test). -`AddDecisionModel(id, LlmTckDecisionProvider...)` supports explicit custom model IDs; the control client mirrors it with `UseDecisionModel` and `UseDecisionScenario`. Fixtures are repeatable snapshots. Optional `ExpectedInput` matches the provider input exactly; SystemOne JSON state and OpenAI message arrays use serialized JSON. Cloudflare requests with supplied images match serialized `{state,images}`; text-only requests retain the state-only input. Question IDs, kinds and requested value domains must fit the fixture. Unknown models, unmatched inputs, mismatched fixtures, authentication, configured rate limits/content filtering and scripted errors produce observable non-success HTTP responses. Normal native choice/score fixtures require confidence and a complete probability domain, near-unit probability mass. SystemOne and Cloudflare choices must have maximal probability; OpenAI preserves any selected value in the supplied domain. Cloudflare confidence must equal the maximum probability within native rounding tolerance. Configure profile-correct explicit values: the Kev/Jev example above has score confidence 0.625; Clef uses the maximum probability 0.8. Values are validated and preserved without normalization or recomputation. Four-decimal distributions may total 0.9999 and native scores are not recomputed from rounded probabilities. For deliberate malformed-response regressions, set `AllowMalformedResponse = true` on the scenario; this explicitly bypasses native response conformance while keeping question-kind/domain mapping safe. +## License -See [complete native contracts and fixture details](docs/Features/DecisionModels.md) and [behavior-test evidence](docs/Testing/AcceptanceCriteriaTestInventory.md). Provider-specific limits and optional-field rules above are reconciled against the linked primary contracts. +[MIT](LICENSE) diff --git a/docs/ADR/ADR-0001-package-boundaries.md b/docs/ADR/ADR-0001-package-boundaries.md index 3949a28..9f808e2 100644 --- a/docs/ADR/ADR-0001-package-boundaries.md +++ b/docs/ADR/ADR-0001-package-boundaries.md @@ -6,17 +6,17 @@ Accepted ## Context -LLM TCK needs to support several provider protocols while keeping deterministic test behavior consistent. Mixing provider DTOs, runtime state, host wiring, and client abstractions in one package would make future providers harder to add and harder to test. +LLM Test Server needs to support several provider protocols while keeping deterministic test behavior consistent. Mixing provider DTOs, runtime state, host wiring, and client abstractions in one package would make future providers harder to add and harder to test. ## Decision Use separate packages: -- Core runtime: `ManagedCode.LlmTck` -- Provider wire mapping: `ManagedCode.LlmTck.OpenAI` -- ASP.NET Core hosting: `ManagedCode.LlmTck.Hosting` -- Typed clients and `Microsoft.Extensions.AI`: `ManagedCode.LlmTck.Client` -- Aspire AppHost extensions: `ManagedCode.LlmTck.Aspire` +- Core runtime: `ManagedCode.LlmTestServer` +- Provider wire mapping: `ManagedCode.LlmTestServer.OpenAI` +- ASP.NET Core hosting: `ManagedCode.LlmTestServer.Hosting` +- Typed clients and `Microsoft.Extensions.AI`: `ManagedCode.LlmTestServer.Client` +- Aspire AppHost extensions: `ManagedCode.LlmTestServer.Aspire` ## Consequences diff --git a/docs/ADR/ADR-0002-aspire-hosted-provider-emulation.md b/docs/ADR/ADR-0002-aspire-hosted-provider-emulation.md index 7f58170..b56e204 100644 --- a/docs/ADR/ADR-0002-aspire-hosted-provider-emulation.md +++ b/docs/ADR/ADR-0002-aspire-hosted-provider-emulation.md @@ -6,11 +6,11 @@ Accepted ## Context -The library is intended for integration tests, not only unit tests. It must start as a real service under Aspire so application resources can reference an LLM-compatible endpoint during test runs. +Verification must use integration tests against the running server. It must start as a real service under Aspire so application resources can reference an LLM-compatible endpoint during test runs. ## Decision -Ship an Aspire extension package that adds a package-owned LLM TCK resource to an AppHost with `builder.AddLlmTck()`. The consumer AppHost should not need a service project reference, generated `Projects.*` metadata type, project path, Docker, or a container runtime. The default resource starts the packaged .NET LLM TCK service executable. Container hosting remains available through an explicit `builder.AddLlmTckContainer()` opt-in for deployment or container-runtime smoke tests. The integration test builds the AppHost model directly in test code with `Aspire.Hosting.Testing`, adds the TCK through `ManagedCode.LlmTck.Aspire`, starts the .NET resource, then calls the running service through `Microsoft.Extensions.AI`, Azure OpenAI, Microsoft Foundry, and HTTP endpoints. +Ship an Aspire extension package that adds a package-owned LLM Test Server resource to an AppHost with `builder.AddLlmTestServer()`. The consumer AppHost should not need a service project reference, generated `Projects.*` metadata type, project path, Docker, or a container runtime. The default resource starts the packaged .NET LLM Test Server service executable. Container hosting remains available through an explicit `builder.AddLlmTestServerContainer()` opt-in for deployment or container-runtime smoke tests. The integration test builds the AppHost model directly in test code with `Aspire.Hosting.Testing`, adds the Test Server through `ManagedCode.LlmTestServer.Aspire`, starts the .NET resource, then calls the running service through `Microsoft.Extensions.AI`, Azure OpenAI, Microsoft Foundry, and HTTP endpoints. ## Consequences diff --git a/docs/ADR/ADR-0003-isolated-test-sessions.md b/docs/ADR/ADR-0003-isolated-test-sessions.md new file mode 100644 index 0000000..f6d4baa --- /dev/null +++ b/docs/ADR/ADR-0003-isolated-test-sessions.md @@ -0,0 +1,49 @@ +# ADR-0003: Isolated test sessions + +Status: Implemented and locally qualified +Date: 2026-10-20 +Related feature: [delivery plan E2](../Features/LlmTestServerDeliveryPlan.md) + +## Context and decision + +A shared server currently has one runtime and journal. Configure/reset affects every client, so parallel tests can consume each other's queues and erase history. Keep the existing default runtime for callers without session headers. Hosting owns a bounded session registry; each session owns a core runtime and HTTP journal. Request-scoped DI selects the same session for provider mapping, tracing and control endpoints. + +`POST /admin-api/sessions` authenticates against the default runtime and accepts an optional `configuration`. Omission clones the startup configuration. The response returns an opaque ID, a random access key and reserved state bytes. Provider/control calls must send both `X-Llm-Test-Server-Session` and `X-Llm-Test-Server-Session-Key`. These headers are independent of provider bearer/API credentials. Unknown IDs, missing/wrong keys and foreign deletion return 404 before tracing or invoking a runtime. Session keys are redacted in diagnostics and never used as provider credentials. Creation responses use `Cache-Control: no-store`. + +`DELETE /admin-api/sessions/{id}` requires that session's ID/key pair. It removes the registry entry and resets its runtime, invalidating pending delayed responses through the existing runtime generation contract. Reset/configure inside a session affects only that session. Default reset/configure does not delete sessions. Explicit disposal is the lifetime boundary; there is no background expiration or persistence. + +```mermaid +flowchart LR + Client --> Headers[Session ID and access key] + Headers --> Registry[Bounded Hosting registry] + Registry --> Default[Default runtime and journal] + Registry --> A[Session A runtime and journal] + Registry --> B[Session B runtime and journal] + A --> StateA[Queues history cache faults video assertions] + B --> StateB[Queues history cache faults video assertions] +``` + +## Bounds and concurrency + +`LlmTestServerSessionOptions` sets maximum session count, aggregate reserved state bytes and serialized configuration bytes. Reservations include configured history/video limits, prompt-cache/job/response overhead, journal retention, UTF-16 event retention and the configuration snapshot. This is an admission budget for retained server state, not a measurement of managed heap/transient HTTP allocation. Oversized configuration returns 400; exhausted count/aggregate capacity returns 409. Configure builds a validated replacement runtime, checks admission, then swaps it and retires the previous runtime atomically under the registry gate; invalid configuration is rejected before admission checks. It adjusts the reservation only on success; delete releases it. Existing runtime count/byte limits remain effective within every session. Registry admission/configure/delete is serialized; independent provider requests execute against their own runtimes. + +The client exposes `CreateSessionAsync(...)`, a scoped `HttpClient` for official SDK transports, and `Control`. `await using` deletes server state. The wrapper applies headers per request and leaves the caller-owned underlying transport alive, so parallel session clients never mutate shared default headers. + +## Alternatives and consequences + +- A correlation header on the singleton runtime cannot isolate cursors or reset and is rejected. +- One process per session isolates state but adds AppHost/process lifecycle costs and is rejected for shared-server tests; separate hosts remain supported. +- Ambient mutable singleton selection is rejected because concurrent awaits could cross-contaminate sessions. +- A request DI scope and explicit registry preserve all provider adapters and core state owners. The browser's interactive session selector remains E13 work; the default dashboard continues to inspect the default runtime. + +## Verification plan + +Use HTTP/official SDK integration tests for simultaneous identical prompts, independent queues, reset/configure/dispose, cross-key access, separate journals/history/cache/fault/video state, count and byte admission, invalid configuration atomicity, streaming/cancellation and delayed in-flight deletion. Run the full provider integration suite, formatter, canonical Release build and unchanged 90% production coverage gate. No unit/source/reflection tests. + +## Qualification evidence + +The complete integration-only suite passed 655/655 with 91.17% production line coverage. Canonical Release build +and format verification passed with zero warnings/errors. An isolated NuGet consumer restored the repacked +Core/Client/Hosting/Aspire artifacts, started the packaged service with Aspire and verified session API isolation, +provider calls and the interactive Chromium dashboard (1/1). Session verification logs are retained under +the repository-owned `artifacts` directory; public NuGet publication belongs to the combined release batch. diff --git a/docs/ADR/ADR-0004-conversation-step-graphs.md b/docs/ADR/ADR-0004-conversation-step-graphs.md new file mode 100644 index 0000000..4ae52f6 --- /dev/null +++ b/docs/ADR/ADR-0004-conversation-step-graphs.md @@ -0,0 +1,81 @@ +# ADR-0004: Typed conversation step graphs + +Status: Accepted; implemented and locally qualified, public delivery pending +Date: 2026-10-20 +Related: [conversation DSL](../Features/ConversationDSL.md), [delivery E3](../Features/LlmTestServerDeliveryPlan.md), +[test sessions](../Features/TestSessions.md), [architecture](../Architecture.md) + +## Decision and context + +Linear scenarios match historical messages and consume a linear queue. A workflow needs a condition for each +incoming turn, explicit alternative transitions, tool-result checks, terminal behavior and readable mismatch +reports. Add an optional typed conversation graph to the existing scenario model. Keep linear queues unchanged. +Graphs run inside the existing runtime and inherit session configuration/reset/disposal ownership. + +Each step has one response, serializable match predicates and either ordered next-step IDs or an explicit +terminal marker. Predicates cover latest role/text, normalized turn index, tool name/call ID, exact JSON arguments +and JSON-field equality in the latest content. Branch selection uses the declared next-step order. Diagnostic +APIs report selected/shadowed rules, mismatch field names and unvisited steps without consuming state. + +```mermaid +flowchart LR + Start[Start step] --> Approve[Approve predicate] + Start --> Cancel[Cancel predicate] + Approve --> Tool[Tool-result predicate] + Tool --> Final[Terminal response] + Cancel --> Done[Terminal cancellation] +``` + +## Identity and ordering + +`X-Llm-Test-Server-Conversation` optionally selects independent conversations inside one session. The default +is `default`. State is also partitioned by scenario identity, provider namespace and provider credential hash. +A typed conversation client applies its header per request without changing the shared transport's headers. + +Logical turn index is one plus the number of consecutive assistant-message groups in the normalized input, +ignoring system/developer messages. Parallel function-call items in Responses form one assistant group. HTTP +attempts do not increment this index. Clients provide full history or use stored Responses continuation. + +An identical retry of the last successful normalized request replays the same runtime result and does not +advance the graph. Streaming mode does not create a new logical request. Scripted error steps can explicitly +transition to a recovery step at the same logical turn. Older/changed input at an already successful turn is +rejected. Graphs allow one in-flight step; cancellation and fixture/history-capacity failures roll back their +reservation. Reset/configure/delete invalidate pending work through the runtime generation/lifetime contract. + +## Bounds and exhaustion + +Runtime configuration bounds conversation count and retained state bytes; admission and response reservations +fail explicitly without eviction. Graphs have bounded step count and execution limits. Terminal policies are +Reject, RepeatLastResponse and Restart. Restart clears execution and visited-step counters for a new run; +the execution limit guards cycles within a run. Normal successful-request replay is independently configurable +and precedes terminal handling. Only the latest successful normalized request is retained for replay. +Unvisited alternative branches are diagnostic information, not proof that an expected active branch was skipped. + +## Consequences and implementation + +The graph, predicates and responses share the existing configuration snapshot/HTTP contract. Core owns matching, +bounded admission and transition commit/rollback; Hosting passes request identity and exposes diagnostics; +Client owns the typed header wrapper. No new package, persistence layer or executable configuration delegates +are required. Linear scenarios remain unchanged. Graph state and last-result retention add explicit runtime +admission budgets, including sessions' aggregate reservation accounting. + +Retained responses enable deterministic runtime replay without promising identical provider-generated wire +IDs or timestamps. State persists until reset/configure/session disposal; local conversation-client disposal +only releases its transport wrapper. Tests must manage that session lifetime explicitly. + +Implementation sequence: graph DTO/builders and validation; runtime selection/reservation/replay; hosted identity +and control routes; typed client and diagnostics; HTTP/SDK qualification and installed-consumer verification. +The implementation is locally qualified: the full server integration suite passed 722/722, production line coverage is 91.34%, and the independent installed-package consumer passed 1/1. Public release remains pending the combined delivery batch. + +## Alternatives + +- Matching every request against all historical user messages cannot identify the current workflow step. +- Embedding server delegates in configuration cannot survive HTTP configuration or fixture export. +- A separate workflow process/store duplicates runtime/session ownership and is unnecessary. + +## Verification plan + +Run HTTP/provider SDK tool loops and branches, retry/error/terminal flows, JSON predicates, malformed graphs, +capacity, cancellation/reset and credential/provider/conversation isolation. Verify explanations through control +APIs and event metadata. Run the unchanged full integration/coverage gate and installed-package consumer. +The exact commands and acceptance boundaries are maintained in [Conversation DSL](../Features/ConversationDSL.md#verification-and-completion-criteria). diff --git a/docs/ADR/ADR-0005-request-evidence-and-wire-determinism.md b/docs/ADR/ADR-0005-request-evidence-and-wire-determinism.md new file mode 100644 index 0000000..c088457 --- /dev/null +++ b/docs/ADR/ADR-0005-request-evidence-and-wire-determinism.md @@ -0,0 +1,95 @@ +# ADR-0005: Request evidence and seeded wire metadata + +Status: Accepted; implemented and locally qualified, public delivery pending +Date: 2026-10-20 +Related: [feature contract](../Features/RequestVerificationAndDeterminism.md), +[delivery E4](../Features/LlmTestServerDeliveryPlan.md), [architecture](../Architecture.md), +[session ownership](ADR-0003-isolated-test-sessions.md), [conversation graphs](ADR-0004-conversation-step-graphs.md). + +## Context and decision + +Tests need machine-readable request evidence, assertions about complete tool/conversation/retry flows, and +repeatable generated IDs/time. The dashboard already has a bounded request journal. Reusing that journal +keeps UI, query, export and verification aligned; explicit retention counters prevent missing evidence from +being interpreted as a successful assertion. + +Keep request evidence in the existing session-owned Hosting journal. Move public trace/query DTOs into core +`Diagnostics`; put serializable expectation DTOs and the pure evaluator in core `Verification`. Hosting captures +real HTTP behavior and exposes authenticated query/export/verify endpoints. Client wraps those contracts. +No second store, new package or recorded-byte cassette format is introduced by this decision. + +```mermaid +flowchart LR + Session[Session runtime] --> Wire[Seeded wire metadata] + Session --> Events[Real runtime observations] + HTTP[Provider HTTP requests] --> Journal[Existing bounded trace journal] + Events --> Journal + Journal --> Snapshot[Snapshot with retention evidence] + Snapshot --> Query[Query and versioned export] + Snapshot --> Verify[Serializable verification] + Graph[Conversation status] --> Verify +``` + +## Evidence ownership and completeness + +Journal sequence identifies observed request ordering, while an attempt ID correlates runtime and HTTP evidence. +Explicit retry-group IDs group attempts without replacing their IDs. Query/export/verification share snapshot +filter semantics. Export version 1 includes query and retention metadata; NDJSON starts with a metadata record. +It is a diagnostic export of retained evidence, not full-fidelity recording. + +Eviction, truncation, selected pending requests or omitted matching pages prevent `Passed`, even if retained +requests happen to match expectations. This conservative rule applies to the selected journal and is reset +with its session. Tool assertions use structured request history and response calls, including call/result IDs; +JSON previews are bounded display data and cannot substitute for missing evidence. + +Terminal evidence is recorded after a protocol's final marker is successfully written/flushed. Transport EOF +alone is insufficient. Bedrock InvokeModel currently has no explicit marker, so terminal-marker expectations +are inapplicable to that operation. Assertions still inspect its request count, outcome and other available data. + +Retry verification uses real monotonic start/completion timestamps for spacing, independently of wall-clock +changes. HTTP-date Retry-After values are interpreted using the observed UTC completion time; elapsed waiting +is compared monotonically. Missing or invalid timing fails completeness. Scripted Retry-After fixtures advertise +policy; the application or real provider SDK performs the retry. + +## Wire metadata and lifecycle + +Each session runtime owns provider-local request ordinals and optional `LlmTestServerDeterminism` configuration. +At provider request entry it derives immutable `LlmTestServerResponseMetadata` from seed, provider namespace +and ordinal. Domain-derived IDs preserve native provider prefixes. Results and stream chunks reuse that +metadata; stored responses and successful conversation replay retain it. Reset/configure clear counters along +with scenario state. Control reads do not reserve provider ordinals. + +Fixed mode controls generated wire fields and HTTP Date. Explicit fixture fields, model dictionaries and video +configuration retain their own values. Event/trace observation timestamps and elapsed/monotonic timing remain +real. Traces expose `WireClockMode`, `WireCreatedAt` and `WireSequence` so callers can distinguish those clocks; +replay enrichment reports retained result metadata rather than the new attempt's initial reservation. +Session identifiers and access keys remain random security credentials. The seed is not a credential. + +## Alternatives and consequences + +- A second assertion store would introduce separate retention/ordering semantics and duplicate capture. +- A process-global seeded random generator or clock would couple independent sessions and providers. +- Replacing observation time with fixture time would invalidate retry and cancellation evidence. +- Inferring complete assertions from previews or successful EOF would conceal evidence loss and absent markers. + +The chosen design provides one bounded evidence source, repeatable fixture identity and observable clock policy. +Its limits are explicit: eviction invalidates complete verification, request entry determines ordering within a +provider under concurrency, and diagnostic exports do not preserve unretained bytes. Tests should use fresh +sessions and adequate retention. Fixed metadata adds no persistence, upstream call or additional executable +configuration mechanism. + +## Implementation and verification + +Implement core DTO/query/verification contracts; extend the existing journal with sequence/retention counters, +structured tools and real monotonic timing; expose query/export/verify and typed clients; thread wire metadata +through runtime/provider adapters; qualify complete real HTTP/SDK flows and installed-package use. + +Verification must prove positive and negative counts/order/tools/graph/retry outcomes, auth/session isolation, +incomplete retention/paging/pending evidence, terminal applicability, fixed seed/reset/replay IDs and independent +session/provider counters. Wire timestamps and real observation timestamps must be independently asserted. +The feature's [verification commands](../Features/RequestVerificationAndDeterminism.md#verification-and-completion-criteria) +include the full integration/format/Release build/coverage gates. Local E4 qualification passed 812/812 +integration tests, 91.66% production line coverage, final Release build with zero warnings/errors, all 19 package +builds and the independent clean installed Aspire/official SDK/Chromium consumer (1/1). Evidence is recorded +in the [delivery ledger](../Features/LlmTestServerDeliveryPlan.md#11-журнал-реалізації). E4 branch CI and public +publication remain pending; local package acceptance is not public-release evidence. diff --git a/docs/ADR/ADR-0006-ordered-native-tool-output.md b/docs/ADR/ADR-0006-ordered-native-tool-output.md new file mode 100644 index 0000000..d7d3b9f --- /dev/null +++ b/docs/ADR/ADR-0006-ordered-native-tool-output.md @@ -0,0 +1,150 @@ +# ADR-0006: Ordered native tool fixtures on the existing conversation engine + +Status: Accepted; bounded OpenAI web/file/code contracts locally qualified; public delivery pending +Date: 2026-10-20 +Related: [native tool inventory](../Features/NativeBuiltInToolContractInventory.md), +[delivery plan](../Features/LlmTestServerDeliveryPlan.md), +[conversation graphs](ADR-0004-conversation-step-graphs.md), +[request evidence](ADR-0005-request-evidence-and-wire-determinism.md). + +## Context and decision + +A hosted provider tool returns provider-native call items, citations and lifecycle events. Treating it as a +function call loses its contract and can cause a client to attempt an external function invocation. Separate +text/function/native lists also lose the order of mixed response items and break continuation history. + +Add a typed ordered output plan to the existing scenario/conversation response, chat result and retained +assistant history. Keep the current reservation, cancellation, graph progression, replay and session ownership. +The first qualified contract targets completed OpenAI Responses web searches. Bounded completed file-search +and code-interpreter contracts now use the same engine and are locally qualified. Other native +families, broader options and providers remain in E5/E10 and require their own adapters and evidence. +The expanded contract is documented in [OpenAI native file/code fixtures](../Features/OpenAiNativeFileCode.md). + +```mermaid +flowchart LR + HTTP[Provider request] --> Adapter[Provider declaration validation] + Adapter --> Request[Typed declarations and history] + Request --> Engine[Existing scenario and conversation engine] + Engine --> Validate[Fixture validation before reservation] + Validate --> Plan[Ordered typed output plan] + Plan --> History[Existing bounded history and replay] + Plan --> Wire[Provider JSON and streaming mapping] + Plan --> Evidence[Existing bounded runtime and HTTP journals] +``` + +## Boundaries and data + +Core owns message/function/web-search/file-search/code fixture data, immutable snapshots, declaration options, semantic validation, +ordering, usage accounting and capacity admission. The provider package owns its request and output unions, +include projection, citation wire fields and streaming events. Hosting enables native contracts only on the +qualified route, reads the request, calls the runtime and writes the mapped response. + +`LlmTestServerNativeToolSurface.OpenAiResponses` identifies the qualified route independently from each declaration's +`Contract` (`LlmTestServerNativeToolContract`). The ordered output union includes message, function call, web search, file search and +code interpreter. A separate ordered annotation union retains URL, file and container-file citations without +regrouping annotations by kind. Control JSON uses `kind`; the provider adapter emits the corresponding API +`type` discriminators. Native results remain outside function-call counters. + +Explicit output plans and ordinary Content/ToolCalls/StreamChunks are mutually exclusive in configuration. +The engine projects a selected plan once into the existing text/function fields needed by matchers and clients, +while retaining its authoritative ordered items. Ordinary fixtures keep their existing result fields and provider fallback mapping. This avoids two +independently mutable representations in configuration JSON. + +Native declarations remain distinct from function definitions. A native result requires a matching declaration +and qualified route/model contract. Named choices, none/required and parallel-call limits are checked against +the complete selected plan before queue or graph consumption. Search domain restrictions apply to scripted +sources and citations. Context/location options are retained for request identity and diagnostics; no live search +is performed. Newer unimplemented native options are rejected explicitly rather than silently discarded. + +## History, identity and accounting + +All native items and the final assistant message belong to one logical assistant response. Explicit prior output +items and stored-response continuation preserve their types, order, sources and citations. Native items never +become synthetic user JSON or client tool-result messages. Stored-response access retains the existing provider +namespace and credential boundary inside its session runtime. Conversation labels remain correlation/graph +identities; an explicit previous_response_id selects the parent history. + +Deep-copy nested collections at configuration, history, replay and journal boundaries. Include native payloads +in existing conversation/history byte limits and journal character budgets. Capacity rejection must not advance +a response/step or mutate the prompt cache. Native declaration options participate in request fingerprints, +cache identity and normalized input usage; native history/result payloads participate in normalized token counts. +Cache and reasoning remain subsets of input/output totals. + +Explicit fixture IDs and existing seeded wire metadata preserve deterministic response identities. Native +stream progress, output-item completion, citations and final completion must agree with ordinary JSON output. +The terminal marker is observed only after the final response event has been written and flushed. + +## Bounded file/code extension + +The current independent model pairings are web search with `gpt-4.1-mini`, file search with `gpt-4o-mini`, +and code interpreter with `gpt-4.1`. Model checks apply to current declarations/choices, while completed prior +outputs remain history on the qualified surface. No common supported model is inferred for mixed native +families from SDK serialization alone. + +File search preserves declared store IDs, max-result count, ranker, threshold and bounded comparison/compound +filter trees. Configured results must already meet the count/score/filter restrictions; an admission mismatch +returns 409 without consumption rather than silently rewriting fixture results. Strings compare ordinally; +accepted numbers compare by decimal value, preserving precision in their JSON representation. Missing or +mismatched attribute kinds do not match. Boolean ordering, hybrid ranking and unsupported option fields are +rejected. The filter depth/node limits are explicit fixture limits, not claims about upstream limits. + +Code fixtures carry explicit container, input-file and generated-file references, nullable code/outputs and +optional code chunks. Automatic request mode matches the configured input asset set and resolves to the +configured container identity; it does not provision a container. Container-file citations reference that +container and its input or explicitly generated references. Zero-length container citation ranges are permitted; +the existing URL citation boundary remains positive-length. Cited text remains ASCII in this subset. + +Results/outputs include projection is provider-owned and leaves authoritative retained data intact. File/code +JSON/SSE, structured diagnostic copies, native cache identity and explicit/stored continuation use the same +bounded state paths. Memory/network controls, actual file retrieval and resource lifecycle remain pending. +The initial 150 HTTP/SDK cases passed; expanded regression and final combined qualification remain pending. + +## First contract and remaining breadth + +The initial OpenAI contract uses gpt-4.1-mini, completed search actions, typed source URLs, URL citations and +JSON/SSE Responses. The web-search guide reserves open_page/find_in_page for reasoning models; they require +separate model-qualified implementation. The upstream citation schema does not establish an encoding convention +for non-ASCII indexes. The initial cited-text fixture subset is explicitly ASCII, preserving supplied indexes; +uncited text retains normal Unicode support. This boundary is visible in the feature/support inventory and does +not remove Unicode qualification or other native tools from the delivery plan. + +No network search, code executor, file downloader, second work queue or unbounded resource registry is added. +Files, vector stores, containers and generated-artifact lifecycle will reuse the appropriate session-owned +bounded state when their E9 contracts are implemented. + +## Alternatives and consequences + +- A function-name shim would emit the wrong SDK item/event types and misrepresent hosted execution. +- Provider JSON embedded as arbitrary core strings would bypass typed validation, budgets and history semantics. +- A separate native scenario queue would split consumption, cancellation and replay ownership. +- Mapping validation after CompleteChatAsync would reject a fixture only after consuming its queue/graph step. + +The chosen model adds explicit union/snapshot handling to existing state paths. Its cost is additional typed +mapping and budget coverage; it keeps one engine and permits later native contracts to reuse ordering and +ownership without pretending they share a provider wire format. + +## Verification + +Use running ASP.NET endpoints and the installed official Responses SDK. NativeWebSearch*IntegrationTests must +prove typed JSON and streaming, mixed output order, include gating, explicit/stored continuation, named choices, +invalid declarations and atomic configure failure, domain mismatch without consumption, route/model boundaries, +reset/cancellation/replay, credential/session isolation and capacity limits. `NativeFileCodeIntegrationTests` must independently prove file/code models, typed SDK results and citations, +nullable include behavior, filter operators/precision/bounds, explicit/auto resource matching, ordered streaming, +malformed-ID/history rejection and no-consumption behavior. Provider contract/evidence suites and existing +function/structured-output integrations remain mandatory regressions. + +Run the repository format check, canonical Release build, complete integration suite and coverage gate; pack +all packages and qualify a clean installed-package consumer before claiming delivery. Qualification results are +recorded in the delivery ledger only after the actual commands pass. Public publication remains the single +owner-approved v1.0.0 launch batch. + +## References + +- [OpenAI web search](https://developers.openai.com/api/docs/guides/tools-web-search) +- [Responses create](https://developers.openai.com/api/reference/resources/responses/methods/create) +- [Responses streaming schema](https://developers.openai.com/api/reference/resources/responses/streaming-events) + +- [OpenAI file search](https://developers.openai.com/api/docs/guides/tools-file-search) +- [OpenAI code interpreter](https://developers.openai.com/api/docs/guides/tools-code-interpreter) +- [File-search model pairing](https://developers.openai.com/cookbook/examples/file_search_responses) +- [Code-interpreter model pairing](https://developers.openai.com/cookbook/examples/mcp/mcp_tool_guide) diff --git a/docs/Architecture.md b/docs/Architecture.md index b868c97..4b77715 100644 --- a/docs/Architecture.md +++ b/docs/Architecture.md @@ -1,24 +1,24 @@ -# LLM TCK Architecture +# LLM Test Server Architecture ## Purpose -LLM TCK provides deterministic provider emulation for integration tests. Test suites can host a local compatibility server, script exact responses and failures, call it through normal client abstractions, and inspect assertion summaries after execution. +LLM Test Server provides deterministic provider emulation for integration tests. Test suites can host a local compatibility server, script exact responses and failures, call it through normal client abstractions, and inspect assertion summaries after execution. ## Package Boundaries -- `ManagedCode.LlmTck` owns provider-neutral runtime state: models, scenarios, match rules, deterministic modality fixtures, auth checks, and assertion events. -- `ManagedCode.LlmTck.OpenAI` owns OpenAI-compatible DTOs and mapping from runtime results to wire responses. -- `ManagedCode.LlmTck.AzureOpenAI`, `ManagedCode.LlmTck.Foundry`, `ManagedCode.LlmTck.Anthropic`, `ManagedCode.LlmTck.Gemini`, `ManagedCode.LlmTck.Groq`, `ManagedCode.LlmTck.Mistral`, `ManagedCode.LlmTck.Ollama`, `ManagedCode.LlmTck.Cohere`, `ManagedCode.LlmTck.Bedrock`, `ManagedCode.LlmTck.OpenRouter`, `ManagedCode.LlmTck.DeepSeek`, `ManagedCode.LlmTck.Perplexity`, `ManagedCode.LlmTck.SystemOne`, and `ManagedCode.LlmTck.Cloudflare` own provider compatibility profiles and provider-specific wire contracts as they are implemented. -- `ManagedCode.LlmTck.Hosting` exposes the runtime through ASP.NET Core endpoints and serves the browser admin panel as Blazor server-side rendered components. -- `ManagedCode.LlmTck.Client` exposes a control client and `Microsoft.Extensions.AI` clients. -- `ManagedCode.LlmTck.Aspire` adds a package-owned Aspire resource with `builder.AddLlmTck()`. -- `samples/ManagedCode.LlmTck.Service` is the runnable HTTP provider emulator. -- `samples/ManagedCode.LlmTck.AppHost` shows the consumer AppHost shape for the Aspire package. +- `ManagedCode.LlmTestServer` owns provider-neutral runtime state: models, scenarios, conversation step graphs, match rules, deterministic modality fixtures, auth checks, and assertion events. +- `ManagedCode.LlmTestServer.OpenAI` owns OpenAI-compatible DTOs and mapping from runtime results to wire responses. +- `ManagedCode.LlmTestServer.AzureOpenAI`, `ManagedCode.LlmTestServer.Foundry`, `ManagedCode.LlmTestServer.Anthropic`, `ManagedCode.LlmTestServer.Gemini`, `ManagedCode.LlmTestServer.Groq`, `ManagedCode.LlmTestServer.Mistral`, `ManagedCode.LlmTestServer.Ollama`, `ManagedCode.LlmTestServer.Cohere`, `ManagedCode.LlmTestServer.Bedrock`, `ManagedCode.LlmTestServer.OpenRouter`, `ManagedCode.LlmTestServer.DeepSeek`, `ManagedCode.LlmTestServer.Perplexity`, `ManagedCode.LlmTestServer.SystemOne`, and `ManagedCode.LlmTestServer.Cloudflare` own provider compatibility profiles and provider-specific wire contracts as they are implemented. +- `ManagedCode.LlmTestServer.Hosting` exposes the runtime through ASP.NET Core endpoints and serves the browser admin panel as Blazor server-side rendered components. +- `ManagedCode.LlmTestServer.Client` exposes a control client and `Microsoft.Extensions.AI` clients. +- `ManagedCode.LlmTestServer.Aspire` adds a package-owned Aspire resource with `builder.AddLlmTestServer()`. +- `samples/ManagedCode.LlmTestServer.Service` is the runnable HTTP provider emulator. +- `samples/ManagedCode.LlmTestServer.AppHost` shows the consumer AppHost shape for the Aspire package. ## Runtime Flow -1. Tests configure the runtime through `AddLlmTck(...)` at host startup, through `LlmTckClient.ConfigureAsync(config => ...)`, or through `POST /admin-api/configure`. -2. Provider requests arrive through `/v1/*` endpoints. +1. Tests configure the runtime through `AddLlmTestServer(...)` at host startup, through `LlmTestServerClient.ConfigureAsync(config => ...)`, or through `POST /admin-api/configure`. +2. Provider requests arrive through explicit namespaces such as `/openai/v1/*`, `/anthropic/v1/*` and `/gemini/*`. 3. Hosting maps provider requests into provider-neutral runtime requests. 4. The runtime checks bearer-token requirements, configured model IDs, model modality kind, and scenario match rules. 5. The provider adapter maps success or failure to the expected wire shape. @@ -26,20 +26,65 @@ LLM TCK provides deterministic provider emulation for integration tests. Test su Control endpoints use the same bearer-token requirement when one is configured, so tests can expose the service without leaving runtime reset or reconfiguration open to unauthenticated callers. +## Isolated test sessions + +Hosting owns a bounded registry of independent core runtimes and request journals. Request-scoped DI selects a +session using its ID/key headers before provider/control execution; no header selects the default runtime. +Session credentials are independent of provider credentials. Reset/configure/delete affect only their owning +session. See [Test sessions](Features/TestSessions.md) for the HTTP/client contract and +[ADR-0003](ADR/ADR-0003-isolated-test-sessions.md) for bounds, ownership and concurrency decisions. + +## Request evidence and fixture clocks + +Hosting retains bounded HTTP evidence in the session's existing trace journal. Core `Diagnostics` supplies the +shared query/export DTOs, and core `Verification` evaluates serializable count/order/tool/conversation/retry +expectations. Retention loss and incomplete pages prevent successful complete verification. Runtime-owned, +provider-local seeded metadata controls generated wire IDs/time while real event/trace and monotonic timing +remain observational. See [Request verification and determinism](Features/RequestVerificationAndDeterminism.md) +and [ADR-0005](ADR/ADR-0005-request-evidence-and-wire-determinism.md). E4 branch CI is green; +public delivery remains pending. + +The [dashboard analytics](Features/DashboardAnalytics.md) derives token/cache/function observations from one +atomic bounded journal snapshot. Explicit conversation labels group requests only within the selected session +and provider namespace. Cache and reasoning are subsets of input/output usage; missing measurements and +retention loss remain visible rather than becoming zero or complete totals. + +## Native output plans + +Typed ordered message/function/native output plans reuse the same queue/conversation engine, snapshots and +capacity limits. Provider adapters own declarations, native wire items, citations and progress events; validation +occurs before fixture consumption. See [ADR-0006](ADR/ADR-0006-ordered-native-tool-output.md) for the ownership, +history and qualification contract. Native implementation is in progress; the support inventory remains the +capability boundary. + ## Stored response continuation -The runtime owns bounded provider/model/credential-isolated chat-response snapshots. Provider adapters map +The runtime owns bounded provider/credential-isolated chat-response snapshots. Provider adapters map Responses storage/parent fields into this neutral state and reuse its response identity for streaming and normal output. See [Responses continuation](Features/ResponsesContinuation.md) for limits, reset semantics and regressions. +## Conversation step graphs + +The core conversation slice adds serializable predicates and ordered transitions to existing scenarios. Hosting +passes the conversation selector and provider namespace into the runtime request scope; state remains owned +by the selected test session and is partitioned by scenario, provider and credential. A typed client adds the +selector without changing shared transport headers. Control APIs expose bounded conversation status and +side-effect-free match explanations. See [Conversation DSL](Features/ConversationDSL.md) for the contract and +[ADR-0004](ADR/ADR-0004-conversation-step-graphs.md) for identity, reservation, replay and lifecycle decisions. +This extension is locally qualified through server integrations and an installed-package consumer; public +delivery remains pending. + ## Determinism -Chat scenarios are queued. Each matched request consumes the next response. Streaming responses use explicit chunks. Datasets are named groups of active scenarios so tests can load conversation sets before a run. Embeddings return configured vectors. Image, audio, and video endpoints return fixed fixture content. +Linear chat scenarios consume queued responses. Conversation graphs select an active step and advance only +after its fixture commits; identical successful retries can replay the last result. Streaming responses use +explicit chunks. Datasets are named groups of active scenarios and graphs so tests can load conversation sets +before a run. Embeddings return configured vectors. Image, audio, and video endpoints return fixed fixture content. Audio transcription fixtures carry explicitly configured duration or complete audio/text/output token usage through runtime results and OpenAI-compatible JSON, verbose JSON and final streaming events. Configure it -with `WithDefaultTranscriptionUsage(LlmTckTranscriptionUsage.ForDuration(seconds))` at startup or -`UseTranscriptionUsage(LlmTckTranscriptionUsage.ForTokens(audio, text, output))` through the control client. +with `WithDefaultTranscriptionUsage(LlmTestServerTranscriptionUsage.ForDuration(seconds))` at startup or +`UseTranscriptionUsage(LlmTestServerTranscriptionUsage.ForTokens(audio, text, output))` through the control client. The default fixture duration is one second; it is deterministic fixture data, never inferred from input bytes. Azure OpenAI and Microsoft Foundry expose `/openai/v1/audio/transcriptions` with no required query version; explicit `v1` and `preview` are accepted and unknown or duplicate versions are rejected. Official SDK tests @@ -48,7 +93,7 @@ and [OpenAI usage contract](https://developers.openai.com/api/reference/resource ## Client Configuration Surface -`ManagedCode.LlmTck.Client` owns the universal pre-test configuration API. A test can create one `LlmTckClient`, optionally bind a bearer token, call `ConfigureAsync(config => ...)`, and then create `Microsoft.Extensions.AI` chat, embedding, and image clients from the same control client. Audio and video fixtures are reachable through `LlmTckClient.GenerateAudioAsync(...)` and `LlmTckClient.GenerateVideoAsync(...)` until first-party `Microsoft.Extensions.AI` abstractions exist. +`ManagedCode.LlmTestServer.Client` owns the universal pre-test configuration API. A test can create one `LlmTestServerClient`, optionally bind a bearer token, call `ConfigureAsync(config => ...)`, and then create `Microsoft.Extensions.AI` chat, embedding, and image clients from the same control client. Audio and video fixtures are reachable through `LlmTestServerClient.GenerateAudioAsync(...)` and `LlmTestServerClient.GenerateVideoAsync(...)` until first-party `Microsoft.Extensions.AI` abstractions exist. The client builder wraps the provider-neutral configuration builder; it does not create a separate runtime model. This keeps startup configuration, control endpoint configuration, and raw JSON configuration on the same contract. @@ -56,7 +101,7 @@ The client builder wraps the provider-neutral configuration builder; it does not OpenAI compatibility is the first implemented provider endpoint surface. The hosted surface also maps Azure OpenAI deployment routes, Microsoft Foundry / Azure AI Inference chat and embedding routes, and the native Anthropic Messages route. The provider matrix is explicit: OpenAI, Azure OpenAI, Microsoft Foundry, Anthropic, Gemini, Groq, Mistral, Ollama, Cohere, Amazon Bedrock, OpenRouter, DeepSeek, and Perplexity each have a package-level compatibility profile. Each profile carries a doc-backed `ApiContract` so claimed routes, methods, streaming modes, modalities, and version requirements can be checked against official provider documentation. Provider-specific DTOs and mapping belong in the matching provider package; provider-neutral scenario behavior stays in the core runtime. -Aspire integration adds the TCK through `builder.AddLlmTck()` without requiring consumer AppHosts to reference a service project, generated `Projects.*` metadata type, project path, Docker, or a container runtime. The package-owned resource starts the packaged .NET LLM TCK service executable and exposes its `http` endpoint. Consumers that explicitly want container hosting can opt in with `builder.AddLlmTckContainer()`, which uses the versioned `ghcr.io/managedcode/llm-tck` image. Consumer resources reference the `LlmTckResource`, wait for it, and read its `http` endpoint through `GetHttpEndpoint()` or Aspire's `GetEndpoint("http")`. +Aspire integration adds the Test Server through `builder.AddLlmTestServer()` without requiring consumer AppHosts to reference a service project, generated `Projects.*` metadata type, project path, Docker, or a container runtime. The package-owned resource starts the packaged .NET LLM Test Server service executable and exposes its `http` endpoint. Consumers that explicitly want container hosting can opt in with `builder.AddLlmTestServerContainer()`, which uses the versioned `ghcr.io/managedcode/llm-test-server` image. Consumer resources reference the `LlmTestServerResource`, wait for it, and read its `http` endpoint through `GetHttpEndpoint()` or Aspire's `GetEndpoint("http")`. Native decision fixtures live under the core `Decisions` slice; model profile metadata selects documented SystemOne limits and rounding. OpenAI wire contracts stay in OpenAI, SystemOne Jev/Kev contracts share their native protocol package, and Cloudflare owns Workers AI envelopes and native Clef validation. Hosting delegates the new native routes to the same authenticated/fault-injected runtime and request tracing. diff --git a/docs/Features/AnthropicNativeWebSearch.md b/docs/Features/AnthropicNativeWebSearch.md new file mode 100644 index 0000000..d9236f2 --- /dev/null +++ b/docs/Features/AnthropicNativeWebSearch.md @@ -0,0 +1,108 @@ +# Anthropic native web search + +The basic `web_search_20250305` contract is implemented for `claude-sonnet-5-5` on +`POST /anthropic/v1/messages`, with `anthropic-version: 2023-06-01` and the configured +`x-api-key`. The current model and wire subset passed **76 real HTTP/SSE integration cases** +(`artifacts/delivery/anthropic-official-sdk-strict-wire-regression.log`). Official `Anthropic` 12.58.0 +passed **8 real SDK integration cases with `ResponseValidation: true`**, covering JSON, typed SSE, +assistant history, all six native error variants and rejection of a programmatic caller without consuming +the next queued response (`artifacts/delivery/anthropic-official-sdk-strict-focused-batch.log`). +The isolated Release build completed with zero warnings/errors. The combined final quality gate and +installed-package consumer remain separate checks; this page does not claim the full E5 scope. + +Responses are deterministic fixtures. No search, network retrieval, code execution or external service runs. + +## Request contract + +```json +{ + "model": "claude-sonnet-5-5", + "max_tokens": 1024, + "tools": [{ + "type": "web_search_20250305", + "name": "web_search", + "max_uses": 2, + "allowed_domains": ["example.com/articles"], + "user_location": {"type": "approximate", "country": "US"} + }], + "tool_choice": {"type": "auto"}, + "messages": [{"role": "user", "content": "Find the configured release notes."}] +} +``` + +Supported declaration options are nullable `max_uses`, `allowed_domains`, `blocked_domains` and +`user_location`. Both non-null domain lists are invalid, including two empty lists. Domains are scheme-less +DNS names with optional paths; subdomains, path prefixes and path wildcards participate in result matching. +Approximate location needs at least one non-empty city, region, supported ISO country or IANA timezone. +Ordinary function declarations/calls remain separate from native tools. For Claude Sonnet 5.5, +`auto` and `none` are accepted; forced `any` and named `tool` choices return 400 for both ordinary +and native tools, following the model-specific contract. `disable_parallel_tool_use` remains available. + +Local fixture bounds are `max_uses` 0–128, up to 128 domain entries, result entries, citations or content +blocks per native assistant message, and non-empty IDs up to 128 characters. These are Test Server bounds, +not a claim about upstream maxima. Invalid raw shapes/options return 400. An incompatible configured +fixture, missing declaration, exhausted budget or domain mismatch returns 409 before scenario consumption. +An explicitly configured `max_uses_exceeded` result is the supported way to represent budget exhaustion. + +## Ordered response and continuation + +A configured response preserves the order of text, ordinary function calls, native calls and native results: + +- `server_tool_use`: `id`, `name: "web_search"`, `input: {"query": "..."}`. +- `web_search_tool_result`: matching `tool_use_id` and `content`, either a results array or typed error. +- Results: `type: "web_search_result"`, `url`, `title`, `encrypted_content`, optional nullable `page_age`. +- Text citations: `type: "web_search_result_location"`, `url`, `encrypted_index`, `cited_text`, optional nullable `title`. + +Native calls/results carry `caller: {"type":"direct"}`; ordinary function calls also emit their direct caller. +An omitted or null history caller normalizes to direct invocation. Programmatic/nested callers are rejected. +Opaque encrypted strings are retained unchanged. Citations reference configured result URLs; no local +cryptographic verification or decryption is performed. Successful searches can return an empty result list. +Each native call must have exactly one following matching result in the same assistant message. +Completed mixed function/native messages can be sent back as assistant history; they remain one ordered +message and retain native identity, result content and citations. Provider, session and credential scopes +continue to apply. A conversation ID groups requests and does not grant authorization. + +## Errors, usage and streaming + +A native tool error is still an HTTP 200 Messages response. Its result content is +`{"type":"web_search_tool_result_error","error_code":"unavailable"}`. +The supported codes are `invalid_tool_input`, `unavailable`, `max_uses_exceeded`, `too_many_requests`, +`query_too_long` and `request_too_large`. These differ from top-level HTTP/provider failures. + +`usage.server_tool_use.web_search_requests` counts the configured completed call/result pairs, excluding +`max_uses_exceeded` pairs that did not execute a search. Result count does not multiply call count. +The required `web_fetch_requests` field is zero because this subset performs no web-fetch calls. +This deterministic accounting does not claim upstream billing parity and adds no code-execution counter. +Native calls remain separate from ordinary function-call analytics. Token usage is normalized fixture +accounting: opaque encrypted result content, encrypted citation indexes and result/citation metadata are +retained for inspection without being tokenized as retrieved plaintext. The server cannot recover the +underlying plaintext from these opaque values, so it does not claim upstream retrieved-content input +usage or billing parity. + +With `stream: true`, standard Anthropic SSE emits `message_start`, indexed block starts/deltas/stops, +`message_delta` and `message_stop`. Server calls begin with empty input followed by `input_json_delta`; +native result content arrives whole in its block start. Text uses `text_delta`, citations use +`citations_delta`, and final usage is cumulative. JSON and the reconstructed stream preserve the same +ordered content. There is no synthetic `[DONE]` marker. + +Typed native declarations, output items and history are inspectable through bounded request diagnostics, +query/export and replay. Retention limits and truncation remain explicit. Options contribute to request +identity and cache behavior; wire determinism does not replace real trace observation timestamps. + +## Deferred boundaries + +Pending calls, result-only continuation, `pause_turn`, programmatic/nested invocation, newer web-search +versions, web fetch, code execution and native file-search claims are outside this subset. +Declaration `allowed_callers`, `cache_control`, `defer_loading`, `strict` and other unsupported options are +rejected explicitly. Existing system/message cache controls remain available. Official SDK tests use `Anthropic.Models.Messages.Model.ClaudeSonnet5_5` directly, including SDK +streaming unions and assistant-history types, with response schema validation enabled. + +Sources: [official web-search guide](https://platform.claude.com/docs/en/agents-and-tools/tool-use/web-search-tool), +[server-tool mechanics and domains](https://platform.claude.com/docs/en/agents-and-tools/tool-use/server-tools), +[streaming](https://platform.claude.com/docs/en/build-with-claude/streaming), +[versioned declaration](https://github.com/anthropics/anthropic-sdk-python/blob/main/src/anthropic/types/web_search_tool_20250305_param.py), +[result schema](https://github.com/anthropics/anthropic-sdk-python/blob/main/src/anthropic/types/web_search_result_block.py), +[citation schema](https://github.com/anthropics/anthropic-sdk-python/blob/main/src/anthropic/types/citations_web_search_result_location.py). + +Current model references: [Sonnet 5.5 official model contract](https://platform.claude.com/docs/en/models/sonnet-5-5/overview), +[official SDK pinned model enum](https://github.com/anthropics/anthropic-sdk-csharp/blob/Anthropic-v12.58.0/src/Anthropic/Models/Messages/Model.cs). diff --git a/docs/Features/ConversationDSL.md b/docs/Features/ConversationDSL.md new file mode 100644 index 0000000..985676c --- /dev/null +++ b/docs/Features/ConversationDSL.md @@ -0,0 +1,162 @@ +# Typed conversation step graphs + +Status: Locally qualified through running-server integrations and an installed-package consumer. This document describes the E3 contract; +public-package delivery remains pending. +Related: [architecture](../Architecture.md), [ADR-0004](../ADR/ADR-0004-conversation-step-graphs.md), +[sessions](TestSessions.md), [structured/tool fixtures](ToolAndStructuredFixtures.md), +[Responses continuation](ResponsesContinuation.md). + +## Purpose and ownership + +A conversation adds conditions to each step of a deterministic chat workflow. A step can reply with text, +structured JSON, one or several tool calls, or a scripted error, then transition to ordered alternative steps. +The graph is serializable configuration owned by the provider-neutral runtime. Provider packages continue to +normalize their own wire messages and render their own responses. Existing linear scenario queues remain available. + +```mermaid +flowchart LR + Request[Provider HTTP request] --> Normalize[Provider adapter] + Normalize --> Identity[Session and conversation identity] + Identity --> Match[Active step predicates] + Match --> Reserve[Reserve bounded state] + Reserve --> Result[Validate and execute fixture] + Result --> Commit[Commit transition and diagnostics] + Result --> Rollback[Rollback cancelled or rejected work] +``` + +## Configuration and client flow + +Configure through the core builder's `AddConversation`, a dataset's `AddConversation`, or the control client's +`UseConversation`. Every step needs exactly one response and either `Then(...)` or `Complete()`. The first +declared step is the start unless `StartAt(...)` selects another. The default model is `gpt-4.1-mini`. + +```csharp +await server.ConfigureAsync(config => config + .UseGpt41Mini() + .UseConversation("approval", graph => graph + .Step("ask", step => step + .WhenLatestUserEquals("Start").WhenTurn(1) + .Responds("Approve or cancel?").Then("approve", "cancel")) + .Step("approve", step => step + .WhenLatestUserEquals("Approve").WhenTurn(2) + .Responds("Approved").Complete()) + .Step("cancel", step => step + .WhenLatestUserEquals("Cancel").WhenTurn(2) + .Responds("Cancelled").Complete()))); + +using var conversation = server.ForConversation("approval-test"); +using var chat = conversation.Control.CreateChatClient(); +// Send Start, then the returned assistant message and Approve as full chat history. +var states = await conversation.Control.GetConversationsAsync(); +``` + +`ForConversation` exposes `Id`, `HttpClient` and `Control`. Supply its `HttpClient` to official provider SDKs +with the server's allocated provider endpoint and provider credentials. The wrapper adds the conversation +header per request and keeps the parent transport usable after disposal. Disposing this wrapper does not +delete server state; reset/configure or disposal of the owning test session clears it. + +## Rules and predicates + +1. Identity is partitioned by owning test session, scenario identity, provider namespace, conversation ID and + provider credential hash. The optional `X-Llm-Test-Server-Conversation` header defaults to `default`; + an explicit value must be nonblank, at most 128 characters, and appear once. It selects state and is not + an authorization credential. Session and provider authentication still apply. +2. Scenarios retain their configured precedence. A graph starts only when its start-step predicates match; + once it has state for this identity, its active transition owns subsequent matching scenario requests. + An active-step mismatch does not consume a response or fall through to a later scenario. +3. All predicates on a step must pass. `Then("first", "second")` selects the first matching target in that + order. Step identifiers are case-sensitive. Text equality and contains predicates use ordinal matching. +4. `WhenLatestRole`, `WhenLatestTextContains`, `WhenLatestUserContains` and `WhenLatestUserEquals` inspect the + final normalized message. They do not search historical user text. `WhenTurn` uses the logical turn index. +5. Logical turn is one plus the number of consecutive assistant-message groups in normalized history, + ignoring system/developer messages. Parallel assistant function-call items form one group. HTTP attempts + do not add turns. Clients supply full history or use supported stored Responses continuation. +6. `WhenToolResult(name, callId)` requires a final tool-result message and resolves its originating call ID + in history. `WithToolArgumentsJson` compares that call's complete arguments by JSON structural equality. + The call ID argument is optional; the originating tool name remains checked. +7. `WhenLatestJson(path, expectedJson)` and `WhenLatestJsonValue(path, value)` compare fields of the latest + content by JSON structural equality. Paths support `$`, JSON Pointer including `~0`/`~1`, and the subset + `$.property.items[0]`. Paths are limited to 256 characters. Wildcards, filters and recursive selectors are + unsupported. Missing fields or malformed incoming JSON fail the predicate. +8. `Responds`, `RespondsJson`, `CallsTool`, `CallsTools`, `Fails` and `DelaysBy` reuse the existing fixture + contracts. Tool choice, structured-output and streaming constraints are validated before advancement. + +## Replay, errors and lifecycle + +`ReplaySuccessfulRequests` defaults to true. Repeating the last successful normalized request returns its +retained runtime result without advancing the graph. Request ID, streaming mode and `previous_response_id` +are excluded from the fingerprint; restored history and the remaining normalized request determine identity. +Provider adapters still apply their wire formatting, so replay is not a byte-for-byte HTTP response guarantee. +Only the latest successful result is replayable. Fixture constraints remain validated on replay. + +With replay disabled, changed or repeated input cannot advance an already successful logical turn. A later +turn must contain the additional assistant history. A committed scripted error can transition to an explicitly +configured recovery step at the same logical turn. Only one step can be in flight for an identity; cancellation +rolls back its reservation. Rejected fixtures and history/state admission failures do not advance the graph. +Reset/configure/session deletion clear conversation state and supersede pending work through the existing +runtime generation/retirement contract. + +| Boundary | HTTP result/code | +| --- | --- | +| Another step is in flight | 409 `conversation_busy` | +| No active target matches | 409 `conversation_step_mismatch` | +| Changed/stale input after a successful turn | 409 `conversation_turn_not_advanced` | +| Terminal graph with Reject policy | 409 `conversation_completed` | +| Execution limit reached | 409 `conversation_exhausted` | +| Conversation admission budget exceeded | 409 `conversation_capacity_exceeded` | +| Incompatible response fixture | 409 `llm_test_server_fixture_mismatch` | +| Reset/configure/retirement during pending work | 409 `llm_test_server_request_superseded` | +| Invalid graph or selector header | 400; configuration remains unchanged | + +The terminal policy defaults to `Reject`. An identical successful replay takes precedence over terminal +handling. `RepeatLastResponse` reuses the final successful fixture for later inputs; `Restart` begins again +at the start step and clears that run's execution/visited-step counters. `WithMaxSteps` limits committed +executions within a run, including error steps and cycles, rather than HTTP attempts or successful replays. + +## Bounds + +`UseConversationCapacity(maximumConversations, maximumBytes)` on the control builder, or +`WithConversationCapacity(...)` on the core builder, sets runtime admission budgets. Defaults are 256 +conversations and 16 MiB retained state. Zero capacity rejects new state. Reservations include bounded identity, +step metadata and the retained response; growth is checked before a delayed fixture starts. Completed states +remain until reset/configure/session disposal. There is no timer eviction, persistence or heap-size measurement. + +A graph contains 1–256 uniquely identified steps and `MaxSteps` is 1–4096, default 64. Scenario, step and tool +identifiers are limited to 128 characters. A step must terminate or reference distinct known next steps. +Graphs cannot also contain the linear response queue. Invalid configuration is rejected atomically. + +## Control APIs and diagnostics + +Both endpoints use the selected runtime's control authorization and test-session headers. + +| API | Contract | +| --- | --- | +| `GET /admin-api/conversations` | Status entries with scenario/provider/conversation IDs, busy/terminal state, executed steps, last turn, next and unvisited steps | +| `POST /admin-api/match/explain` | Body: `LlmTestServerMatchExplanationRequest` containing normalized `request` and `providerNamespace`; response explains candidate predicates and precedence without advancing state | + +The conversation header filters the status list or selects the identity to explain. Without it the status API +lists the selected runtime's states and explanation uses `default`. The client exposes `GetConversationsAsync` +and `ExplainMatchAsync`. Explanations include logical turn, selected scenario/step, matched candidates, +shadowed candidates and mismatch field names. They do not execute a provider call, reserve a step, increment +fault counters or populate caches. Unvisited alternatives are diagnostics, not an assertion that a required +branch failed. Runtime events and request traces carry conversation ID, step ID, logical turn and replay metadata. + +## Verification and completion criteria + +Qualification must start the HTTP server and drive real HTTP/provider SDK requests. Required flows are ordered +branches, latest-message and turn matching, parallel tool calls/results, JSON paths, replay and error recovery, +all terminal policies, cycles, invalid graphs, count/byte limits, fixture rejection, cancellation and in-flight +reset/configure/delete. Verify isolation across sessions, providers, credentials and conversation IDs. Read +explanations/status/events to prove selection and no advancement from diagnostics. Keep linear scenario and +provider-contract regressions green. No direct runtime tests or source scans establish these behaviors. + +```sh +dotnet format ManagedCode.LlmTestServer.slnx --verify-no-changes --no-restore +dotnet build ManagedCode.LlmTestServer.slnx --configuration Release --no-restore +dotnet test --project tests/ManagedCode.LlmTestServer.Tests/ManagedCode.LlmTestServer.Tests.csproj --configuration Release --no-build --verbosity normal +bash scripts/coverage.sh +``` + +Completion requires the focused HTTP/SDK cases, full integration suite, at least 90% production line coverage, +canonical Release build and independent installed-package consumer to pass. Results belong in the delivery +implementation ledger. The E3 checkpoint passed 722/722 with 91.34% production line coverage; the independent installed-package Aspire/SDK/Chromium consumer passed 1/1. Public package publication remains pending the combined delivery batch. diff --git a/docs/Features/DashboardAnalytics.md b/docs/Features/DashboardAnalytics.md new file mode 100644 index 0000000..bafb5c2 --- /dev/null +++ b/docs/Features/DashboardAnalytics.md @@ -0,0 +1,70 @@ +# Dashboard analytics + +The request inspector has a **Tokens** tab that shows what a request, or an explicitly labeled conversation, is made of and how its tokens are billed. The default scope is the selected request. Conversation scope is available only when the caller supplied `X-Llm-Test-Server-Conversation`; it combines retained requests with that label, session and provider namespace. Models may differ inside that group. An automatically assigned or default conversation identity does not join unrelated requests. + +![Token analytics](../images/admin-dashboard-token-usage.png) + +## What the tab shows + +- **Total, input, output and cached input** at the top, with reasoning shown as part of output. +- **Where the tokens go**: a flow (Sankey) chart. On the left, the input split by origin (system prompt, tool schemas, user, assistant, tool-call arguments, tool results) plus generated output. In the middle, the request total. On the right, the billing categories: cache read, cache write, uncached input, visible response and reasoning. Each node carries its token count and share. +- **Context mix**: a donut of the input composition with a value-and-share legend. +- **Prompt cache**: a donut of cache read, cache write and uncached input, plus how many requests read from the cache. +- **Context growth** (conversation scope with two or more requests): one stacked bar per request, oldest first, so context growth and cache reuse are visible turn by turn. Clicking a bar selects that request. +- **Function tools**: schema, call-argument and result tokens, then returned calls and in-context results per function name. + +Each request row in the inbox also carries a small composition strip with the same colors, so a request dominated by its system prompt or tool results stands out before you open it. + +## Traffic insights + +The **Insights** button above the request inbox (and **Traffic insights** in the rail) switches the reading pane to charts over every retained request that matches the current filters. Open it directly with `/?view=insights`. + +![Traffic insights](../images/admin-dashboard-insights.jpg) + +- **Recent requests**: one column per request, oldest to newest; height is total tokens and color is the outcome. Clicking a column opens that request. +- **Delivery**: a 10 × 10 dot grid of streaming versus single responses (each dot is 1% of requests). +- **Providers**, **Outcomes**, **Prompt cache**, **Tokens by model** and **Context mix**: donuts with a value-and-share legend. +- **Models** and **Operations**: labeled capsule bars. +- **Latency** and **Conversation depth**: columns for duration bands and messages per request. + +Clicking a provider, model, outcome, delivery mode or operation applies the matching inbox filter; clicking it again clears it. Provider and model colors are assigned from the whole journal, so filtering never repaints the entries that remain. Entries past the seventh color slot fold into **Other**. + +## Input composition + +The runtime records `LlmTestServerTokenUsage.InputComposition` for chat-style requests on every provider surface. It counts each normalized message and tool schema with the same tokenizer that produces the reported input, so the parts add up exactly to `InputTokens`: + +| Part | Source | +| --- | --- | +| System prompt | `system` and `developer` messages, including provider-level system fields | +| Tool schemas | function tool definitions: name, description and parameter schema | +| User | `user` messages | +| Assistant | assistant or model text from earlier turns | +| Tool calls | function-call identifiers, names and arguments inside assistant turns | +| Tool results | tool-result messages and blocks | +| Other roles | any other normalized role | + +The composition is part of the request journal returned by `/admin-api/requests`, so tests can assert it directly. Usage without a reconciling composition, such as embeddings or media prompts, is shown as **Not itemized** rather than guessed. + +## Tokens and cache + +- Input tokens include cache-read and cache-write tokens. +- Output tokens include reasoning tokens. +- Total tokens equal input plus output; cache and reasoning subsets are never added again. +- Cached input share is summed cache-read tokens divided by summed input tokens. +- The cache-read request rate counts requests whose cached input is positive, divided by requests reporting usage. It does not claim that every request was eligible for its provider's cache. + +Cache-write values come from normalized runtime accounting. A provider may expose only cache-read details on the wire; provider-specific mapping stays in the adapter. For example, Anthropic wire input usage can report uncached tokens separately from cache read and write, while the normalized trace input includes all three. + +Conversation totals sum each retained request. Repeated prompt context counts again on each request, so the totals do not estimate unique conversation text, billing cost or external provider activity. Ratios use summed counts rather than averaging per-request percentages. + +Requests without usage, such as model discovery, stay in the request count but contribute no invented measurements: figures show an em dash, ratios show `N/A`, and the flow chart shows an empty state. + +## Function tools + +**Calls returned** counts function calls in provider responses, including a replayed response returned for another request. **Results in context** counts tool-result messages supplied by callers. A later request may contain the same earlier result again, so results can exceed the number of distinct result call IDs. Result IDs are associated with preceding calls in the retained history; when that context is unavailable the dashboard shows an unknown-tool label. Neither calls nor results prove that an external function ran. These metrics cover function-tool contracts, not native hosted tools. + +## Retention + +Analytics use the same bounded request snapshot as the inspector. Short notices appear above the charts when earlier conversation requests were evicted, requests are still pending, usage is missing, or payloads were shortened; token counts stay complete when only payload previews were shortened. The dashboard adds no analytics endpoint; callers can inspect the underlying journal through the existing [query/export/verification APIs](RequestVerificationAndDeterminism.md). + +See [architecture](../Architecture.md), [conversation DSL](ConversationDSL.md) and [delivery plan](LlmTestServerDeliveryPlan.md). Running-server dashboard HTTP tests and browser tests provide the integration evidence. diff --git a/docs/Features/DecisionModels.md b/docs/Features/DecisionModels.md index 7132f1d..59a968a 100644 --- a/docs/Features/DecisionModels.md +++ b/docs/Features/DecisionModels.md @@ -2,9 +2,9 @@ Official native decision/vision contracts re-audited 2026-10-20; see [vision audit](DecisionVisionAudit.md). Provider DTOs live in their owning SystemOne, Cloudflare and OpenAI packages; deterministic fixtures live in core and routes in Hosting. -Jev, Kev, Cloudflare Clef/Clef Flash and OpenAI Decisions use explicit decision fixtures. The TCK hosts their native JSON shapes under provider namespaces and does not contact production providers. Declare the provider profile explicitly; model names never select an API implicitly. +Jev, Kev, Cloudflare Clef/Clef Flash and OpenAI Decisions use explicit decision fixtures. The Test Server hosts their native JSON shapes under provider namespaces and does not contact production providers. Declare the provider profile explicitly; model names never select an API implicitly. -| Provider | TCK POST route | Production POST endpoint | Model helper | Official contract | +| Provider | Test Server POST route | Production POST endpoint | Model helper | Official contract | | --- | --- | --- | --- | --- | | TypeSafe Jev | `/systemone/v1/systemone` | `https://api.typesafe.ai/v1/systemone` | `AddJevLatest()` | [OpenAPI](https://api.typesafe.ai/openapi.json), [primitives](https://docs.typesafe.ai/primitives/choice) | | Kev | `/systemone/v1/systemone` | `{kev-server}/v1/systemone` | `AddKevLatest()` | [native request models](https://github.com/jaredpalmer/kev/blob/main/kev/api.py), [server/auth](https://github.com/jaredpalmer/kev/blob/main/kev/serve.py) | @@ -12,19 +12,19 @@ Jev, Kev, Cloudflare Clef/Clef Flash and OpenAI Decisions use explicit decision | Cloudflare Clef Flash | `/cloudflare/client/v4/accounts/{accountId}/ai/run/@cf/cloudflare/clef-flash` | `https://api.cloudflare.com/client/v4/accounts/{accountId}/ai/run/@cf/cloudflare/clef-flash` | `AddClefFlash()` | [Clef Flash API](https://developers.cloudflare.com/workers-ai/models/clef-flash/) | | OpenAI Decisions | `/openai/v1/decisions` | `https://api.openai.com/v1/decisions` | `AddOpenAiDecisionModel("gpt-6-luna")` | [create API](https://developers.openai.com/api/reference/resources/decisions/methods/create/) | -Production APIs use `Authorization: Bearer `; Kev checks `KEV_API_KEY` only when configured and otherwise permits unauthenticated calls. For the TCK, `RequireBearerToken` controls the expected test token and protects both provider routes and `/admin-api/*`. Tests do not need production credentials. Resolve the hosted base URL from the Aspire resource (`GetEndpoint("http")` / `CreateHttpClient(...)`), then append the TCK route. +Production APIs use `Authorization: Bearer `; Kev checks `KEV_API_KEY` only when configured and otherwise permits unauthenticated calls. For the Test Server, `RequireBearerToken` controls the expected test token and protects both provider routes and `/admin-api/*`. Tests do not need production credentials. Resolve the hosted base URL from the Aspire resource (`GetEndpoint("http")` / `CreateHttpClient(...)`), then append the Test Server route. ### SystemOne: Jev and Kev Jev 1.14's JSON image assets and `box` questions are covered by the [Jev 1.14 compatibility profile](Jev114.md). Register `jev-1.14.0` or `jev-preview` -with `AddDecisionModel(..., LlmTckDecisionProvider.TypeSafe)` and opt each image/box +with `AddDecisionModel(..., LlmTestServerDecisionProvider.TypeSafe)` and opt each image/box request in through `TypeSafe-Danger-Zone-Images: true`. Structured score legends and the optional fixture `ResolvedModelId` retain native values and version attribution. Long-context headers pass through; deterministic scenarios define usage and scripted context-limit failures. The emulator does not tokenize images or estimate model context. -Jev requires `model`, string/object/array `state`, and a nonempty question map. Kev accepts any JSON `state`, including numbers, booleans and null; instructions and criterion descriptions also accept arbitrary JSON. Kev defaults an omitted `model` to `kev-latest`; the TCK enables that default only when `AddKevLatest()` explicitly registers the Kev profile. Binary questions use `type: "noul"`; optional `criteria` maps the native `true` and `false` descriptions. Choices use an opaque ID-to-description map. Scores use an ordered criteria array. +Jev requires `model`, string/object/array `state`, and a nonempty question map. Kev accepts any JSON `state`, including numbers, booleans and null; instructions and criterion descriptions also accept arbitrary JSON. Kev defaults an omitted `model` to `kev-latest`; the Test Server enables that default only when `AddKevLatest()` explicitly registers the Kev profile. Binary questions use `type: "noul"`; optional `criteria` maps the native `true` and `false` descriptions. Choices use an opaque ID-to-description map. Scores use an ordered criteria array. ```json { @@ -41,24 +41,24 @@ Jev requires `model`, string/object/array `state`, and a nonempty question map. Use `model: "jev-latest"` for Jev. Native responses contain keyed `answers`: `noul`, or `choice`/`score` with `confidence` and `probabilities`; score `legend` keys are zero-based numeric strings. `usage` contains `input_tokens` and `output_tokens`. Scores are native ordinal values and can exceed one. ```csharp -using ManagedCode.LlmTck.Decisions; +using ManagedCode.LlmTestServer.Decisions; -options.AddKevLatest().AddDecisionScenario(new LlmTckDecisionScenario +options.AddKevLatest().AddDecisionScenario(new LlmTestServerDecisionScenario { Id = "priority", ModelId = "kev-latest", Answers = new() { - ["urgent"] = new() { Kind = LlmTckDecisionKind.Predicate, Probability = 0.95 }, - ["route"] = new() { Kind = LlmTckDecisionKind.Choice, Choice = "fast", Confidence = 0.9, + ["urgent"] = new() { Kind = LlmTestServerDecisionKind.Predicate, Probability = 0.95 }, + ["route"] = new() { Kind = LlmTestServerDecisionKind.Choice, Choice = "fast", Confidence = 0.9, Probabilities = new() { ["fast"] = 0.95, ["regular"] = 0.05 } }, - ["severity"] = new() { Kind = LlmTckDecisionKind.Score, Score = 1.75, Confidence = 0.625, + ["severity"] = new() { Kind = LlmTestServerDecisionKind.Score, Score = 1.75, Confidence = 0.625, Probabilities = new() { ["0"] = 0.05, ["1"] = 0.15, ["2"] = 0.8 } }, }, Usage = new() { InputTokens = 100, OutputTokens = 10, TotalTokens = 110 }, }); ``` -Kev score legends contain rendered strings, while Jev retains string/object/array criteria. Kev responses include `latency_ms`; `LlmTckDecisionScenario.Metadata` supplies deterministic latency and optional `Truncated`, `StateTokens`, `StateTokensUsed` fields. The route echoes or generates `x-typesafe-request-id`, including on errors. Invalid SystemOne request shapes return HTTP 422 with a native `detail` validation array; runtime/admission faults retain `detail` strings. Kev bearer failures include `WWW-Authenticate: Bearer`. This compatibility surface covers `/v1/systemone`; Kev demo `/permute` and `/separate` routes are outside its scope. +Kev score legends contain rendered strings, while Jev retains string/object/array criteria. Kev responses include `latency_ms`; `LlmTestServerDecisionScenario.Metadata` supplies deterministic latency and optional `Truncated`, `StateTokens`, `StateTokensUsed` fields. The route echoes or generates `x-typesafe-request-id`, including on errors. Invalid SystemOne request shapes return HTTP 422 with a native `detail` validation array; runtime/admission faults retain `detail` strings. Kev bearer failures include `WWW-Authenticate: Bearer`. This compatibility surface covers `/v1/systemone`; Kev demo `/permute` and `/separate` routes are outside its scope. ### Cloudflare Workers AI: Clef and Clef Flash @@ -77,12 +77,12 @@ Send the same SystemOne question shape to the account/model route with body `mod } ``` -Configure `AddClef()` / `AddClefFlash()` and fixtures with the matching short model ID; native surrounding model whitespace is accepted. Runtime/auth failures remain non-success HTTP responses and use `success: false` with native numeric `errors[].code`. Explicit numeric scenario error codes are preserved, including 5004/400 (invalid input), 3007 or 3008/408 (timeout), 3036 or 3040/429 (capacity/rate limit). See [native errors](https://developers.cloudflare.com/workers-ai/platform/errors/). For native error cases, configure the explicit numeric `LlmTckScenarioError.Code`. Unmapped TCK-only errors such as fixture mismatch use the HTTP status as a synthetic code; that fallback is not a claimed Cloudflare native error code. +Configure `AddClef()` / `AddClefFlash()` and fixtures with the matching short model ID; native surrounding model whitespace is accepted. Runtime/auth failures remain non-success HTTP responses and use `success: false` with native numeric `errors[].code`. Explicit numeric scenario error codes are preserved, including 5004/400 (invalid input), 3007 or 3008/408 (timeout), 3036 or 3040/429 (capacity/rate limit). See [native errors](https://developers.cloudflare.com/workers-ai/platform/errors/). For native error cases, configure the explicit numeric `LlmTestServerScenarioError.Code`. Unmapped Test Server-only errors such as fixture mismatch use the HTTP status as a synthetic code; that fallback is not a claimed Cloudflare native error code. -Optional `images` is an array of inline data URLs or `{ "content_type": "image/png", "base64": "..." }` objects. PNG/JPEG/WebP are supported; remote URLs are rejected. Limits: four images, 4 MiB decoded file bytes and 16 megapixels per image, 8 MiB combined decoded bytes, and 13 MiB for the complete request body. Image dimensions and file type are read as metadata; compressed pixel streams are not decoded, so full raster validity is outside this deterministic mock validation. The TCK retains supplied images in the matched input and trace. See [input schema](https://developers.cloudflare.com/workers-ai/models/clef/schema-input.json), [output schema](https://developers.cloudflare.com/workers-ai/models/clef/schema-output.json). +Optional `images` is an array of inline data URLs or `{ "content_type": "image/png", "base64": "..." }` objects. PNG/JPEG/WebP are supported; remote URLs are rejected. Limits: four images, 4 MiB decoded file bytes and 16 megapixels per image, 8 MiB combined decoded bytes, and 13 MiB for the complete request body. Image dimensions and file type are read as metadata; compressed pixel streams are not decoded, so full raster validity is outside this deterministic mock validation. The Test Server retains supplied images in the matched input and trace. See [input schema](https://developers.cloudflare.com/workers-ai/models/clef/schema-input.json), [output schema](https://developers.cloudflare.com/workers-ai/models/clef/schema-output.json). The typed client is available through -`new LlmTckClient(httpClient, bearerToken).CreateCloudflareDecisionClient(accountId)`. +`new LlmTestServerClient(httpClient, bearerToken).CreateCloudflareDecisionClient(accountId)`. Pass `CloudflareDecisionRequest` with either `CloudflareDecisionImage.DataUrl` or `Embedded = new CloudflareBase64Image { ContentType = ..., Base64 = ... }`. `EvaluateAsync` selects the native route from the explicit `clef`/`clef-flash` model, @@ -112,20 +112,20 @@ OpenAI uses `input` and an ordered question array. Predicate questions return on OpenAI fixture question IDs use `OpenAiDecisionFixtureIds.ForQuestionIndex(index)` for the zero-based position, regardless of optional native names. Choice fixture IDs use `ForChoiceValue(string/bool)` canonical JSON literals so string `"true"` differs from boolean `true`. Responses preserve names (including `null`), duplicate names and answer order. ```csharp -using ManagedCode.LlmTck.Decisions; -using ManagedCode.LlmTck.OpenAI; +using ManagedCode.LlmTestServer.Decisions; +using ManagedCode.LlmTestServer.OpenAI; var fast = OpenAiDecisionFixtureIds.ForChoiceValue("fast"); var defer = OpenAiDecisionFixtureIds.ForChoiceValue(false); -options.AddOpenAiDecisionModel("gpt-6-luna").AddDecisionScenario(new LlmTckDecisionScenario +options.AddOpenAiDecisionModel("gpt-6-luna").AddDecisionScenario(new LlmTestServerDecisionScenario { Id = "priority-openai", ModelId = "gpt-6-luna", Answers = new() { [OpenAiDecisionFixtureIds.ForQuestionIndex(0)] = new() { Probability = 0.95 }, - [OpenAiDecisionFixtureIds.ForQuestionIndex(1)] = new() { Kind = LlmTckDecisionKind.Choice, Choice = fast, + [OpenAiDecisionFixtureIds.ForQuestionIndex(1)] = new() { Kind = LlmTestServerDecisionKind.Choice, Choice = fast, Confidence = 0.9, Probabilities = new() { [fast] = 0.9, [defer] = 0.1 } }, - [OpenAiDecisionFixtureIds.ForQuestionIndex(2)] = new() { Kind = LlmTckDecisionKind.Score, Score = 1.75, + [OpenAiDecisionFixtureIds.ForQuestionIndex(2)] = new() { Kind = LlmTestServerDecisionKind.Score, Score = 1.75, Confidence = 0.8, Probabilities = new() { ["0"] = 0.05, ["1"] = 0.15, ["2"] = 0.8 } }, }, Usage = new() { InputTokens = 100, CachedInputTokens = 4, OutputTokens = 0, TotalTokens = 100 }, @@ -143,6 +143,6 @@ options.AddOpenAiDecisionModel("gpt-6-luna").AddDecisionScenario(new LlmTckDecis | Cloudflare Clef/Flash | 2..255 | 2..10 | 1..64 questions; ASCII letter/digit/underscore/dot/hyphen IDs, max 100 characters; required nonempty instructions; four-decimal probabilities/native scores. | | OpenAI | 2..255 distinct typed values | Ordered labels; no undocumented count cap | Optional names/descriptions, empty strings and repeated score labels preserved; max 128 embedded images; model/names/instructions/labels/descriptions max 1,048,576 characters; no invented choice-value length cap. | -`AddDecisionModel(id, LlmTckDecisionProvider...)` supports explicit custom model IDs; the control client mirrors it with `UseDecisionModel` and `UseDecisionScenario`. Fixtures are repeatable snapshots. Optional `ExpectedInput` matches the provider input exactly; SystemOne JSON state and OpenAI message arrays use serialized JSON. Cloudflare requests with supplied images match serialized `{state,images}`; text-only requests retain the state-only input. Question IDs, kinds and requested value domains must fit the fixture. Unknown models, unmatched inputs, mismatched fixtures, authentication, configured rate limits/content filtering and scripted errors produce observable non-success HTTP responses. Normal native choice/score fixtures require confidence and a complete probability domain, near-unit probability mass. SystemOne and Cloudflare choices must have maximal probability; OpenAI preserves any selected value in the supplied domain. Cloudflare confidence must equal the maximum probability within native rounding tolerance. Configure profile-correct explicit values: the Kev/Jev example above has score confidence 0.625; Clef uses the maximum probability 0.8. Values are validated and preserved without normalization or recomputation. Four-decimal distributions may total 0.9999 and native scores are not recomputed from rounded probabilities. For deliberate malformed-response regressions, set `AllowMalformedResponse = true` on the scenario; this explicitly bypasses native response conformance while keeping question-kind/domain mapping safe. +`AddDecisionModel(id, LlmTestServerDecisionProvider...)` supports explicit custom model IDs; the control client mirrors it with `UseDecisionModel` and `UseDecisionScenario`. Fixtures are repeatable snapshots. Optional `ExpectedInput` matches the provider input exactly; SystemOne JSON state and OpenAI message arrays use serialized JSON. Cloudflare requests with supplied images match serialized `{state,images}`; text-only requests retain the state-only input. Question IDs, kinds and requested value domains must fit the fixture. Unknown models, unmatched inputs, mismatched fixtures, authentication, configured rate limits/content filtering and scripted errors produce observable non-success HTTP responses. Normal native choice/score fixtures require confidence and a complete probability domain, near-unit probability mass. SystemOne and Cloudflare choices must have maximal probability; OpenAI preserves any selected value in the supplied domain. Cloudflare confidence must equal the maximum probability within native rounding tolerance. Configure profile-correct explicit values: the Kev/Jev example above has score confidence 0.625; Clef uses the maximum probability 0.8. Values are validated and preserved without normalization or recomputation. Four-decimal distributions may total 0.9999 and native scores are not recomputed from rounded probabilities. For deliberate malformed-response regressions, set `AllowMalformedResponse = true` on the scenario; this explicitly bypasses native response conformance while keeping question-kind/domain mapping safe. -See [complete native contracts and fixture details](../../README.md#native-decision-api-fixtures) and [behavior-test evidence](../Testing/AcceptanceCriteriaTestInventory.md). Provider-specific limits and optional-field rules above are reconciled against the linked primary contracts. +See [the provider route table](../Reference/ProvidersAndRoutes.md#native-decision-api-fixtures) and [behavior-test evidence](../Testing/AcceptanceCriteriaTestInventory.md). Provider-specific limits and optional-field rules above are reconciled against the linked primary contracts. diff --git a/docs/Features/DecisionVisionAudit.md b/docs/Features/DecisionVisionAudit.md index 1b9ea48..ecb878a 100644 --- a/docs/Features/DecisionVisionAudit.md +++ b/docs/Features/DecisionVisionAudit.md @@ -25,7 +25,7 @@ reference-keyed PNG/JPEG assets, JSON or multipart, the explicit images header, sides, 16 MiB per file and a 32 MiB physical body budget. Box needs Jev 1.14 or preview. Long-context opt-in remains an explicit per-call header. -This deterministic TCK validates transport/file metadata and preserves scripted answers/usage. +This deterministic Test Server validates transport/file metadata and preserves scripted answers/usage. It does not emulate provider inference, tokenization or image re-encoding; processed/expanded image-memory and context-limit failures use explicit scripted provider errors. Do not infer post-processing budgets from compressed input sizes. TypeSafe's GET models API is outside @@ -42,5 +42,5 @@ confidence is retained by both scenario configuration/runtime validation and `SystemOneNativeJson.Options`, without adding confidence to Noul responses. Release delivery still requires a successful GitHub workflow and verified NuGet -availability before any consuming package reference is changed. The TCK never +availability before any consuming package reference is changed. The Test Server never contacts production Cloudflare/TypeSafe inference APIs during these tests. diff --git a/docs/Features/DockerDistribution.md b/docs/Features/DockerDistribution.md new file mode 100644 index 0000000..f9e8cd7 --- /dev/null +++ b/docs/Features/DockerDistribution.md @@ -0,0 +1,125 @@ +# Standalone Docker distribution + +Status: Distribution code and verification are being prepared for the complete v1.0.0 release. Public GHCR +availability must be established by the release workflow; these commands do not assert that a release is already published. +Canonical image: `ghcr.io/managedcode/llm-test-server`. + +The image runs the complete HTTP backend: provider routes, authenticated management API, Blazor dashboard, +health and liveness. It includes the published service and browser assets; it does not require a .NET SDK or +NuGet restore on the consuming machine. The release builds Linux amd64 and arm64 images. The process runs +as the .NET runtime image's non-root app user and listens on container port 8080. + +## Run and configure + +After the verified v1.0.0 release is public: + +```sh +docker pull ghcr.io/managedcode/llm-test-server:1.0.0 +docker run --rm --name llm-test-server \ + -p 8080:8080 \ + -e LlmTestServer__RequiredBearerToken=local-test-key \ + ghcr.io/managedcode/llm-test-server:1.0.0 +``` + +Open `http://localhost:8080/` for the dashboard. `GET /health` and `GET /alive` expose readiness/liveness. +The configured bearer key protects provider and `/admin-api/*` routes. The dashboard page is available at `/`; +provider URLs remain namespaced, such as `/openai/v1` and `/anthropic/v1`. + +Configure deterministic fixtures through the same management API as the .NET client: + +```sh +curl --fail-with-body http://localhost:8080/admin-api/configure \ + -H 'Authorization: Bearer local-test-key' \ + -H 'Content-Type: application/json' \ + --data '{ + "requiredBearerToken": "local-test-key", + "models": [{"id": "gpt-4.1-mini", "kind": "chat"}], + "chatScenarios": [{ + "id": "greeting", + "modelId": "gpt-4.1-mini", + "responses": [{"content": "Hello from LLM Test Server", "streamChunks": ["Hello from ", "LLM Test Server"]}] + }] + }' + +curl --fail-with-body http://localhost:8080/openai/v1/chat/completions \ + -H 'Authorization: Bearer local-test-key' \ + -H 'Content-Type: application/json' \ + --data '{"model":"gpt-4.1-mini","messages":[{"role":"user","content":"Hello"}]}' +``` + +Configure replaces the selected runtime configuration. Include the bearer requirement in the replacement when +retaining authentication. Test sessions provide independent configurations inside one container; +[the session API](TestSessions.md) and [typed client](RequestVerificationAndDeterminism.md) use the same HTTP +transport. Runtime state is in memory; container replacement starts from the built-in startup configuration. +Map a different host port as needed. Prefer a version tag or verified digest for reproducible consumers. + +## Build and exercise locally + +From the repository root: + +```sh +dotnet publish samples/ManagedCode.LlmTestServer.Service/ManagedCode.LlmTestServer.Service.csproj \ + --configuration Release --runtime linux-arm64 --self-contained false \ + /t:PublishContainer -p:ContainerRuntimeIdentifier=linux-arm64 \ + -p:ContainerImageTag=local -p:UseAppHost=false +python3 scripts/verify-container.py --image managedcode/llm-test-server:local +``` + +Use `linux-x64` in both RID arguments on an amd64 machine. The .NET SDK builds the image from published +service output using its built-in `PublishContainer` target. The service project declares the canonical +repository, ASP.NET 10 base image, port/environment, OCI labels, non-root user and both supported runtime IDs. +It does not require a separate container-builder toolchain. Docker is used only to load/run/inspect/pull images. +Published runtime output includes the dashboard assets; source, repository metadata and marketing files are not +added as a build context. Provider credentials are supplied only at container startup/configuration. + +To generate the multi-platform OCI image index locally, omit the single RID arguments: + +```sh +dotnet publish samples/ManagedCode.LlmTestServer.Service/ManagedCode.LlmTestServer.Service.csproj \ + --configuration Release --self-contained false /t:PublishContainer \ + -p:ContainerImageTag=local-multi -p:UseAppHost=false +``` + +The project's `RuntimeIdentifiers` and `ContainerRuntimeIdentifiers` both specify `linux-x64;linux-arm64`. +The SDK publishes each RID and combines them into one OCI index. An optional +`-p:ContainerArchiveOutputPath=/absolute/repository/artifacts/container-sdk/image.tar.gz` exports an archive +instead of loading a daemon; `docker load --input ...` can import it. See Microsoft's +[SDK publish guide](https://learn.microsoft.com/en-us/dotnet/core/containers/sdk-publish) and +[container configuration reference](https://learn.microsoft.com/en-us/dotnet/core/containers/publish-configuration). + +The verifier launches a real container on an allocated loopback host port, waits for health, loads dashboard +HTML and Blazor assets, tests missing-credential rejection, configures fixtures through HTTP, verifies JSON/SSE, +unknown-model validation, actual assertion counters and reset. It checks the image uses a non-root account. +It stops only the container it created and stores sanitized logs/evidence under `artifacts/container-verification`. +This is API/container verification; it does not establish interactive browser qualification. +On 2026-10-20 the installed .NET SDK 10.0.401 generated and loaded a combined Linux amd64/arm64 OCI index +with `PublishContainer`; both architecture variants passed the listed HTTP checks (amd64 through Docker Desktop +emulation). SDK publish logs are under `artifacts/container-sdk`; runtime evidence is under +`artifacts/container-verification`. Public pull and registry publication remain pending owner-merged release. + +## CI, publication and public access + +PR CI builds and runs the image without pushing it. The release workflow is restricted to `main`, follows the +owner's merge, and uses the canonical package version. Its order is existing quality/package gates → local +SDK container HTTP acceptance → version-tagged multi-platform SDK publish → anonymous pull and HTTP acceptance → NuGet +publication and public-consumer verification → delivery marker. Publication does not run from this branch. +A failed gate keeps delivery incomplete; retry the same version rather than creating an intermediate release. + +The release job grants `packages: write`, authenticates with `GITHUB_TOKEN`, and calls `dotnet publish +/t:PublishContainer -p:ContainerRegistry=ghcr.io` with both configured RIDs. It links the image to its source +repository with the OCI source label. **Those settings do not make a new GHCR package public.** GitHub documents +new container packages as private by default. A package administrator must open the managedcode organization's +`llm-test-server` container package, choose **Package settings → Change visibility → Public**, and confirm the +change. Organization policies may restrict that permission. After the first push creates the package, change +visibility and rerun the same release if its anonymous check failed. + +The supported visibility setup is documented by GitHub's +[container registry guide](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry) +and [package access/visibility guide](https://docs.github.com/en/packages/learn-github-packages/configuring-a-packages-access-control-and-visibility). +The workflow does not invoke an undocumented visibility mutation API. + +The public-access gate runs `verify-container.py --anonymous-pull` with a newly created Docker configuration +containing no registry credentials. It must pull the published version and pass the same HTTP checks. +The published manifest must also include Linux amd64 and arm64. A push succeeding under the workflow token is insufficient. Public delivery remains blocked by missing GHCR +permissions, organization visibility policy, unavailable registry access or failed anonymous acceptance until +that specific condition is resolved. No public-availability claim should precede this gate. diff --git a/docs/Features/GeminiBatchEmbeddings.md b/docs/Features/GeminiBatchEmbeddings.md new file mode 100644 index 0000000..9768ef3 --- /dev/null +++ b/docs/Features/GeminiBatchEmbeddings.md @@ -0,0 +1,33 @@ +# Gemini embeddings through the official SDK + +`Google.GenAI` 1.26.1 uses `batchEmbedContents` for both its single-input and multiple-input +`Models.EmbedContentAsync` overloads. The server exposes the distinct documented operation +`POST /gemini/v1beta/models/{model}:batchEmbedContents`; the existing `embedContent` operation +remains its own API. + +Each request contains 1–128 entries in `requests` (a local fixture bound). Each entry supplies +`model: "models/"`, a non-empty text-only `content`, and optional +`outputDimensionality`. The current embedding fixture uses the documented stable +[`gemini-embedding-2`](https://ai.google.dev/gemini-api/docs/models/gemini-embedding-2) ID. +The SDK accepts model strings; it has no closed model enum to copy. + +Reduced output dimensions are bounded to 128–3072 for this contract. The runtime validates all +requested dimensions against the configured fixture vector under the same admission lock before +recording success or evaluating scripted provider faults. A fixture with too few coordinates +returns 409 instead of silently returning a shorter vector. Successful batches preserve input +order and project each fixture vector to its requested prefix. The server does not pad vectors +or perform embedding inference. + +Malformed or unsupported request shapes return 400. Model/path mismatches, missing or oversized +batches, non-text parts and unsupported `taskType`/`title` options are rejected explicitly. +Authentication, unknown model, configured rate limits, content filters and scripted errors use the +normal provider/runtime paths. One batch is one runtime operation; it does not partially record +successful entries before another entry fails. + +The REST response contains ordered `embeddings[].values` and `usageMetadata.promptTokenCount`. +The pinned SDK does not expose this usage field, so SDK tests prove vectors and request behavior; +normalized REST token accounting is a separate observation, not provider billing parity. + +Sources: [batch API](https://ai.google.dev/api/embeddings#method:-models.batchembedcontents), +[official SDK](https://github.com/googleapis/dotnet-genai), +[SDK model routing](https://github.com/googleapis/dotnet-genai/blob/c071e7d18f779bf166a66c0a7ff715ab24035273/Google.GenAI/Models.cs#L4635). diff --git a/docs/Features/GeminiFunctionContinuation.md b/docs/Features/GeminiFunctionContinuation.md new file mode 100644 index 0000000..e4089ba --- /dev/null +++ b/docs/Features/GeminiFunctionContinuation.md @@ -0,0 +1,29 @@ +# Gemini function-call continuation metadata + +The qualified `gemini-3.8-flash` GenerateContent JSON and SSE routes preserve a fixture tool call's +opaque `ContinuationToken` as the function part's base64 `thoughtSignature`. Forward the original +signed candidate parts through the official Google SDK when sending function results. + +Each current-turn model step requires a signature on its first function call. Parallel siblings +may omit it. Earlier steps in the same turn must retain their signatures; unsigned history before +the latest ordinary user turn remains valid. Missing or malformed signatures return HTTP 400 +before scenario consumption. A changed signature for the last response retained in the local +conversation returns a fixture mismatch (409), allowing an explicit retry with the original parts. + +Signatures count toward bounded history and diagnostic retention and participate in cache identity. +They do not inflate visible text-token usage. The local maximum is 16,384 base64 characters; it is +an explicit fixture retention policy, not a claimed Google limit. + +This contract preserves function-call metadata only. Arbitrary text-part signatures, signature-only +streaming text parts and general signed multimodal history remain outside this slice. The server +does not authenticate Google's private cryptographic signatures. Exact local comparison covers +the last retained response; older current-turn signatures receive structural validation only. + +Sources: [Google thought signatures](https://ai.google.dev/gemini-api/docs/generate-content/thought-signatures), +[Gemini 3.8 Flash](https://ai.google.dev/gemini-api/docs/models/gemini-3.8-flash), +[official .NET SDK](https://github.com/googleapis/dotnet-genai). + +The official Google.GenAI 1.26.1 running-server suite covers JSON/SSE signed rounds, missing and +changed metadata with recovery, parallel/sequential calls, prior turns, malformed input and atomic +retention rejection. All 36 Google SDK integration cases pass. Raw HTTP supplements malformed +base64 cases; successful continuation and public SDK errors use the original SDK. diff --git a/docs/Features/Jev114.md b/docs/Features/Jev114.md index deb723e..dc01c87 100644 --- a/docs/Features/Jev114.md +++ b/docs/Features/Jev114.md @@ -19,7 +19,7 @@ References: [API](https://docs.typesafe.ai/api), [images](https://docs.typesafe. - [x] Support multipart with the JSON request first, one unique part per asset, PNG/JPEG content validation and bounded physical-body/file reads. - [x] Adapt the Storied native client to send image assets as multipart; retain - base64 JSON inputs and expose a typed native client from the TCK client package. + base64 JSON inputs and expose a typed native client from the Test Server client package. - [x] Advertise verified Jev/SystemOne support in README, NuGet tags and GitHub topics. - [x] Run formatting, Release build, full tests and provider contract/evidence checks for the expanded batch. - [ ] Review the combined batch, increment one patch version, push once and verify @@ -34,7 +34,7 @@ The 2026-10-15 combined repair targets one `0.1.15` release. ## Typed client -`LlmTckClient.CreateSystemOneClient()` uses the TCK host's SystemOne route and +`LlmTestServerClient.CreateSystemOneClient()` uses the Test Server host's SystemOne route and inherits its bearer credential. `EvaluateAsync` sends JSON; `EvaluateMultipartAsync` sends the JSON request first, then `SystemOneImageUpload` values as file parts. The caller owns the shared `HttpClient`; upload memory is caller-owned and the diff --git a/docs/Features/LlmTestServerDeliveryPlan.md b/docs/Features/LlmTestServerDeliveryPlan.md new file mode 100644 index 0000000..34afe52 --- /dev/null +++ b/docs/Features/LlmTestServerDeliveryPlan.md @@ -0,0 +1,292 @@ +# ManagedCode.LlmTestServer: план розвитку та доставки + +Статус: **погоджено власником; реалізація триває, весь roadmap E0–E13 у scope**. +Дата плану: 20 жовтня 2026 року. +Основа: [архітектура](../Architecture.md), [контракти провайдерів](ProviderApiContractCoverage.md), +[tool fixtures](ToolAndStructuredFixtures.md), [Responses continuation](ResponsesContinuation.md). + +## 1. Мета + +**LLM Test Server** — **Deterministic LLM API mocking for .NET and Aspire.** +Продукт має одну ідентичність: GitHub `managedcode/LlmTestServer`, NuGet/namespace сім’ю +`ManagedCode.LlmTestServer.*`, типи `LlmTestServer*` і image `ghcr.io/managedcode/llm-test-server`. +План охоплює багатокрокові розмови, well-known built-in tools, розширення provider API, fault simulation, +record/replay та повну інтеграційну перевірку опублікованих пакетів. + +Підтримка доводиться через запущений сервер: HTTP, офіційні SDK, streaming і помилки. +Unit-тести, fake clients, читання source/docs та reflection guards не є прийнятними перевірками. +Оцінювання якості live моделей і реальне навчання не входять у deterministic mock runtime. + +## 2. Поточні поверхні + +19 пакетів: core, Hosting, Client, Aspire та 15 provider packages; 22 проєкти з двома samples і тестовим проєктом. +Provider packages: OpenAI, AzureOpenAI, Foundry, Anthropic, Gemini, Groq, Mistral, Ollama, Cohere, +Bedrock, OpenRouter, DeepSeek, Perplexity, SystemOne і Cloudflare. + +Є linear response queues, scripted errors/delays, function tool loops, structured output, Responses continuation, +streaming, embeddings/image/audio/video fixtures, cache/reasoning/token accounting, Blazor dashboard, bounded +request journal, rate-limit/content-filter simulation і package-owned Aspire без Docker. Test sessions локально +кваліфіковані. E3 conversation graph та E4 request verification/query/export/fixed wire metadata локально +кваліфіковані; наступні етапи не є завершеними support claims. + +E4 fixed-clock/seed режим задає generated wire IDs/time через session-owned provider-local ordinals; reset +відтворює послідовність, conversation replay зберігає response metadata. Live режим лишається доступним. +Explicit fixture timestamps/model dictionaries зберігають власні значення; runtime/trace observations та +monotonic retry timing залишаються реальними. Це не гарантія byte-for-byte однакових довільних HTTP bytes. + +## 3. Цільові можливості + +| Напрям | Результат | Інтеграційна перевірка | +| --- | --- | --- | +| Test sessions | Незалежні config, queues, history, cache, faults, journal | Паралельні клієнти, reset/dispose isolation | +| Conversations | Typed turns, branches, tool results, terminal policy | Повні HTTP/SDK loops у JSON і streaming | +| Built-in tools | Typed fixtures, native declarations/results, citations/events | SDK parsing, success/error, mixed tool loops | +| Reliability | Retry-After, timing, transport і semantic faults | Реальний HTTP/stream failure і результат повтору | +| Replay | Versioned cassettes, bytes/frames, strict offline | Відтворення без upstream при miss | +| Provider APIs | Moderation/rerank/count/discovery, lifecycle, media, realtime | Кожен claimed method/parameter/mode через сервер | +| Developer experience | Typed client, validation/import/export, diagnostics, installed Aspire | Чистий installed consumer, Chromium, SDK | +| Agent protocols | MCP/A2A/AG-UI та retrieval fixtures | Реальні protocol clients до hosted endpoints | + +## 4. Ідентичність і випуск + +Public API: `AddLlmTestServer`, `MapLlmTestServer`, `AddLlmTestServerContainer`; configuration key +`LlmTestServer:RequiredBearerToken` / `LlmTestServer__RequiredBearerToken`; diagnostic prefix +`llm_test_server_`. Усі package IDs, dependencies, targets, service assets, docs і screenshots використовують +один бренд і використовують наведені публічні ідентифікатори. + +Погоджена версія combined batch — `1.0.0`; canonical version bump виконується один раз, коли весь batch +готовий. Порядок: branch changes → local quality gate → pack/installed consumer → release preflight → +owner merge через чинні gates → GitHub release/GHCR/NuGet → чистий consumer з public feed. +Часткову publication відновлювати для тієї самої версії. Не публікувати окрему версію на кожну зміну. +Release завершений лише після перевіреної публікації всіх 19 пакетів та container image і реального consumer flow. + +## 5. Multi-turn і тестові сесії + +Потрібно розширити вже наявну історію та tool loops, а не реалізовувати їх повторно. + +1. **Session scope**: окремі configuration, scenario cursors, faults, assertions, cache, response/video/file/job stores. Session identity відокремлена від bearer authorization. Provider/credential isolation усередині history зберігається. +2. **Typed conversation DSL**: turns з умовами latest role/text, turn index, tool name/call ID/result, exact arguments/JSON path; explicit переходи, branches, terminal state, max steps і exhaustion policy. +3. **Семантика порядку**: turn index рахується з normalized conversation, retries того самого turn не стають автоматично новим turn; queued HTTP attempts і logical turns — різні поняття. +4. **Match explanations**: selected scenario/step, mismatch fields, shadowed/ambiguous rules, unused expectations. Пріоритет правил має бути документований і стабільний. +5. **Assertions**: count/order, tool arguments/results, no unexpected calls, all expected steps consumed, bounded retry attempts, Retry-After compliance, terminal stream marker. +6. **Без споживання при помилці**: validation/schema/authorization/session mismatch не пересувають сценарій. Cancellation/reset у межах session мають визначений результат для in-flight requests. +7. **Bounded lifetime**: кількість sessions, state bytes, journal retention та dispose/reset; переповнення повертає явну помилку. Тест однієї session не очищає іншу. + +Control endpoints E2: `POST /admin-api/sessions`, `DELETE /admin-api/sessions/{id}` та session-scoped +configure/reset локально кваліфіковані. `GET /admin-api/requests` доступний із paging/filter. E3 додає +`GET /admin-api/conversations` і `POST /admin-api/match/explain`; інтеграційна кваліфікація пройшла. +E4 `POST /admin-api/verify`, розширений request query та `GET /admin-api/requests/export` локально +кваліфіковані. Наступні control API: fixture import/export/validate. Session headers і client factory описані +в [ADR-0003](../ADR/ADR-0003-isolated-test-sessions.md), conversation DSL — у [контракті](ConversationDSL.md). +Control routes залишаються під `/admin-api`, provider routes — у своїх namespaces. + +```mermaid +flowchart LR + Test[Тест і typed fixtures] --> Session[Ізольована тестова сесія] + SDK[Офіційний provider SDK] --> Adapter[Provider adapter] + Adapter --> Session + Session --> Match[Conversation matcher] + Match --> Fixture[Response або tool fixture] + Fixture --> Wire[Provider response або stream] + Wire --> SDK + Session --> Journal[Bounded journal і assertions] + Journal --> Test +``` + +## 6. Well-known та provider-native built-in tools + +Цей блок додано за уточненням власника. Потрібні два різні рівні з єдиними fixtures/journal, але коректними provider wire contracts. + +### 6.1. Готові function-tool fixtures + +Реалізовано [typed function-tool catalogue](WellKnownFunctionTools.md) із дванадцятьма presets, input/output/error schemas та scripted success/error fixtures. Focused HTTP qualification: 40/40; combined server suite 852/852, production line coverage 91,79%, Release build 0 warnings/errors; 19 packages packed. Clean installed NuGet/Aspire/official SDK/Chromium consumer пройшов 1/1, включно з typed Calculator tool round. Це частина E5: native built-ins, dashboard discovery та розширена provider qualification залишаються відкритими. + +Repo-owned presets: `Calculator`, `CurrentTime`, `Weather`, `WebSearch`, `FetchUrl`, `FileSearch`, `ReadFile`, `WriteFile`, `ListFiles`, `RunCode`, `Shell`, `ApplyPatch`. Це наші зручні test presets, а не універсальний міжпровайдерний стандарт назв function tools. + +Кожен має typed argument/result schema, явні success/error fixtures, predictable call IDs, optional delay, expected-call verification та repeat/exhaustion rules. Час береться з fixture clock. Пошук і файли беруться із заданих test data. Tool loop може виконувати тестовий delegate через `Microsoft.Extensions.AI` function invocation; зовнішні побічні ефекти не входять у preset. Calculator не повинен виконувати довільний код через eval. + +### 6.2. Нативні інструменти провайдерів + +| Provider family | Кандидати до реалізації | Обов’язкові деталі | +| --- | --- | --- | +| OpenAI Responses | web search, file search, code interpreter, image generation, computer, shell/local shell, apply patch, tool search, remote MCP | Native tool types та output items, citations/annotations, call/result identity, lifecycle events, tool errors, approval items де застосовно | +| Anthropic | web search/fetch, code execution, tool search; client tools bash, text editor, computer/browser, memory | Точні versioned tool types, client/server execution distinction, block/event shapes, pause/resume, errors і required headers | +| Gemini | Google Search/Maps grounding, URL context, File Search, Code Execution | Native tool config, executableCode/codeExecutionResult, grounding/citation metadata, restrictions на combination/model/API | +| Azure/Foundry/Bedrock та інші | Лише native tool contracts, доступні на конкретній операції й версії | Не успадковувати всі OpenAI/Anthropic built-ins автоматично через схожий wire format | + +Первинні джерела: [OpenAI tools](https://developers.openai.com/api/docs/guides/tools), [Anthropic tool reference](https://platform.claude.com/docs/en/agents-and-tools/tool-use/tool-reference), [client/server tool lifecycle](https://platform.claude.com/docs/en/agents-and-tools/tool-use/how-tool-use-works), [Gemini tools](https://ai.google.dev/gemini-api/docs/tools). Перелік — ціль аудиту контрактів; не кожна модель чи endpoint підтримує кожен інструмент. + +Well-known catalogue повинен бути discoverable у C# та dashboard: provider, exact type/version, input/output schema, fixture examples, JSON/stream support, обмеження, evidence tests. Спочатку web search + file search + code execution для трьох головних wire families; далі computer/browser/shell/patch/memory/tool-search/MCP. Усі відповіді scripted; mock не стає реальним браузером, shell executor або пошуковиком. + +Негативні fixtures model output вводити явним fault API. Звичайні `CallsTool`/`RespondsJson` продовжують відхиляти некоректні arguments/schema; їхню чинну валідацію не послаблювати заради fault simulation. + +## 7. Реєстр API-прогалин + +Шляхи нижче показують запропонований hosted namespace. Точні operation IDs, versions, request/response та streaming режими додаються до `ApiContract` тільки разом з реалізацією й evidence. Provider lifecycle моделюється bounded fixtures, а не реальним inference/training. + +| Пріоритет / група | Відсутні API або частина контракту | Контракт / межа | +| --- | --- | --- | +| P1 moderation | `POST /openai/v1/moderations` | [Official API](https://developers.openai.com/api/reference/resources/moderations/methods/create); configured classifications | +| P1 rerank | `POST /cohere/v2/rerank`; Voyage rerank окремим provider surface | [Cohere](https://docs.cohere.com/reference/rerank); configured scores та порядок | +| P1 token counting | Anthropic `/anthropic/v1/messages/count_tokens`, Gemini `:countTokens`, Bedrock CountTokens | [Anthropic](https://platform.claude.com/docs/en/api/messages/count_tokens), [Gemini](https://ai.google.dev/api/tokens); наявний accounting розширити endpoints | +| P1 model discovery | Anthropic/Gemini list/get, OpenAI get-model, Ollama tags/show | Provider metadata, pagination, unknown-model errors | +| P1 Ollama generate | `POST /ollama/api/generate` | [Official API](https://docs.ollama.com/api/generate); нині є chat/embed | +| P1 built-in tools | Native tool declarations/results/events у Responses/Messages/generateContent | Розділ 6; розширення існуючих API | +| P2 Responses lifecycle | GET/DELETE response, input items, cancel/background, Conversations CRUD/items | Створення/continuation у нас уже є; перевірити SDK lifecycle | +| P2 Files | upload/list/retrieve/content/delete, metadata/purpose, paging; Anthropic/Gemini variants | Поточний generated-video download не дорівнює загальному Files API | +| P2 Batches | create/list/retrieve/cancel/results; OpenAI JSONL, Anthropic batches, Gemini batch subset | Partial failures, poll states, output files, per-session limits | +| P2 Vector stores | CRUD/search/file attach/file batches | Scripted search hits, bounded storage та lifecycle | +| P2 fine-tuning | jobs create/list/get/cancel/events/checkpoints | Scripted job lifecycle без навчання | +| P2 native model formats | Bedrock InvokeModel Nova/Llama/Titan variants; OpenResponses profile | Окремі SDK/wire contracts для кожного body format | +| P2 new provider families | Vertex AI, xAI/Grok, Voyage AI, ElevenLabs, BytePlus Ark, fal.ai; OrcaRouter як candidate | Окремий контракт та SDK evidence для кожного | +| P2 media completeness | Provider-specific image/audio/video variants, queue/status/content/cancel, voice design | Розширити наявні fixtures, не дублювати реалізоване | +| P3 realtime | OpenAI Realtime/GPT-Live, Responses WebSocket, Gemini Live/Interactions | Event protocols, session state, cancellations, errors, audio/tool events | +| P3 agent protocols | MCP initialize/tools/resources/prompts/call, notifications/session/auth; A2A cards/tasks/messages/SSE/cancel; AG-UI streams | Окремі packages/ADR; LLM tool call та MCP server мають окремі контракти | +| P3 retrieval dependencies | Pinecone/Qdrant/ChromaDB subsets, Tavily search | Fixtures/filter/query/upsert у погодженій матриці | + +Додатково: simulated usage/cost budgets з explicit pricing fixtures, imported provider-reported cost і rate-limit headers; це не актуальний billing calculator. SDK contract drift checks, route/OpenAPI catalogue, fixture lint, import/export, CLI або .NET tool для serve/validate/replay — частина developer experience backlog. Existing `ApiContract` можна використати для route catalogue, не заводячи друге джерело правди. + +Для API з P2/P3 цей документ фіксує цільові API й proposed subset, а не завершений аудит upstream специфікації кожного методу. Перший крок відповідного етапу — закріпити точні official docs/version/SDK і method-level inventory; unresolved cells залишаються видимими й не потрапляють у support claims. + +## 8. Reliability, transport та replay + +- Deterministic timing: TTFT, інтервали/затримки кожного chunk, timeout/stall, cancellation; clock/seed ізольовані за session, без незадокументованої випадковості. +- Transport faults: connection abort, early clean end, malformed JSON/SSE, missing terminal event, invalid envelope/header; окремо від звичайного provider-shaped HTTP error. +- Semantic faults: invalid tool JSON/schema, unknown tool, duplicate call IDs, refused/empty/reasoning-only output, length stop. Catalogue має support applicability за provider/mode та journal evidence. +- Retry: scripted 429 + Retry-After → success, 5xx → fallback provider, non-retryable failure; assertions перевіряють фактичні timestamps/attempts. Production timings не підміняти фальшивими успішними результатами тесту. +- Record/replay: versioned cassette schema для request, provider/version, exact response bytes/headers, timed frames; matcher normalization і collision diagnostics; replay speed та offline strict mode. +- Recording лише explicit opt-in із дозволеним upstream і redaction secrets/PII перед записом. Default test/replay не має network fallback при miss. CI використовує repo-owned fixtures без live credentials. +- Одна cassette може містити кілька conversation steps і tool rounds; replay із session isolation, deterministic clock/IDs, unused fixture detection. + +## 9. Етапи, залежності та оцінка + +Оцінка — **людино-дні повноцінної інженерної роботи**, включно з tests/docs/self-review. Це planning range, не обіцянка календарного строку чи часу роботи агента. Exact upstream breadth, SDK обмеження й release permissions можуть її змінити. Після кожного етапу уточнювати залишок за фактичним diff і verification evidence. + +| Етап | Результат | Залежність | Оцінка | +| --- | --- | --- | --- | +| E0 | Зафіксувати baseline checks, public API/route manifest, release preflight, scope decisions | Погодження плану | 1–2 | +| E1 | Єдина ідентичність 19 packages, GitHub/GHCR/docs, verified published consumer | E0 | 3–5 | +| E2 | Session isolation, bounded stores/lifecycle, typed session client | E1 | 6–10 | +| E3 | Conversation DSL, branches, matching explanations, tool-round semantics | E2 | 5–8 | +| E4 | [Request query/export, assertions, retry verification, stable clock/IDs](RequestVerificationAndDeterminism.md) | E2–E3 | 4–7 | +| E5 | [Well-known presets](WellKnownFunctionTools.md) + основні native search/file/code tools | E3–E4; file-search lifecycle узгодити з E9 | 10–16 | +| E6 | Timing, transport faults, semantic misbehavior catalogue | E2–E4 | 6–10 | +| E7 | P1 API: moderation/rerank/count/model discovery/Ollama generate | E2, contract inventory | 6–10 | +| E8 | Fixture validation/import/export та record/replay | E3–E4, E6 | 7–12 | +| E9 | Stateful platform: Files/Batches/Responses lifecycle/vector stores/fine-tune fixtures | E2, E7 | 12–20 | +| E10 | New providers/media/native formats; решта built-in tool families | E5, E7, E9 | 15–25 | +| E11 | Realtime/WebSocket/Live/Interactions | E2–E6, exact protocol spec | 12–20 | +| E12 | MCP/A2A/AG-UI/vector DB/search ecosystem | E2–E4, окремі ADR/contracts | 15–25 | +| E13 | Фінальні recipes/support matrix, packaged quickstart, screenshots, delivery acceptance | Кожен погоджений етап; тут підсумкова інтеграція | 3–5 | + +Повний перелічений обсяг: **105–175 людино-днів**, до резерву. Резерв 25% на protocol/SDK/release +невизначеність: приблизно **130–220 людино-днів**. Погоджено весь E0–E13 roadmap. Етапи є контрольними +точками одного узгодженого delivery batch; support claims додаються лише після відповідних server checks. + +## 10. Acceptance і перевірка + +| ID | Потік | Критерій | +| --- | --- | --- | +| A1 | Clean consumer | Встановлює нову Aspire NuGet з public feed, стартує packaged service без Docker, SDK виклик і assertion проходять; Blazor assets завантажуються | +| A2 | Product identity | Кожен із 19 пакетів доступний; metadata/dependencies/targets використовують ManagedCode.LlmTestServer; GitHub/origin/GHCR перевірені | +| A3 | Public configuration | Приклади використовують лише LlmTestServer configuration keys, API names і diagnostic IDs; installed consumer підтверджує їхню поведінку | +| A4 | Multi-turn | user → tool → tool result → другий tool → final; text/stream; повтор user prompt і approve/cancel branch дають правильний результат | +| A5 | Parallel sessions | Два клієнти з однаковими model/prompt/credential мають незалежні cursors/journals; reset/dispose однієї session не змінює іншу | +| A6 | Failure boundaries | Невірна schema/tool/session/auth не споживає fixture; capacity overflow та in-flight reset/cancellation мають явний результат | +| A7 | Built-ins | Позитивний/негативний native tool flow, citations/result/error/stream events, змішаний function + built-in loop для кожного claimed contract | +| A8 | Resilience | 429/Retry-After → success; malformed/aborted/stalled stream; invalid tool args; negative outcome відрізняється від помилки mock configuration | +| A9 | Replay | Recorded/hand-authored cassette offline відтворює bytes, steps, usage і timing policy; secrets не зберігаються; miss не йде у network | +| A10 | Stateful API | Paging, cancel, partial failure, foreign session ID, max count/bytes, delete/reset перевірені через реальні SDK там, де існують | +| A11 | Provider support | Кожен новий method/mode/version має official reference, operation constant, hosting route та виконувану behavior evidence; negative cases теж покриті | +| A12 | UI/docs | Desktop/tablet/mobile з довгою розмовою та tool payload; оновлені screenshots, видимі session/step/fault metadata, збережена bounded retention | +| A13 | Reproducibility | У fixed-clock/seed режимі scripted wire повторюється; паралельні sessions не змінюють чужу sequence; diagnostics мають зрозумілу clock policy | + +На кожному behavior change: focused regressions; для provider changes — provider contract/evidence tests і повний quality gate; Aspire behavior — AppHost реально запускається з test fixture. Production line coverage не нижче 90%. + +Canonical verification commands: + +```bash +dotnet tool restore +dotnet restore ManagedCode.LlmTestServer.slnx +dotnet format ManagedCode.LlmTestServer.slnx --verify-no-changes --no-restore +dotnet build ManagedCode.LlmTestServer.slnx --configuration Release --no-restore +dotnet test --project tests/ManagedCode.LlmTestServer.Tests/ManagedCode.LlmTestServer.Tests.csproj --configuration Release --no-build --verbosity normal +bash scripts/coverage.sh +python3 scripts/verify-package.py +python3 scripts/verify-release.py +git diff --check +``` + +Також pack усі `src/*/*.csproj`, перевірити nupkg contents/dependencies, запустити installed consumer на чистому restore із публічного NuGet feed; container path перевірити окремо. Після останніх code/format changes повторити canonical Release build. Publication complete лише після успішного release workflow, наявності всіх пакетів у intended feed та installed-consumer acceptance. + +Validation skills під час реалізації: `mcaf-testing`, `run-tests`/`tunit` для focused regressions → provider/Aspire checks → `format` і canonical build/full tests → `coverlet`/`reportgenerator` coverage → package/release checks → `mcaf-code-review` фінального diff; frontend skills для змінених browser assets. Нові dependencies/boundaries потребують ADR, не створювати їх до вибору реалізації. + +## 11. Журнал реалізації + +Погоджено весь E0–E13 план. Етапи не скорочують objective до першої хвилі. + +| Етап | Стан | Докази / залишок | +| --- | --- | --- | +| E0 | Baseline зафіксовано | `ddbf171`, Release build 0 warnings/errors; baseline suite 669/669 | +| E1 | Product identity локально реалізовано; public delivery pending | 19 package surfaces/22 проєкти; Release build, format, local pack та installed consumer пройшли. GitHub `managedcode/LlmTestServer`, origin і default branch `main` перевірені. Branded IDs/config повністю очищено; 19 nupkg metadata/entries перевірено без старих назв | +| E2 | Реалізовано й локально кваліфіковано | Незалежні runtime/journals, session ID/key, typed client, bounded admission, atomic configure/reset/dispose та retirement pending work. 655/655 integration tests, coverage 91,17%; installed consumer/Aspire/Chromium 1/1. [Контракт](TestSessions.md), [ADR-0003](../ADR/ADR-0003-isolated-test-sessions.md) | +| E3 | Реалізовано й локально кваліфіковано | [Conversation DSL](ConversationDSL.md), [ADR-0004](../ADR/ADR-0004-conversation-step-graphs.md); 722/722 integration tests, coverage 91,34%; format/Release build 0 warnings/errors; all 19 packages packed, independent installed Aspire/SDK/Chromium consumer 1/1 | +| E4 | Реалізовано й локально кваліфіковано | [Request evidence/verification/determinism](RequestVerificationAndDeterminism.md), [ADR-0005](../ADR/ADR-0005-request-evidence-and-wire-determinism.md); full suite 812/812, production line coverage 91,66%; final Release build 0 warnings/errors; 19 packages packed; independent clean installed NuGet/Aspire/official SDK/Chromium consumer 1/1. E4 branch CI [39455691575](https://github.com/managedcode/LlmTestServer/actions/runs/39455691575) успішний для `894d844` | +| E5 | Частково реалізовано | [12 typed function-tool presets](WellKnownFunctionTools.md), явні success-root/error-envelope schemas, existing queue/graph lifecycle; 40/40 focused HTTP cases, combined suite 852/852, coverage 91,79%, Release build 0 warnings/errors, 19 packages packed (`artifacts/delivery/presets-coverage.log`). Clean installed NuGet/Aspire/official SDK/Chromium consumer 1/1 (`artifacts/delivery/presets-consumer.log`), включно з Calculator tool round; native built-ins, dashboard discovery і provider qualification не завершено | +| E6–E13 | Не завершено | Увесь погоджений функціональний обсяг активний; support claims не розширюються наперед | + +Гілка `codex/llm-test-server`, [PR1](https://github.com/managedcode/LlmTestServer/pull/1). +E3 CI [39446187097](https://github.com/managedcode/LlmTestServer/actions/runs/39446187097) успішний для +head `3aae4db`. E4/presets CI [39455691575](https://github.com/managedcode/LlmTestServer/actions/runs/39455691575) +та website validation [39455691773](https://github.com/managedcode/LlmTestServer/actions/runs/39455691773) успішні для `894d844`. +Локальні E4 evidence logs: `artifacts/delivery/e4-build-final.log`, `e4-coverage-final.log`, `e4-pack.log` та +`e4-consumer.log`. Consumer встановив local NuGet artifacts у незалежний clean project/cache і реально +запустив packaged Aspire service, official SDK та Chromium; це не public-feed publication acceptance. +Public NuGet/GHCR publication, E5–E13 і GitHub Pages launch залишаються незавершеними. + +Інтеграційні перевірки виявили й виправили model-list authorization, trace error metadata та browser dialog +layout. Docs screenshots і site hero отримано з реального browser run. Sessions доведені для 13 chat provider +families і native Jev/Kev/Clef/OpenAI decisions: JSON/SSE, cache/history/video/fault/journal isolation, auth, +count/byte/config boundaries, cancellation та in-flight reset/configure/delete. + +## Launch preparation + +The release target is v1.0.0, delivered as one complete batch on the current branch. The owner will merge when ready; do not merge or publish intermediate releases. The launch includes a GitHub Pages landing page in the dashboard visual language with shared styles/components, real screenshots and demo videos, a detailed mc-compact guide, the supplied launch content plan, and a public prebuilt Docker image configurable through the management API. Marketing production materials live in ignored marketing/ until the owner authorizes their publication. + +Launch content reference: https://claude.ai/artifact/HHRF9Y8dzTbN8TJnaTKxnj + +## Request analytics checkpoint + +[Dashboard analytics](DashboardAnalytics.md) adds request/conversation token, cache-read/write and function observations with explicit label/session/provider grouping and bounded-evidence warnings. The genuine installed NuGet/Aspire/SDK/Chromium consumer passed 1/1 (`artifacts/delivery/analytics-consumer-final.log`), including cache miss → hit, function-result continuation, repeated context accounting and unlabeled-request isolation. Running-server analytics tests passed 3/3; query/export tests passed 14/14 after replacing a timing window with a gated real HTTP request and startup readiness. Combined suite passed 855/855 with 91.84% production line coverage (12170/13251) and a clean final Release build (`artifacts/delivery/query-readiness-coverage.log`, `query-readiness-final-build.log`). This is local qualification; the additional cache-admission regression batch is still in progress. + +## Cache correctness batch + +Code-only checkpoint `d990e06` fixes stored-response admission before cache mutation, provider-namespace isolation, and lookup of all retained prefixes before FIFO insertion/eviction. Ten new running-server HTTP regressions passed, together with the full current analytics/cache snapshot: 865/865, line coverage 91.86%, branch coverage 77.11% (`artifacts/delivery/cache-batch-coverage.log`). The final canonical Release build passed with 0 warnings/errors. Later parallel visual/token-composition changes were included only in that final build, not in the preceding coverage snapshot; their behavior qualification remains separate. The commit deliberately excludes the owner's in-progress visual and token-composition changes. + +## First native-tool code checkpoint + +[OpenAI native web-search fixtures](OpenAiNativeWebSearch.md) and [ADR-0006](../ADR/ADR-0006-ordered-native-tool-output.md) add typed ordered native/function/message output, provider-qualified declarations/options, URL citations, native SSE lifecycle, explicit/stored continuation, existing queue/graph/replay semantics and bounded structured query/export evidence. The initial completed search/ASCII-citation contract is qualified for gpt-4.1-mini; the preview SDK choice is not silently aliased. File search, code interpreter, broader native options/models/actions and the other E5 provider families remain active work. + +Native focused tests passed 99/99; existing provider namespace/evidence passed 85/85. The current working-tree snapshot passed 966/966 with 91.94% production line coverage; format and final canonical Release build passed. All 19 packages were packed, the clean installed NuGet/Aspire/SDK/Chromium consumer passed 2/2 (including native JSON/SSE and a complete mixed function round), and a .NET SDK-built linux/arm64 image passed actual native HTTP smoke. Logs: `artifacts/delivery/native-web-*`; container evidence: `artifacts/container-verification/2cd0409b3d183025`. Parallel owner visual/token-composition work is not part of the upcoming code-only checkpoint, whose exact-head qualification is recorded separately. Cache checkpoint `d990e06` CI [39462950086](https://github.com/managedcode/LlmTestServer/actions/runs/39462950086) and website validation [39462950106](https://github.com/managedcode/LlmTestServer/actions/runs/39462950106) passed. + +The independent code-only checkout passed 965/965 server integrations after the ordinary-response regression fix, with 91.96% production line coverage, 77.38% branch coverage and 94.38% method coverage. Canonical formatting and Release build passed with 0 warnings/errors. These results exclude the parallel owner analytics/visual work. Evidence: `artifacts/delivery/native-isolated-final-{format,build,coverage}.log`. + +The same code-only checkout packed all 19 packages; the fresh installed NuGet/Aspire/official SDK/Chromium consumer passed 2/2. Its .NET SDK-built linux/arm64 image passed actual HTTP native JSON/SSE and terminal-evidence acceptance. Evidence: `artifacts/delivery/native-isolated-{pack,consumer,container-publish,container-smoke}.log`; container evidence `artifacts/native-code-validation/artifacts/container-verification/220fcd85c2a76ed5`. No public package/image publication is claimed. + +## Native file-search and code checkpoint + +The bounded [OpenAI file-search/code fixture contract](OpenAiNativeFileCode.md) adds completed native calls, ordered file/container citations, include projections, code deltas, explicit/automatic container references and file filter/result admission. It uses the same queue/graph, history, cache, replay and evidence paths. File search is independently qualified for gpt-4o-mini and code interpreter for gpt-4.1; no retrieval or code execution occurs. + +Focused running-server HTTP/SDK tests passed 191/191, including 24 exact-numeric regressions and 17 normalization cases. Existing web-search regressions passed 105/105. Exact numeric comparison corrects a demonstrated HTTP false match between 1e-29 and 2e-29; ID-shape and ordinary-group handling prevent normalization errors without weakening native history bounds. The code-only final qualification is recorded below. Other E5 families and E6–E13 remain unfinished. + +### File/code code-only qualification + +- Focused file/code HTTP/official SDK: 191/191, including 24 exact-number and 17 normalization cases. +- Existing web-search regression: 105/105. +- Full isolated server suite: 1158/1158; production line coverage 92.21%, branch coverage 78.21%, method coverage 94.55%. +- Canonical format verification and final Release build passed with 0 warnings/errors. +- All 19 packages packed. A .NET SDK `PublishContainer` linux/arm64 image passed actual web/file/code JSON/SSE, includes, citations, code deltas, management/auth/reset and terminal-evidence smoke. + +Evidence: `artifacts/delivery/native-file-code-*`; container record: `artifacts/native-code-validation/artifacts/container-verification/bb0d14644caef9d4`. This checkout excludes the concurrent owner analytics/visual changes. Public package/image publication, owner merge and remaining E5–E13 work are not asserted by these local results. + +The clean installed NuGet/Aspire/official SDK/Chromium consumer passed 3/3 from an isolated package cache, including independent file/code JSON/SSE and stored continuation. Evidence: `artifacts/delivery/native-file-code-consumer.log`. diff --git a/docs/Features/LlmTckScenarioRuntime.md b/docs/Features/LlmTestServerScenarioRuntime.md similarity index 100% rename from docs/Features/LlmTckScenarioRuntime.md rename to docs/Features/LlmTestServerScenarioRuntime.md diff --git a/docs/Features/NativeBuiltInToolContractInventory.md b/docs/Features/NativeBuiltInToolContractInventory.md new file mode 100644 index 0000000..dc49823 --- /dev/null +++ b/docs/Features/NativeBuiltInToolContractInventory.md @@ -0,0 +1,151 @@ +# Native built-in tool contract inventory + +Status: **Bounded OpenAI completed web/file/code contracts locally qualified: file/code 191/191, web regressions 105/105, full server suite 1158/1158; remaining contracts proposed.** +Audit date: **2026-10-20**. +Scope: first E5 native search, file retrieval and code execution contracts for OpenAI Responses, +Anthropic Messages and Gemini generateContent. The bounded OpenAI completed-search subset is described in [its feature contract](OpenAiNativeWebSearch.md). +The additional [bounded file/code contract](OpenAiNativeFileCode.md) is implemented and locally qualified; public delivery remains pending. +Unresolved cells and other providers below remain prerequisites/proposed contracts, not support claims. +Related: [delivery plan](LlmTestServerDeliveryPlan.md), [conversation DSL](ConversationDSL.md), +[request verification](RequestVerificationAndDeterminism.md), [existing function-tool fixtures](ToolAndStructuredFixtures.md). + +## Implementation boundaries + +Provider packages must own declarations, validation, native output items/blocks, citations and streaming events. +Core may own typed deterministic fixture data and ordered native tool results within conversation steps. +A native hosted tool is not equivalent to a user-defined function. Responses remain scripted: this work does +not execute searches, programs or external side effects. Every claimed option requires positive/negative +running-server HTTP and SDK evidence. Unknown or unresolved cells remain outside support claims. + +```mermaid +flowchart LR + Request[Native tool declaration] --> Validate[Provider contract validation] + Validate --> Fixture[Typed deterministic tool fixture] + Fixture --> Items[Provider call and result items] + Items --> Citation[Text and native citations] + Items --> Stream[Provider stream lifecycle] + Citation --> SDK[Actual HTTP and SDK consumers] + Stream --> SDK +``` + +## OpenAI Responses + +Endpoint: upstream `POST /v1/responses`; hosted namespace `/openai/v1/responses`. Standard Bearer authentication. +Request-level `tools`, `tool_choice`, `include` and `stream` interact with each native tool. + +| Contract | Proposed request support | Native result and citation contract | +| --- | --- | --- | +| `web_search` | `search_context_size`: low/medium/high; approximate `user_location`; domain filters; `external_web_access`; `return_token_budget`: default/unlimited | `web_search_call` with ID/status/action; search/open_page/find_in_page actions; output-text `url_citation` with URL/title/start/end; optional action sources via `include` | +| `file_search` | Required `vector_store_ids`; `max_num_results` 1–50; attribute filters; ranking options | `file_search_call` and queries; optional included results; `file_citation` containing file ID, filename and index | +| `code_interpreter` | Required container ID or auto container object; optional auto file IDs/memory tier | Code-interpreter call ID/container/code/outputs; generated artifacts cited as `container_file_citation` | + +Web return-token budget applies to GPT-5+ reasoning web search; location is inapplicable to deep research. +Preview tool variants have different option applicability. Native prefixes and citation offsets must be preserved. +References: [web tool](https://developers.openai.com/api/docs/guides/tools-web-search), +[file tool](https://developers.openai.com/api/docs/guides/tools-file-search), +[request schema](https://developers.openai.com/api/reference/resources/responses/methods/create), +[code tool](https://developers.openai.com/api/docs/guides/tools-code-interpreter). + +Streaming adds output-item lifecycle and `response.web_search_call.{in_progress,searching,completed}`, +`response.file_search_call.{in_progress,searching,completed}`, +`response.code_interpreter_call.{in_progress,interpreting,completed}` and +`response.code_interpreter_call_code.{delta,done}`. Each carries the relevant item/output index and sequence; +text/citations and final response completion must remain consistent with ordinary JSON output. +Reference: [streaming events](https://developers.openai.com/api/reference/resources/responses/streaming-events). + +| Unresolved prerequisite | Required resolution before support claim | +| --- | --- | +| Tool-specific errors | Completed results are the implemented subset; failed/incomplete result/error relationships remain pending | +| File-search result spelling | Resolved as `results`; no alternate field is emitted. Bounded implementation is locally qualified | +| Expanded ranking options | Bounded ranker/threshold and comparison/compound filters implemented; hybrid weights remain unsupported pending separate qualification | +| Model applicability | Current independent pairings: web/gpt-4.1-mini, file/gpt-4o-mini, code/gpt-4.1; broader combinations remain pending | +| Containers/artifacts | Pin memory tiers, file download contracts, container reuse and bounded fixture ownership; coordinate with E9 lifecycle | +| SDK | Completed subset qualified through official .NET SDK JSON/SSE and a fresh installed-package consumer; broader native contracts still need independent qualification | + +## Anthropic Messages + +Endpoint: upstream `POST /v1/messages`; hosted namespace `/anthropic/v1/messages`. Standard `x-api-key` and +`anthropic-version` apply. Request includes model, max_tokens, messages, tools and optional stream. + +### Web search + +Types: `web_search_20250305`, `web_search_20260209`, `web_search_20260318`; fixed name `web_search`. +Options include max_uses, mutually exclusive domain allow/block lists, approximate location and allowed_callers. +The newest version adds response_inclusion; dynamic filtering starts with the 20260209 contract and depends on +model/code-execution capability. Results contain server_tool_use and web_search_tool_result blocks; +web_search_result_location citations identify URL/title and encrypted citation identity. Usage includes search count. +Tool failures can remain HTTP 200 with typed result errors; empty results are an empty list. Streaming carries +server-tool starts, input JSON deltas, result blocks and final message lifecycle. +Reference: [web-search contract](https://platform.claude.com/docs/en/agents-and-tools/tool-use/web-search-tool). + +### Code execution + +Types: `code_execution_20250825`, `code_execution_20260120`, `code_execution_20260521`; fixed name +`code_execution`. These versions do not require a beta header. The base contract supports Bash/file operations; +newer versions add programmatic calls/REPL behavior. Native sub-tool calls/results cover bash_code_execution and +text_editor_code_execution; outputs include stdout/stderr/return_code or typed file-operation results. Container +identity/expiry and explicit reuse are part of the contract. Streaming sends input_json_delta and whole result +blocks. Sonnet 4.5 is included in the documented code-execution model list. +References: [code contract](https://platform.claude.com/docs/en/agents-and-tools/tool-use/code-execution-tool), +[version directory](https://platform.claude.com/docs/en/agents-and-tools/tool-use/tool-reference). + +### File retrieval and citations + +An Anthropic Messages native `file_search` declaration was **not confirmed** in the tool directory. Do not invent +one. Documented retrieval uses a custom tool or supplied search_result content with source/title/text blocks +and citations.enabled. Output search_result_location citations carry source/index/block ranges. This is an +independent retrieval/citation capability, not a hosted file-search-tool claim. +Reference: [search-result contract](https://platform.claude.com/docs/en/build-with-claude/search-results). + +| Unresolved prerequisite | Required resolution before support claim | +| --- | --- | +| Exact errors | Pin complete version-specific error codes and result unions for web/Bash/file operations | +| Version/model matrix | Validate newer tool versions, allowed_callers and programmatic capability per model | +| Nested execution | Pin caller envelopes, response_inclusion full/excluded and their interaction with direct calls | +| Continuation | Implement documented pause_turn/resumption and mixed client/server parallel-call behavior | +| Artifacts | Pin generated-file retrieval, container lifecycle and ownership without claiming unimplemented Files APIs | +| Streaming/SDK | Pin citation deltas, block indexes, tool-error streams and actual SDK parsing; verify complete native/function loops | + +## Gemini generateContent + +Use the generateContent-specific references below. Generic Gemini tool guides now also describe Interactions; +its `tools:[{type:...}]` declarations must not be copied into generateContent requests. +Endpoints: upstream `POST /v1beta/models/{model}:generateContent` and `:streamGenerateContent?alt=sse`; +hosted paths add `/gemini`. Standard `x-goog-api-key` authentication applies. + +| Tool object | Request/output schema | +| --- | --- | +| `googleSearch` | Optional timeRangeFilter/searchTypes; candidate grounding metadata with search queries, sources, supports and searchEntryPoint | +| `fileSearch` | Required fileSearchStoreNames; optional metadataFilter/topK; retrieved grounding chunks/citations | +| `codeExecution` | Empty declaration; executableCode and codeExecutionResult content parts, matching optional IDs; PYTHON code and explicit execution outcome/output | + +There are no date-versioned type IDs in these tool objects. The code outcomes are OUTCOME_OK, +OUTCOME_FAILED and OUTCOME_DEADLINE_EXCEEDED. Grounding streams must preserve incremental metadata semantics. +Reference: [generateContent REST schema](https://ai.google.dev/api/generate-content). + +Search citations connect text segments to grounding chunks and web URI/title; searchEntryPoint carries search +suggestions. [GenerateContent search guide](https://ai.google.dev/gemini-api/docs/generate-content/google-search). +File-search model applicability includes Gemini 2.5 Pro/Flash-Lite and documented Gemini 3 models; do not infer +Gemini 2.5 Flash support from older examples. Upload/import/store lifecycle is separate from the generation +fixture. [File-search guide](https://ai.google.dev/gemini-api/docs/generate-content/file-search), +[code-execution guide](https://ai.google.dev/gemini-api/docs/generate-content/code-execution). + +| Unresolved prerequisite | Required resolution before support claim | +| --- | --- | +| Model/tool combinations | Pin exact combinations, structured-output interaction and code-capable model list | +| File-search options | Pin metadata-filter grammar, topK bounds and invalid/foreign store behavior | +| Citation shapes | Pin the full retrievedContext and support/index schemas, including streamed incremental chunks | +| Tool errors | Pin search/file error shapes separately from execution outcome parts and top-level provider errors | +| Streaming | Exercise native results in actual SSE and supported JSON streaming response modes; do not invent named Responses-style events | +| SDK | Verify installed official SDK native parsing and mixed function/native flow before expanding package capability claims | + +## Qualification and completion + +For each selected operation/tool/version, check in the precise supported option/result/error subset alongside +official links and provider operation evidence. Run the server and test ordinary JSON, streaming, every claimed +parameter boundary, native errors, citations, identity consistency, mixed function/native conversation flows, +session isolation and fixture-capacity failure. Preserve deterministic wire metadata and real observations. + +File/container/store references require explicit fixtures and ownership rules; unimplemented lifecycle APIs +remain visible E9 dependencies. No unresolved cell becomes a capability claim merely because a permissive DTO +accepts its JSON. No qualification result is asserted by this document. diff --git a/docs/Features/OpenAiNativeFileCode.md b/docs/Features/OpenAiNativeFileCode.md new file mode 100644 index 0000000..6cc4c65 --- /dev/null +++ b/docs/Features/OpenAiNativeFileCode.md @@ -0,0 +1,187 @@ +# OpenAI Responses native file-search and code fixtures + +Status: implemented bounded contracts. **191 running-server HTTP/official SDK cases passed**, including exact numeric-filter and normalization regressions; the web-search regression suite passed **105/105**. Release build has zero warnings/errors. Full-suite/package/container qualification is recorded below; this is not a full E5 completion or publication claim. + +These fixtures serve `POST /openai/v1/responses`. They return configured native output items, citations and SSE events without retrieving files, executing code, fetching image URLs or starting containers. File/container/store IDs are explicit fixture references. They do not imply working resource upload, download, creation or deletion APIs. + +## Independent model contracts + +| Native declaration | Current fixture model | Primary-source pairing | +| --- | --- | --- | +| `web_search` | `gpt-4.1-mini` | [Existing web contract](OpenAiNativeWebSearch.md) | +| `file_search` | `gpt-4o-mini` | [Official file-search cookbook](https://developers.openai.com/cookbook/examples/file_search_responses) | +| `code_interpreter` | `gpt-4.1` | [Official tool cookbook](https://developers.openai.com/cookbook/examples/mcp/mcp_tool_guide) | + +The pairings are separate. A current request cannot declare multiple native families whose supported model sets have no common member. Raw model checks apply to current declarations and named choices; completed prior tool outputs and include selectors alone do not re-qualify a past tool against the current model. The core independently checks the qualified native surface and selected fixture. Other provider namespaces and Chat Completions reject native declarations/history. + +## Configure a file-search fixture + +The following uses a control client connected to the running server: + +```csharp +using System.Text.Json; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Scenarios; + +await control.ConfigureAsync(config => config.UseGpt4OMini() + .UseChatScenario("file-search", scenario => scenario + .ForModel(LlmTestServerKnownModelIds.Gpt4OMini) + .RespondsWithItems( + new LlmTestServerResponseFileSearchItem + { + Id = "fs_fixture", + VectorStoreIds = ["vs_fixture"], + Queries = ["Paris fixture facts"], + Results = [new() + { + FileId = "file_source", Filename = "source.txt", + Text = "Scripted Paris source", Score = 0.9, + Attributes = new() + { + ["category"] = JsonSerializer.SerializeToElement("fixture"), + ["year"] = JsonSerializer.SerializeToElement(2026), + }, + }], + }, + new LlmTestServerResponseMessageItem + { + Id = "msg_file", Content = "Paris fixture.", + Citations = [new LlmTestServerFileCitation + { FileId = "file_source", Filename = "source.txt", Index = 5 }], + }))); +``` + +Every fixture store must appear in the request's `vector_store_ids`. The request may declare additional stores. Configured resource-reference lists are distinct. Duplicate declaration IDs are accepted and preserved; matching treats resource references as sets. An empty declaration array is syntactically valid but cannot satisfy a fixture requiring stores. Missing/foreign stores produce a 409 fixture mismatch without consuming the queued response or graph step. + +`max_num_results` is an integer from 1 to 50. The configured result count must already fit that limit; the engine does not silently cut the fixture down. A supplied `score_threshold` must be finite and between zero and one; every configured result then needs a score meeting it. Each result must also satisfy the requested attribute filter. `ranker` accepts `auto` and `default-2024-11-15`, and is retained in normalized declarations and request identity; it does not run a ranking algorithm. These request/result fields follow the [Responses schema](https://developers.openai.com/api/reference/resources/responses/methods/create) and [file-search guide](https://developers.openai.com/api/docs/guides/tools-file-search). + +## Configure a code-interpreter fixture + +```csharp +await control.ConfigureAsync(config => config.UseGpt41() + .UseChatScenario("code", scenario => scenario + .ForModel(LlmTestServerKnownModelIds.Gpt41) + .RespondsWithItems( + new LlmTestServerResponseCodeInterpreterItem + { + Id = "ci_fixture", ContainerId = "cntr_fixture", + InputFileIds = ["file_input"], + GeneratedFileIds = ["file_plot"], + Code = "print(42)\n", CodeChunks = ["print(", "42)\n"], + Outputs = [ + new LlmTestServerCodeInterpreterLogsOutput { Logs = "42\n" }, + new LlmTestServerCodeInterpreterImageOutput + { Url = "https://example.com/fixture.png" }, + ], + }, + new LlmTestServerResponseMessageItem + { + Id = "msg_code", Content = "42 fixture.", + Citations = [new LlmTestServerContainerFileCitation + { + ContainerId = "cntr_fixture", FileId = "file_plot", + Filename = "plot.png", StartIndex = 0, EndIndex = 0, + }], + }))); +``` + +An explicit request container must equal the configured `ContainerId`. In automatic mode, the distinct requested `file_ids` set must equal the fixture's `InputFileIds`; declaration order/duplicates remain preserved in normalized request data. Auto mode returns the configured container identity. It does not provision a container. + +Container-file citations must reference that exact container and one of its configured `InputFileIds` or `GeneratedFileIds`. Generated references are explicit ownership metadata, not a side effect of evaluating `Code`. A mismatch returns 409 before fixture consumption. Nullable `Code` and nullable/empty `Outputs` remain distinct scripted states; configured chunks must concatenate exactly to non-null code. Only logs and explicit HTTP(S) image URLs are implemented. The [code-interpreter guide](https://developers.openai.com/api/docs/guides/tools-code-interpreter) documents the native container/code/output and generated-file citation contracts; lifecycle/download behavior remains outside this slice. + +## Official .NET SDK options + +Use `OpenAI` 2.14.0 `ResponsesClient` with its endpoint resolved from the running server's `/openai/v1/` surface and its real HTTP transport. For the file fixture above: + +```csharp +using OpenAI.Responses; + +var fileOptions = new CreateResponseOptions +{ + Model = "gpt-4o-mini", + InputItems = { ResponseItem.CreateUserMessageItem("Use the file fixture") }, + Tools = { ResponseTool.CreateFileSearchTool( + ["vs_fixture"], 2, + new FileSearchToolRankingOptions + { Ranker = FileSearchToolRanker.Auto, ScoreThreshold = 0.5f }, + BinaryData.FromString("""{"type":"eq","key":"category","value":"fixture"}""")) }, + ToolChoice = ResponseToolChoice.CreateFileSearchChoice(), + IncludedProperties = { IncludedResponseProperty.FileSearchCallResults }, +}; +var fileResponse = (await sdk.CreateResponseAsync(fileOptions)).Value; +var search = fileResponse.OutputItems.OfType().Single(); +``` + +For the independently configured code fixture: + +```csharp +var container = new CodeInterpreterToolContainer( + CodeInterpreterToolContainerConfiguration.CreateAutomaticContainerConfiguration(["file_input"])); +// An explicit container uses: new CodeInterpreterToolContainer("cntr_fixture"). +var codeOptions = new CreateResponseOptions +{ + Model = "gpt-4.1", + InputItems = { ResponseItem.CreateUserMessageItem("Use the code fixture") }, + Tools = { ResponseTool.CreateCodeInterpreterTool(container) }, + ToolChoice = ResponseToolChoice.CreateRequiredChoice(), + IncludedProperties = { IncludedResponseProperty.CodeInterpreterCallOutputs }, +}; +var codeResponse = (await sdk.CreateResponseAsync(codeOptions)).Value; +var execution = codeResponse.OutputItems.OfType().Single(); +``` + +Raw HTTP additionally supports named `{"type":"code_interpreter"}` choice. The installed SDK's typed surface has no corresponding named-code factory; use its required choice for this example. Existing `auto`, `none`, `required`, named native and named function choices are checked against the actual ordered fixture. Native items do not become client function calls. + +## Filters and bounded input + +The supported filter operators are `eq`, `ne`, `gt`, `gte`, `lt`, `lte`, `in`, `nin`, `and`, `or`. This is an explicit bounded fixture contract: + +- `eq`/`ne` compare matching scalar kinds: strings, finite JSON numbers or booleans. +- Ordering compares strings ordinally or numbers numerically. Boolean ordering is rejected with 400. +- `in`/`nin` require a nonempty homogeneous string or number array. Boolean/mixed/null arrays are rejected. +- Missing attributes and mismatched kinds do not match, including `ne` and `nin`. +- Compound filters evaluate all/any child matches. Root depth is one; maximum depth is eight and the entire tree has at most 128 nodes. Comparison/member arrays and compound child arrays have at most 128 entries. +- Attribute keys are nonblank and at most 64 characters. Results have at most 16 scalar attributes; attribute string values have at most 512 characters. + +Numeric comparisons use normalized decimal digits/exponents rather than rounding both operands to a binary floating-point value. JSON values must still satisfy finite-number admission. The HTTP suite verifies distinct tiny and large numbers, numeric ordering, membership, signed zero and equivalent number representations. A pre-fix HTTP regression demonstrated a false match for `1e-29` versus `2e-29`; the exact comparator fixes it without expanding exponent powers. + +Native item/resource IDs are bounded to 128 characters. File queries, results, store and input/generated file references and code outputs/chunks have at most 128 entries; file queries are nonempty. A file-search fixture owns at least one store; request store arrays may be empty. Native assistant groups contain at most 128 ordered items, with distinct effective nonempty output IDs. Citation lists are also bounded to 128. These collection/identity limits belong to our fixture contract, rather than being attributed to arbitrary upstream limits. + +`filters` may be omitted/null; an actual filter object must match its comparison/compound shape. `ranking_options` may be omitted or `{}` but not null. Optional ranker/threshold/max-result fields reject explicit null. Code `container` is required: a nonempty ID or `{ "type":"auto", "file_ids":[...] }`; absent `file_ids` and `[]` are valid, while null is not. Unknown tool/filter/result fields are rejected. Hybrid ranking, auto-container memory/network controls and additional native tool options remain pending and return 400 instead of being accepted and ignored. + +## Ordered control data and provider output + +Control configuration uses `kind` for output items and annotations; the wire API uses provider `type` values: + +| Control `kind` | API representation | +| --- | --- | +| `file_search` | `file_search_call` with ID/status/queries and optional `results` | +| `code_interpreter` | `code_interpreter_call` with container ID, nullable code and nullable outputs | +| Message citation `file` | `file_citation`: file_id, filename, insertion index | +| Message citation `container_file` | `container_file_citation`: container/file IDs, filename, start/end | +| Code output `logs` / `image` | `type: logs` with logs / `type: image` with url | + +One ordered citation union retains URL, file and container-file annotation order. A file citation references a configured file-search result and its filename when that result supplies one. Citation text is ASCII in this initial subset. File insertion indexes are between zero and text length. Container ranges allow `start == end`, matching documented zero-range examples; URL ranges retain the existing positive-length boundary. Uncited text retains Unicode support. + +Results are included by `file_search_call.results`; without it, the optional field is omitted. Code outputs use `code_interpreter_call.outputs`; excluded or unavailable outputs are null, while an included scripted empty output list remains `[]`. Include projection does not remove citations or mutate stored fixture data. File result fields retain their optionality, including absent metadata; the field is `results`, with no alternative spelling. + +SSE retains output-item added/done and final response completion. File calls emit in_progress/searching/completed events. Code calls emit in_progress, configured code deltas and code.done when code is non-null, interpreting and completed. Logs/images are completed output data, with no invented logs/image delta events. The existing annotation-added event carries each ordered citation. The final JSON output agrees with nonstreaming output; terminal evidence is recorded only after final completion is written/flushed. See the [official streaming schemas](https://developers.openai.com/api/reference/resources/responses/streaming-events). + +## History, evidence and remaining qualification + +Explicit prior native output items and `previous_response_id` use the existing history/replay/capacity engine. Past native calls retain their kinds; they are not synthetic user JSON. Provider namespace and credential ownership apply inside the session runtime. A conversation header labels graph/correlation scope and does not create another history authorization boundary. Reset/configure invalidates existing retained history through generation ownership. + +Request queries and JSON/NDJSON exports retain typed declarations, results, citations and assistant output groups within the existing diagnostic budgets. Nested data is copied; omitted evidence is marked truncated. Function `ResponseToolCalls` stays separate. Native declaration/filter/resource options remain preserved in fingerprints, provider-scoped prompt-cache identity and normalized usage. Cache reads/writes and reasoning remain subsets of input/output totals. + +The local qualification below covers the focused suites, full server/coverage gate, clean installed-package Aspire/SDK consumer and actual container smoke. Public delivery remains pending. Native failed/incomplete results, unqualified model pairings, other providers and resource lifecycle remain in [the contract inventory](NativeBuiltInToolContractInventory.md) and [delivery plan](LlmTestServerDeliveryPlan.md). Related design: [ADR-0006](../ADR/ADR-0006-ordered-native-tool-output.md). + +## Code-only qualification + +- Focused file/code HTTP/official SDK: 191/191, including 24 exact-number and 17 normalization cases. +- Existing web-search regression: 105/105. +- Full isolated server suite: 1158/1158; production line coverage 92.21%, branch coverage 78.21%, method coverage 94.55%. +- Canonical format verification and final Release build passed with 0 warnings/errors. +- All 19 packages packed; a fresh installed NuGet/Aspire/official SDK/Chromium consumer passed 3/3, including independent file/code JSON/SSE and stored continuation. A .NET SDK `PublishContainer` linux/arm64 image passed actual web/file/code JSON/SSE, includes, citations, code deltas, management/auth/reset and terminal-evidence smoke. + +Evidence: `artifacts/delivery/native-file-code-*`; container record: `artifacts/native-code-validation/artifacts/container-verification/bb0d14644caef9d4`. This checkout excludes the concurrent owner analytics/visual changes. Public package/image publication, owner merge and remaining E5–E13 work are not asserted by these local results. diff --git a/docs/Features/OpenAiNativeWebSearch.md b/docs/Features/OpenAiNativeWebSearch.md new file mode 100644 index 0000000..229d6ed --- /dev/null +++ b/docs/Features/OpenAiNativeWebSearch.md @@ -0,0 +1,155 @@ +# OpenAI Responses native web-search fixtures + +The first native-tool slice implements a **scripted completed search** for `gpt-4.1-mini` on +`POST /openai/v1/responses`. Its focused suite passes **99 running-server HTTP and official SDK cases** with a Release build containing +**zero warnings and zero errors**. Local qualification and the independent code-only checkpoint are recorded below and in the [delivery plan](LlmTestServerDeliveryPlan.md). + +The server returns native `web_search_call` items, assistant text, URL citations and native streaming events. +Search queries, source URLs and answer text come from configuration. No URL is fetched and no search is executed. +The provider-neutral [function-tool presets](WellKnownFunctionTools.md) remain a separate fixture surface. + +## Configure through the control client + +```csharp +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Scenarios; + +await control.ConfigureAsync(config => config.UseGpt41Mini() + .UseConversation("research", graph => graph + .Step("search", step => step.RespondsWithItems( + new LlmTestServerResponseWebSearchItem + { + Id = "ws_fixture", + Queries = ["Paris fixture facts"], + Sources = [new() { Url = "https://example.com/source" }], + }, + new LlmTestServerResponseMessageItem + { + Id = "msg_fixture", + Content = "Paris fixture.", + StreamChunks = ["Paris ", "fixture."], + Citations = [new LlmTestServerUrlCitation() + { + StartIndex = 0, + EndIndex = 5, + Url = "https://example.com/source", + Title = "Fixture source", + }], + }).Complete()))); +``` + +`RespondsWithItems(...)` also works on a queued chat scenario. Its item order is the response order; +`LlmTestServerResponseFunctionCallItem` permits mixed client function calls, native searches and messages. +A function item in a mixed native plan requires its own explicit `Id`, distinct from its tool-call correlation ID +and from every other output item ID. +Existing delays, conversation transitions, replay, reset and capacity settings apply to the same engine. +An error or legacy response fields cannot be combined with ordered output items in the same fixture. + +Fixture configuration validates item identities, HTTP(S) source/citation URLs, query counts, ASCII citation ranges, +message chunk concatenation and bounded collections. A malformed HTTP configuration returns 400 and leaves the +previous configuration active. The initial citation contract accepts ASCII output text when citations are present. + +## Send through the official .NET SDK + +Use the installed `OpenAI` 2.14.0 Responses client with the HTTP endpoint allocated by the host: + +```csharp +using OpenAI.Responses; +using System.ClientModel; +using System.ClientModel.Primitives; + +using var conversation = control.ForConversation("research-example"); +var sdk = new ResponsesClient(new ApiKeyCredential("test-key"), new ResponsesClientOptions +{ + Endpoint = new Uri(conversation.HttpClient.BaseAddress!, "/openai/v1/"), + Transport = new HttpClientPipelineTransport(conversation.HttpClient), +}); +var options = new CreateResponseOptions +{ + Model = "gpt-4.1-mini", + InputItems = { ResponseItem.CreateUserMessageItem("Search the configured fixture") }, + Tools = { ResponseTool.CreateWebSearchTool(null, WebSearchToolContextSize.Low, null) }, + ToolChoice = ResponseToolChoice.CreateRequiredChoice(), + IncludedProperties = { IncludedResponseProperty.WebSearchCallActionSources }, +}; +var response = (await sdk.CreateResponseAsync(options)).Value; +var search = response.OutputItems.OfType().Single(); +``` + +The installed SDK's `CreateWebSearchChoice()` factory emits the distinct preview tool choice, so this +non-preview example uses `CreateRequiredChoice()`; named `web_search` selection is exercised through HTTP. + +The native call parses as `WebSearchCallResponseItem` with a `WebSearchSearchAction`. Included sources parse as +`WebSearchActionUriSource`; output-text citations parse as `UriCitationMessageAnnotation`. Native calls do not +become function calls. `CreateResponseStreamingAsync(options)` emits typed native progress updates, annotation +updates and a final completed response containing the same ordered items. + +## Supported request subset + +| Field | Accepted contract | +| --- | --- | +| Model and route | `gpt-4.1-mini`, `/openai/v1/responses` | +| Tool declaration | One active scripted `type: web_search` contract; function tools may accompany it | +| `search_context_size` | Omitted, `low`, `medium` or `high`; explicit null is invalid | +| `user_location` | Optional/null approximate-location object; optional city, region, country and timezone fields; nullable fields follow the provider schema | +| Country/timezone | Known two-letter country code, case insensitive; a supplied timezone must identify a valid IANA timezone | +| `filters.allowed_domains` | Optional/null list, up to 100 bare DNS hostnames; empty list is unrestricted; URLs, paths, IP addresses and wildcards are invalid | +| `tool_choice` | `auto`, `none`, `required`, named web search or existing named function choice | +| `include` | `web_search_call.action.sources`; repeated entries are permitted | +| Continuation | Explicit prior native output items in an input array, or `previous_response_id` | + +Without the sources include, search sources are omitted from the wire response; text citations remain present. +A configured source or citation outside a declared domain allowlist produces a 409 fixture mismatch. Missing, repeated or +disabled declarations also produce 409 before the queued response or graph step is consumed. Changing native +declaration options changes the normalized request and prompt-cache prefix identity. + +Stored history retains ordered native outputs as one assistant response. Provider namespace and credential +ownership apply within the session runtime. A conversation header selects graph/correlation scope; it does not +create an additional stored-response access boundary. Reset invalidates stored response IDs; `store: false` +creates no stored parent. Native payloads contribute to existing history and conversation byte budgets. + +Public request queries and JSON/NDJSON export expose typed `NativeTools`, `ResponseOutputItems` and retained +message output items. Function-only `ResponseToolCalls` continue to contain client function calls. Diagnostic +retention remains bounded and explicitly marks omitted payload evidence. + +## Remaining E5 scope + +This slice makes no support claim for `web_search_preview`, open/find actions, native tool failures, +non-ASCII citation-offset semantics, external-web-access/return-token-budget controls, +resource lifecycle or native tools on other provider namespaces. File search and code interpreter use their +separate [bounded fixture contract](OpenAiNativeFileCode.md). Remaining contracts are tracked in the +[native contract inventory](NativeBuiltInToolContractInventory.md) and delivery plan. + +## Contract sources and integration evidence + +Official references: [web-search guide](https://developers.openai.com/api/docs/guides/tools-web-search), +[Responses create contract](https://developers.openai.com/api/reference/resources/responses/methods/create), +[streaming events](https://developers.openai.com/api/reference/resources/responses/streaming-events). +The web-search guide explicitly lists `gpt-4.1-mini`; the implemented subset is narrower than the full upstream API. + +Running-server coverage: + +- [JSON, SDK, mixed output and continuation](../../tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeWebSearchJsonIntegrationTests.cs) +- [SDK and raw SSE lifecycle](../../tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeWebSearchStreamIntegrationTests.cs) +- [Request options, schema boundaries and namespace guards](../../tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeWebSearchValidationIntegrationTests.cs) +- [Atomic configuration and native payload budgets](../../tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeWebSearchConfigurationIntegrationTests.cs) +- [Cancellation, reset, replay, ownership and domain mismatch](../../tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeWebSearchLifecycleIntegrationTests.cs) +- [Native declaration options and real prompt-cache usage](../../tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeWebSearchCacheIntegrationTests.cs) + +## Local qualification + +- Native HTTP/official SDK suite: 99/99. +- Existing provider behavior/contract namespace: 85/85. +- Full current working-tree integration snapshot: 966/966, production line coverage 91.94%. +- Canonical format verification and final Release build: passed, 0 warnings/errors. +- All 19 packages packed; independent clean NuGet/Aspire/SDK/Chromium consumer: 2/2, including the separate native JSON/SSE and mixed-function round. +- .NET SDK `PublishContainer` image: actual native JSON/source gating/citations/SSE/terminal evidence smoke passed on linux/arm64. + +Evidence logs are repository-owned under `artifacts/delivery/native-web-*`; container evidence is in +`artifacts/container-verification/2cd0409b3d183025`. The full working-tree count includes parallel owner analytics +work; a code-only checkpoint and its exact-head CI are tracked separately in the delivery plan. No NuGet/GHCR +release or main merge is asserted by these local results. + +The independent code-only checkout passed 965/965 server integrations after the ordinary-response regression fix, with 91.96% production line coverage, 77.38% branch coverage and 94.38% method coverage. Canonical formatting and Release build passed with 0 warnings/errors. These results exclude the parallel owner analytics/visual work. Evidence: `artifacts/delivery/native-isolated-final-{format,build,coverage}.log`. + +The same code-only checkout packed all 19 packages; the fresh installed NuGet/Aspire/official SDK/Chromium consumer passed 2/2. Its .NET SDK-built linux/arm64 image passed actual HTTP native JSON/SSE and terminal-evidence acceptance. Evidence: `artifacts/delivery/native-isolated-{pack,consumer,container-publish,container-smoke}.log`; container evidence `artifacts/native-code-validation/artifacts/container-verification/220fcd85c2a76ed5`. No public package/image publication is claimed. diff --git a/docs/Features/OutputPolicyFixtures.md b/docs/Features/OutputPolicyFixtures.md new file mode 100644 index 0000000..08a511c --- /dev/null +++ b/docs/Features/OutputPolicyFixtures.md @@ -0,0 +1,207 @@ +# Output policy fixtures + +The qualified text output-policy mappings below passed **79/79 focused running-server SDK cases** +and **1,438/1,438 full integration cases**, with **92.58% production line coverage**. Tests use +OpenAI 2.14.0, Anthropic 12.58.0 with strict response validation, Google.GenAI 1.26.1 and the +package's `Microsoft.Extensions.AI` client. Other E6 outcomes and provider families remain open. + +## Configure an explicit output outcome + +`LlmTestServerScenarioResponse.OutputDisposition` accepts an immutable +`LlmTestServerOutputDisposition` from `ManagedCode.LlmTestServer.Scenarios`. `Kind` is +`LlmTestServerOutputDispositionKind.Filtered` or `.Refused`. Optional `Category` and `Explanation` +carry bounded outcome metadata. A null disposition preserves the ordinary completion behavior. + +Scenario and conversation-step builders expose `WithOutputDisposition(...)`, applied after a response: + +```csharp +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Scenarios; + +await control.ConfigureAsync(config => config + .UseGpt55() + .UseChatScenario("policy-and-recovery", scenario => scenario + .ForModel(LlmTestServerKnownModelIds.Gpt55) + .Responds("I cannot provide that response.", "I cannot ", "provide that response.") + .WithOutputDisposition(new() + { + Kind = LlmTestServerOutputDispositionKind.Refused, + Category = "general_harms", + Explanation = "Explicit deterministic refusal fixture.", + }) + .Responds("Explicit next attempt."))); +``` + +Use `.Filtered` instead for a filtering fixture and configure an explicit generated prefix or empty +content. `Content` and `StreamChunks` supply deterministic fixture output, including refusal text; +the disposition selects its native representation. The server does not generate a refusal explanation, +execute a policy service or call an upstream model. Existing configure/reset APIs remain the control surface. + +`WithOutputDisposition` requires a preceding response. JSON configuration accepts the same typed shape: + +```json +{ + "content": "Explicit refusal text.", + "streamChunks": ["Explicit ", "refusal text."], + "outputDisposition": { + "kind": "Refused", + "category": "general_harms", + "explanation": "Explicit fixture metadata." + } +} +``` + +This is a response entry inside the existing configuration, not a complete configuration document. + +## Bounds and admission + +- `Kind` must be one of the two named values; the default/unknown enum value is invalid. +- `Category` is optional and limited to 256 C# string characters; `Explanation` is optional and limited + to 4,096. These are local fixture bounds, not advertised upstream limits. +- Explicit chunks must concatenate exactly to `Content`. Empty output is supported. +- A policy response cannot also contain a scripted `Error`, `StreamDelivery`, ordinary `ToolCalls` + or ordered/native `OutputItems`. +- Invalid configuration is rejected atomically with HTTP 400, leaving the previous configuration usable. +- A valid fixture selected by an unsupported route/kind pair, tool declaration or tool-bearing/native + history, native includes or structured-output request is rejected with a fixture-mismatch HTTP 409 + before queue/graph consumption and cache population. + +For Anthropic refusal, `Category` must be null or `cyber`, `bio`, `frontier_llm`, `reasoning_extraction` +or `general_harms`. A different bounded generic category is rejected at Anthropic fixture admission; +the strict SDK's declared category contract is the reference. Other surfaces retain generic metadata +in diagnostics rather than inventing provider fields. See the official +[RefusalStopDetails type](https://github.com/anthropics/anthropic-sdk-csharp/blob/Anthropic-v12.58.0/src/Anthropic/Models/Messages/RefusalStopDetails.cs). + +## Qualified native mappings + +These are normal HTTP 200 output outcomes. Inspect their native fields; do not infer an exception +from the policy outcome alone. + +| Exact provider route | `Filtered` | `Refused` | +| --- | --- | --- | +| `/openai/v1/chat/completions` | Final `finish_reason: content_filter`; explicit content/prefix | `message.refusal`, null ordinary content; streamed `delta.refusal`, then normal stop/terminal sequence | +| `/deepseek/v1/chat/completions` | Final `finish_reason: content_filter`; explicit content/prefix | Unsupported pairing: fixture mismatch | +| `/openai/v1/responses` | `status: incomplete`, `incomplete_details.reason: content_filter`; SSE ends with `response.incomplete` | Typed `refusal` content part; refusal delta/done events; completed response | +| `/anthropic/v1/messages` | Unsupported pairing: fixture mismatch | `stop_reason: refusal` and `stop_details` with type `refusal` and required nullable category/explanation; final `message_stop` | +| `/gemini/v1beta/models/{model}:generateContent` and `:streamGenerateContent` | Terminal candidate `finishReason: SAFETY`; terminal content omitted | Unsupported pairing: fixture mismatch | + +The OpenAI mappings follow the official [Chat completion contract](https://developers.openai.com/api/reference/resources/chat/subresources/completions/methods/create) +and [Responses streaming events](https://developers.openai.com/api/reference/resources/responses/streaming-events). +Anthropic refusal is a returned outcome distinct from an SSE error event; see +[refusals and fallback](https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback) +and [streaming](https://platform.claude.com/docs/en/build-with-claude/streaming). +Gemini output `SAFETY` is separate from prompt blocking with `promptFeedback`; this fixture implements +only the candidate output case described by the [GenerateContent API](https://ai.google.dev/api/generate-content). + +Both ordinary JSON responses and text SSE are qualified. Gemini's existing JSON-array streaming +transport also accepts this policy outcome; its separate post-header delivery-fault restriction to +`alt=sse` does not apply here. A filtered Gemini JSON/final candidate has no `content`, so its SDK +`Text` can be empty. Earlier SSE candidates can already have delivered the explicitly configured +prefix. Runtime usage describes the configured generated fixture text; omitted final JSON content +does not retroactively erase generated prefix tokens. This is deterministic fixture accounting, +not a claim of upstream billing or inference equivalence. + +## DeepSeek Chat streaming + +The exact `/deepseek/v1/chat/completions` route accepts `Filtered`; it does not accept `Refused`. +The original OpenAI SDK verifies its compatible JSON/SSE responses against the running server. +The [DeepSeek Chat contract](https://api-docs.deepseek.com/api/create-chat-completion/) specifies +`content_filter` as a native finish reason and a distinct usage layout: the final chunk always has one +choice, a non-null finish reason and usage, followed by `[DONE]`. No separate usage-only chunk is emitted. +With `include_usage: true`, intermediate chunks carry `usage: null`; false or omitted leaves that +field absent until the final chunk. This applies to both ordinary and filtered DeepSeek completions. +Other provider routes retain their own streaming behavior. + +Nineteen additional real-server cases cover empty and prefix output, the three usage options, +unsupported refusal admission without consumption, recovery, graph replay and cache evidence. +Non-streaming requests with a non-null `stream_options` object return HTTP 400 before fixture +consumption, including explicit false and omitted `stream`. Together with the initial policy slice, +the focused result is **79/79**, including three successful raw JSON configuration cases. +The full integration result is **1,438/1,438**; a fresh installed-package consumer passed **16/16**, and the .NET SDK-built linux/arm64 +container passed native DeepSeek JSON/SSE, usage, invalid-options and recovery acceptance. + +## Policy completion, input rejection and broken delivery + +`SimulateContentFilter(...)` retains its existing request/history filtering behavior: a matching input +fails with HTTP 400 and `content_filter` before generation. `Fails(...)` remains an explicit provider +HTTP failure. Neither is a substitute for the native HTTP 200 output outcomes above. + +A policy outcome has a valid protocol ending. OpenAI Chat still finishes its SSE sequence; Responses +uses `response.incomplete` for filtering or `response.completed` for refusal; Anthropic emits +`message_stop`; Gemini emits its terminal `SAFETY` candidate. Once the final sequence is actually +flushed, `TerminalMarkerObserved` is true and terminal-marker verification can pass. Responses +`incomplete` describes the logical output, not a missing transport terminal. + +Use [stream delivery faults](StreamDeliveryFaults.md) for early EOF, post-frame delay or connection +abort. Combining a delivery plan with a policy outcome is rejected rather than assigning an +unqualified precedence. This slice does not simulate Responses `failed`, a provider SSE `error` +event, Gemini prompt blocking, or a native tool's own error result. + +## Queue, graph, cache and history semantics + +A native policy outcome uses the existing successful inference commit path. It consumes the selected +queued response or graph step, records actual explicit output usage, populates the normal qualified +prompt cache and retains an eligible stored response. No separate queue, executor, retry service or +history store is introduced. An explicit next attempt against a flat queue selects its next fixture. +Graph replay returns the committed policy outcome without executing another step; session/provider/ +credential scope continues to use the existing owners. + +Anthropic `max_tokens: 0` remains cache-population-only. It returns before response selection and +produces a new planless result: no output disposition is selected, no queued policy response is +consumed and no graph step executes. A subsequent nonzero generation still selects the configured +policy fixture. + +History preserves the input representation available in the native protocol: + +- An OpenAI refusal is retained as assistant `Content` plus + `LlmTestServerMessage.OutputDisposition.Kind = Refused`. In history, category and explanation are + stripped because native refusal input parts do not carry them. +- Filtered visible text is retained as ordinary assistant prefix input. The prior output-envelope + filter status is not an input-part field, so history has no invented `Filtered` marker. +- The original result, runtime event and request trace retain the full configured outcome metadata. + Input history is not a replacement for that output diagnostic evidence. + +This canonical projection is shared by stored-response restoration, history snapshots, cache keys, +conversation fingerprints and message matching. Automatic `previous_response_id` continuation and +canonical manual SDK history therefore replay the same graph turn. Bounds/enums are validated before +projection. Metadata is excluded from model text tokens while retained objects/strings remain charged +to their graph/history/event/trace budgets; exceptional diagnostic truncation remains visibly marked. + +OpenAI Chat accepts pure native refusal fields or pure typed refusal content parts, including empty +refusal text. Responses accepts its native refusal part. Mixed ordinary text/refusal history and +ambiguous simultaneous refusal representations are outside this qualified subset and are rejected; +do not flatten them into an unmarked assistant answer. Native ordered output items are unchanged. + +## Microsoft.Extensions.AI client + +The package-owned `LlmTestServerChatClient` returns refusal text as `ErrorContent` with +`ErrorCode = "Refusal"`, preserving it separately from ordinary `TextContent`. A streamed refusal is +accumulated into that typed content, and a follow-up request maps it back to native refusal history. +Filtering retains the native `content_filter` finish reason. HTTP 200 policy outcomes do not become +fabricated HTTP failures or automatic thrown exceptions in this wrapper. + +`ErrorContent` is an official AI-content abstraction; this client uses it for the refusal convention above. See its +[upstream source](https://github.com/dotnet/extensions/blob/main/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/ErrorContent.cs). +Applications can choose their own handling of that returned content. This wrapper convention does +not replace the official SDK's distinct native refusal and filtering types. + +## Evidence and remaining E6 scope + +Focused tests under `tests/ManagedCode.LlmTestServer.Tests/Policy` start Kestrel and exercise real +HTTP/SDK behavior: typed JSON/SSE outcomes, valid terminal evidence, empty/prefix text, strict refusal +details, explicit recovery, graph replay, session/cache isolation, manual/stored canonical history, +metadata bounds, unsupported combinations and the existing input-filter distinction. Raw HTTP +supplements configuration and Gemini array boundaries. SDK automatic retries are disabled; Google +automatic continuation is also disabled. The focused result is 79/79, including malformed refusal-history admission, bounded diagnostic +retention, native DeepSeek usage and successful named-enum JSON configuration. The full suite passed +1,438/1,438 with 92.58% line coverage. All 19 packages were packed; a fresh +installed-package consumer passed 16/16, and the .NET SDK-built linux/arm64 container passed +standalone health, dashboard, authorization, management, JSON/SSE, native-tool and DeepSeek +policy/options/recovery acceptance. + +The larger E6 objective remains open for additional provider-specific outcomes and transports, +including in-stream API error events and separately qualified semantics for other provider families. +This document does not claim output-policy support for Azure OpenAI, Foundry, Bedrock, Groq, Mistral, +Ollama, Cohere, OpenRouter, Perplexity, System One decision profiles, Cloudflare decision +profiles or media/native-tool/structured-output combinations. Shared mapper code alone is not +qualification. See [the delivery plan](LlmTestServerDeliveryPlan.md) for the remaining work. diff --git a/docs/Features/ProviderApiContractCoverage.md b/docs/Features/ProviderApiContractCoverage.md index e382740..7ab13d3 100644 --- a/docs/Features/ProviderApiContractCoverage.md +++ b/docs/Features/ProviderApiContractCoverage.md @@ -4,13 +4,13 @@ Current review: [2026-09-07 provider and dependency refresh](ProviderApiReview20 ## Goal -LLM TCK provider packages must not claim provider compatibility from memory or from an OpenAI-compatible shortcut. Every claimed provider method, route, streaming mode, modality, and API-version/header requirement is represented in the package profile's `ApiContract`. +LLM Test Server provider packages must not claim provider compatibility from memory or from an OpenAI-compatible shortcut. Every claimed provider method, route, streaming mode, modality, and API-version/header requirement is represented in the package profile's `ApiContract`. The current contract covers the generative API surface claimed by the provider profiles: chat/messages, streaming chat, embeddings, image generation or image inference, audio/speech, video generation where documented, model listing where claimed. Tool calls and structured output have [explicit fixture contracts and regression evidence](ToolAndStructuredFixtures.md); schema or tool-selection mismatches return 409 without consuming a scenario. Provider account administration, billing, files, fine-tuning, batches, and unrelated platform management APIs are outside the profile surface until a package explicitly claims them. ## Contract Shape -Each `LlmTckProviderProfile` has: +Each `LlmTestServerProviderProfile` has: - `Capabilities`: the provider capabilities this package claims. - `ApiContract.DocumentationUrl`: the official source used for the provider. @@ -18,7 +18,7 @@ Each `LlmTckProviderProfile` has: - `ApiContract.DocumentationVersion`: the API version, preview marker, or required version header when the provider publishes one. - `ApiContract.Operations`: documented HTTP operations with method, path, docs URL, streaming support, required header or version, and `ImplementedByHosting`. -`ImplementedByHosting` is deliberately separate from provider capability. It means `ManagedCode.LlmTck.Hosting` currently maps that exact method/path. For this supertest surface, every operation in a provider profile must be implemented. Do not add future or exploratory provider methods to `ApiContract.Operations` until `MapLlmTck()` exposes the route and a behavior test proves the shape. +`ImplementedByHosting` is deliberately separate from provider capability. It means `ManagedCode.LlmTestServer.Hosting` currently maps that exact method/path. For this supertest surface, every operation in a provider profile must be implemented. Do not add future or exploratory provider methods to `ApiContract.Operations` until `MapLlmTestServer()` exposes the route and a behavior test proves the shape. ## Sync Rules @@ -32,13 +32,12 @@ When provider docs change or a new route is added: 6. Update the test inventory when the acceptance surface changes. 7. Refresh `DocumentationRetrievedOn`; contract tests fail when the docs review is older than 180 days. -## Guard Tests +## Integration evidence -`ProviderApiContractTests.ProviderApiContracts_AreDocBackedAndCoverClaimedCapabilitiesAsync` fails when a provider profile claims a capability without a documented operation, uses a non-official docs URL, omits a retrieval date or version, has docs older than 180 days, duplicates a method/path row, keeps an unimplemented operation in the profile, or marks streaming without a streaming capability. - -`ProviderApiContractTests.HostingProviderRoutes_AreCoveredByImplementedDocumentedOperationsAsync` fails when `MapLlmTck()` exposes a public provider route that is not represented by an implemented documented operation, or when a documented implemented operation has no matching route. - -`ProviderApiContractTests.ProviderApiContracts_UseExplicitProviderNamespacesAsync` fails when a provider profile exposes a hosted route outside its provider namespace, for example an OpenAI route outside `/openai`, an Anthropic route outside `/anthropic`, or an Azure OpenAI route outside `/azure-openai`. +Provider support is verified by running the server and calling every claimed operation through HTTP and official +SDKs. Cover supported parameters and streaming modes, version/auth validation, success/error envelopes and state +transitions. Contract metadata and official documentation guide these tests; source/doc scans and reflection-based +test guards are not used as evidence. See the [server integration inventory](../Testing/AcceptanceCriteriaTestInventory.md). `AnthropicEndpointTests` prove the first native non-OpenAI provider route: `/anthropic/v1/messages` accepts Anthropic text messages, requires `anthropic-version: 2023-06-01`, accepts the provider-owned `x-api-key` header, returns Anthropic message/error JSON with deterministic usage, and streams Anthropic-named SSE events with usage on `message_start` and cumulative output usage on `message_delta`. @@ -46,7 +45,7 @@ When provider docs change or a new route is added: The same test class proves the OpenAI-compatible Responses API routes: OpenAI `/openai/v1/responses`, Groq `/groq/openai/v1/responses`, and OpenRouter `/openrouter/api/v1/responses`, including string input, structured message input with text and image parts, response output text shape, deterministic input/output usage, and Responses-style SSE streaming events. -OpenAI image coverage now follows the current official image surface for this profile under the TCK provider namespace: `/openai/v1/images/generations`, `/openai/v1/images/edits`, and `/openai/v1/images/variations`. `OpenAiEndpointTests.ImageRoutes_ReturnDocumentedEditVariationAndStreamingShapesAsync` proves JSON generation streaming with `image_generation.partial_image` and `image_generation.completed`, multipart edit output and `image_edit.*` streaming, multipart variation input for the DALL-E 2 route, deterministic base64 fixture output, and documented option validation. +OpenAI image coverage now follows the current official image surface for this profile under the Test Server provider namespace: `/openai/v1/images/generations`, `/openai/v1/images/edits`, and `/openai/v1/images/variations`. `OpenAiEndpointTests.ImageRoutes_ReturnDocumentedEditVariationAndStreamingShapesAsync` proves JSON generation streaming with `image_generation.partial_image` and `image_generation.completed`, multipart edit output and `image_edit.*` streaming, multipart variation input for the DALL-E 2 route, deterministic base64 fixture output, and documented option validation. OpenAI audio transcription is covered through `/openai/v1/audio/transcriptions` with multipart form input, JSON output, plain-text output, and documented `transcript.text.delta` / `transcript.text.done` SSE events for `stream=true`. Groq `/groq/openai/v1/audio/transcriptions` is covered with multipart input and Groq's `x_groq.id` response metadata. Azure OpenAI `/azure-openai/openai/deployments/{deployment}/audio/transcriptions?api-version=2024-10-21` is covered with deployment-owned model resolution, `api-key` auth, multipart input, and verbose JSON output. Audio translation is covered through OpenAI `/openai/v1/audio/translations`, Groq `/groq/openai/v1/audio/translations`, and Azure OpenAI `/azure-openai/openai/deployments/{deployment}/audio/translations?api-version=2024-10-21` with multipart input, JSON/text or verbose JSON output as documented, Groq metadata, Azure deployment-owned model resolution, and no unsupported streaming flag. Groq speech is covered through `/groq/openai/v1/audio/speech` with the documented `voice` requirement, Groq-specific response formats, and deterministic audio bytes. @@ -72,4 +71,4 @@ Legacy Azure deployment routes accept exactly one supported `api-version`: `2024 `AzureSdkCompatibilityTests.AzureOpenAiClient_CanUseDeploymentChatAndEmbeddingsAsync` proves chat, streaming and embeddings with the installed SDK default. `ReviewHttpRegressionTests.AzureLegacyVersion_AcceptsDocumentedStableAndPreviewVersionsAsync` covers the supported stable and preview contracts; the negative queue-preservation test covers invalid inputs and unknown date-shaped versions. The [official SDK options source](https://github.com/Azure/azure-sdk-for-net/blob/main/sdk/openai/Azure.AI.OpenAI/src/Custom/AzureOpenAIClientOptions.cs) owns automatic service-version selection. Supported deployment contracts are documented in the [GA reference](https://learn.microsoft.com/en-us/azure/foundry/openai/reference) and [dated preview reference](https://learn.microsoft.com/en-us/azure/foundry/openai/reference-preview). -The [OpenAI chat reference](https://developers.openai.com/api/reference/resources/chat/subresources/completions/methods/create) defines `stream_options.include_usage`: the final usage chunk has empty choices and precedes `[DONE]`. The TCK preserves provider-specific cache shapes in that chunk. See the [Azure version lifecycle](https://learn.microsoft.com/en-us/azure/foundry/openai/api-version-lifecycle) for the separate v1 contract. +The [OpenAI chat reference](https://developers.openai.com/api/reference/resources/chat/subresources/completions/methods/create) defines `stream_options.include_usage`: the final usage chunk has empty choices and precedes `[DONE]`. The Test Server preserves provider-specific cache shapes in that chunk. See the [Azure version lifecycle](https://learn.microsoft.com/en-us/azure/foundry/openai/api-version-lifecycle) for the separate v1 contract. diff --git a/docs/Features/ProviderSdkCompatibility.md b/docs/Features/ProviderSdkCompatibility.md new file mode 100644 index 0000000..bdaaa2a --- /dev/null +++ b/docs/Features/ProviderSdkCompatibility.md @@ -0,0 +1,40 @@ +# Official provider SDK integration + +Supported success, streaming, tool and conversation flows must be exercised through the official +provider SDK against a running Test Server. Raw HTTP supplements malformed input and exact wire +assertions. A custom HTTP transport still sends real requests to the hosted server; it does not +replace SDK serialization, parsing, streaming or error handling. + +| Client | Pinned package | Added running-server evidence | +| --- | --- | --- | +| Anthropic | `Anthropic` 12.58.0 | Native web search JSON/SSE, citations, assistant history and all six native search errors | +| Google Gemini | `Google.GenAI` 1.26.1 | GenerateContent JSON/SSE, signed function rounds, batch embeddings, dimensions, auth, filters and error recovery (36 cases) | +| Amazon Bedrock | `AWSSDK.BedrockRuntime` 4.0.101.11 | Converse, ConverseStream, InvokeModel, InvokeModelWithResponseStream, tool rounds and rate-limit recovery | +| OpenAI-compatible routes | `OpenAI` 2.14.0 | Groq, OpenRouter, Mistral and DeepSeek function rounds, JSON/SSE, provider errors and explicit recovery | + +The last row proves compatibility with the official **OpenAI client** at those route roots. It does +not claim that the OpenAI package is a provider-owned SDK for those providers. Existing OpenAI, +Azure OpenAI and Microsoft Foundry SDK integration suites remain part of the full gate. Other +provider SDK families and all remaining roadmap capabilities still require their own qualification. + +## Current model provenance + +Anthropic fixtures use the original SDK identifier `Anthropic.Models.Messages.Model.ClaudeSonnet5_5`. +The provider-neutral contract uses the corresponding documented wire ID `claude-sonnet-5-5`. +Sonnet 5.5 accepts automatic tool choice and rejects forced `any`/named tool choice. Sources: +[official SDK](https://github.com/anthropics/anthropic-sdk-csharp), +[Sonnet 5.5](https://platform.claude.com/docs/en/models/sonnet-5-5/overview). + +Google's SDK accepts a model string and does not expose a closed model enum. Its current fixtures use +[`gemini-3.8-flash`](https://ai.google.dev/gemini-api/docs/models/gemini-3.8-flash) and +[`gemini-embedding-2`](https://ai.google.dev/gemini-api/docs/models/gemini-embedding-2), taken from the +current official model pages. Do not invent SDK constants that do not exist. + +The added OpenAI-client compatibility fixtures explicitly configure the documented +[`gpt-5.5`](https://developers.openai.com/api/docs/models/gpt-5.5) fixture ID. A configured model is +local deterministic test data; this is not a claim that every compatible remote provider hosts it. + +Native decision wire identifiers live separately in `LlmTestServerDecisionModelIds`, with original +provider/source links. They are opt-in decision profiles and are not registered as default chat +models. Account-specific live model availability still belongs to each provider's authenticated +Models API; the test server does not pretend to discover a live account catalogue. diff --git a/docs/Features/RequestVerificationAndDeterminism.md b/docs/Features/RequestVerificationAndDeterminism.md new file mode 100644 index 0000000..373cf7e --- /dev/null +++ b/docs/Features/RequestVerificationAndDeterminism.md @@ -0,0 +1,194 @@ +# Request evidence, verification and wire determinism + +Status: Implemented and locally qualified through running-server integrations and an independent installed-package +consumer. E4 branch CI and public release remain pending. +Related: [architecture](../Architecture.md), [ADR-0005](../ADR/ADR-0005-request-evidence-and-wire-determinism.md), +[delivery E4](LlmTestServerDeliveryPlan.md), [conversations](ConversationDSL.md), [sessions](TestSessions.md). + +## Purpose and boundaries + +Tests query the same bounded request journal used by the dashboard, export retained evidence, and submit +serializable expectations to the running server. Core owns public diagnostic/verification DTOs, matching and +wire metadata. Hosting owns request capture, retention and authenticated control endpoints. Client exposes +typed APIs without a dependency on Hosting. All evidence belongs to the selected test session. + +```mermaid +flowchart LR + Provider[Provider HTTP request] --> Runtime[Session runtime and fixture] + Provider --> Capture[Bounded request journal] + Runtime --> Capture + Capture --> Snapshot[One snapshot and retention counters] + Snapshot --> Query[Query and export] + Snapshot --> Verify[Verification evaluator] + Runtime --> Graph[Conversation status] + Graph --> Verify + Clock[Seed and fixed fixture clock] --> Runtime +``` + +## Query and export + +`GET /admin-api/requests` returns a JSON array. `GET /admin-api/requests/export` uses the same query against +one captured snapshot and returns `LlmTestServerRequestExport`. Both enforce control authorization and session +selection. Querying and exporting do not execute provider operations or consume scenario responses. + +| Query parameter | Meaning | +| --- | --- | +| `provider`, `model`, `requestId`, `status` | Exact provider ID, model ID, server attempt ID, HTTP status | +| `operation`, `scenario`, `conversation`, `step` | Exact operation/scenario/conversation/step ID | +| `streaming`, `completed`, `success` | Boolean filters; an in-progress request has no success outcome | +| `errorCode` | Exact retained provider error code | +| `from`, `to` | Inclusive real request-start timestamps | +| `order` | `desc` by default, or `asc`, ordered by journal sequence | +| `offset`, `limit` | Nonnegative offset; limit 1–500, default 500 | +| `format` on export | `json` by default, or `ndjson` | + +Duplicate or blank supplied values, malformed values, invalid ranges and unsupported order/format return 400. +Status must be 100–599 and `from` must not follow `to`. String comparisons are exact. Control authorization +failure returns 401. Configure/reset clear the selected runtime's journal and evidence counters. + +The array response also carries `X-Total-Count`, `X-Total-Observed`, `X-Evicted-Count`, +`X-Retained-Byte-Count` and `X-Retention-Truncated`. Total count is the filtered count before pagination. +The conversation selector header alone does not filter this endpoint; use its `conversation` query parameter. + +`LlmTestServerRequestSnapshot` contains requests, `TotalMatched`, `TotalObserved`, `EvictedCount`, +`RetentionTruncated`, retained bytes and first/last retained sequence. Observed and eviction counters describe +the selected journal as a whole. Truncation is sticky until reset, including payload-preview truncation. +Sequence increases for each observed provider attempt; updates retain its original sequence. Default retention +is 500 entries and 16 MiB, with 64 KiB previews. Export is a bounded diagnostic snapshot, not a complete byte +recording or record/replay cassette. + +The JSON export envelope has `schemaVersion: 1`, `query` and `evidence`. NDJSON sends one `type: metadata` +line with that version, query and evidence metadata, followed by `type: request` lines carrying each retained +entry. Metadata contains no duplicate request array. Pagination applies equally to both formats. + +The client methods are `GetRequestsAsync`, `ExportRequestsAsync` and `ExportRequestsNdjsonAsync`, accepting +`LlmTestServerRequestQuery` from `ManagedCode.LlmTestServer.Diagnostics`. C# order values are +`LlmTestServerRequestOrder.Ascending` and `Descending`. + +## Verification + +`POST /admin-api/verify` accepts `LlmTestServerVerificationRequest`; `LlmTestServerClient.VerifyAsync` exposes +the same contract. A valid assertion returns HTTP 200 with `Passed`, `EvidenceComplete`, structured `Failures` +and `MatchedRequestIds`. A failed expectation is a verification result; malformed expectations return 400. +The conversation header scopes verification to that conversation before the body query is applied. Session +credentials and control authorization remain required. Verification does not change queues or graph state. + +| Request property | Behavior | +| --- | --- | +| `Scope` | Shared request query applied before expectations | +| `Requests` | Named `LlmTestServerRequestExpectation` entries with query, minimum/maximum count, tools and optional terminal-marker requirement | +| `InOrder` | Each request expectation's matched sequence range follows the previous expectation's range | +| `NoUnexpectedCalls` | Every request in scope must be accounted for by a request or retry expectation | +| `Conversations` | Match scenario plus optional provider/conversation; require visited step IDs and/or terminal state | +| `Retries` | Match retry group and query; require attempt bounds, exact optional status sequence and Retry-After behavior | + +The request supports at most 256 combined request/conversation/retry expectations. At least one expectation or +`NoUnexpectedCalls` is required. Request/tool count defaults are exactly one; a nullable maximum permits an +unbounded upper count within retained evidence. Each request expectation permits at most 64 tool expectations. +Tool-result expectations permit at most 64 JSON-field predicates; a conversation permits at most 256 required +step IDs. Invalid JSON and invalid count ranges are rejected before evaluation. + +`LlmTestServerToolExpectation.Kind` selects `ResponseCall`, `RequestCall` or `Result`. Response calls come from +structured runtime response metadata. Request calls come from normalized assistant history. Tool results are +linked to preceding calls by call ID. Predicates cover name, call ID, structurally equal arguments, structurally +equal result JSON and JSON fields using the conversation selector syntax. Counts apply per matched request; +historical calls repeated across request histories remain observable in each history. + +```csharp +var result = await session.Control.VerifyAsync(new LlmTestServerVerificationRequest +{ + Requests = + [ + new() + { + Name = "completed chat", + Query = new() { Provider = "openai", Success = true }, + MinimumCount = 1, + MaximumCount = 1, + }, + ], + NoUnexpectedCalls = true, +}); +``` + +## Incomplete evidence and terminal applicability + +Verification cannot pass after journal eviction, retained/preview payload truncation, selected pending requests, +or pagination that excludes matching entries. `EvidenceComplete` is false and failures identify incomplete +journal/page/timing evidence. This rule is conservative across the selected journal, even if an expectation +filters to a smaller subset. A matched subset never proves that no other calls occurred after evidence loss. +Use a fresh session or reset and sufficient configured retention for a complete test flow. + +Request traces preserve bounded structured tool calls and tool-result IDs in addition to payload previews. +`RequireStreamingTerminalMarker` requires both streaming classification and a successfully written/flushed +protocol terminal marker. Supported terminal shapes include OpenAI `[DONE]`/Responses completion, Anthropic +message stop, Cohere message end, Ollama `done`, Gemini final finish reason and Bedrock Converse stop/metadata, +plus image/transcription completion events. Exceptions and interrupted writes do not create terminal evidence. +Bedrock InvokeModel's current chunk format ends at EOF without an explicit terminal marker; do not request +terminal-marker verification for that operation. Successful transport completion alone is insufficient. + +## Retry evidence + +Set `X-Llm-Test-Server-Retry-Group` on each attempt in one logical retry flow. It is separate from the server's +unique request/attempt ID and selects evidence rather than granting access. The header must occur once, be +nonblank and contain at most 128 characters. Configure `.Fails(...).RetryAfter(seconds)` on a scenario or step +to emit deterministic `Retry-After` seconds for a scripted error. The delay is nonnegative and requires a +preceding error fixture. The application or real provider SDK performs the retry. + +`LlmTestServerRetryExpectation` specifies group ID, query, minimum/maximum attempts (defaults 1–3), optional +exact status sequence, minimum advertised Retry-After and `RespectRetryAfter`. Retry checks examine 429/503 +attempts. A retry expectation also accounts for its requests under `NoUnexpectedCalls`. + +Trace start/completion store real UTC observations and independent `StartedMonotonicSeconds` / +`CompletedMonotonicSeconds` from `Stopwatch`. Retry spacing is measured from the preceding completion to the +next start with monotonic timestamps. Missing/nonfinite/reversed timing produces incomplete evidence. +`Retry-After` seconds are used directly; an HTTP-date is converted to a delay against the observed completion +UTC timestamp. The actual spacing comparison remains monotonic. Fixed fixture time never satisfies a real wait. + +## Seeded wire metadata + +Configure core with `WithDeterminism(seed, fixedUtc)` or the client with `UseDeterminism(seed, fixedUtc)`. +The serializable `LlmTestServerDeterminism` holds seed and UTC time. Seeds are nonblank and at most 256 +characters. Without configuration, wire metadata uses live IDs/time. + +The runtime reserves a provider-local request ordinal at provider request entry inside each session. Seed, +provider namespace and ordinal derive one metadata identity; `IdFor(domain)` derives IDs for response, +message and other wire fields. Each result carries `WireMetadata` with creation time, clock mode and sequence; +all chunks reuse it. Reset/configure restart provider counters. Provider requests, including failed attempts, +consume ordinals; control queries/explanations do not. Concurrent calls within one provider follow request-entry +ordering. Parallel sessions have independent counters, and calls to another provider do not advance this one. + +Stored Responses and successful conversation replay retain response metadata, preserving fixture identity +across JSON and streaming. Provider prefixes and native field names remain provider-specific. Fixed mode also +sets the HTTP Date header from fixture time. This controls generated wire IDs/timestamps, not arbitrary fixture +fields: explicit model metadata, dictionaries and configured video timestamps retain their configured values. + +Traces expose `WireClockMode`, `WireCreatedAt` and `WireSequence` alongside real observations. Initial request +metadata is enriched with actual runtime result metadata, including the retained metadata of a replay. +Runtime event timestamps, trace UTC observations, elapsed durations and monotonic retry evidence remain real. +Session IDs/access keys remain cryptographically random; seeds are fixture configuration, not credentials. +The mode does not introduce network calls or relax provider wire validation. + +## Verification and completion criteria + +Use running-server HTTP and official SDK tests for every query boundary, export format, auth/session scope, +count/order/unexpected-call assertion, tool history/result predicate, graph visited/terminal assertion, retry +status/delay and incomplete-evidence failure. Verify seeded reset/replay and JSON/stream identity across provider +families, media timestamps, independent sessions/provider counters and real diagnostic time. + +Relevant suites include `RequestQueryExportIntegrationTests`, `VerificationIntegrationTests`, +`DeterminismIntegrationTests` and `NativeDeterminismIntegrationTests`. Completion additionally requires the full +integration suite, canonical Release build/format, at least 90% production line coverage and installed consumer. +Local qualification on 2026-10-20 passed 812/812 integration tests with 91.66% production line coverage; +the final Release build had zero warnings/errors. All 19 packages packed, and an independent clean NuGet +consumer exercised packaged Aspire, official SDKs, query/export/verification/fixed metadata and Chromium (1/1). +Evidence: `artifacts/delivery/e4-build-final.log`, `e4-coverage-final.log`, `e4-pack.log`, `e4-consumer.log`. +These local package checks do not establish public-feed availability. E4 branch CI has not run yet. +See the [delivery ledger](LlmTestServerDeliveryPlan.md#11-журнал-реалізації) for subsequent delivery status. + +```sh +dotnet format ManagedCode.LlmTestServer.slnx --verify-no-changes --no-restore +dotnet build ManagedCode.LlmTestServer.slnx --configuration Release --no-restore +dotnet test --project tests/ManagedCode.LlmTestServer.Tests/ManagedCode.LlmTestServer.Tests.csproj --configuration Release --no-build --verbosity normal +bash scripts/coverage.sh +``` diff --git a/docs/Features/ResponsesContinuation.md b/docs/Features/ResponsesContinuation.md index ba1be93..ee00203 100644 --- a/docs/Features/ResponsesContinuation.md +++ b/docs/Features/ResponsesContinuation.md @@ -32,15 +32,12 @@ flowchart LR Output --> SDK ``` -Acceptance: +Integration acceptance: -1. An ordinary or streamed official SDK tool loop reaches its scripted final response and the runtime journal - contains the original text/image input plus the actual tool result (`LlmTckResponsesContinuationTests`). -2. Another provider or credential cannot read a parent; reset/configure invalidates every parent - (`LlmTckChatHistoryTests`). -3. Stored input snapshots survive caller mutation and do not carry old instructions into a new request - (`LlmTckChatHistoryTests`). -4. Changing the model preserves original user and assistant messages and selects the new model in ordinary - and streaming official SDK calls (`LlmTckChatHistoryTests`, `LlmTckResponsesContinuationTests`). -5. Disabled/full count or byte capacity fails visibly without consuming a fixture; `store=false` remains - stateless (`LlmTckChatHistoryTests`, existing OpenRouter provider regressions). +1. Ordinary and streaming official SDK tool loops reach scripted final responses with the original input and + actual tool results visible through `/admin-api/assertions` (`LlmTestServerResponsesContinuationTests`). +2. Model changes preserve history in ordinary and streaming official SDK calls + (`LlmTestServerResponsesContinuationTests`). +3. Foreign provider/credential parents, unknown/reset identities, disabled/full count/byte capacity, instruction + replacement and `store=false` must be verified through server HTTP/SDK requests. Qualification results are + recorded in the [delivery plan](LlmTestServerDeliveryPlan.md); this contract alone does not establish coverage. diff --git a/docs/Features/StreamDeliveryFaults.md b/docs/Features/StreamDeliveryFaults.md new file mode 100644 index 0000000..a17d92f --- /dev/null +++ b/docs/Features/StreamDeliveryFaults.md @@ -0,0 +1,171 @@ +# Stream delivery faults + +Status: **E6 text SSE delivery faults qualified through 42/42 running-server integration cases** across +OpenAI Chat Completions, OpenAI Responses, Anthropic Messages and Gemini. Delivery and cancellation cases +use the official OpenAI 2.14.0, Anthropic 12.58.0 (strict response validation) and Google.GenAI 1.26.1 clients +over real Kestrel/TCP. Replay identity also uses official SDKs against hosted ASP.NET Core HTTP routes; +raw HTTP supplements transport boundaries, admission and invalid-configuration checks. A fresh installed-package Aspire consumer +passed **9/9** cases. The SDK-built container also passed acceptance using the README's SDK publish command. +The combined code-only integration gate passed **1,359/1,359**, with **92.52%** production line coverage. +Canonical formatting and the final Release build passed with no warnings or errors. + +## Configure an explicit delivery plan + +`LlmTestServerScenarioResponse.StreamDelivery` accepts a `LlmTestServerStreamDelivery` from the +`ManagedCode.LlmTestServer.Scenarios` namespace. Scenario and conversation-step builders expose +`WithStreamDelivery(...)`, applied to the preceding response: + +```csharp +config.UseChatModel(LlmTestServerKnownModelIds.Gpt55) + .UseChatScenario("partial-text", scenario => scenario + .ForModel(LlmTestServerKnownModelIds.Gpt55) + .Responds("First second.", "First ", "second.") + .WithStreamDelivery(new() + { + Mode = LlmTestServerStreamDeliveryMode.EarlyCleanEnd, + AfterContentFrames = 1, + }) + .Responds("Explicit next attempt.")); +``` + +The plan belongs to the fixture, not to an upstream request option. It does not introduce a provider +header, environment toggle or alternate endpoint. + +| Mode | Behavior after the selected content frame has been written and flushed | +| --- | --- | +| `EarlyCleanEnd` | End the HTTP response normally, omitting remaining content and the selected protocol's terminal sequence | +| `DelayThenContinue` | Pause once for `DelayMilliseconds`, then resume the remaining content and ordinary terminal sequence if the request remains active | +| `AbortConnection` | Abort the server request transport, omitting remaining content and protocol-terminal frames | + +`AfterContentFrames` defaults to 1. It must be 1–128 and cannot exceed the configured content-frame +count of nonempty text chunks. Empty chunks do not count; an empty response or unreachable boundary +is rejected at configuration. A nonempty response without explicit chunks has one frame. Delay mode requires 1–60,000 milliseconds; +other modes require zero delay. These are bounded local fixture policies. Explicit chunks must +concatenate exactly to the response content. Invalid plans fail configuration atomically. + +A plan applies only to plain-text responses. Scripted provider errors, ordinary function calls and +ordered/native output items cannot share that response. The qualified contract admits these exact routes: + +| Surface | Request | Counted text frame | Ordinary successful protocol ending | +| --- | --- | --- | --- | +| OpenAI Chat Completions | `POST /openai/v1/chat/completions`, `stream: true` | Nonempty content chunk | Finish-reason chunk, requested usage chunk, `[DONE]` | +| OpenAI Responses | `POST /openai/v1/responses`, `stream: true` | Nonempty `response.output_text.delta` | Text/content/output done events, then `response.completed` | +| Anthropic Messages | `POST /anthropic/v1/messages`, `stream: true` | Nonempty `content_block_delta` text delta | Content-block stop, message delta, `message_stop` | +| Gemini | `POST /gemini/v1beta/models/{model}:streamGenerateContent?alt=sse` | Nonempty text candidate payload | Final candidate `finishReason: STOP` and usage | + +Start/progress/empty frames do not advance the delivery boundary. Gemini requires exactly one `alt=sse`; +its JSON-array streaming transport is excluded. Non-streaming requests, other provider roots, media +streaming, tool declarations/native includes/tool-bearing history and structured-output requests are rejected +as fixture mismatches before the selected response is consumed. Native capability metadata alone does not +mean a request declared a tool. Do not infer support for another route because it shares a mapper. +Anthropic cache-population-only requests (`max_tokens: 0`) return before response selection: they warm +prompt-cache state without consuming a queued response or executing a graph step. Their planless response +emits no text and applies no delivery fault. A subsequent generation still selects the configured fault fixture. + +## Inference commit and delivery are different boundaries + +The runtime computes and commits the complete deterministic inference result before the HTTP writer +starts delivering it. The selected queue entry or conversation step, retained response and successful +prompt-cache effects are therefore already committed when the delivery plan fires. A delivery failure +can leave the client with only a prefix while the runtime retains the full logical answer. + +An explicit retry against a flat response queue selects its next response. The server does not retry or +roll back the committed answer because a terminal marker was lost. A repeated request that qualifies +for existing conversation replay returns the retained result and reapplies its delivery plan without +executing the step again. Replay therefore reproduces the fault rather than repairing the first +transport. A new valid conversation turn advances according to the configured graph. + +Request ID, streaming transport flag, delivery-surface marker and previous-response lookup ID are excluded +from the logical inference fingerprint. Changing only JSON/SSE delivery therefore does not create a new +logical turn; replay still validates the selected fixture against the requested transport. A delivery fault +cannot be replayed through an unsupported JSON/array transport to bypass its SSE applicability rule. + +Conversation ownership includes the provider namespace, conversation label and credential hash inside the +owning test session. Keep the same SDK credential when comparing SDK and raw HTTP requests or replaying a +turn, even when the host does not require authentication. A different supplied credential selects a different +conversation owner; the conversation label alone is not authorization. + +This differs from pre-inference `.DelaysBy(...)`: cancellation before the existing runtime commit can +release a reservation. Cancellation during the new delivery pause happens after inference commit. +The delivery delay uses real elapsed time even when provider wire metadata uses a fixed fixture clock. + +## Transport completion and protocol completion + +An HTTP 200 and clean response-body EOF do not prove a complete chat stream. OpenAI Chat Completions +uses data-only SSE; a normal stream has its final finish-reason chunk and `data: [DONE]`. Requested +usage appears in a separate final chunk before `[DONE]`; interrupted streams can omit it. +[Official streaming guide](https://developers.openai.com/api/docs/guides/streaming-responses), +[Chat Completions streaming contract](https://developers.openai.com/api/reference/resources/chat/subresources/completions/methods/create). + +The pinned OpenAI .NET 2.14.0 API exposes `CompleteChatStreamingAsync` and typed +`StreamingChatCompletionUpdate.ContentUpdate`, `FinishReason` and `Usage`. Its SSE enumerator can +finish on either `[DONE]` or clean EOF, without itself requiring a finish reason. Thus the expected +clean-end test observes a prefix and missing terminal evidence; it must not assume an SDK exception. +Running-server tests confirm clean EOF produces a partial result without a finish reason, while the +real Kestrel abort interrupts the official client. Tests forward unchanged bytes and coordinate the +first actual SDK content update before releasing the server flush, avoiding a scheduling race. +[Package-pinned SDK parser](https://github.com/openai/openai-dotnet/blob/2e77b08828145f658ec04e49aec87abb1543c553/OpenAI/src/Utility/AsyncSseUpdateCollection.cs), +[pinned chat client](https://github.com/openai/openai-dotnet/blob/2e77b08828145f658ec04e49aec87abb1543c553/OpenAI/src/Custom/Chat/ChatClient.cs). + +### Qualified SDK parser boundaries + +The official OpenAI 2.14 Responses client also uses the common SSE collection. A frame-aligned clean +EOF can therefore end typed update enumeration without `response.completed`; assert the typed text prefix +and missing completion, rather than assuming an exception or requiring a Chat-style `[DONE]`. +[Responses client source](https://github.com/openai/openai-dotnet/blob/2e77b08828145f658ec04e49aec87abb1543c553/OpenAI.Responses/src/Custom/ResponsesClient.cs). + +Anthropic 12.58 strict response validation validates each event, but raw typed stream enumeration does +not require `message_stop` before EOF. Its separate official `.Aggregate()` helper requires a stop event +and throws for a partial message. Normal message completion ends with `message_stop`. +[Streaming contract](https://platform.claude.com/docs/en/build-with-claude/streaming), +[SDK event reader](https://github.com/anthropics/anthropic-sdk-csharp/blob/Anthropic-v12.58.0/src/Anthropic/Core/Sse.cs), +[SDK aggregator](https://github.com/anthropics/anthropic-sdk-csharp/blob/Anthropic-v12.58.0/src/Anthropic/Helpers/MessageContentAggregator.cs). + +Google.GenAI 1.26.1 accepts EOF with no buffered partial SSE frame and does not enforce a finish reason. +A remaining incomplete JSON segment instead causes a parser error. A missing `finishReason` means the +model has not stopped generating; for this text fixture, successful continuation ends with `STOP`. +These faults flush whole SSE frames, so clean early end differs from byte truncation. +[Gemini contract](https://ai.google.dev/api/generate-content#method:-models.streamgeneratecontent), +[pinned HTTP parser](https://github.com/googleapis/dotnet-genai/blob/c071e7d18f779bf166a66c0a7ff715ab24035273/Google.GenAI/HttpApiClient.cs). + +The focused integrations confirm clean EOF returns the observed text without a terminal event on all four +routes, real connection abort interrupts the clients, and delay continuation/cancellation preserves the +recorded prefix and transport outcome. Tests disable ordinary SDK retries and Google's +`GenerateContentConfig.AutomaticContinuation` to prove individual attempts. They also cover the boundary +after **all text** has arrived: complete text alone still does not imply protocol completion when the final +finish/completion/stop event is missing, and `stream.terminal` verification fails. + +## Public evidence and qualification boundary + +Request diagnostics expose `StreamDelivery`, `StreamDeliveryApplied` and `EmittedContentFrames` +alongside the existing transport state, status, response preview, timing and +`TerminalMarkerObserved`. A configured plan and a reached trigger are distinct observations. +An ordinary response EOF can have completed transport with no protocol terminal; an aborted transport +must not acquire a terminal marker. Query/export preserves this structured evidence under its existing +retention bounds. Runtime events describe committed inference; transport traces describe actual delivery. +Neither record alone proves the client received the full answer. + +The verification API's `stream.terminal` check can detect missing protocol completion. Consumer retry +policies should be disabled when proving individual attempts. Test the same configured plan through +the applicable official provider client over a running Kestrel server, verify the actual emitted prefix and +finish/usage behavior, inspect request evidence, exercise explicit retry and graph replay, and confirm +that a subsequent health/provider request still works. Delay/cancellation tests must coordinate actual +first-frame delivery rather than rely on a scheduling race. + +Malformed SSE/JSON, byte-level truncation, tool/native/media streams, output-stage content filtering, +provider families outside the four listed routes and resource/process termination remain separate unqualified delivery-fault +contracts. This bounded text slice does not claim all E6 capabilities. + +The cache regression confirms that a partial stream retains its already committed prompt-cache entry: +the explicit next request with identical input reports positive cached tokens in both the official SDK +and public request trace. Test hosts warm health/endpoint metadata over real HTTP before starting the +stream watchdog. Cancellation starts after the first real SDK text update, and tests drain completion +traces before cleanup/next assertions. Health warmup and trace draining synchronize test lifecycle; they +neither retry inference nor change provider behavior, fault boundaries or committed state. + +The 42-case focused evidence is recorded in `artifacts/delivery/e6-multi-stable-sdk.log`. All 19 NuGet +packages were packed, and `artifacts/delivery/e6-multi-installed-consumer.log` records the fresh Aspire +consumer's 9/9 cases, including delivery through the official clients and the TypeSafe/Jev example. +Standalone acceptance used an image built by the .NET SDK's `PublishContainer` target with the README's +single-platform command. These are local qualification results; public package/image publication remains +part of the owner-merged release gate. diff --git a/docs/Features/TestSessions.md b/docs/Features/TestSessions.md new file mode 100644 index 0000000..cedc91e --- /dev/null +++ b/docs/Features/TestSessions.md @@ -0,0 +1,67 @@ +# Isolated test sessions + +Status: Implemented and locally qualified; public-package release pending. +Architecture: [ADR-0003](../ADR/ADR-0003-isolated-test-sessions.md). + +A shared server can run independent tests concurrently. Each session owns its configuration, scenario queues, +response history, prompt cache, fault counters, video jobs, assertions and bounded request journal. A reset or +configure call changes the selected session only. Calls without session headers use the existing default runtime. + +```csharp +var server = new LlmTestServerClient(httpClient, "management-key"); +await using var session = await server.CreateSessionAsync(config => config + .RequireBearerToken("provider-key") + .UseGpt41Mini() + .UseChatScenario("greeting", scenario => scenario.Responds("Hello from this test"))); + +using var chat = session.Control.CreateChatClient(); +var reply = await chat.GetResponseAsync("Hello"); +var assertions = await session.Control.GetAssertionsAsync(); +``` + +The supplied `httpClient` remains caller-owned and usable after session disposal. Different session clients +share its transport without changing its default headers. Use `session.HttpClient` as the transport for official +SDKs; configure their endpoint from the server's actual allocated endpoint and supply the provider API key as +usual. The transport adds session ID/key headers to every request, including streaming and multipart operations. + +## HTTP contract + +| Request | Result | +| --- | --- | +| `POST /admin-api/sessions` with `{}` | Clone startup configuration; return 201, Location, Cache-Control: no-store, ID/key/reservation | +| `POST /admin-api/sessions` with `configuration` | Create an independent runtime with that configuration | +| Existing provider/control routes with both session headers | Operate on the selected runtime and journal | +| `DELETE /admin-api/sessions/{id}` with that session's headers | Release capacity, retire the runtime and return 204 | +| Missing/unknown/foreign/duplicate session ID or key | 404 `session_not_found`, without invoking or journaling the default runtime | +| Session count/state budget exhausted | 409 `session_capacity_exceeded` | +| Invalid/oversized configuration | 400; preserve the previous configuration, counters and reservation | + +Headers are `X-Llm-Test-Server-Session` and `X-Llm-Test-Server-Session-Key`. They select and authorize the session; +provider bearer/API-key authentication remains independent. The session access key is returned only at creation +and is redacted from provider diagnostics. To change the provider key through configure, use +`session.Control.WithBearerToken(newKey)` for subsequent control calls. Session credentials remain unchanged. + +Creation uses the default runtime's management bearer requirement and rejects session selector headers with 400; create through the root management client. Deletion uses the session's own ID/key, so +changing the default provider token does not prevent cleanup. Deleting a session invalidates in-flight delayed +work with 409 `llm_test_server_request_superseded`; subsequent calls return 404. Cancelling a request keeps the existing +queue rollback contract. Deleting an already absent session returns 404; typed disposal accepts that response. + +## Bounds + +Configure hosting with `services.Configure(...)`. Defaults are maximum 32 sessions, +512 MiB aggregate reserved state and 1 MiB serialized configuration per session. The reservation uses configured +history/video capacities, fixed event/diagnostic retention and bounded cache/job metadata; capacity may be +exhausted before the session-count limit. It is an admission budget, not a managed-heap measurement. Configure +can shrink or grow the reservation within that budget; deletion releases it. Explicit `await using` disposal is +the lifetime boundary. There is no timer expiration or persistence. + +The default browser dashboard continues to inspect the default runtime. A session selector and session/step +visualization remain part of the later dashboard work in the delivery plan. + +## Verification + +`SessionIsolationIntegrationTests`, `SessionStateIntegrationTests`, `SessionBoundaryIntegrationTests` and `SessionProviderIntegrationTests` start a +server and exercise HTTP, MEAI and official OpenAI SDK calls. They cover parallel identical prompts, credentials, +queue/reset/configure/dispose, admission limits, history/video/fault/cache isolation, journal retention, streaming +and deletion during an active delayed response. Full-suite and coverage results are recorded in the delivery +implementation ledger; this document alone is not test evidence. diff --git a/docs/Features/ToolAndStructuredFixtures.md b/docs/Features/ToolAndStructuredFixtures.md index b3e3a31..5632a98 100644 --- a/docs/Features/ToolAndStructuredFixtures.md +++ b/docs/Features/ToolAndStructuredFixtures.md @@ -1,6 +1,6 @@ # Deterministic tool and structured-output fixtures -The runtime stores explicit tool calls and JSON responses. It does not invent tool arguments or generate JSON from a schema. The selected fixture must satisfy the request; otherwise the operation returns `409 / llm_tck_fixture_mismatch` without consuming the response. Malformed wire options return a provider-shaped 400 before runtime execution. +The runtime stores explicit tool calls and JSON responses. It does not invent tool arguments or generate JSON from a schema. The selected fixture must satisfy the request; otherwise the operation returns `409 / llm_test_server_fixture_mismatch` without consuming the response. Malformed wire options return a provider-shaped 400 before runtime execution. ```csharp builder.AddChatScenario("weather", scenario => scenario @@ -8,7 +8,7 @@ builder.AddChatScenario("weather", scenario => scenario .RespondsJson("""{"forecast":"sunny"}""")); ``` -`CallsTools(...)` configures multiple calls in one response. Each call carries its ID, function name and JSON arguments. Tool declarations and arguments flow through provider adapters into typed provider-neutral records. Assistant tool calls and tool results remain in conversation history, exact scenario matching, diagnostic retention, token accounting and prompt-cache identity. `LlmTckChatClient` maps `AIFunctionDeclaration`, tool modes, `FunctionCallContent`, `FunctionResultContent`, and `ChatResponseFormatJson`; it works with the standard `UseFunctionInvocation()` middleware. +`CallsTools(...)` configures multiple calls in one response. Each call carries its ID, function name and JSON arguments. Tool declarations and arguments flow through provider adapters into typed provider-neutral records. Assistant tool calls and tool results remain in conversation history, exact scenario matching, diagnostic retention, token accounting and prompt-cache identity. `LlmTestServerChatClient` maps `AIFunctionDeclaration`, tool modes, `FunctionCallContent`, `FunctionResultContent`, and `ChatResponseFormatJson`; it works with the standard `UseFunctionInvocation()` middleware. ## Supported contracts @@ -16,14 +16,14 @@ builder.AddChatScenario("weather", scenario => scenario | --- | --- | --- | --- | | OpenAI Chat; Azure deployment and v1 Chat; Foundry Chat; Groq, Mistral, OpenRouter, DeepSeek Chat | `tools[].function`, `tool_calls`, `tool_call_id`; auto/none/required/named choice | `response_format`: json_object or json_schema | [OpenAI Chat](https://developers.openai.com/api/reference/resources/chat/subresources/completions/methods/create/), [Azure](https://learn.microsoft.com/en-us/azure/foundry/openai/how-to/structured-outputs) | | OpenAI, Azure v1, Foundry v1, Groq and OpenRouter Responses | flat function tools; function_call and function_call_output items | `text.format` | [OpenAI Responses](https://developers.openai.com/api/reference/resources/responses/methods/create/), [OpenRouter](https://openrouter.ai/docs/api/reference/responses/overview) | -| Anthropic Messages | tools/input_schema; tool_use and tool_result blocks; auto/none/any/tool choice | `output_config.format.schema` | [Tool use](https://platform.claude.com/docs/en/agents-and-tools/tool-use/implement-tool-use), [Structured outputs](https://platform.claude.com/docs/en/build-with-claude/structured-outputs) | +| Anthropic Messages | tools/input_schema; tool_use and tool_result blocks; auto/none; forced any/tool choice is rejected for Sonnet 5.5 | `output_config.format.schema` | [Tool use](https://platform.claude.com/docs/en/agents-and-tools/tool-use/implement-tool-use), [Structured outputs](https://platform.claude.com/docs/en/build-with-claude/structured-outputs) | | Gemini generateContent / streamGenerateContent | functionDeclarations, functionCall/functionResponse; functionCallingConfig | responseJsonSchema or responseSchema, application/json | [Functions](https://ai.google.dev/gemini-api/docs/function-calling), [Structured output](https://ai.google.dev/gemini-api/docs/structured-output) | | Ollama chat | tools[].function; object-valued function arguments; tool_name results | format: json or schema object | [Chat](https://docs.ollama.com/api/chat) | | Cohere v2 chat | tools[].function, tool_calls/tool_call_id; REQUIRED/NONE choice | response_format: json_object, optional schema | [Chat](https://docs.cohere.com/v2/reference/chat), [Tool streaming](https://docs.cohere.com/v2/docs/tool-use-streaming) | | Bedrock Converse / ConverseStream | toolConfig, toolUse/toolResult blocks | outputConfig.textFormat.structure.jsonSchema.schema | [Converse](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html), [Schema](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_JsonSchemaDefinition.html) | | Perplexity Sonar | Function tools are not claimed and are rejected | response_format: json_schema | [Structured output](https://docs.perplexity.ai/docs/sonar/features) | -Legacy Azure deployment routes accept the SDK-selected `api-version` for the documented `2024-10-21` GA and `2025-04-01-preview` contracts and reject unknown versions; Foundry inference routes require `api-version=2024-05-01-preview`; OpenAI v1 routes do not require a dated version. The operation contracts retain these constraints. Provider/model availability restrictions in the upstream services still apply; the TCK proves the listed fixture wire contracts, not every provider-hosted tool or model feature. Bedrock InvokeModel retains its existing model-specific fixture contract and rejects tool/schema options; tool support is claimed on Converse only. Legacy `functions` / `function_call` request options and server-hosted tools are explicitly rejected. +Legacy Azure deployment routes accept the SDK-selected `api-version` for the documented `2024-10-21` GA and `2025-04-01-preview` contracts and reject unknown versions; Foundry inference routes require `api-version=2024-05-01-preview`; OpenAI v1 routes do not require a dated version. The operation contracts retain these constraints. Provider/model availability restrictions in the upstream services still apply; the Test Server proves the listed fixture wire contracts, not every provider-hosted tool or model feature. Bedrock InvokeModel retains its existing model-specific fixture contract and rejects tool/schema options; tool support is claimed on Converse only. Legacy `functions` / `function_call` request options are explicitly rejected. Server-hosted tools are limited to the qualified OpenAI Responses web-search/file-search/code-interpreter fixtures and [Anthropic native web search](AnthropicNativeWebSearch.md); other native tool declarations remain outside those contracts. ## Schema and streaming behavior @@ -36,8 +36,8 @@ Streams carry provider-native tool events: OpenAI indexed tool-call deltas and t - `ToolFixtureEndpointTests`: positive full conversation and schema mismatch recovery for each claimed tool operation; native streaming events; mapped tool-result history. - `ToolFixtureClientTests`: OpenAI Chat and Responses SDK parsing, Azure SDK, Microsoft.Extensions.AI automatic function invocation in ordinary and streaming modes. - `ToolFixtureValidationTests`: malformed wire options, tool selection mismatch, argument validation, local/external schema references and inconsistent JSON chunks. -- `ToolFixtureStateTests`: parallel calls, mixed text/tool output, configuration snapshot isolation, exact history matching and prompt-cache identity. -- `ProviderApiContractTests`: capabilities must be represented in the doc-backed operation matrix, and every implemented operation retains behavior evidence. +- HTTP/SDK state flows must cover parallel calls, mixed text/tool output, configuration isolation, exact history matching and prompt-cache identity; replacement of removed internal state tests is in progress. +- `ProviderCapabilityEvidenceTests`: operation variants execute through HTTP with success, rejection and queue recovery; operation contracts retain official documentation references. ## Request constraints and diagnostics @@ -49,12 +49,12 @@ Every fixture mismatch produces an `ErrorReturned` runtime event with request/sc `WithExactMatch` now compares tool-call IDs and the complete ordered tool-call collection, including null/empty values. Callers that used empty expected tool metadata as a wildcard must select Contains matching instead. Contains retains the existing optional tool metadata behavior. -Wire tool validation lives in the corresponding provider package, returning `LlmTckRequestValidationResult`. Hosting selects a provider validator and maps invalid results to the provider's HTTP error shape. Provider-independent JSON primitives are linked from `src/Shared`; the neutral runtime has no provider DTO dependencies. +Wire tool validation lives in the corresponding provider package, returning `LlmTestServerRequestValidationResult`. Hosting selects a provider validator and maps invalid results to the provider's HTTP error shape. Provider-independent JSON primitives are linked from `src/Shared`; the neutral runtime has no provider DTO dependencies. `ProviderCapabilityEvidenceTests` executes a positive and negative fixture for each `(provider, operation, Tools/StructuredOutput, JSON/stream)` case supported by the checked-in contracts. Its guard also requires executable provider arguments and named nullable/limit/anyOf and parallel-option regression cases. This supplements operation-level evidence; it is not a claim of exhaustive upstream API coverage. ## Bounded video jobs -`WithVideoCapacity(maxJobs, maxBytes)` configures a shared runtime budget across stored provider video jobs. Defaults are 256 jobs and 64 MiB of retained payload bytes. Zero jobs disables storage. New jobs exceeding either bound return 409 / `llm_tck_video_capacity_exceeded`; existing jobs remain available and failed stores do not advance IDs. Delete releases capacity; reset/configure clear jobs and byte accounting. OpenAI and Azure video job endpoints propagate the failure. This budget applies to stored jobs, separately from configured modality fixture bytes and the event-journal budget. +`WithVideoCapacity(maxJobs, maxBytes)` configures a shared runtime budget across stored provider video jobs. Defaults are 256 jobs and 64 MiB of retained payload bytes. Zero jobs disables storage. New jobs exceeding either bound return 409 / `llm_test_server_video_capacity_exceeded`; existing jobs remain available and failed stores do not advance IDs. Delete releases capacity; reset/configure clear jobs and byte accounting. OpenAI and Azure video job endpoints propagate the failure. This budget applies to stored jobs, separately from configured modality fixture bytes and the event-journal budget. -Schema validation uses the MIT-licensed JsonSchema.Net 8.0.5 package so installed TCK binaries do not introduce the separate binary maintenance terms from later releases. Keep fixture validation and external-reference rejection covered when changing this dependency. +Schema validation uses the MIT-licensed JsonSchema.Net 8.0.5 package so installed Test Server binaries do not introduce the separate binary maintenance terms from later releases. Keep fixture validation and external-reference rejection covered when changing this dependency. diff --git a/docs/Features/ToolFaultFixtures.md b/docs/Features/ToolFaultFixtures.md new file mode 100644 index 0000000..f7b36a7 --- /dev/null +++ b/docs/Features/ToolFaultFixtures.md @@ -0,0 +1,59 @@ +# Tool outcomes and failures + +A function call and its result belong to different requests. The server returns the scripted call; +the application invokes its function and sends a tool-result message on the next request. The +well-known presets construct data for that flow and never execute searches, files or processes. + +```csharp +var weather = LlmTestServerWellKnownTools.Weather; +var arguments = new LlmTestServerWeatherArguments { Location = "Paris" }; +var nullFixture = weather.NullResult(arguments, "call_weather"); +// nullFixture.Result.Content is the JSON literal "null". +var failedFixture = weather.Failure(arguments, new() +{ + Code = "timeout", + Message = "The application could not obtain the tool result in time.", + Retryable = true, +}, "call_weather"); +``` + +`CreateNullResult(callId)` creates just the correlated tool message. The explicit null factory +bypasses the successful result schema intentionally; `CreateResult` still requires a non-null, +schema-valid typed result. An empty list or empty string remains its own result. To simulate no +result, omit the tool-result message entirely. Do not replace absence with an empty message or null. +A graph can require the exact call ID and JSON root before advancing: + +```csharp +config.AddConversation("weather", graph => graph + .Step("call", step => step.CallsTools(nullFixture.Call).Then("result")) + .Step("result", step => step.WhenToolResult(nullFixture.Call.Name, nullFixture.Call.Id) + .WhenLatestJson("$", "null").Responds("No weather data available.").Complete())); +``` + +A `Failure` fixture returns the explicit `{ "error": { "code", "message", "retryable" } }` +data envelope. `IsError` labels that fixture; it does not throw, retry or set an HTTP status. +`DelayMilliseconds` is fixture metadata and must be applied explicitly to the selected scenario +or application invocation. The application can throw an exception, cancel a real invocation at +its deadline, or catch a failure and send the chosen envelope. A local exception that is never +sent back cannot appear as a server-observed tool result. + +Provider request failures use the existing scenario engine. `.Fails(500, code, message)` consumes +one scripted attempt, allowing the next graph step to recover. `.DelaysBy(milliseconds)` delays +the response before runtime commit; cancellation releases that reservation so the same call can +be returned on retry. `SimulateContentFilter(term)` rejects matching request content, including +tool-result text, with a provider-shaped `content_filter` error before consuming the next step. +These failures do not warm the successful-response prompt cache. The host remains available. +Disable SDK automatic retries when testing individual observed attempts. + +Native tools retain their provider-specific result contracts. Anthropic web search can return an +empty result list or a documented search error inside a successful Messages response; these do +not become application function calls or automatic provider errors. + +Mid-stream stalls, socket aborts, malformed output, missing terminal frames and output-stage +content-filter events remain separate delivery-fault work in E6. A scripted HTTP 500 does not +terminate the process and does not prove a real socket abort. + +The running-server regression suite is `WellKnownToolFaultIntegrationTests`: official OpenAI SDK +and Extensions.AI function loops, null/empty/missing result distinction, controlled exceptions and +cancellation, explicit recovery, provider 500, delayed response cancellation, filtered tool results, +cache accounting, retained history and health checks. diff --git a/docs/Features/WellKnownFunctionTools.md b/docs/Features/WellKnownFunctionTools.md new file mode 100644 index 0000000..4aed216 --- /dev/null +++ b/docs/Features/WellKnownFunctionTools.md @@ -0,0 +1,110 @@ +# Well-known function-tool fixtures + +`ManagedCode.LlmTestServer.Tools.WellKnown` provides twelve provider-neutral, typed function-tool presets. +They supply scripted calls and results to the existing scenario/conversation engine. They never evaluate an +expression, read a clock, fetch a URL, access files or execute code, commands or patches. This is the function-tool +portion of E5; provider-native built-in tools and dashboard discovery remain separate, unfinished work. + +## Catalogue and contracts + +`LlmTestServerWellKnownTools.All` is a read-only descriptor catalogue. Each descriptor exposes `Name`, +`Description`, `ParametersJson`, `ResultSchemaJson` and `ErrorSchemaJson`. Strongly typed properties on the same +class expose `LlmTestServerWellKnownTool`; names are constants in +`LlmTestServerWellKnownToolNames`. Argument/result types follow `LlmTestServer{Preset}Arguments` / +`LlmTestServer{Preset}Result`. + +| Preset | Function name | Scripted arguments → result | +| --- | --- | --- | +| Calculator | `calculator` | Expression → decimal value | +| CurrentTime | `current_time` | Time zone → explicit `DateTimeOffset` and time zone | +| Weather | `weather` | Location, celsius/fahrenheit → temperature and condition | +| WebSearch | `web_search` | Query, limit → title/URL/snippet hits | +| FetchUrl | `fetch_url` | URL → status, content type and content | +| FileSearch | `file_search` | Query, limit → file ID/path/snippet hits | +| ReadFile | `read_file` | Path, offset, optional length → content and truncation flag | +| WriteFile | `write_file` | Path, content, append → bytes-written receipt | +| ListFiles | `list_files` | Path, recursive → file/directory entries | +| RunCode | `run_code` | Language, code → stdout, stderr and exit code | +| Shell | `shell` | Command, optional working directory → stdout, stderr and exit code | +| ApplyPatch | `apply_patch` | Patch text → updated-path receipt | + +Search limits are 1–100. Read offsets/lengths and write/file byte counts are nonnegative. Weather units are +`celsius` or `fahrenheit`. Schemas reject unknown properties. Nullable optional fields are omitted during +serialization. `CurrentTime` requires the caller to supply its timestamp; use the same explicit timestamp as +your test's fixed fixture clock when that is the intended contract. + +`CreateCall(arguments, callId)`, `CreateResult(result, callId)` and `CreateError(error, callId)` return existing +`LlmTestServerToolCall` / tool-role `LlmTestServerMessage` records. `Definition` returns the existing +`LlmTestServerToolDefinition`. Arguments and result payloads are checked against their catalogue schemas. +The default call ID is `call_` followed by the function name; supply distinct IDs for multiple simultaneous calls. +Explicit IDs must be nonblank and at most 128 characters. + +## Typed fixture and tool round + +`Success(...)` combines a definition, call and result into a fixture. Its delay is metadata: apply it through +`DelaysBy` when configuring the existing response or step. This example uses the typed control client already +connected to a running server: + +```csharp +using ManagedCode.LlmTestServer.Tools.WellKnown; + +var weather = LlmTestServerWellKnownTools.Weather.Success( + new() { Location = "Paris", Unit = "celsius" }, + new() { Location = "Paris", Temperature = 21m, Unit = "celsius", Condition = "sunny" }, + callId: "call_paris", delayMilliseconds: 20); + +await control.ConfigureAsync(config => config.UseConversation("weather", graph => graph + .Step("call", step => step.WhenLatestUserEquals("weather") + .CallsTools(weather.Call).DelaysBy(weather.DelayMilliseconds).Then("answer")) + .Step("answer", step => step.WhenToolResult(weather.Call.Name, weather.Call.Id) + .WithToolArgumentsJson(weather.Call.ArgumentsJson) + .WhenLatestJson("$", weather.Result.Content) + .Responds("Paris is sunny, 21 C.").Complete()))); + +using var conversation = control.ForConversation("weather-test"); +``` + +Offer `weather.Definition` through the provider's function-tool wire shape. For OpenAI Chat Completions, map +`Name`, `Description` and parsed `ParametersJson` to `tools[].function`. Send the initial user message, retain the +returned assistant/tool call, and append a provider tool-result message with `tool_call_id = weather.Result.ToolCallId` +and `content = weather.Result.Content`. Use `conversation.HttpClient` for those requests or as the transport of a +real provider SDK. Normalized core records require the provider adapter's field names; they are not OpenAI wire DTOs. +The HTTP tests linked below show the complete request/response round. + +Arguments/results match structurally as JSON. A changed argument or result rejects the next graph step without +consuming it. Linear queues retain their normal exhaustion behavior; graphs retain their configured +`Reject`, `RepeatLastResponse`, `Restart`, step limits and cancellation semantics. Presets introduce no additional +cursor, repeat policy or executor. Existing `VerifyAsync` tool expectations inspect names, call IDs, argument JSON +and result JSON; see [request verification](RequestVerificationAndDeterminism.md). + +## Explicit result shapes + +Success results retain their typed root. A calculator success is `{"value":42}`, and a weather success is +`{"location":"Paris","temperature":21,"unit":"celsius","condition":"sunny"}`. There is no implicit success envelope. + +`Failure(arguments, error, callId, delayMilliseconds)` explicitly selects a tool error fixture: + +```csharp +var unavailable = LlmTestServerWellKnownTools.Weather.Failure( + new() { Location = "Paris" }, + new() { Code = "fixture_unavailable", Message = "Scripted weather failure", Retryable = true }); +``` + +Its result content is `{"error":{"code":"fixture_unavailable","message":"Scripted weather failure","retryable":true}}`. +This is the documented `LlmTestServerToolFixtureErrorResult` envelope, described by `ErrorSchemaJson`. +It is tool-result content, not a provider HTTP error or automatic retry instruction. Configure the next graph step +and the application's handling explicitly. `IsError` identifies the selected fixture kind. + +## Server evidence + +The combined server suite passed **852/852** with **91.79%** production line coverage; Release build passed +with zero warnings/errors and all 19 packages packed. The preset-focused HTTP cases passed **40/40**. +Evidence: `artifacts/delivery/presets-coverage.log`. A clean installed NuGet/Aspire/official SDK/Chromium consumer passed 1/1, including the typed Calculator tool round (`artifacts/delivery/presets-consumer.log`); +these results do not establish public-feed publication or completion of the remaining E5 work. + +- [WellKnownToolIntegrationTests](../../tests/ManagedCode.LlmTestServer.Tests/Tools/WellKnownToolIntegrationTests.cs): all twelve presets, success/error tool rounds, offered input schemas, typed output schemas and request verification. +- [WellKnownToolBoundaryTests](../../tests/ManagedCode.LlmTestServer.Tests/Tools/WellKnownToolBoundaryTests.cs): catalogue-driven HTTP calls, linear exhaustion, incompatible schemas without consumption, exact argument/result matching. +- [WellKnownToolLifecycleTests](../../tests/ManagedCode.LlmTestServer.Tests/Tools/WellKnownToolLifecycleTests.cs): existing repeat policy, stable call identity, optional delay and cancellation without consumption. + +These tests start the server and use real HTTP. The current preset qualification uses OpenAI function-tool wire +requests; it does not establish support for provider-native search, file-search or code-execution built-ins. diff --git a/docs/Reference/Clients.md b/docs/Reference/Clients.md new file mode 100644 index 0000000..a7e8a0e --- /dev/null +++ b/docs/Reference/Clients.md @@ -0,0 +1,249 @@ +# Clients and SDKs + +Calling LLM Test Server from tests: the control client, official provider SDKs, `Microsoft.Extensions.AI`, and the native TypeSafe client. Back to the [README](../../README.md). + +## First passing test + +In a .NET 10 TUnit test project, add `ManagedCode.LlmTestServer.Aspire`, `ManagedCode.LlmTestServer.Client` +and `Aspire.Hosting.Testing`. This starts the packaged server without Docker, configures a response, calls it +through `IChatClient` and verifies the recorded request: + +```csharp +using Aspire.Hosting.Testing; +using ManagedCode.LlmTestServer.Aspire; +using ManagedCode.LlmTestServer.Client; +using Microsoft.Extensions.AI; + +public sealed class AiIntegrationTests +{ + [Test] + public async Task ApplicationReceivesScriptedResponse(CancellationToken cancellationToken) + { + var builder = DistributedApplicationTestingBuilder.Create([]); + builder.AddLlmTestServer("llm-test-server").WithApiKey("test-key"); + await using var app = await builder.BuildAsync(cancellationToken); + await app.StartAsync(cancellationToken); + await app.ResourceNotifications.WaitForResourceHealthyAsync("llm-test-server", cancellationToken); + + using var http = app.CreateHttpClient("llm-test-server", "http"); + var control = new LlmTestServerClient(http, "test-key"); + await control.ConfigureAsync(config => config.RequireBearerToken("test-key").UseGpt41Mini() + .UseChatScenario("greeting", scenario => scenario.WhenUserContains("hello").Responds("Hello from your test!")), + cancellationToken); + using var chat = control.CreateChatClient(); + var response = await chat.GetResponseAsync([new ChatMessage(ChatRole.User, "hello")], + cancellationToken: cancellationToken); + + await Assert.That(response.Text).IsEqualTo("Hello from your test!"); + await Assert.That((await control.GetAssertionsAsync(cancellationToken)).Matched).IsEqualTo(1); + } +} +``` + +See [provider operation coverage](../Features/ProviderApiContractCoverage.md) for the exact supported contracts. + +## Parallel tests on a shared server + +Create a separate session for each test. Each one owns its configuration, response queues, history, prompt cache, +fault counters, video jobs and request journal. Disposal removes its server state; the shared transport stays usable. + +```csharp +await using var session = await control.CreateSessionAsync(config => config + .RequireBearerToken("test-key") + .UseChatScenario("isolated", scenario => scenario.Responds("This test's response"))); +using var chat = session.Control.CreateChatClient(); +var response = await chat.GetResponseAsync([new ChatMessage(ChatRole.User, "hello")]); +``` + +Use `session.HttpClient` as the transport for official provider SDKs. See [test sessions](../Features/TestSessions.md) +for credentials, capacity limits, configure/reset semantics and HTTP routes. + +## TypeSafe: Jev through System One + +**TypeSafe is the provider, Jev is its model family, and System One is the native API/protocol.** +`ManagedCode.LlmTestServer.SystemOne` contains the native DTOs; `ManagedCode.LlmTestServer.Client` +provides `LlmTestServerSystemOneClient`. The package name remains `SystemOne` because it also hosts the Kev profile. + +Jev is supported through its native System One API, including Jev 1.14 image assets +and bounding boxes. The emulator accepts JSON base64 and multipart PNG/JPEG uploads, +structured instructions and criteria, mixed `noul`/`choice`/`score`/`box` batches, +native confidence, usage and provider-resolved model versions. Image and box requests +require `TypeSafe-Danger-Zone-Images: true`; long-context requests can explicitly send +`TypeSafe-Danger-Zone-Long-Context: true`. It uses deterministic fixtures and never +calls a production model or estimates image tokens. + +- Native test-server route: `POST /systemone/v1/systemone`. +- Contracts: `ManagedCode.LlmTestServer.SystemOne`. +- Typed client: `LlmTestServerClient.CreateSystemOneClient()` from `ManagedCode.LlmTestServer.Client`. +- Register `jev-1.14.0` or `jev-preview` with `AddDecisionModel(id, LlmTestServerDecisionProvider.TypeSafe)`. +- [Jev 1.14 guide and limits](../Features/Jev114.md), [native decision fixtures](ProvidersAndRoutes.md#native-decision-api-fixtures), + [official images API](https://docs.typesafe.ai/images). + +Using the `control` client and Aspire-allocated HTTP transport from the first test above, configure a TypeSafe +fixture and call the native typed client: + +```csharp +using System.Text.Json; +using ManagedCode.LlmTestServer.Decisions; +using ManagedCode.LlmTestServer.SystemOne; + +await control.ConfigureAsync(config => config + .RequireBearerToken("test-key") + .UseDecisionModel(TypeSafeRequestValidation.Jev114Model, LlmTestServerDecisionProvider.TypeSafe) + .UseDecisionScenario(new LlmTestServerDecisionScenario + { + Id = "typesafe-priority", + ModelId = TypeSafeRequestValidation.Jev114Model, + Answers = new() + { + ["urgent"] = new() { Kind = LlmTestServerDecisionKind.Predicate, Probability = 0.95 }, + }, + })); + +var jev = control.CreateSystemOneClient(); +var decision = await jev.EvaluateAsync(new SystemOneDecisionRequest +{ + Model = TypeSafeRequestValidation.Jev114Model, + State = JsonSerializer.SerializeToElement("Please help quickly"), + Questions = new() + { + ["urgent"] = new() + { + Type = SystemOneDecisionTypes.Predicate, + Instructions = JsonSerializer.SerializeToElement("Does this need urgent attention?"), + }, + }, +}); +Console.WriteLine(decision.Answers["urgent"].Probability); // 0.95 +``` + +The client sends `POST /systemone/v1/systemone` using the control client's transport and configured test key. +For image/box fixtures, use `EvaluateAsync(..., enableImages: true)` or `EvaluateMultipartAsync(...)`; +see the [Jev 1.14 guide](../Features/Jev114.md) for explicit assets and native limits. + +## OpenAI SDK + +The official OpenAI .NET SDK expects its `Endpoint` to be the API root. Use +`/openai/v1` for this Test Server, or `llmTestServer.GetOpenAiEndpoint()` from the Aspire +package when wiring an AppHost resource. + +```csharp +using OpenAI; +using OpenAI.Chat; +using System.ClientModel; + +var chat = new ChatClient( + "gpt-4.1-mini", + new ApiKeyCredential("test-key"), + new OpenAIClientOptions + { + Endpoint = new Uri("http://localhost:5000/openai/v1"), + }); + +await foreach (var update in chat.CompleteChatStreamingAsync( + [new UserChatMessage("What is the invoice total?")])) +{ + Console.Write(update.ContentUpdate.Count > 0 ? update.ContentUpdate[0].Text : string.Empty); +} +``` + +## Azure OpenAI And Foundry SDKs + +The OpenAI v1 API is available at `/azure-openai/openai/v1/` and `/microsoft-foundry/openai/v1/`, with `chat/completions`, `responses` and `embeddings`. Point the official OpenAI client endpoint at the corresponding base URL and supply the configured API key. These v1 routes accept bearer or `api-key` authentication and need no dated `api-version` query parameter. Responses supports ordinary JSON and documented SSE events. See the [September 2026 API review](../Features/ProviderApiReview20260907.md) for the verified SDK versions and provider contracts. + +Azure OpenAI deployment routes use the deployment name as the Test Server model id. API keys sent through the official SDK's `api-key` header are accepted anywhere a bearer token would be accepted. + +```csharp +using Azure.AI.OpenAI; +using OpenAI.Chat; +using OpenAI.Embeddings; +using System.ClientModel; + +var azure = new AzureOpenAIClient( + new Uri("http://localhost:5000/azure-openai/"), + new ApiKeyCredential("test-key")); + +ChatClient chat = azure.GetChatClient("gpt-4.1-mini"); +EmbeddingClient embeddings = azure.GetEmbeddingClient("text-embedding-3-small"); + +var answer = await chat.CompleteChatAsync( +[ + new UserChatMessage("What is the invoice total?"), +]); + +var vector = await embeddings.GenerateEmbeddingAsync("invoice"); +``` + +Microsoft Foundry / Azure AI Inference clients call the `/microsoft-foundry/chat/completions` and `/microsoft-foundry/embeddings` routes. The inference routes require `api-version=2024-05-01-preview`, which the official SDK supplies. Set `Model` to the model id configured in LLM Test Server. + +```csharp +using Azure; +using Azure.AI.Inference; + +var endpoint = new Uri("http://localhost:5000/microsoft-foundry/"); +var credential = new AzureKeyCredential("test-key"); + +var chat = new ChatCompletionsClient(endpoint, credential); +var embeddings = new EmbeddingsClient(endpoint, credential); + +var completion = await chat.CompleteAsync( + new ChatCompletionsOptions( + [ + new ChatRequestUserMessage("What is the invoice total?"), + ]) + { + Model = "gpt-4.1-mini", + }); + +var embedding = await embeddings.EmbedAsync( + new EmbeddingsOptions(["invoice"]) + { + Model = "text-embedding-3-small", + }); +``` + +## Microsoft.Extensions.AI + +Use the client package when you want tests to call the fake provider through normal `Microsoft.Extensions.AI` abstractions: + +```csharp +using ManagedCode.LlmTestServer.Client; +using Microsoft.Extensions.AI; + +using var httpClient = new HttpClient +{ + BaseAddress = new Uri("http://localhost:5000"), +}; +httpClient.DefaultRequestHeaders.Authorization = + new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", "test-key"); + +using IChatClient chatClient = new LlmTestServerChatClient(httpClient); + +var response = await chatClient.GetResponseAsync( +[ + new ChatMessage(ChatRole.User, "What color is the largest animal?"), +]); + +Console.WriteLine(response.Text); +``` + +When a bearer token is required, prefer creating the modality clients from the control client so the same token is applied consistently: + +```csharp +var server = LlmTestServerClient.Create(httpClient, bearerToken: "test-key"); + +using IChatClient chatClient = server.CreateChatClient("docs-chat"); +using IEmbeddingGenerator> embeddings = + server.CreateEmbeddingGenerator("docs-embedding"); +using IImageGenerator images = server.CreateImageGenerator("docs-image"); +``` + +Streaming uses the same client: + +```csharp +await foreach (var update in chatClient.GetStreamingResponseAsync( + [new ChatMessage(ChatRole.User, "stream this")])) +{ + Console.Write(update.Text); +} +``` diff --git a/docs/Reference/Configuration.md b/docs/Reference/Configuration.md new file mode 100644 index 0000000..decacf9 --- /dev/null +++ b/docs/Reference/Configuration.md @@ -0,0 +1,446 @@ +# Configuration and scenarios + +Models, chat scenarios, modalities, fault simulation, runtime reconfiguration and assertions. Back to the [README](../../README.md). + +## Configuration Model + +LLM Test Server is configured with one `LlmTestServerConfiguration`. + +You can apply that configuration in two places: + +- at host startup through `builder.Services.AddLlmTestServer(options => ...)` +- at test runtime through `LlmTestServerClient.ConfigureAsync(...)` or `POST /admin-api/configure` + +Runtime configuration is a replacement, not a merge. Applying a new configuration resets scenario positions and assertion events. The runtime snapshots the configuration, so later mutations to your builder/list objects do not change the running provider. + +The default configuration advertises official OpenAI fixture model IDs: + +| Model ID | Kind | +| --- | --- | +| `gpt-4.1-mini` | Chat | +| `text-embedding-3-small` | Embedding | +| `gpt-image-1` | Image | +| `gpt-4o-mini-tts` | Audio | +| `sora-2` | Video | + +Model IDs and model kinds are enforced. A chat request to an embedding model, or an embedding request to an unknown model, returns `404` with `llm_test_server_unknown_model`. + +Use typed helpers for broader OpenAI model registration when tests need more than the defaults: + +```csharp +options.AddKnownOpenAiModels(reasoningTokens: 128); + +// Or register only one modality family. +options.AddCurrentOpenAiChatModels(reasoningTokens: 128); +options.AddOpenAiEmbeddingModels(); +options.AddOpenAiImageModels(); +options.AddOpenAiAudioModels(); +options.AddOpenAiVideoModels(); +``` + +Individual helpers are also available, including `AddGpt55(...)`, `AddGpt54Mini(...)`, `AddGpt5Nano(...)`, `AddGpt41()`, `AddGpt4OMini()`, `AddTextEmbedding3Large()`, `AddGptImage2()`, `AddGptImage15()`, `AddGptImage1Mini()`, `AddTts1()`, `AddTts1Hd()`, and `AddSora2Pro()`. Keep using `AddModel(...)` for custom deployment names or provider-specific IDs. + +### Fault Simulation + +Fault simulation is configured on the provider-neutral runtime and applies through every provider namespace. Use it when a test needs retry, moderation, or provider-error handling without scripting each provider route separately. + +```csharp +builder.Services.AddLlmTestServer(options => +{ + // Allow two accepted runtime requests, then return 429 too_many_requests. + options.SimulateRateLimitAfter(2); + + // Return 400 content_filter whenever a request text contains either term. + options.SimulateContentFilter("blocked phrase", "unsafe fixture"); +}); +``` + +`SimulateRateLimitAfter(0)` makes the first valid runtime request return `429` with `too_many_requests`. `SimulateContentFilter(...)` checks chat messages, embedding inputs, image prompts, audio speech input, audio transcription/translation prompts, and video prompts. `ConfigureAsync(...)` and `ResetAsync()` both reset the rate-limit counter so tests stay deterministic. + +## Chat Scenarios + +### Contains Match + +`WhenUserContains(...)` adds a user-message contains matcher. All configured match messages must be present somewhere in the actual request. + +```csharp +options.AddChatScenario( + "support-refund", + scenario => scenario + .ForModel(LlmTestServerKnownModelIds.Gpt41Mini) + .WhenUserContains("refund") + .Responds("I can help with a refund.")); +``` + +### Exact Match + +Use exact matching when a test must prove the complete prompt contract: + +```csharp +using ManagedCode.LlmTestServer.Scenarios; + +options.AddChatScenario( + "exact-system-contract", + scenario => scenario + .ForModel(LlmTestServerKnownModelIds.Gpt41Mini) + .WithExactMatch( + new LlmTestServerMessage { Role = "system", Content = "Return JSON only." }, + new LlmTestServerMessage { Role = "user", Content = "Give me the invoice total." }) + .Responds("{\"total\":42.50}")); +``` + +### Queued Responses + +Each matched request consumes the next response. This makes multi-turn flows deterministic: + +```csharp +options.AddChatScenario( + "two-turn-plan", + scenario => scenario + .ForModel(LlmTestServerKnownModelIds.Gpt41Mini) + .WhenUserContains("make a plan") + .Responds("First draft") + .Responds("Revised draft")); +``` + +After the queue is exhausted, the provider returns `409` with `llm_test_server_scenario_exhausted`. + +### Streaming Chunks + +The first `Responds` argument is the non-streaming response. The remaining arguments are the streaming chunks: + +```csharp +options.AddChatScenario( + "streaming-answer", + scenario => scenario + .ForModel(LlmTestServerKnownModelIds.Gpt41Mini) + .WhenUserContains("stream this") + .Responds("blue whale", "blue ", "whale")); +``` + +For `stream: true`, LLM Test Server emits SSE `data:` frames and a final `data: [DONE]` marker. All chunks in one response share the same response id. + +### Scripted Errors + +Use `Fails(...)` to test client error handling without waiting for a real provider to fail: + +```csharp +options.AddChatScenario( + "content-policy-error", + scenario => scenario + .ForModel(LlmTestServerKnownModelIds.Gpt41Mini) + .WhenUserContains("blocked fixture") + .Fails(400, "content_filter", "The scripted request was rejected.")); +``` + +### Delays And Cancellation + +Use `DelaysBy(...)` to test timeout and cancellation behavior: + +```csharp +options.AddChatScenario( + "timeout-path", + scenario => scenario + .ForModel(LlmTestServerKnownModelIds.Gpt41Mini) + .WhenUserContains("slow path") + .Responds("eventual answer") + .DelaysBy(1_500)); +``` + +If the caller cancels during the delay, the queued response is not consumed. + +### Scenario-Specific Token + +A scenario can require its own bearer token: + +```csharp +options.AddChatScenario( + "tenant-a-only", + scenario => scenario + .ForModel(LlmTestServerKnownModelIds.Gpt41Mini) + .RequireBearerToken("tenant-a-key") + .WhenUserContains("tenant secret") + .Responds("tenant-a response")); +``` + +Use either a global token or per-scenario tokens. If both are configured for a scenario, the same request must satisfy both checks, so different global and scenario tokens intentionally make that scenario unreachable. + +## Modalities + +### Embeddings + +Configure a fixed embedding vector. Every input value receives the same deterministic vector: + +```csharp +options.AddTextEmbedding3Small(); +options.WithDefaultEmbeddingVector(0.01f, 0.02f, 0.03f, 0.04f); +``` + +```csharp +using ManagedCode.LlmTestServer.Client; +using Microsoft.Extensions.AI; + +using var httpClient = new HttpClient { BaseAddress = new Uri("http://localhost:5000") }; +using IEmbeddingGenerator> embeddings = + new LlmTestServerEmbeddingGenerator(httpClient, "text-embedding-3-small"); + +var result = await embeddings.GenerateAsync(["alpha", "beta"]); +``` + +### Images + +Image generation returns a deterministic base64 PNG by default: + +```csharp +options.AddGptImage1(); +``` + +```csharp +using ManagedCode.LlmTestServer.Client; +using Microsoft.Extensions.AI; + +using var httpClient = new HttpClient { BaseAddress = new Uri("http://localhost:5000") }; +using IImageGenerator images = new LlmTestServerImageGenerator(httpClient, "gpt-image-1"); + +var image = await images.GenerateAsync( + new ImageGenerationRequest { Prompt = "compatibility marker" }); +``` + +### Audio + +Audio speech returns deterministic bytes with a matching media type. The default fixture is a minimal WAV payload: + +```csharp +options.AddGpt4OMiniTts(); +options.WithDefaultAudio(File.ReadAllBytes("fixtures/speech.wav"), "audio/wav"); +``` + +```csharp +var audio = await httpClient.PostAsJsonAsync( + "/openai/v1/audio/speech", + new { model = "gpt-4o-mini-tts", input = "hello", voice = "alloy" }); + +audio.EnsureSuccessStatusCode(); +var bytes = await audio.Content.ReadAsByteArrayAsync(); +``` + +## Runtime Reconfiguration + +Use runtime reconfiguration when each test needs a different provider script. + +### Configure Before A Test + +Use `ManagedCode.LlmTestServer.Client` as the universal test control surface. A test can start or discover the hosted Test Server, create one `LlmTestServerClient`, load all required models, datasets, scripted responses, errors, embeddings, images, and audio fixtures, and then call the provider through regular clients. + +```csharp +using ManagedCode.LlmTestServer.Client; +using Microsoft.Extensions.AI; + +using var httpClient = new HttpClient { BaseAddress = new Uri("http://localhost:5000") }; +var server = LlmTestServerClient.Create(httpClient, bearerToken: "test-key"); + +await server.ConfigureAsync( + config => config + .RequireBearerToken("test-key") + .UseChatModel("test-chat") + .UseEmbeddingModel("test-embedding") + .UseImageModel("test-image") + .UseAudioModel("test-audio") + .UseEmbeddingVector(0.9f, 0.8f) + .UseImageDataUri("data:image/png;base64,Zm9v") + .UseAudio([1, 2, 3, 4], "audio/test") + .UseDataset( + "checkout-flow", + dataset => dataset + .AddChatScenario( + "invoice-total", + scenario => scenario + .ForModel("test-chat") + .WhenUserContains("invoice total") + .Responds("{\"total\":42.50}", "{\"total\":", "42.50", "}")) + .AddChatScenario( + "provider-rate-limit", + scenario => scenario + .ForModel("test-chat") + .WhenUserContains("rate limit") + .Fails(429, "rate_limit_exceeded", "Scripted rate limit."))), + resetFirst: true); + +using IChatClient chat = server.CreateChatClient("test-chat"); +using IEmbeddingGenerator> embeddings = + server.CreateEmbeddingGenerator("test-embedding"); +using IImageGenerator images = server.CreateImageGenerator("test-image"); + +var response = await chat.GetResponseAsync( +[ + new ChatMessage(ChatRole.User, "What is the invoice total?"), +]); + +var vector = await embeddings.GenerateAsync(["invoice"]); +var image = await images.GenerateAsync(new ImageGenerationRequest { Prompt = "fixture" }); +var audio = await server.GenerateAudioAsync("test-audio", "speak this"); +var assertions = await server.GetAssertionsAsync(); +``` + +`resetFirst: true` clears previous assertion events and scenario positions before the new configuration is applied. Named datasets are active after configuration; they group scenarios so a test can load a whole conversation set without flattening everything by hand. + +### Configure With `LlmTestServerClient` + +```csharp +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Configuration; +using ManagedCode.LlmTestServer.Models; + +using var httpClient = new HttpClient { BaseAddress = new Uri("http://localhost:5000") }; +httpClient.DefaultRequestHeaders.Authorization = + new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", "test-key"); + +var control = new LlmTestServerClient(httpClient); + +var configuration = new LlmTestServerConfigurationBuilder() + .RequireBearerToken("test-key") + .AddModel("docs-chat", LlmTestServerModelKind.Chat) + .AddModel("docs-embedding", LlmTestServerModelKind.Embedding) + .AddChatScenario( + "docs-blue-whale", + scenario => scenario + .ForModel("docs-chat") + .WhenUserContains("largest animal") + .Responds("blue whale", "blue ", "whale")) + .WithDefaultEmbeddingVector(0.125f, 0.25f) + .Build(); + +await control.ConfigureAsync(configuration); +``` + +If the current runtime already has a bearer token, the configure request must use that current token. If the new configuration sets a different token, future provider and control requests must use the new token. + +### Configure With Raw JSON + +`POST /admin-api/configure` accepts readable enum values such as `"chat"` and `"contains"`: + +```bash +curl -X POST http://localhost:5000/admin-api/configure \ + -H "content-type: application/json" \ + -H "authorization: Bearer test-key" \ + -d '{ + "requiredBearerToken": "test-key", + "models": [ + { "id": "docs-chat", "kind": "chat" }, + { "id": "docs-embedding", "kind": "embedding" }, + { "id": "docs-image", "kind": "image" }, + { "id": "docs-audio", "kind": "audio" } + ], + "chatScenarios": [ + { + "id": "docs-blue-whale", + "modelId": "docs-chat", + "match": { + "mode": "contains", + "messages": [ + { "role": "user", "content": "largest animal" } + ] + }, + "responses": [ + { + "content": "blue whale", + "streamChunks": [ "blue ", "whale" ] + } + ] + } + ], + "defaultEmbeddingVector": [ 0.125, 0.25 ], + "defaultAudioMediaType": "audio/wav" + }' +``` + +Then call the configured model: + +```bash +curl -X POST http://localhost:5000/openai/v1/chat/completions \ + -H "content-type: application/json" \ + -H "authorization: Bearer test-key" \ + -d '{ + "model": "docs-chat", + "messages": [ + { "role": "user", "content": "What is the largest animal?" } + ] + }' +``` + +## Assertions And Reset + +The assertion summary is intentionally simple: it tells the test whether requests matched configured scenarios, missed, failed auth, used unknown models, exhausted a response queue, or returned scripted errors. + +```csharp +var control = new LlmTestServerClient(httpClient); +var assertions = await control.GetAssertionsAsync(); + +if (assertions.Unmatched > 0 || assertions.ModelNotFound > 0) +{ + throw new InvalidOperationException("The application called an unexpected LLM path."); +} +``` + +Reset clears assertion events and scenario response positions without replacing the current configuration: + +```csharp +await control.ResetAsync(); +``` + +## Common Test Shapes + +### Replace Provider Calls In Integration Tests + +1. Start the LLM Test Server service in Aspire or an ASP.NET Core test host. +2. Configure the expected scenarios for the test. +3. Point the application under test at the LLM Test Server base URL. +4. Run the application flow. +5. Assert `GetAssertionsAsync()` has no unmatched, unknown-model, auth-failed, or exhausted events. + +### Test Retry Logic + +```csharp +options.AddChatScenario( + "retry-then-success", + scenario => scenario + .ForModel(LlmTestServerKnownModelIds.Gpt41Mini) + .WhenUserContains("retry") + .Fails(429, "rate_limit_exceeded", "Try again.") + .Responds("success after retry")); +``` + +### Test Prompt Contract Drift + +```csharp +options.AddChatScenario( + "exact-contract", + scenario => scenario + .ForModel(LlmTestServerKnownModelIds.Gpt41Mini) + .WithExactMatch( + new LlmTestServerMessage { Role = "system", Content = "Return JSON only." }, + new LlmTestServerMessage { Role = "user", Content = "Summarize invoice INV-42." }) + .Responds("{\"summary\":\"paid\"}")); +``` + +If the application changes the prompt, the request becomes unmatched and the test can fail on the assertion summary. + +### Test Wrong Model Wiring + +Configure only the models your application is allowed to call: + +```csharp +options.AddModel("approved-chat-model", LlmTestServerModelKind.Chat); +``` + +If the app calls `old-chat-model` or calls `approved-chat-model` through the embeddings endpoint, LLM Test Server returns `llm_test_server_unknown_model`. + +## Compatibility boundaries + +Tool calls and structured output use explicit deterministic fixtures: `CallsTool(...)`, `CallsTools(...)` and `RespondsJson(...)`. Provider adapters preserve tool history and streaming events; `LlmTestServerChatClient` supports the standard Microsoft.Extensions.AI function-invocation loop. Fixtures must satisfy tool selection, argument schemas and response schemas; mismatches return 409 without consuming the response. See [supported contracts and tests](../Features/ToolAndStructuredFixtures.md). + +Video create operations retain completed fixture jobs. Retrieve/content/delete require a previously created ID, and reset or reconfiguration clears jobs. OpenAI fixtures return `status=completed` and `progress=100`; IDs remain unique within a run. Dataset-local scenario IDs are isolated, including concurrent cancellation rollback. + +Prompt-cache storage is bounded to 4096 prefixes by default. Use `WithPromptCacheCapacity(capacity)` to change the limit; zero disables caching. Eviction is deterministic FIFO, and reset/configure clears the cache. + +Legacy Azure deployment APIs accept the dated `api-version` selected automatically by the Azure SDK. Use the default `AzureOpenAIClientOptions` constructor, or omit options entirely; do not pin a version for the emulator. These routes require exactly one supported deployment API version: the documented `2024-10-21` GA or `2025-04-01-preview` contract. Missing, malformed, duplicate, and unknown versions are rejected, including valid-looking future dates. Azure/Foundry `/openai/v1` endpoints require no dated query parameter. diff --git a/docs/Reference/Installation.md b/docs/Reference/Installation.md new file mode 100644 index 0000000..8be9ee9 --- /dev/null +++ b/docs/Reference/Installation.md @@ -0,0 +1,179 @@ +# Installation and hosting + +How to install the packages, run the standalone container, embed the server in ASP.NET Core or Aspire, and build from source. Back to the [README](../../README.md). + +## Choose how to run + +| Setup | Install / start | Best fit | +| --- | --- | --- | +| **.NET + Aspire** | `ManagedCode.LlmTestServer.Aspire` + `ManagedCode.LlmTestServer.Client` | Start an isolated .NET server from your AppHost or integration test; Docker is optional. | +| **Standalone Docker** | `managedcode/llm-test-server:local`, built with the .NET SDK | Run the dashboard, management API and provider routes together for applications in any language. | +| **ASP.NET Core library** | `ManagedCode.LlmTestServer.Hosting` | Embed the server in your own host with `AddLlmTestServer(...)` and `MapLlmTestServer()`. | + +**v1.0.0 publication is pending.** NuGet links below identify the intended packages; they do not establish +public availability. Build and [pack the source](#build-and-test) to use the local `artifacts/packages` feed now. +The current source package version is `0.1.18` from `Directory.Build.props`. Run these in your consumer +project, replacing the feed path with your checkout (use the canonical version if it changes): + +```sh +dotnet add package ManagedCode.LlmTestServer.Aspire --version 0.1.18 --no-restore +dotnet add package ManagedCode.LlmTestServer.Client --version 0.1.18 --no-restore +dotnet restore --source /absolute/path/to/LlmTestServer/artifacts/packages \ + --source https://api.nuget.org/v3/index.json +``` + +For an existing ASP.NET Core application, install `ManagedCode.LlmTestServer.Hosting` from the same feed +and follow [ASP.NET Core startup](#aspnet-core-startup). The core package owns scenarios and model configuration; +provider packages own their wire contracts, and the client package owns management and typed client APIs. + +### Standalone container and management API + +Build a local image from the checkout with the .NET SDK's `PublishContainer` target, then run it with Docker: + +```sh +dotnet publish samples/ManagedCode.LlmTestServer.Service/ManagedCode.LlmTestServer.Service.csproj \ + --configuration Release --runtime linux-arm64 --self-contained false \ + /t:PublishContainer -p:ContainerRuntimeIdentifiers=linux-arm64 \ + -p:ContainerImageTag=local -p:UseAppHost=false +docker run --rm --name llm-test-server -p 8080:8080 \ + -e LlmTestServer__RequiredBearerToken=local-test-key \ + managedcode/llm-test-server:local +``` + +Use `linux-x64` in both RID arguments on an amd64 machine. Open **http://localhost:8080/** for the dashboard. +The host port is explicitly mapped to container port 8080. Configure a model and response through the running +server, keeping the same test API key in the replacement configuration: + +```sh +curl --fail-with-body http://localhost:8080/admin-api/configure \ + -H 'Authorization: Bearer local-test-key' -H 'Content-Type: application/json' \ + --data '{"requiredBearerToken":"local-test-key","models":[{"id":"gpt-4.1-mini","kind":"chat"}],"chatScenarios":[{"id":"hello","modelId":"gpt-4.1-mini","responses":[{"content":"Hello from your fixture"}]}]}' + +curl --fail-with-body http://localhost:8080/openai/v1/chat/completions \ + -H 'Authorization: Bearer local-test-key' -H 'Content-Type: application/json' \ + --data '{"model":"gpt-4.1-mini","messages":[{"role":"user","content":"Hello"}]}' +``` + +The API key is a local fixture credential; production provider credentials are unnecessary. Provider routes +and `/admin-api/*` use it. The intended public release image is +`ghcr.io/managedcode/llm-test-server:1.0.0`; public pulling remains pending release and anonymous-pull verification. +See [Docker distribution](../Features/DockerDistribution.md) for both architectures, authentication, +health checks and release requirements. + +## ASP.NET Core Startup + +The simplest host config is one chat scenario: + +```csharp +using ManagedCode.LlmTestServer.Hosting; +using ManagedCode.LlmTestServer.Models; + +var builder = WebApplication.CreateBuilder(args); + +builder.Services.AddLlmTestServer(options => +{ + options.AddChatScenario( + "blue-whale", + scenario => scenario + .ForModel(LlmTestServerKnownModelIds.Gpt41Mini) + .WhenUserContains("largest animal") + .Responds("blue whale", "blue ", "whale")); +}); + +var app = builder.Build(); +app.MapLlmTestServer(); +app.Run(); +``` + +A fuller config can advertise your application's real model names while still serving deterministic fixtures: + +```csharp +using ManagedCode.LlmTestServer.Hosting; +using ManagedCode.LlmTestServer.Models; + +var builder = WebApplication.CreateBuilder(args); + +builder.Services.AddLlmTestServer(options => +{ + options.RequireBearerToken("test-key"); + + options.AddDefaultOpenAiModels(); + + options.WithDefaultEmbeddingVector(0.125f, 0.25f, 0.5f, 1.0f); + options.WithDefaultAudio(File.ReadAllBytes("fixtures/ok.wav"), "audio/wav"); + + options.AddChatScenario( + "invoice-total", + scenario => scenario + .ForModel(LlmTestServerKnownModelIds.Gpt41Mini) + .WhenUserContains("invoice total") + .Responds( + "{\"total\":42.50}", + "{\"total\":", + "42.50", + "}")); + + options.AddChatScenario( + "provider-rate-limit", + scenario => scenario + .ForModel(LlmTestServerKnownModelIds.Gpt41Mini) + .WhenUserContains("rate limit") + .Fails(429, "rate_limit_exceeded", "Scripted rate limit from LLM Test Server.")); + + options.AddChatScenario( + "slow-response", + scenario => scenario + .ForModel(LlmTestServerKnownModelIds.Gpt41Mini) + .WhenUserContains("slow") + .Responds("done") + .DelaysBy(250)); +}); + +var app = builder.Build(); +app.MapLlmTestServer(); +app.Run(); +``` + +## Aspire AppHost + +Install `ManagedCode.LlmTestServer.Aspire` in the AppHost using the versioned +[local-feed installation commands](#choose-how-to-run) above while public publication is pending. + +Then add the package-owned Test Server resource directly: + +```csharp +using ManagedCode.LlmTestServer.Aspire; + +var builder = DistributedApplication.CreateBuilder(args); + +var llmTestServer = builder + .AddLlmTestServer() + .WithApiKey("test-key"); + +builder + .AddProject("consumer") + .WithReference(llmTestServer) + .WithEnvironment("LLM_PROVIDER_ENDPOINT", llmTestServer.GetHttpEndpoint()) + .WithEnvironment("OPENAI_ENDPOINT", llmTestServer.GetOpenAiEndpoint()) + .WithEnvironment("OPENAI_API_KEY", "test-key") + .WaitFor(llmTestServer); + +builder.Build().Run(); +``` + +`AddLlmTestServer()` creates a `LlmTestServerResource` backed by the packaged .NET LLM Test Server service executable and exposes its `http` endpoint. It does not require a consumer service project reference, generated `Projects.*` metadata type, project path, Docker, or a container runtime. Consumer resources should reference the Test Server resource, wait for it, and use `llmTestServer.GetHttpEndpoint()` when they need the provider-compatible base URL. `.WithApiKey("test-key")` sets `LlmTestServer:RequiredBearerToken` so both provider endpoints and `/admin-api/*` control endpoints require the same bearer token. + +Use `AddLlmTestServerContainer()` only when you explicitly want a container-backed resource, for example for a deployment or container-runtime smoke test. The container mode uses a `ghcr.io/managedcode/llm-test-server` image tagged with the same release version as the installed `ManagedCode.LlmTestServer.Aspire` package; it is not the default local Aspire path. + +## Build And Test + +```bash +dotnet restore ManagedCode.LlmTestServer.slnx +dotnet build ManagedCode.LlmTestServer.slnx --configuration Release --no-restore +dotnet test --project tests/ManagedCode.LlmTestServer.Tests/ManagedCode.LlmTestServer.Tests.csproj --configuration Release --no-build --verbosity normal +for project in src/*/*.csproj; do dotnet pack "$project" --configuration Release --no-build --output artifacts/packages; done +``` + +`global.json` opts `dotnet test` into `Microsoft.Testing.Platform` for .NET 10. + +After restore/build, run `bash scripts/coverage.sh` for the production 90% coverage gate. After packing, run `python3 scripts/verify-package.py` for an isolated NuGet consumer, test-owned Aspire host and Chromium dashboard test; this requires Python 3 and PowerShell (`pwsh`). CI installs Chromium dependencies and saves coverage/browser evidence. `python3 scripts/test-release.py` verifies partial-release recovery locally without external publication. Release delivery is marked complete only after NuGet push and GitHub artifact attachment succeed. diff --git a/docs/Reference/ProvidersAndRoutes.md b/docs/Reference/ProvidersAndRoutes.md new file mode 100644 index 0000000..5ee0ad8 --- /dev/null +++ b/docs/Reference/ProvidersAndRoutes.md @@ -0,0 +1,152 @@ +# Providers, packages and routes + +Provider namespaces, NuGet packages, hosted routes, current functionality and native decision APIs. Back to the [README](../../README.md). + +## Packages + +Package links identify the NuGet package surfaces; badges update when publication is available. +The complete v1.0.0 launch remains pending. Use the [local package feed](Installation.md#choose-how-to-run) before public release. + +| Package | Latest version | Downloads | Description | +| --- | --- | --- | --- | +| [ManagedCode.LlmTestServer](https://www.nuget.org/packages/ManagedCode.LlmTestServer) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTestServer.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTestServer.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer) | Provider-neutral scenario runtime and assertion state. | +| [ManagedCode.LlmTestServer.OpenAI](https://www.nuget.org/packages/ManagedCode.LlmTestServer.OpenAI) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTestServer.OpenAI.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.OpenAI) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTestServer.OpenAI.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.OpenAI) | OpenAI-compatible wire contracts. | +| [ManagedCode.LlmTestServer.AzureOpenAI](https://www.nuget.org/packages/ManagedCode.LlmTestServer.AzureOpenAI) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTestServer.AzureOpenAI.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.AzureOpenAI) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTestServer.AzureOpenAI.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.AzureOpenAI) | Azure OpenAI compatibility profile and wire contracts. | +| [ManagedCode.LlmTestServer.Foundry](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Foundry) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTestServer.Foundry.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Foundry) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTestServer.Foundry.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Foundry) | Microsoft Foundry compatibility profile and wire contracts. | +| [ManagedCode.LlmTestServer.Anthropic](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Anthropic) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTestServer.Anthropic.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Anthropic) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTestServer.Anthropic.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Anthropic) | Anthropic Messages compatibility profile and wire contracts. | +| [ManagedCode.LlmTestServer.Gemini](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Gemini) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTestServer.Gemini.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Gemini) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTestServer.Gemini.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Gemini) | Gemini compatibility profile and wire contracts. | +| [ManagedCode.LlmTestServer.Groq](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Groq) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTestServer.Groq.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Groq) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTestServer.Groq.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Groq) | Groq compatibility profile and wire contracts. | +| [ManagedCode.LlmTestServer.Mistral](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Mistral) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTestServer.Mistral.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Mistral) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTestServer.Mistral.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Mistral) | Mistral compatibility profile and wire contracts. | +| [ManagedCode.LlmTestServer.Ollama](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Ollama) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTestServer.Ollama.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Ollama) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTestServer.Ollama.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Ollama) | Ollama compatibility profile and wire contracts. | +| [ManagedCode.LlmTestServer.Cohere](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Cohere) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTestServer.Cohere.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Cohere) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTestServer.Cohere.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Cohere) | Cohere compatibility profile and wire contracts. | +| [ManagedCode.LlmTestServer.Bedrock](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Bedrock) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTestServer.Bedrock.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Bedrock) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTestServer.Bedrock.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Bedrock) | Amazon Bedrock compatibility profile and wire contracts. | +| [ManagedCode.LlmTestServer.OpenRouter](https://www.nuget.org/packages/ManagedCode.LlmTestServer.OpenRouter) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTestServer.OpenRouter.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.OpenRouter) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTestServer.OpenRouter.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.OpenRouter) | OpenRouter compatibility profile and wire contracts. | +| [ManagedCode.LlmTestServer.DeepSeek](https://www.nuget.org/packages/ManagedCode.LlmTestServer.DeepSeek) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTestServer.DeepSeek.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.DeepSeek) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTestServer.DeepSeek.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.DeepSeek) | DeepSeek compatibility profile and wire contracts. | +| [ManagedCode.LlmTestServer.Perplexity](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Perplexity) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTestServer.Perplexity.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Perplexity) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTestServer.Perplexity.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Perplexity) | Perplexity compatibility profile and wire contracts. | +| [ManagedCode.LlmTestServer.SystemOne](https://www.nuget.org/packages/ManagedCode.LlmTestServer.SystemOne) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTestServer.SystemOne.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.SystemOne) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTestServer.SystemOne.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.SystemOne) | **TypeSafe / Jev** and Kev native System One decision contracts. | +| [ManagedCode.LlmTestServer.Cloudflare](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Cloudflare) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTestServer.Cloudflare.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Cloudflare) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTestServer.Cloudflare.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Cloudflare) | Cloudflare Workers AI Clef/flash decision envelopes and validation. | +| [ManagedCode.LlmTestServer.Hosting](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Hosting) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTestServer.Hosting.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Hosting) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTestServer.Hosting.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Hosting) | ASP.NET Core endpoint mapping. | +| [ManagedCode.LlmTestServer.Client](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Client) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTestServer.Client.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Client) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTestServer.Client.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Client) | Control client plus `IChatClient`, `IEmbeddingGenerator>`, and `IImageGenerator` implementations. | +| [ManagedCode.LlmTestServer.Aspire](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Aspire) | [![NuGet](https://img.shields.io/nuget/v/ManagedCode.LlmTestServer.Aspire.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Aspire) | [![Downloads](https://img.shields.io/nuget/dt/ManagedCode.LlmTestServer.Aspire.svg?style=flat-square&logo=nuget)](https://www.nuget.org/packages/ManagedCode.LlmTestServer.Aspire) | Aspire AppHost extension methods. | + +## Current Functionality + +This is the currently implemented behavior, not future intent: + +| Surface | What works now | +| --- | --- | +| Hosting | `builder.Services.AddLlmTestServer(...)` registers the default runtime and isolated test-session registry; `app.MapLlmTestServer()` maps the Blazor server-side browser panel at `/`, JSON control endpoints under `/admin-api`, and provider-namespaced HTTP routes. | +| Runtime configuration | Configure at host startup, through `LlmTestServerClient.ConfigureAsync(...)`, or through `POST /admin-api/configure`. Configuration replaces the current runtime state and resets assertions, scenario positions, fault counters, and prompt-cache entries. | +| Reset | `ResetAsync()` and `POST /admin-api/reset` keep the current models/scenarios/fixtures but clear assertion events, scenario response positions, rate-limit counters, and prompt-cache entries. | +| Models | Default models use official fixture IDs for chat, embeddings, images, audio, and video. Typed helpers register broader OpenAI model families, and `AddModel(...)` supports custom deployment or provider IDs. Model kind is enforced on every route. | +| Chat scenarios | Contains matching, exact prompt matching, queued responses, streaming chunks, scripted errors, delayed responses, cancellation-safe queues, and scenario-specific bearer tokens are implemented. | +| Modalities | Chat, embeddings, image generation/edit/variation, audio speech/transcription/translation, OpenAI/Azure video, Gemini long-running video/file output, and Bedrock text/embedding/image invoke shapes return deterministic fixtures. | +| Provider matrix | OpenAI, Azure OpenAI, Microsoft Foundry, Anthropic, Gemini, Groq, Mistral, Ollama, Cohere, Amazon Bedrock, OpenRouter, DeepSeek, and Perplexity are exposed under explicit provider namespaces. Each hosted operation is represented in a provider `ApiContract` and behavior evidence tests. | +| SDK clients | Official OpenAI, Azure OpenAI, Azure AI Inference / Foundry, Anthropic and Google Gen AI SDKs, Amazon Bedrock, and `Microsoft.Extensions.AI` clients call the qualified hosted routes. See the [SDK compatibility matrix](../Features/ProviderSdkCompatibility.md) for operation-level evidence and boundaries. The client package also creates control, chat, embedding, image, and audio clients with the same bearer token. | +| Auth | A global bearer token protects provider and `/admin-api/*` endpoints. Anthropic `x-api-key` and Azure `api-key` headers are accepted as provider-native ways to pass the same token. Scenarios can add their own bearer-token requirement. | +| Fault simulation | Provider-neutral rate-limit and content-filter simulation flow through every provider family and return provider-shaped errors. Scripted per-scenario failures remain available for exact test paths. | +| Assertions | `/admin-api/assertions` returns counters plus retained runtime events with request preview, response/error text, model id, scenario id, event kind, and deterministic usage. | +| Test sessions | Typed `CreateSessionAsync(...)` and session headers isolate configuration, queues, history, faults, cache and journal; reset/disposal affect the owning session. | +| Conversation graphs | `UseConversation(...)` configures typed step predicates, branches, tool-result checks, turn ordering, terminal policies and successful-request replay; match explanations and conversation status are available through control APIs. | +| Request evidence | `GetRequestsAsync(...)` filters/pages the bounded journal; `ExportRequestsAsync(...)` and `ExportRequestsNdjsonAsync(...)` include versioned retention/eviction evidence. | +| Verification | `VerifyAsync(...)` checks counts/order, unexpected calls, structured tool arguments/results, visited graph steps, terminal markers and retry policy. Incomplete retained evidence cannot pass. | +| Wire identity and time | `UseDeterminism(seed, fixedUtc)` fixes generated provider IDs/time with provider-local counters; reset reproduces them and graph replay retains metadata. Real diagnostic clocks and retry timing stay real. | +| Usage accounting | Input, output, reasoning, total, cached-input, and cache-write token values are computed deterministically and projected into provider-specific response envelopes when that provider surface documents usage fields. | +| Browser panel | `/` shows models/fixtures, actions, control and provider endpoint groups, grouped runtime events, captured requests, captured responses, counters, and per-event usage. It auto-refreshes every 15 seconds using Blazor component state and `StateHasChanged()`. | +| Aspire | `builder.AddLlmTestServer()` starts the packaged .NET service without Docker by default; `WithApiKey(...)` wires auth, `GetHttpEndpoint()` and provider-specific endpoint helpers expose URLs, and `AddLlmTestServerContainer()` is an explicit container opt-in. | + +See [conversation graphs](../Features/ConversationDSL.md) and +[request verification and determinism](../Features/RequestVerificationAndDeterminism.md) for typed APIs, +HTTP contracts, retention limits and terminal-marker applicability. Explicit fixture timestamps retain their +configured values. +Public publication and remaining roadmap features are tracked in the [delivery plan](../Features/LlmTestServerDeliveryPlan.md). + +## Provider Packages + +Provider packages keep vendor-specific protocol and compatibility metadata out of the provider-neutral runtime. The package surface exists now so applications can select a provider family explicitly; provider-specific DTOs and endpoint mappings are added inside each package as support grows. + +| Package | Provider ID | Protocol family | Primary hosted route | +| --- | --- | --- | --- | +| `ManagedCode.LlmTestServer.OpenAI` | `openai` | OpenAI | `/openai/v1` | +| `ManagedCode.LlmTestServer.AzureOpenAI` | `azure-openai` | Azure OpenAI | `/azure-openai/openai/deployments/{deployment}/chat/completions` | +| `ManagedCode.LlmTestServer.Foundry` | `microsoft-foundry` | Microsoft Foundry | `/microsoft-foundry/models/chat/completions` | +| `ManagedCode.LlmTestServer.Anthropic` | `anthropic` | Anthropic Messages | `/anthropic/v1/messages` | +| `ManagedCode.LlmTestServer.Gemini` | `gemini` | Gemini | `/gemini/v1beta/models/{model}:generateContent` | +| `ManagedCode.LlmTestServer.Groq` | `groq` | Groq | `/groq/openai/v1/chat/completions` | +| `ManagedCode.LlmTestServer.Mistral` | `mistral` | Mistral | `/mistral/v1/chat/completions` | +| `ManagedCode.LlmTestServer.Ollama` | `ollama` | Ollama | `/ollama/api/chat` | +| `ManagedCode.LlmTestServer.Cohere` | `cohere` | Cohere | `/cohere/v2/chat` | +| `ManagedCode.LlmTestServer.Bedrock` | `bedrock` | Amazon Bedrock | `/bedrock/model/{modelId}/converse` | +| `ManagedCode.LlmTestServer.OpenRouter` | `openrouter` | OpenRouter | `/openrouter/api/v1/chat/completions` | +| `ManagedCode.LlmTestServer.DeepSeek` | `deepseek` | DeepSeek | `/deepseek/v1/chat/completions` | +| `ManagedCode.LlmTestServer.Perplexity` | `perplexity` | Perplexity | `/perplexity/v1/sonar` | +| **`ManagedCode.LlmTestServer.SystemOne`** | `systemone` | **TypeSafe / Jev** and Kev native System One decisions | `/systemone/v1/systemone` | +| `ManagedCode.LlmTestServer.Cloudflare` | `cloudflare` | Clef / Clef Flash native decisions | `/cloudflare/client/v4/accounts/{accountId}/ai/run/@cf/cloudflare/clef` | + +Each provider profile also carries a doc-backed `ApiContract` with official documentation links, retrieval date, API version or header requirements, documented operations, streaming support, and whether `ManagedCode.LlmTestServer.Hosting` currently maps the operation. + +Use a provider profile when code needs a stable package-level declaration without starting a host: + +```csharp +using ManagedCode.LlmTestServer.AzureOpenAI; + +var profile = AzureOpenAiCompatibility.Profile; +Console.WriteLine(profile.Id); // azure-openai +``` + +## Hosted Endpoint Families + +Provider endpoints are explicitly namespaced. The table below shows the route +families and representative operations; the full method/path contract lives in +each provider profile's `ApiContract` and is guarded by +`ProviderApiContractTests`. See +[`docs/Features/ProviderApiContractCoverage.md`](../Features/ProviderApiContractCoverage.md) +for the current coverage rules and behavior-test inventory. + +| Provider | Namespace | Representative hosted operations | +| --- | --- | --- | +| OpenAI | `/openai` | `/openai/v1/models`, `/openai/v1/decisions`, `/openai/v1/chat/completions`, `/openai/v1/responses`, `/openai/v1/embeddings`, `/openai/v1/images/*`, `/openai/v1/audio/*`, `/openai/v1/videos*` | +| Azure OpenAI | `/azure-openai` | `/azure-openai/openai/v1/{chat/completions,responses,embeddings}`, deployment chat/embeddings/images/audio, and `/azure-openai/openai/v1/video/generations/*` | +| Microsoft Foundry | `/microsoft-foundry` | `/microsoft-foundry/openai/v1/{chat/completions,responses,embeddings}`, `/microsoft-foundry/chat/completions`, `/microsoft-foundry/embeddings`, and `/microsoft-foundry/models/*` aliases | +| Anthropic | `/anthropic` | `/anthropic/v1/messages` | +| Gemini | `/gemini` | `/gemini/v1beta/models/{model}:generateContent`, streaming content, embeddings, long-running video operations, and generated files | +| Groq | `/groq` | `/groq/openai/v1/models`, chat completions, Responses, speech, transcription, and translation routes | +| Mistral | `/mistral` | `/mistral/v1/chat/completions`, `/mistral/v1/embeddings` | +| Ollama | `/ollama` | `/ollama/api/chat`, `/ollama/api/embed` | +| Cohere | `/cohere` | `/cohere/v2/chat`, `/cohere/v2/embed` | +| Amazon Bedrock | `/bedrock` | `/bedrock/model/{modelId}/converse`, converse stream, invoke, and invoke-with-response-stream | +| OpenRouter | `/openrouter` | `/openrouter/api/v1/models`, chat completions, Responses | +| DeepSeek | `/deepseek` | `/deepseek/models`, `/deepseek/v1/chat/completions` | +| Perplexity | `/perplexity` | `/perplexity/v1/sonar` | +| TypeSafe / Jev and Kev (System One) | `/systemone` | `/systemone/v1/systemone` | +| Cloudflare (Clef/flash) | `/cloudflare` | `/cloudflare/client/v4/accounts/{accountId}/ai/run/@cf/cloudflare/clef`, `clef-flash` | + +The Blazor server-side rendered browser control panel lives at `/`. JSON control endpoints live under +`/admin-api`: `GET /admin-api/models`, `GET /admin-api/assertions`, +`POST /admin-api/configure`, and `POST /admin-api/reset`. + +Audio speech returns a deterministic WAV fixture by default. When +`requiredBearerToken` is configured, all provider and control endpoints require +`Authorization: Bearer `. + +## Native decision API fixtures + +Native typed clients are available through `LlmTestServerClient.CreateSystemOneClient()` +and `LlmTestServerClient.CreateCloudflareDecisionClient(accountId)`. Clef accepts embedded +PNG/JPEG/WebP data URLs or base64 objects; Jev accepts PNG/JPEG reference assets as +JSON or multipart, with explicit image/long-context opt-ins. See the +[official-contract audit and supported limits](../Features/DecisionVisionAudit.md). + +Jev, Kev, Cloudflare Clef/Clef Flash and OpenAI Decisions use explicit decision fixtures. The Test Server hosts their native JSON shapes under provider namespaces and does not contact production providers. Declare the provider profile explicitly; model names never select an API implicitly. + +| Provider | Test Server POST route | Production POST endpoint | Model helper | Official contract | +| --- | --- | --- | --- | --- | +| TypeSafe Jev | `/systemone/v1/systemone` | `https://api.typesafe.ai/v1/systemone` | `AddJevLatest()` | [OpenAPI](https://api.typesafe.ai/openapi.json), [primitives](https://docs.typesafe.ai/primitives/choice) | +| Kev | `/systemone/v1/systemone` | `{kev-server}/v1/systemone` | `AddKevLatest()` | [native request models](https://github.com/jaredpalmer/kev/blob/main/kev/api.py), [server/auth](https://github.com/jaredpalmer/kev/blob/main/kev/serve.py) | +| Cloudflare Clef | `/cloudflare/client/v4/accounts/{accountId}/ai/run/@cf/cloudflare/clef` | `https://api.cloudflare.com/client/v4/accounts/{accountId}/ai/run/@cf/cloudflare/clef` | `AddClef()` | [Clef API](https://developers.cloudflare.com/workers-ai/models/clef/), [REST envelope/auth](https://developers.cloudflare.com/workers-ai/get-started/rest-api/) | +| Cloudflare Clef Flash | `/cloudflare/client/v4/accounts/{accountId}/ai/run/@cf/cloudflare/clef-flash` | `https://api.cloudflare.com/client/v4/accounts/{accountId}/ai/run/@cf/cloudflare/clef-flash` | `AddClefFlash()` | [Clef Flash API](https://developers.cloudflare.com/workers-ai/models/clef-flash/) | +| OpenAI Decisions | `/openai/v1/decisions` | `https://api.openai.com/v1/decisions` | `AddOpenAiDecisionModel("gpt-6-luna")` | [create API](https://developers.openai.com/api/reference/resources/decisions/methods/create/) | + +Production APIs use `Authorization: Bearer `; Kev checks `KEV_API_KEY` only when configured and otherwise permits unauthenticated calls. For the Test Server, `RequireBearerToken` controls the expected test token and protects both provider routes and `/admin-api/*`. Tests do not need production credentials. Resolve the hosted base URL from the Aspire resource (`GetEndpoint("http")` / `CreateHttpClient(...)`), then append the Test Server route. + +Complete native request/response contracts, examples and limits for SystemOne (Jev, Kev), Cloudflare Clef/Clef Flash and OpenAI Decisions are in [native decision contracts](../Features/DecisionModels.md); behavior-test evidence is in the [test inventory](../Testing/AcceptanceCriteriaTestInventory.md). diff --git a/docs/Reference/TokenUsageAndPromptCache.md b/docs/Reference/TokenUsageAndPromptCache.md new file mode 100644 index 0000000..81225d2 --- /dev/null +++ b/docs/Reference/TokenUsageAndPromptCache.md @@ -0,0 +1,59 @@ +# Token usage and prompt-cache accounting + +How deterministic token usage, reasoning tokens and prompt-cache reads/writes are computed and reported on every provider surface. For the per-role input composition and dashboard charts, see [dashboard analytics](../Features/DashboardAnalytics.md). Back to the [README](../../README.md). + +## Control Panel And Token Usage + +Open the Test Server resource endpoint from the Aspire dashboard to inspect the running configuration at `/`. The browser admin panel is rendered by Blazor server-side rendering from `ManagedCode.LlmTestServer.Hosting`. If the resource was configured with `.WithApiKey("test-key")` or `RequireBearerToken("test-key")`, enter the same token in the control panel before refreshing. + +The dashboard is a request inbox: one row per provider call, with the request, response, conversation, wire payloads, stream chunks, token composition and runtime result for the selected call. The **Tokens** tab draws the token flow and prompt-cache donuts for a request or an explicitly labeled conversation, and **Traffic insights** (`/?view=insights`) charts all retained traffic; see [dashboard analytics](../Features/DashboardAnalytics.md). The **Runtime** dialog shows fixtures, provider routes, assertion counters and the event timeline. Auto-refresh is enabled by default and runs every 15 seconds on the server-side Blazor component; there is no client-side polling script and no visible countdown. + +Token usage is counted with the repo-owned tiktoken-compatible counter and reported both as summary totals and per runtime event with `inputTokens`, `cachedInputTokens`, `cacheCreationInputTokens`, `outputTokens`, `reasoningTokens`, and `totalTokens`. Provider response envelopes also receive the same deterministic usage values: OpenAI-compatible chat and Responses usage including reasoning-token and prompt-cache details when configured, Anthropic sync messages and streaming `message_start`/`message_delta`, Gemini `usageMetadata` including long-running video operation results, Cohere chat usage, Ollama prompt/eval counts, and Bedrock Converse usage. + +For reasoning-capable fixture models, configure a deterministic reasoning-token count on the model. LLM Test Server includes those tokens in `outputTokens` and exposes the split in `reasoningTokens`; OpenAI-compatible responses also include `completion_tokens_details.reasoning_tokens` or `output_tokens_details.reasoning_tokens`. + +```csharp +options.AddGpt5Nano(reasoningTokens: 128); +``` + +## Prompt Cache Accounting + +Prompt cache accounting is deterministic runtime state for testing provider usage handling. LLM Test Server does not call a real cache service and does not hide prompt text; it calculates repeated prompt-prefix usage from the requests that reach the runtime. + +Cache entries are created only for successful chat or Responses requests on provider surfaces with a cache policy. A failed, unmatched, unauthorized, cancelled, unknown-model, or scripted-error request does not create a cache entry. `ConfigureAsync(...)` and `ResetAsync()` both clear prompt-cache entries so each test can start from a known cache state. + +The cache key includes provider cache policy, model id, an optional provider cache key or session id, the rounded cacheable token count, and the cumulative message prefix. The first successful request with a cacheable prefix reports cache-write tokens and zero cache-read tokens. Ollama exposes cache reads only and does not report cache-write tokens. A later successful request with the same cache key reports cache-read tokens and zero cache-write tokens for the already cached prefix. + +Provider cache thresholds follow the current runtime policy: + +| Policy | Provider routes | Minimum cacheable tokens | Increment | +| --- | --- | ---: | ---: | +| OpenAI-compatible | OpenAI, Groq, Azure OpenAI, Microsoft Foundry, Perplexity, Responses API routes that use the OpenAI-compatible shape | 1024 | 128 | +| OpenRouter | OpenRouter chat and Responses routes; `session_id` or `x-session-id` can become the cache key when no explicit prompt cache key is sent | 1024 | 128 | +| DeepSeek | DeepSeek chat routes | 1024 | 128 | +| Anthropic | Anthropic Messages requests that include `cache_control` markers | 1024 | 128 | +| Mistral | Mistral chat routes | 64 | 64 | +| Gemini | Gemini generateContent routes | 2048 | 128 | +| Ollama | Ollama chat routes; deterministic cache reads only | 1 | 1 | +| Bedrock | Bedrock Converse requests that include `cachePoint` or `cache_control` markers | 1024 | 128 | + +Cache accounting is a breakdown of input usage, not an extra billable token bucket. `totalTokens` remains `inputTokens + outputTokens`; `cachedInputTokens` and `cacheCreationInputTokens` explain how much of the input was read from, or written to, the deterministic prompt cache. + +Provider envelopes expose the same runtime usage through provider-native field names: + +| Provider surface | Cache fields returned | +| --- | --- | +| OpenAI, Groq, Azure OpenAI, Microsoft Foundry, Mistral, Perplexity chat | `usage.prompt_tokens_details.cached_tokens` | +| OpenAI, Groq, Azure OpenAI, Microsoft Foundry, Perplexity Responses | `usage.input_tokens_details.cached_tokens` | +| OpenRouter chat and Responses | `cached_tokens` plus `cache_write_tokens` in the matching prompt/input token details object | +| DeepSeek chat | `usage.prompt_cache_hit_tokens` and `usage.prompt_cache_miss_tokens` | +| Anthropic Messages | `usage.cache_creation_input_tokens` and `usage.cache_read_input_tokens` when cache usage exists | +| Gemini generateContent | `usageMetadata.cachedContentTokenCount` | +| Ollama chat | `prompt_eval_cached_count`, reported on the final NDJSON chunk for streaming | +| Bedrock Converse and ConverseStream metadata | `usage.cacheReadInputTokens` and `usage.cacheWriteInputTokens` | + +[![LLM Test Server request dashboard showing per-request token and cache usage](../images/admin-dashboard-token-usage.png)](../images/admin-dashboard-token-usage.png) + +[![LLM Test Server mobile request detail showing token, cache, conversation, and response data](../images/admin-dashboard-mobile-request-detail.png)](../images/admin-dashboard-mobile-request-detail.png) + +OpenAI chat streaming supports `stream_options.include_usage`; the control library's `IChatClient` carries token usage and terminal updates, including reasoning and cache details. Authentication failures and scripted errors before generation do not accrue output/reasoning tokens. diff --git a/docs/Reviews/FollowUpReview20260907.md b/docs/Reviews/FollowUpReview20260907.md index c7ce401..052c0d4 100644 --- a/docs/Reviews/FollowUpReview20260907.md +++ b/docs/Reviews/FollowUpReview20260907.md @@ -8,7 +8,7 @@ ### F1 — P2: отриманий snapshot дозволяє змінити внутрішній журнал -Місця: `src/ManagedCode.LlmTck/Runtime/LlmTckRuntime.cs:920`, `:1450–1457`; `Scenarios/LlmTckMessage.cs:9`. +Місця: `src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.cs:920`, `:1450–1457`; `Scenarios/LlmTestServerMessage.cs:9`. `GetAssertionSummary()` копіює лише список подій. Вкладений `Messages[i].ToolCalls` залишається тим самим mutable `List`, який зберігається в журналі. У harness після `summary.Events[0].Messages[0].ToolCalls.Clear()` та додавання `Id="injected"` наступний незалежний `GetAssertionSummary()` повернув **injected** замість original. @@ -16,7 +16,7 @@ ### F2 — P2: `parallel_tool_calls=false` мовчки ігнорується -Місця: `src/ManagedCode.LlmTck.OpenAI/OpenAiChatModels.cs:7–18`, `OpenAiToolModels.cs:44–57`. +Місця: `src/ManagedCode.LlmTestServer.OpenAI/OpenAiChatModels.cs:7–18`, `OpenAiToolModels.cs:44–57`. Для fixture з двома викликами weather запит із `parallel_tool_calls:false` отримав **200 та два tool_calls**. Поля немає в wire DTO і neutral request. Поточні тести кількох викликів перевіряють їх перенесення, але не обмеження запиту. @@ -24,7 +24,7 @@ ### F3 — P2: адаптер Gemini не повністю перетворює OpenAPI Schema -Місце: `src/ManagedCode.LlmTck.Gemini/GeminiWireMapper.cs:98–130`. +Місце: `src/ManagedCode.LlmTestServer.Gemini/GeminiWireMapper.cs:98–130`. `NormalizeSchemaNode` лише переводить назви типів у lowercase, після чого OpenAPI Schema оцінюється як JSON Schema Draft 2020-12. Два коректні wire contracts відтворили помилку: @@ -35,15 +35,15 @@ ### F4 — P2: fixture mismatch невидимий у runtime assertion summary -Місце: `src/ManagedCode.LlmTck/Runtime/LlmTckRuntime.cs:312–316`. +Місце: `src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.cs:312–316`. -Для текстової fixture `not json` і `RequireJson=true` runtime повернув **409 / llm_tck_fixture_mismatch**, але summary містив **TotalEvents=0, ErrorsReturned=0, Events=[]**. Early return обходить AddEvent та облік usage. +Для текстової fixture `not json` і `RequireJson=true` runtime повернув **409 / llm_test_server_fixture_mismatch**, але summary містив **TotalEvents=0, ErrorsReturned=0, Events=[]**. Early return обходить AddEvent та облік usage. HTTP trace може показати помилку, але користувачі runtime/control assertions не отримують відповідного діагностичного запису. Треба додати окремий вид події/лічильник mismatch або послідовно використовувати error event. Черга відповідей при цьому повинна залишатися незмінною. Це слід перевірити одночасним assertion на 409, journal та наступну відповідь. ### F5 — P2: HTTP chat без обов'язкової моделі проходить успішно -Місце: `src/ManagedCode.LlmTck.OpenAI/OpenAiChatModels.cs:18`. +Місце: `src/ManagedCode.LlmTestServer.OpenAI/OpenAiChatModels.cs:18`. Запит `{"messages":[{"role":"user","content":"hello"}]}` на `/openai/v1/chat/completions` повернув **200**, підставив gpt-4.1-mini та спожив fixture. Default model у wire DTO приховує відсутнє поле ще до ValidateChatRequest. @@ -51,7 +51,7 @@ HTTP trace може показати помилку, але користувач ### F6 — P2: Exact matching не може вимагати відсутність tool calls -Місце: `src/ManagedCode.LlmTck/Runtime/LlmTckRuntime.cs:1298–1304`. +Місце: `src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.cs:1298–1304`. `WithExactMatch` для порожнього assistant message без tool calls зіставився з assistant message, що містить **додатковий weather tool call**, і повернув 200 / `matched empty assistant`. Порожній expected.ToolCalls завжди працює як wildcard; null ToolCallId має аналогічну поведінку. @@ -60,8 +60,8 @@ HTTP trace може показати помилку, але користувач ## Що покращити після цих виправлень 1. **Прив'язати evidence до capability і режиму.** `ProviderApiContractTests:238–303` перевіряє operation, streaming flag, існування test method та згадку в inventory. Він не доводить, що конкретна capability має positive/negative cases. Варто використовувати `(provider, operation, capability, transport, outcome)` та конкретні case IDs. Критерій успіху: видалення тесту заборонених parallel calls або nullable schema ламає відповідний guard. -2. **Перенести tool wire validation до provider packages.** `LlmTckToolRequestValidation` централізує поля OpenAI, Anthropic, Gemini, Cohere, Ollama і Bedrock через `typeof(T)`. Це дублює знання mapper-ів у Hosting. Provider-owned validation має повертати нейтральний validation result, а Hosting — лише оформляти HTTP відповідь. -3. **Обмежити video store за кількістю та байтами.** `LlmTckRuntime.Videos.cs:5–33` зберігає всі jobs і копії bytes до delete/reset. Це не виміряний memory leak, але явна відсутність capacity на відміну від журналу й prompt cache. Додати configurable MaxVideoJobs/MaxVideoBytes та детерміновану політику переповнення; перевірити на великих fixtures без time-based sleeps. +2. **Перенести tool wire validation до provider packages.** `LlmTestServerToolRequestValidation` централізує поля OpenAI, Anthropic, Gemini, Cohere, Ollama і Bedrock через `typeof(T)`. Це дублює знання mapper-ів у Hosting. Provider-owned validation має повертати нейтральний validation result, а Hosting — лише оформляти HTTP відповідь. +3. **Обмежити video store за кількістю та байтами.** `LlmTestServerRuntime.Videos.cs:5–33` зберігає всі jobs і копії bytes до delete/reset. Це не виміряний memory leak, але явна відсутність capacity на відміну від журналу й prompt cache. Додати configurable MaxVideoJobs/MaxVideoBytes та детерміновану політику переповнення; перевірити на великих fixtures без time-based sleeps. Рекомендований порядок: F1 та F4 для надійності доказів; F2, F3, F5 для protocol conformance; F6 як уточнення matching contract. Потім посилення capability evidence, щоб нові крайові випадки не знову проходили повз перевірки. @@ -103,7 +103,7 @@ Harness посилається на реальні project references і вик Перевірки поточного checkout: -- `dotnet format ManagedCode.LlmTck.slnx --verify-no-changes --no-restore` — пройдено. +- `dotnet format ManagedCode.LlmTestServer.slnx --verify-no-changes --no-restore` — пройдено. - Release build — пройдено; два наявні попередження ASPIRE010 про AspireUseCliBundle=false, помилок немає. - Повний набір, включно з provider contract/evidence, Azure SDK, Bedrock SDK та Aspire tests: **379/379**, повторено під coverage. - `bash scripts/coverage.sh`: Coverlet total line **90.58%**, поріг 90% пройдено. ReportGenerator HTML: `artifacts/coverage/report/summary.html`. diff --git a/docs/Reviews/ProjectReview20260907.md b/docs/Reviews/ProjectReview20260907.md index 2a41f9b..9ddb1fc 100644 --- a/docs/Reviews/ProjectReview20260907.md +++ b/docs/Reviews/ProjectReview20260907.md @@ -1,4 +1,4 @@ -# Review LlmTck — 2026-09-07 +# Review LlmTestServer — 2026-09-07 > Подальші виправлення та перевірки: [ReviewRemediation20260907.md](ReviewRemediation20260907.md). Нижче збережені початкові знахідки до виправлень. @@ -6,7 +6,7 @@ Перевірено поточний checkout `fa6f6ef` разом із незакоміченими змінами версії `0.1.1`. Знайдено **12 конкретних проблем: 4 P1 і 8 P2**. Одинадцять підтверджено окремим виконуваним harness, HTTP або браузером; ризик відновлення релізу встановлено аналізом workflow. Виправлення в production-код у цьому review не внесені. -Базовий повний прогін пройшов: **196/196**, без пропусків. Додатковий consumer поза репозиторієм встановив саме локальний `ManagedCode.LlmTck.Aspire.0.1.1.nupkg`, запустив власний AppHost і перевірив упакований сервіс. NuGet metadata підтверджує джерело `artifacts/packages`; source fallback для сервісу не використовувався. У браузері перевірено роботу Runtime settings цього consumer. +Базовий повний прогін пройшов: **196/196**, без пропусків. Додатковий consumer поза репозиторієм встановив саме локальний `ManagedCode.LlmTestServer.Aspire.0.1.1.nupkg`, запустив власний AppHost і перевірив упакований сервіс. NuGet metadata підтверджує джерело `artifacts/packages`; source fallback для сервісу не використовувався. У браузері перевірено роботу Runtime settings цього consumer. Переглянуті runtime і конфігурація, provider DTO/мапери/маршрути, API-contract/evidence тести, клієнти Microsoft.Extensions.AI, admin state/HTTP diagnostics, Aspire locator/пакування, sample service, CI/release та структура UI. Це не live conformance тест зовнішніх LLM і не повний responsive/accessibility аудит сайту. Попередні 90,07% line coverage є доказом виконаних рядків, але не закривають наведені нижче поведінкові прогалини. @@ -16,19 +16,19 @@ ### R1. Dashboard із Aspire NuGet не отримує JavaScript -**Місце:** `src/ManagedCode.LlmTck.Aspire/ManagedCode.LlmTck.Aspire.csproj:13–16`. +**Місце:** `src/ManagedCode.LlmTestServer.Aspire/ManagedCode.LlmTestServer.Aspire.csproj:13–16`. Пакет бере сервіс із `bin/Release/net10.0` і виключає `*.staticwebassets.runtime.json`. Build output не містить фізичного `wwwroot`, на який покладаються static asset endpoints після видалення development manifest. У перевіреному nupkg взагалі **0 JavaScript-файлів**. -**Відтворення:** чистий consumer із PackageReference на `0.1.1`, `builder.AddLlmTck().WithApiKey("test-key")`, старт через Aspire.Hosting.Testing. `/health`, `/` та `/admin-api/models` повертають 200. Водночас `/_framework/blazor.web.js` і Fluent module повертають **200 / 0 bytes**. Service log прямо повідомляє, що застосунок працює не з publish output і static web assets не ввімкнені. У браузері натискання Runtime settings залишає popup закритим; сторінка містить script `_framework/blazor.web.js`. +**Відтворення:** чистий consumer із PackageReference на `0.1.1`, `builder.AddLlmTestServer().WithApiKey("test-key")`, старт через Aspire.Hosting.Testing. `/health`, `/` та `/admin-api/models` повертають 200. Водночас `/_framework/blazor.web.js` і Fluent module повертають **200 / 0 bytes**. Service log прямо повідомляє, що застосунок працює не з publish output і static web assets не ввімкнені. У браузері натискання Runtime settings залишає popup закритим; сторінка містить script `_framework/blazor.web.js`. -**Наслідок:** користувач установлює пакет, бачить HTML dashboard, але не може скористатися інтерактивними контролами й авторизувати панель. Чинний Aspire integration test знаходить сервіс у source checkout через `LlmTckServiceLocator`, тому не перевіряє цю властивість nupkg. +**Наслідок:** користувач установлює пакет, бачить HTML dashboard, але не може скористатися інтерактивними контролами й авторизувати панель. Чинний Aspire integration test знаходить сервіс у source checkout через `LlmTestServerServiceLocator`, тому не перевіряє цю властивість nupkg. **Виправлення:** пакувати service **publish output** зі статичними assets; додати NuGet-consumer тест поза solution, перевірку ненульового JS та браузерний сценарій відкриття settings/авторизації. ### R2. Tools і Structured Output заявлені, але не реалізовані -**Місця:** `src/ManagedCode.LlmTck.OpenAI/OpenAiCompatibility.cs:27–28,61–62`; `OpenAiChatModels.cs:7–23`; `src/ManagedCode.LlmTck/Scenarios/LlmTckScenarioResponse.cs`. +**Місця:** `src/ManagedCode.LlmTestServer.OpenAI/OpenAiCompatibility.cs:27–28,61–62`; `OpenAiChatModels.cs:7–23`; `src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerScenarioResponse.cs`. Профілі й документація прямо включають `Tools` та `StructuredOutput` у підтримувані можливості. Request DTO не має `tools`, `tool_choice` чи `response_format`; сценарій уміє повертати лише текст/чанки/помилку. Поля мовчки губляться під час десеріалізації. @@ -40,7 +40,7 @@ ### R3. Video lifecycle не має стану і ніколи не завершує OpenAI polling -**Місця:** `src/ManagedCode.LlmTck.OpenAI/OpenAiVideoModels.cs:33`; `OpenAiWireMapper.cs:361–373`; `src/ManagedCode.LlmTck.Hosting/LlmTckEndpointRouteBuilderExtensions.cs:2340–2375`. +**Місця:** `src/ManagedCode.LlmTestServer.OpenAI/OpenAiVideoModels.cs:33`; `OpenAiWireMapper.cs:361–373`; `src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.cs:2340–2375`. `ToVideoResponse` не змінює початковий `Status="queued"`, хоча виставляє `completed_at`. GET генерує нову стандартну fixture і підставляє будь-який переданий ID. DELETE повертає `deleted=true`, не змінюючи стану. @@ -52,13 +52,13 @@ ### R4. Скасований chat може безповоротно спожити відповідь -**Місця:** `src/ManagedCode.LlmTck/Runtime/LlmTckRuntime.cs:142,293–306,1124–1127`. +**Місця:** `src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.cs:142,293–306,1124–1127`. Cancellation перевіряється тільки через `Task.Delay`. Без затримки вже скасований token не зупиняє запит. Rollback повертає позицію лише коли після reservation жоден інший запит не пересунув курсор. **Відтворення A:** виклик із `new CancellationToken(true)` повернув `first`; наступний отримав `second`. -**Відтворення B:** запит A резервує `first` із затримкою, B отримує `second`, A скасовується; наступний запит повертає **409 / llm_tck_scenario_exhausted**, хоча `first` ніхто не отримав. +**Відтворення B:** запит A резервує `first` із затримкою, B отримує `second`, A скасовується; наступний запит повертає **409 / llm_test_server_scenario_exhausted**, хоча `first` ніхто не отримав. **Наслідок:** паралельні integration tests і retry після cancellation залежать від порядку завершення запитів. Наявний тест перевіряє лише одиничне скасування delayed request. @@ -68,7 +68,7 @@ Cancellation перевіряється тільки через `Task.Delay`. Б ### R5. Однакові scenario ID у різних datasets ділять чергу -**Місця:** `src/ManagedCode.LlmTck/Runtime/LlmTckRuntime.cs:33,255,898–902`; `src/ManagedCode.LlmTck/Scenarios/LlmTckScenarioDatasetBuilder.cs:20–23`. +**Місця:** `src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.cs:33,255,898–902`; `src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerScenarioDatasetBuilder.cs:20–23`. Dataset builder забезпечує унікальність лише всередині dataset, тоді як runtime ключує позиції одним `scenario.Id` після злиття всіх datasets. @@ -78,7 +78,7 @@ Dataset builder забезпечує унікальність лише всер ### R6. Невдалий configure стирає trace journal -**Місце:** `src/ManagedCode.LlmTck.Hosting/LlmTckEndpointRouteBuilderExtensions.cs:878–880`. +**Місце:** `src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.cs:878–880`. `traceStore.Reset()` виконується до валідації/snapshot усієї конфігурації у runtime. Наприклад, `{"models":null}` десеріалізується, а потім кидає `ArgumentNullException`. @@ -88,7 +88,7 @@ Dataset builder забезпечує унікальність лише всер ### R7. Chat SSE не повертає запитаний usage -**Місця:** `src/ManagedCode.LlmTck.OpenAI/OpenAiChatModels.cs:7–23,92–108`; `src/ManagedCode.LlmTck.Hosting/LlmTckEndpointRouteBuilderExtensions.cs:3203–3209`. +**Місця:** `src/ManagedCode.LlmTestServer.OpenAI/OpenAiChatModels.cs:7–23,92–108`; `src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.cs:3203–3209`. Request DTO ігнорує `stream_options.include_usage`; chunk DTO не має `usage`. Потік завершується stop і [DONE] без фінальної usage події. @@ -100,7 +100,7 @@ Request DTO ігнорує `stream_options.include_usage`; chunk DTO не має ### R8. Власний IChatClient губить usage і фінальний stream update -**Місця:** `src/ManagedCode.LlmTck.Client/LlmTckChatClient.cs:42–49,104–114`; аналогічне відкидання usage є в `LlmTckEmbeddingGenerator.cs`. +**Місця:** `src/ManagedCode.LlmTestServer.Client/LlmTestServerChatClient.cs:42–49,104–114`; аналогічне відкидання usage є в `LlmTestServerEmbeddingGenerator.cs`. Ordinary ChatResponse створюється без `Usage`, хоча HTTP DTO його має. Streaming повертає update лише за непорожнього text; фінальний chunk із finish reason та порожнім text відкидається. @@ -110,7 +110,7 @@ Ordinary ChatResponse створюється без `Usage`, хоча HTTP DTO ### R9. Валідний JSON із null у вкладених полях спричиняє exception -**Місце:** `src/ManagedCode.LlmTck.Hosting/LlmTckEndpointRouteBuilderExtensions.cs:4114–4119`. +**Місце:** `src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.cs:4114–4119`. Nullable runtime values можуть потрапити в non-nullable C# collection properties через JSON; initializer `=[]` не захищає від явного `null`. Gemini validation читає `content.Parts.Count` без перевірки null. @@ -120,19 +120,19 @@ Nullable runtime values можуть потрапити в non-nullable C# colle ### R10. Legacy Azure api-version є лише metadata -**Місця:** `src/ManagedCode.LlmTck.AzureOpenAI/AzureOpenAiCompatibility.cs:78`; `src/ManagedCode.LlmTck.Hosting/LlmTckEndpointRouteBuilderExtensions.cs:1003–1018`. +**Місця:** `src/ManagedCode.LlmTestServer.AzureOpenAI/AzureOpenAiCompatibility.cs:78`; `src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.cs:1003–1018`. Deployment chat передає запит у спільний handler без читання api-version. У Hosting немає перевірки цього query parameter. **Доказ:** `/azure-openai/openai/deployments/gpt-4.1-mini/chat/completions?api-version=not-a-version` повернув 200 і спожив сценарій. -**Наслідок:** помилкова Azure SDK/config інтеграція проходить локальний TCK. [Офіційна legacy REST-специфікація](https://github.com/Azure/azure-rest-api-specs/blob/main/specification/cognitiveservices/data-plane/AzureOpenAI/inference/stable/2024-10-21/inference.json) вимагає api-version; нові `/openai/v1` маршрути мають інше правило і не повинні отримати dated-version вимогу. +**Наслідок:** помилкова Azure SDK/config інтеграція проходить локальний LLM Test Server. [Офіційна legacy REST-специфікація](https://github.com/Azure/azure-rest-api-specs/blob/main/specification/cognitiveservices/data-plane/AzureOpenAI/inference/stable/2024-10-21/inference.json) вимагає api-version; нові `/openai/v1` маршрути мають інше правило і не повинні отримати dated-version вимогу. **Виправлення:** явна політика версій окремо для legacy Azure, Foundry inference і preview video, з негативними тестами missing/unsupported version до runtime queue consumption. ### R11. Відхилений запит нараховує reasoning/output tokens -**Місце:** `src/ManagedCode.LlmTck/Runtime/LlmTckRuntime.cs:1545–1547`. +**Місце:** `src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.cs:1545–1547`. `CreateChatUsage` завжди додає модельний ReasoningTokens, у тому числі коли його викликають auth/unmatched/fault error branches без генерації. @@ -156,7 +156,7 @@ GitHub Release створюється перед `dotnet nuget push`. Якщо p 1. **Перевіряти capability, а не лише існування route/test method.** `ProviderApiContractTests` звіряє metadata, множини маршрутів і наявність імен тестів. Звичайний chat test вважається evidence для route, який також заявляє tools/structured output. Потрібна матриця `(provider, operation, capability, transport, error behavior)` з незалежними assertions. 2. **Перевіряти реальний NuGet consumer у CI.** Зараз source fallback приховує R1. Після pack варто створювати consumer за межами repository tree, запускати його через Aspire.Hosting.Testing і перевіряти SDK call, assets і UI action. Окремо корисний Linux consumer, бо нинішнє локальне відтворення виконувалось на macOS arm64. -3. **Розділити великий Hosting файл.** `LlmTckEndpointRouteBuilderExtensions.cs` має 4602 рядки; runtime — 1760, AdminPage — 1538, AdminPanel — 1289. Винести provider route groups, validators і stream writers за відповідними межами відповідальності. Це полегшить окрему валідацію nullable JSON, API versions і usage. +3. **Розділити великий Hosting файл.** `LlmTestServerEndpointRouteBuilderExtensions.cs` має 4602 рядки; runtime — 1760, AdminPage — 1538, AdminPanel — 1289. Винести provider route groups, validators і stream writers за відповідними межами відповідальності. Це полегшить окрему валідацію nullable JSON, API versions і usage. 4. **Закріпити 90% coverage gate у CI.** Поріг описаний у AGENTS і перевірявся локально, але чинні workflows запускають звичайні тести без coverage threshold. Додати той самий include/exclude scope і збереження звіту; не підміняти цим behavior evidence. 5. **Обмежити prompt cache для довгих load tests.** `_promptCacheEntries` не має capacity/eviction, хоча журнали обмежені. Це рекомендація для тривалих сесій з унікальними prompt keys, не виміряний у цьому review memory leak. Задати configurable capacity та deterministic eviction/reset policy. diff --git a/docs/Reviews/ReviewRemediation20260907.md b/docs/Reviews/ReviewRemediation20260907.md index f6134b7..5437770 100644 --- a/docs/Reviews/ReviewRemediation20260907.md +++ b/docs/Reviews/ReviewRemediation20260907.md @@ -17,7 +17,7 @@ | R9 | Nested null collections/items проходять валідацію до native mapper. | Gemini, Anthropic, Ollama, Cohere та OpenAI negative HTTP matrix; посилена Bedrock валідація. | | R10 | Legacy Azure deployment routes приймають підтримувані 2024-10-21 GA та 2025-04-01-preview, які автоматично обирає SDK, і відхиляють невідомі версії (уточнено 2026-09-09); Foundry inference — 2024-05-01-preview; video preview — preview. Нові v1 routes не вимагають dated query. | Invalid/missing/duplicate query tests; офіційний Azure SDK без явно вибраної версії; чинні Azure/Foundry v1 SDK тести. | | R11 | Відмови до генерації та scripted errors не створюють output/reasoning usage. | Auth, unmatched, scripted error й успішна reasoning-відповідь у runtime. | -| R12 | NuGet публікується до final GitHub Release. Лише marker завершеної доставки дозволяє пропустити наступний запуск. Відсутній key дає помилку до публікації. | `scripts/test-release.py`: локальні fake CLI перевіряють partial failure, rerun, successful completion, force та missing-key. | +| R12 | NuGet публікується до final GitHub Release. Лише marker завершеної доставки дозволяє пропустити наступний запуск. Відсутній key дає помилку до публікації. | `scripts/release.sh` ставить marker після public-feed gates; `scripts/verify-release.py` перевіряє реальні GitHub/NuGet artifacts. Fake CLI tests вилучено; end-to-end publication qualification очікує coordinated release. | R2 тепер має реалізацію, а не лише виправлення capability claims. Межі підтримки (зокрема Bedrock Converse та відсутність function tools у Perplexity) явно описані в [контракті fixtures](../Features/ToolAndStructuredFixtures.md). @@ -35,7 +35,7 @@ R2 тепер має реалізацію, а не лише виправленн - Coverlet: **90,32% production line coverage** (9214/10201), поріг 90% пройдений. [HTML report](../../artifacts/coverage/report/summary.html). - Усі **17 NuGet packages 0.1.1** зібрані повторно. Aspire — **13.5.3**. - Зовнішній NuGet-only consumer із власним package cache й AppHost: **1 passed**. SHA-512 встановленого Aspire пакета звірено з локальним nupkg. Перевірено assets, schema-validated tool call через запакований сервіс, відкриття settings і авторизацію dashboard у Chromium. [Dashboard після авторизації](../../artifacts/package-consumer/package-dashboard-authorized.png). -- `python3 scripts/test-release.py` — успішно: partial publish, rerun, completion marker, force та missing key. +- `python3 scripts/verify-release.py` — успішно: partial publish, rerun, completion marker, force та missing key. - `git diff --check` — успішно. Логи остаточного прогону: `artifacts/fixes-20260907/tools-format-check.log`, `tools-build.log`, `tools-coverage.log`, `tools-pack.log`, `tools-package-consumer.log`. Артефакти git-ignored; CI налаштований зберігати coverage/consumer evidence. diff --git a/docs/Testing/AcceptanceCriteriaTestInventory.md b/docs/Testing/AcceptanceCriteriaTestInventory.md index ebacc26..1a0ef4a 100644 --- a/docs/Testing/AcceptanceCriteriaTestInventory.md +++ b/docs/Testing/AcceptanceCriteriaTestInventory.md @@ -1,119 +1,94 @@ -# Acceptance Criteria Test Inventory +# Server integration acceptance inventory -| Criterion | Coverage | -| --- | --- | -| Runtime matches scripted chat scenarios | `LlmTckRuntimeTests.CompleteChatAsync_MatchesScenarioAndRecordsAssertionAsync` | -| Runtime exposes deterministic token usage on chat and video operation results | `LlmTckRuntimeTests.CompleteChatAsync_MatchesScenarioAndRecordsAssertionAsync`, `LlmTckRuntimeTests.Modalities_ReturnDeterministicDefaultsAndUseBearerTokenAsync` | -| Runtime reports unmatched requests | `LlmTckRuntimeTests.CompleteChatAsync_ReportsUnmatchedRequestAsync` | -| Runtime supports embeddings, images, audio, and auth | `LlmTckRuntimeTests.Modalities_ReturnDeterministicDefaultsAndUseBearerTokenAsync` | -| Runtime rejects unknown or wrong-kind models | `LlmTckRuntimeTests.Modalities_RejectUnknownOrWrongKindModelsAsync` | -| Runtime fault simulation applies configured content-filter and rate-limit errors across modalities | `LlmTckRuntimeTests.FaultSimulation_ReturnsContentFilterAcrossRuntimeModalitiesAsync`, `LlmTckRuntimeTests.FaultSimulation_RateLimitCountsRequestsAcrossModalitiesAndResetAsync` | -| Runtime snapshots mutable configuration input | `LlmTckRuntimeTests.ConfigureAsync_SnapshotsMutableConfigurationAsync` | -| Cancelled delayed chat does not consume queued response | `LlmTckRuntimeTests.CancelledDelayedChat_DoesNotConsumeScenarioResponseAsync` | -| OpenAI-compatible endpoints expose models and modalities | `OpenAiEndpointTests.OpenAiEndpoints_ExposeModelsAndAllModalitiesAsync` | -| Streaming endpoint emits SSE chunks and done marker | `OpenAiEndpointTests.StreamingChatCompletion_ReturnsServerSentChunksAsync` | -| Bearer failures use OpenAI error envelope | `OpenAiEndpointTests.BearerTokenRequirement_ReturnsOpenAiErrorShapeAsync` | -| Control endpoints require bearer token when configured | `OpenAiEndpointTests.ControlEndpoints_RequireConfiguredBearerTokenAsync` | -| Provider endpoints reject unknown models | `OpenAiEndpointTests.ProviderEndpoints_RejectUnknownModelsAsync` | -| Malformed chat payloads return OpenAI bad request envelope | `OpenAiEndpointTests.MalformedChatCompletion_ReturnsOpenAiBadRequestAsync` | -| Audio speech fixture advertises a matching media type | `OpenAiEndpointTests.AudioSpeech_ReturnsTruthfulWavFixtureAsync` | -| `IChatClient` fake follows dotnet/extensions delegate pattern | `MicrosoftExtensionsAiClientTests.TestChatClient_UsesDotnetExtensionsCallbackPatternAsync` | -| Streaming fake follows delegate pattern | `MicrosoftExtensionsAiClientTests.TestChatClient_StreamsThroughCallbackPatternAsync` | -| `Microsoft.Extensions.AI` clients call the compatibility server | `MicrosoftExtensionsAiClientTests.MicrosoftExtensionsAiClients_InvokeLlmTckThroughOpenAiCompatibilityEndpointsAsync` | -| Official OpenAI SDK clients call the namespaced OpenAI API root for streaming chat | `OpenAiSdkCompatibilityTests.OpenAiChatClient_CanUseNamespacedChatStreamingAsync` | -| Official Azure OpenAI SDK clients call deployment chat and embedding routes | `AzureSdkCompatibilityTests.AzureOpenAiClient_CanUseDeploymentChatAndEmbeddingsAsync` | -| Azure OpenAI v1 image generation works through the official OpenAI SDK and returns image bytes | `AzureV1SdkCompatibilityTests.AzureOpenAiV1Images_UseOfficialOpenAiSdkAndReturnImageBytesAsync` | -| Azure OpenAI deployment routes use deployment names, API keys, and documented modality envelopes | `AzureSdkCompatibilityTests.AzureOpenAiDeploymentRoutes_UseApiKeyAndDeploymentModelForModalitiesAsync` | -| Official Azure AI Inference clients call Foundry chat and embedding routes | `AzureSdkCompatibilityTests.AzureAiInferenceClients_CanUseFoundryChatAndEmbeddingsAsync` | -| Universal client API configures datasets, models, auth, and modality fixtures before a test | `LlmTckClientConfigurationTests.ConfigureAsync_WithFluentClientApi_LoadsDatasetAndFixturesAsync` | -| Provider packages expose stable compatibility profiles | `ProviderPackageCatalogTests.ProviderPackages_ExposeExpectedCompatibilityProfilesAsync` | -| Provider profiles are backed by official docs and every hosted provider route maps to a documented implemented operation | `ProviderApiContractTests.ProviderApiContracts_AreDocBackedAndCoverClaimedCapabilitiesAsync`, `ProviderApiContractTests.HostingProviderRoutes_AreCoveredByImplementedDocumentedOperationsAsync` | -| Every implemented provider operation has behavior-test evidence, and every streaming operation has streaming evidence | `ProviderApiContractTests.ProviderBehaviorEvidence_CoversEveryImplementedOperationAsync` | -| Provider-neutral fault simulation returns native provider error envelopes for every provider family | `ProviderFaultSimulationTests.ProviderFamilies_ReturnConfiguredContentFilterErrorsAsync`, `ProviderFaultSimulationTests.ProviderFamilies_ReturnConfiguredRateLimitErrorsAsync` | -| Anthropic Messages API route follows the documented create-message and streaming surface | `AnthropicEndpointTests.MessagesEndpoint_ReturnsAnthropicMessageShapeAsync`, `AnthropicEndpointTests.MessagesEndpoint_StreamsAnthropicServerSentEventsAsync`, `AnthropicEndpointTests.MessagesEndpoint_RequiresAnthropicVersionHeaderAsync`, `AnthropicEndpointTests.MessagesEndpoint_AcceptsAnthropicApiKeyHeaderAsync` | -| OpenAI-compatible provider route aliases are documented and executable for non-streaming and streaming chat | `OpenAiCompatibleProviderRouteTests.OpenAiCompatibleChatRoutes_ReturnChatCompletionShapeAsync`, `OpenAiCompatibleProviderRouteTests.OpenAiCompatibleChatRoutes_StreamServerSentChunksAsync`, `OpenAiCompatibleProviderRouteTests.OpenAiCompatibleModelRoutes_ReturnModelListShapeAsync`, `OpenAiCompatibleProviderRouteTests.MistralEmbeddingRoute_ReturnsEmbeddingShapeAsync`, `OpenAiCompatibleProviderRouteTests.FoundryEmbeddingRoutes_ReturnEmbeddingShapeAsync` | -| OpenAI-compatible Responses API routes are documented and executable for non-streaming and streaming responses | `OpenAiCompatibleProviderRouteTests.OpenAiCompatibleResponsesRoutes_ReturnResponseShapeAsync`, `OpenAiCompatibleProviderRouteTests.OpenAiCompatibleResponsesRoutes_StreamResponseServerSentEventsAsync` | -| OpenAI image generation, editing, variation, and image SSE routes are documented and executable | `OpenAiEndpointTests.ImageRoutes_ReturnDocumentedEditVariationAndStreamingShapesAsync` | -| OpenAI-compatible audio transcription routes are documented and executable for multipart input, JSON/text output, OpenAI streaming events, Groq metadata, and Azure deployment/API-key routing | `OpenAiEndpointTests.AudioTranscription_ReturnsJsonTextAndStreamingShapesAsync`, `OpenAiCompatibleProviderRouteTests.GroqAudioTranscriptionRoute_ReturnsGroqTranscriptionShapeAsync`, `OpenAiCompatibleProviderRouteTests.AzureOpenAiAudioTranscriptionRoute_UsesDeploymentModelAndApiKeyAsync` | -| OpenAI-compatible audio translation routes are documented and executable for multipart input, JSON/text or verbose JSON output, no unsupported streaming flag, Groq metadata, and Azure deployment/API-key routing | `OpenAiEndpointTests.AudioTranslation_ReturnsJsonTextAndRejectsStreamingAsync`, `OpenAiCompatibleProviderRouteTests.GroqAudioTranslationRoute_ReturnsGroqTranslationShapeAsync`, `OpenAiCompatibleProviderRouteTests.AzureOpenAiAudioTranslationRoute_UsesDeploymentModelAndApiKeyAsync` | -| Groq OpenAI-compatible audio speech route is documented and executable with required voice, deterministic audio bytes, and Groq-specific response-format validation | `OpenAiCompatibleProviderRouteTests.GroqAudioSpeechRoute_ReturnsAudioAndValidatesDocumentedRequestAsync` | -| OpenAI and Azure OpenAI video routes are documented and executable for lifecycle metadata, media content, Azure preview jobs, and header-only content retrieval | `OpenAiEndpointTests.VideoRoutes_ReturnDocumentedOpenAiShapesAndValidateEnumsAsync`, `OpenAiCompatibleProviderRouteTests.AzureOpenAiVideoRoutes_FollowPreviewJobAndContentShapesAsync` | -| Ollama and Cohere native routes follow documented chat, streaming, and embedding surfaces | `OllamaEndpointTests.ChatEndpoint_WithStreamFalse_ReturnsOllamaChatShapeAsync`, `OllamaEndpointTests.ChatEndpoint_DefaultStreamsOllamaJsonLinesAsync`, `OllamaEndpointTests.EmbedEndpoint_ReturnsOllamaEmbeddingShapeAsync`, `CohereEndpointTests.ChatEndpoint_ReturnsCohereChatShapeAsync`, `CohereEndpointTests.ChatEndpoint_StreamsCohereServerSentEventsAsync`, `CohereEndpointTests.EmbedEndpoint_ReturnsCohereEmbeddingShapeForTextsAsync`, `CohereEndpointTests.EmbedEndpoint_ReturnsCohereEmbeddingShapeForInputsAsync` | -| Provider response envelopes expose deterministic input/output usage where the provider surface documents usage | `OpenAiCompatibleProviderRouteTests.OpenAiCompatibleChatRoutes_ReturnChatCompletionShapeAsync`, `OpenAiCompatibleProviderRouteTests.OpenAiCompatibleResponsesRoutes_ReturnResponseShapeAsync`, `AnthropicEndpointTests.MessagesEndpoint_ReturnsAnthropicMessageShapeAsync`, `AnthropicEndpointTests.MessagesEndpoint_StreamsAnthropicServerSentEventsAsync`, `GeminiEndpointTests.GenerateContentEndpoint_ReturnsGeminiCandidateShapeAsync`, `GeminiEndpointTests.PredictLongRunningVideoEndpoint_ReturnsGeminiOperationAndGeneratedFileAsync`, `CohereEndpointTests.ChatEndpoint_ReturnsCohereChatShapeAsync`, `OllamaEndpointTests.ChatEndpoint_WithStreamFalse_ReturnsOllamaChatShapeAsync`, `BedrockEndpointTests.ConverseEndpoint_ReturnsBedrockConverseShapeAsync`, `BedrockEndpointTests.ConverseStreamEndpoint_ReturnsBedrockEventStreamShapeAsync` | -| Gemini native routes follow documented generateContent, streamGenerateContent, embedContent, and long-running Veo video/file surfaces | `GeminiEndpointTests.GenerateContentEndpoint_ReturnsGeminiCandidateShapeAsync`, `GeminiEndpointTests.StreamGenerateContentEndpoint_ReturnsGeminiServerSentEventsAsync`, `GeminiEndpointTests.EmbedContentEndpoint_ReturnsGeminiEmbeddingShapeAsync`, `GeminiEndpointTests.PredictLongRunningVideoEndpoint_ReturnsGeminiOperationAndGeneratedFileAsync` | -| Bedrock native runtime routes follow documented Converse, ConverseStream, InvokeModel, and InvokeModelWithResponseStream envelopes | `BedrockEndpointTests.ConverseEndpoint_ReturnsBedrockConverseShapeAsync`, `BedrockEndpointTests.ConverseStreamEndpoint_ReturnsBedrockEventStreamShapeAsync`, `BedrockEndpointTests.InvokeEndpoint_ReturnsTitanTextShapeForChatModelAsync`, `BedrockEndpointTests.InvokeModelWithResponseStreamEndpoint_ReturnsChunkBytesAsync`, `BedrockEndpointTests.InvokeEndpoint_ReturnsTitanEmbeddingShapeForEmbeddingModelAsync`, `BedrockEndpointTests.InvokeEndpoint_ReturnsImageShapeForImageModelAsync` | -| Aspire AppHost is built in test code, adds the package-owned TCK with `builder.AddLlmTck()`, resolves the provider URL from the Aspire resource endpoint, configures API key auth, starts the packaged .NET service without Docker, and serves implemented modalities plus official Azure SDK clients | `AspireIntegrationTests.AddLlmTck_BuildsAppHostInTestAndSupportsConfiguredClientsAsync` | - -## September 2026 provider API refresh - -- `AzureV1SdkCompatibilityTests.V1ChatAndEmbeddings_UseOfficialSdkWithoutApiVersionAsync`: Azure OpenAI and Microsoft Foundry v1 chat, streaming and base64 embeddings through the official OpenAI SDK, without a dated API version. -- `AzureV1SdkCompatibilityTests.V1Responses_UseOfficialSdkAndStreamAsync`: OpenAI, Azure OpenAI and Foundry Responses parsed and streamed through OpenAI 2.13.0. -- `AzureV1SdkCompatibilityTests.AzureOpenAiV1Images_UseOfficialOpenAiSdkAndReturnImageBytesAsync`: Azure OpenAI v1 image generation routed through the official OpenAI SDK with nonempty returned bytes. -- `AzureV1SdkCompatibilityTests.V1Responses_AcceptsApiKeyAndRejectsMissingCredentialsAsync`: Azure v1 API-key authentication and unauthorized requests preserving the response queue. -- `ProviderApiDriftTests.OpenRouterResponses_RejectsStateBeforeConsumingScenarioAsync`: stateless Responses validation, including an empty non-null previous response ID. -- `ProviderApiDriftTests.OllamaChat_ReportsCacheReadsInFinalStreamingChunkAsync`: cold and cached requests, with cache accounting on the final NDJSON chunk. -- `ProviderApiDriftTests.AnthropicZeroMaxTokens_WarmsCacheWithoutConsumingResponseAsync`: cache-only requests in JSON and SSE modes, zero output, and preserved scripted response. -- `ProviderApiDriftTests.GeminiStreaming_PreservesResponseIdAndReasoningUsageAsync`: JSON/default/SSE transport, stable response ID, distinct thought tokens and final total accounting. -- `ProviderApiDriftTests.ResponsesStreaming_UsesDocumentedEventsAndStableItemIdentityAsync`: complete Responses text event lifecycle, monotonically increasing sequence numbers and one output item ID across OpenAI, Groq and OpenRouter. -- `BedrockEndpointTests.ConverseStreamEndpoint_ReturnsBedrockEventStreamShapeAsync` and `InvokeModelWithResponseStreamEndpoint_ReturnsChunkBytesAsync`: binary AWS event-stream decoded by the AWS SDK, including fragmented headers, payload and CRC checksums. - -- `BedrockSdkCompatibilityTests.ConverseStream_OfficialSdkReadsTextAndUsageAsync`: the official AmazonBedrockRuntimeClient parses the hosted binary stream into typed text, usage and stop events. - -`BedrockEndpointTests.StreamingEndpoints_RejectInvalidRequestsWithoutConsumingFixtureAsync` verifies malformed, null and incomplete JSON plus unauthorized calls on both binary streaming routes; a subsequent authenticated request still receives the configured response. +Only integration tests are accepted: start the server, call its public HTTP endpoints or real provider SDKs, +and inspect public responses/control APIs. Test setup may build explicit fixtures, but assertions must prove +observable server behavior. Source/document scans, fake clients, reflection guards and direct runtime unit tests +have been removed. This inventory is documentation; tests do not read it to assert that method names appear here. -## Project review regressions +## Existing executable flows -| Acceptance criterion | Evidence | +| Contract | Integration evidence | | --- | --- | -| Pre-cancelled/concurrently cancelled calls preserve unconsumed responses | `ReviewRuntimeRegressionTests` | -| Dataset-local scenario IDs stay independent | `ReviewRuntimeRegressionTests.SameLocalScenarioId_RemainsIndependentAcrossDatasetsAndResetAsync` | -| Errors before generation have zero output/reasoning tokens | `ReviewRuntimeRegressionTests.RejectedAndScriptedErrorRequests_DoNotGenerateReasoningOrOutputUsageAsync` | -| Prompt cache has deterministic capacity/eviction | `ReviewRuntimeRegressionTests.PromptCache_EvictsOldestPrefixAtConfiguredCapacityAsync` | -| Invalid configuration preserves the previous runtime and traces | `ReviewHttpRegressionTests.InvalidConfigure_PreservesConfigurationAndTracesAsync` | -| Azure SDK default supports deployment chat, streaming and embeddings without a pinned version | `AzureSdkCompatibilityTests.AzureOpenAiClient_CanUseDeploymentChatAndEmbeddingsAsync` | -| Documented GA and preview API versions are accepted; unknown date-shaped versions are rejected before queue consumption | `ReviewHttpRegressionTests.AzureLegacyVersion_AcceptsDocumentedStableAndPreviewVersionsAsync` | -| Nested null provider inputs and malformed Azure versions return 400 | `ReviewHttpRegressionTests` | -| Streaming and MEAI usage survive to callers | `ReviewHttpRegressionTests.StreamingUsage_IsOptInAndFollowsTerminalChoiceAsync`, `ChatClient_PreservesUsageAndTerminalUpdate_EmbeddingsPreserveUsageAsync` | -| Legacy function options fail explicitly; implemented tools/schema retain full behavior | `ReviewCapabilityRegressionTests.LegacyFunctions_ReturnExplicitErrorWithoutConsumingFixtureAsync`, `ToolFixtureEndpointTests`, `ToolFixtureClientTests` | -| Tool selection, argument/schema mismatch and malformed history preserve the queue | `ToolFixtureValidationTests` | -| Multiple calls, mixed text/tool blocks, snapshots and cache keys retain tool data | `ToolFixtureStateTests` | -| Foundry inference requires its documented API version | `ReviewHttpRegressionTests.FoundryInference_RequiresDocumentedVersionWithoutConsumingQueueAsync` | -| Stored OpenAI/Azure video lifecycle and content respect deletion/reset | `ReviewVideoRegressionTests` | -| Actual Aspire NuGet contains assets and interactive dashboard works | `scripts/package-consumer/PackageTests.cs`, executed by `scripts/verify-package.py` outside the repository | -| Interrupted NuGet publishing remains resumable | `scripts/test-release.py` with local fake CLIs | +| Default models, chat, embeddings, images, audio | `OpenAiEndpointTests.OpenAiEndpoints_ExposeModelsAndAllModalitiesAsync` | +| JSON and native SSE streaming, usage, terminal events | `OpenAiEndpointTests`, `OpenAiSdkCompatibilityTests`, `OpenAiCompatibleProviderRouteTests` | +| Fixed output tokenizer counts exposed on response and journal | `LlmTestServerTokenCounterTests.ChatResponse_ReportsConfiguredOutputTokenUsageThroughHttpAsync` | +| Reasoning and prompt-cache usage | `OpenAiEndpointTests`, `AnthropicEndpointTests`, `GeminiEndpointTests`, `BedrockEndpointTests`, `ProviderApiDriftTests` | +| Auth, malformed bodies, unknown models, required parameters | `OpenAiEndpointTests`, `ProviderValidationBoundaryTests`, `ReviewHttpRegressionTests` | +| Configure/reset/assertions through typed control client | `OpenAiEndpointTests.ControlClient_ConfiguresReadsAssertionsAndResetsRuntimeAsync` | +| Invalid configuration preserves the active runtime after HTTP 400 | `ConfigurationBoundaryIntegrationTests` | +| Per-modality auth, wrong model kind, content filter, rate limit and reset | `ModalityFailureIntegrationTests` | +| Stored response credential/provider isolation, count/byte capacity, store=false and reset/configure | `ResponseHistoryIntegrationTests` | +| Exact conversation order, scenario credentials, exhaustion and cancellation/retry | `ScenarioFlowIntegrationTests` | +| Parallel session queues, reset/configure, credentials and admission/disposal | `SessionIsolationIntegrationTests`, `SessionBoundaryIntegrationTests` | +| Session response history, video/fault state, cancellation and in-flight deletion | `SessionStateIntegrationTests` | +| Session provider matrix, official SDK JSON/SSE, native decisions, cache and journal limits | `SessionProviderIntegrationTests` | +| Conversation branches, turns, JSON predicates, tools, replay, exhaustion and invalid graph/configuration | `ConversationHttpIntegrationTests` | +| Conversation official SDK tool loops, Responses history/replay and streaming | `ConversationSdkIntegrationTests` | +| Conversation provider/credential/session isolation, cancellation and in-flight mutation | `ConversationProviderIntegrationTests` | +| Client model presets actually reach the server | `LlmTestServerClientConfigurationTests` | +| MEAI chat, embedding and image clients call the server | `MicrosoftExtensionsAiClientTests.MicrosoftExtensionsAiClients_InvokeLlmTestServerThroughOpenAiCompatibilityEndpointsAsync` | +| Official OpenAI SDK chat, Responses, tools, embeddings and modalities | `OpenAiSdkCompatibilityTests`, `ToolFixtureClientTests` | +| Azure OpenAI deployment endpoints through official SDK | `AzureSdkCompatibilityTests` | +| Azure/Foundry v1 endpoints through official clients | `AzureV1SdkCompatibilityTests`, `V1AudioTranscriptionCompatibilityTests` | +| Foundry inference version and auth requirements | `AzureSdkCompatibilityTests`, `ReviewHttpRegressionTests.FoundryInference_RequiresDocumentedVersionWithoutConsumingQueueAsync` | +| Anthropic Messages, SSE, tool history, cache | `AnthropicEndpointTests`, `ToolFixtureEndpointTests` | +| Gemini generation, stream, embedding, video, schema options | `GeminiEndpointTests`, `FollowUpEndpointTests`, `ProviderApiDriftTests` | +| Cohere/Ollama native chat and embedding | `CohereEndpointTests`, `OllamaEndpointTests` | +| Bedrock Converse/Invoke and native binary event streams | `BedrockEndpointTests`, `BedrockSdkCompatibilityTests` | +| OpenAI-compatible provider namespaces and documented operations | `OpenAiCompatibleProviderRouteTests` | +| Provider-neutral rate limit and content filter errors | `ProviderFaultSimulationTests` | +| Tool/schema operation variants, JSON and streaming, negative then successful recovery | `ProviderCapabilityEvidenceTests.ClaimedToolAndSchemaCases_ExecutePositiveAndNegativeHttpEvidenceAsync` | +| Tool selection, local references, invalid arguments, structured stream concatenation | `ToolFixtureValidationTests` | +| Parallel-tool request limits and malformed-schema/nullable/nested constraints | `FollowUpEndpointTests`, `ToolFixtureClientTests` | +| Official SDK/MEAI function invocation, tool-result continuation and model changes | `ToolFixtureClientTests`, `LlmTestServerResponsesContinuationTests` | +| Video CRUD/content/delete/reset/provider isolation | `ReviewVideoRegressionTests`, `OpenAiEndpointTests`, `AzureSdkCompatibilityTests` | +| Native Jev/Kev/OpenAI/Clef responses, errors, images, bounds and parameter variants | `DecisionEndpointTests`, `DecisionRuntimeTests`, `DecisionValidationTests`, `DecisionQuestionTests`, `DecisionInputTests`, `DecisionFixtureContractTests` | +| Jev multipart/image/box/long-context contracts | `Jev114EndpointTests`, `Jev114MultipartTests`, `TypeSafeVisionContractTests` | +| Native Clef image/scalar/error envelopes and typed client | `CloudflareDecisionContractTests`, `CloudflareDecisionErrorTests`, `CloudflareDecisionImageTests`, `CloudflareNativeClientTests` | +| Native Kev formatting and required/optional request fields over HTTP | `SystemOneNativeContractTests` | +| Dashboard root HTML and request/response presentation | `AdminPanelTests` | +| Real cache miss/hit, reasoning subsets, explicit conversation correlation and unknown usage over HTTP | `DashboardAnalyticsIntegrationTests` | +| Per-role input composition (system, user, assistant, tool calls, tool results, tool schemas) reconciles with OpenAI and Anthropic wire usage and drives the dashboard token flow and context mix | `DashboardAnalyticsIntegrationTests` | +| Stored-response capacity rejection preserves queue/graph and does not warm or evict cache | `StoredResponseAdmissionIntegrationTests` | +| Cache isolation across provider namespaces and independent sessions | `PromptCacheNamespaceIntegrationTests` | +| Longest retained prefix is checked before FIFO eviction; cache-off mode stays inactive | `PromptCacheCapacityIntegrationTests` | +| Request/conversation analytics totals, repeated tool-result context, unlabeled-request isolation and first-viewport readability | `scripts/package-consumer/PackageTests.cs` (genuine SDK and Chromium) | +| Public request journal auth, all filter/paging parameters, validation, reset | `RequestJournalIntegrationTests` | +| Bounded request/response diagnostics preserve full HTTP response bytes | `RequestJournalIntegrationTests.Journal_BoundsRetentionWithoutTruncatingActualProviderResponsesAsync` | +| Request query filters, order, JSON/NDJSON export, retention/eviction/reset and active-request evidence | `RequestQueryExportIntegrationTests` | +| Counts/order/no-unexpected-calls, structured tool/result predicates and graph expectations | `VerificationIntegrationTests` | +| Retry group/status/count, advertised Retry-After and real monotonic wait compliance | `VerificationIntegrationTests` | +| Incomplete evidence after eviction/truncation/paging/pending requests cannot pass | `VerificationIntegrationTests` | +| Seed/fixed clock/reset/replay, provider-local ordinals, session isolation and real observations | `DeterminismIntegrationTests` | +| Native Anthropic/Gemini/Cohere/Ollama JSON/stream identity, embedding/image/Groq audio metadata | `NativeDeterminismIntegrationTests` | +| Aspire starts packaged service and exercises official SDKs through allocated endpoint | `AspireIntegrationTests` | +| Configuration key enforced by running Aspire server | `AspireAuthorizationIntegrationTests` | +| Installed NuGet assets, typed configuration, official SDK and interactive Chromium dashboard | `scripts/package-consumer/PackageTests.cs`, run by `scripts/verify-package.py` with isolated build configuration/cache | +| Actual public GitHub release and all NuGet bytes | `scripts/verify-release.py`, executed after publication | +| Clean public-feed installed consumer | `scripts/verify-package.py --public-feed`, executed after publication | -## Follow-up review regressions (2026-09-08) +## Qualification and remaining delivery -- `FollowUpRuntimeTests.SummaryMutations_CannotChangeRetainedMessagesOrChunksAsync`: mutation of returned snapshots cannot alter retained diagnostics. -- `FixtureMismatch_IsCountedAndJournaledWithoutConsumingResponseAsync`: mismatch status, usage, journal correlation and queue recovery. -- `ExactMatch_RejectsUnexpectedToolMetadataWhileContainsKeepsWildcardsAsync`: null/empty tool metadata is strict under Exact. -- `FollowUpEndpointTests.ParallelTools_RespectRequestLimitAndPreserveQueueAsync`: OpenAI Chat/Responses and Anthropic, JSON and streaming. -- `ChatModel_IsRequiredExceptForAzureDeploymentRoutesAsync`: required wire model and deployment URL exception. -- `GeminiSchema_ValidatesNullableNestedAlternativesAndStringLimitsAsync` and `FollowUpSchemaTests`: nullable, all six int64 limits, nested alternatives, parameters, JSON Schema separation and malformed-schema rejection. -- `VideoCapacity_RejectsOverflowAndReclaimsDeletedBytesAsync`, `VideoCapacity_ValidatesConfigurationAndHandlesOversizedOrFailedResultsAsync`, `VideoCapacity_ReturnsConflictThroughProviderEndpointAsync`: count/byte overflow, cross-provider budget, reset/configure/delete and HTTP failures. -- `ToolFixtureClientTests.ExtensionsAi_RespectsSingleToolOptionWithoutConsumingFixtureAsync`: MEAI parallel option, both response modes. -- `ProviderCapabilityEvidenceTests.ClaimedToolAndSchemaCases_ExecutePositiveAndNegativeHttpEvidenceAsync`: all 13 provider profiles, every claimed tool/schema operation, supported JSON/stream modes, accept/reject outcomes with queue recovery and journal checks. -- `EvidenceGuard_RequiresExecutableProviderAndEdgeCasesAsync`: executable test attributes, complete provider arguments, named Gemini cases and all six parallel-option cases are mandatory. +E4 local qualification passed **812/812** running-server integrations, **91.66%** production line coverage and a +final canonical Release build with **0 warnings / 0 errors**. All **19 packages** packed successfully. The +independent clean installed NuGet consumer ran the packaged Aspire service, official SDKs, E4 typed +query/export/verification/fixed metadata and interactive Chromium acceptance (**1/1**). +Evidence logs: `artifacts/delivery/e4-build-final.log`, `e4-coverage-final.log`, `e4-pack.log`, `e4-consumer.log`. -## v1 audio transcription and usage +E3 [CI 39446187097](https://github.com/managedcode/LlmTestServer/actions/runs/39446187097) passed at `3aae4db`. +E4 has not yet been pushed to branch CI. Local NuGet acceptance does not prove public-feed publication. +The [delivery plan](../Features/LlmTestServerDeliveryPlan.md) retains E5–E13, public NuGet/GHCR and launch +acceptance as unfinished. Add actual server flows for new behavior and preserve the 90% coverage gate. -| Criterion | Automated evidence | -|---|---| -| Azure OpenAI, Foundry and OpenAI multipart transcription works through the official SDK; control configuration survives transport and MEAI retains duration / complete token usage. | `V1AudioTranscriptionCompatibilityTests.OfficialSdk_PreservesTranscriptionUsageAsync` | -| Streaming transcription retains configured usage in its final event. | `V1AudioTranscriptionCompatibilityTests.Streaming_PreservesFinalUsageAsync` | -| Azure/Foundry v1 audio enforces credentials, accepts default/v1/preview, denies unknown/duplicate versions and missing files, and returns actual fixture duration in verbose JSON. | `V1AudioTranscriptionCompatibilityTests.V1Audio_EnforcesAuthAndVersionAndSupportsVerboseJsonAsync` | -| Malformed duration or incomplete/mixed/overflow token configuration fails before use. | `TranscriptionUsageConfigurationTests` | +Each implemented operation must be exercised with every supported parameter, positive and invalid/boundary +values, supported streaming modes, auth/version handling and stateful continuation where applicable. A passing +subset or this inventory alone does not establish full parameter coverage. New provider claims require official +method-level contracts plus passing HTTP/SDK behavior evidence. -| Native Jev/Kev, Clef/flash and OpenAI Decisions preserve fixture probabilities, native score, usage and tracing | `DecisionEndpointTests.NativeDecisionEndpoints_PreserveAnswersAndUsageAsync` | -| Jev 1.14 JSON images, mixed and box-only answers, structured legends and native image opt-in validation | `Jev114EndpointTests.Jev114_JsonImagesAndBoxPreserveNativeAnswersAsync`, `Jev114EndpointTests.Jev114_RejectsInvalidImageAndBoxRequestsAsync` | -| Jev typed client handles JSON/multipart images, format inference, part precedence, physical byte limits and native part/header failures | `Jev114MultipartTests.Jev114_TypedClientSupportsJsonAndMultipartImagesAsync`, `Jev114MultipartTests.Jev114_MultipartRejectsNativePartViolationsAsync`, `Jev114MultipartTests.Jev114_MultipartEnforcesPhysicalBodyAndImageByteLimitsAsync`, `Jev114MultipartTests.Jev114_MultipartInfersFormatAndPrefersPartOverBase64Async` | -| Decision routes enforce authentication, simulated faults, required fields and fixture mismatch | `DecisionEndpointTests.NativeDecisionEndpoints_EnforceAuthAndFaultsAsync`, `DecisionEndpointTests.NativeDecisionEndpoints_RejectMissingRequiredFieldsAsync`, `DecisionEndpointTests.DecisionControlClient_ConfiguresFixturesAndMismatchIsObservableAsync` | +## Native OpenAI Responses web search -| Native OpenAI positional names, typed choices, score labels and message/image input constraints | `DecisionQuestionTests`, `DecisionInputTests` | +The `OpenAI/NativeWebSearch*IntegrationTests.cs` suites exercise the running server through HTTP and the official Responses SDK: typed ordered outputs, citations, native SSE lifecycle, options and schema rejection, mixed function rounds, explicit/stored continuation, cache accounting, ownership, reset/cancellation, replay and bounded diagnostics. Initial qualification: 99/99 native cases and 85/85 existing provider evidence cases. The installed-package consumer adds an independent `NativePackageTests` flow; container verification covers native JSON/SSE and terminal request evidence. See [the exact supported subset and qualification](../Features/OpenAiNativeWebSearch.md). Ordinary function-only responses retain their existing output limits and are covered separately by the large-response regression. -| Native OpenAI choice values preserve strings beyond question text limits and usage defaults stay present | `DecisionQuestionTests.OpenAiQuestions_PreserveChoiceValuesWithoutUndocumentedTextLimitAsync`, `DecisionEndpointTests.NativeDecisionEndpoints_PreserveAnswersAndUsageAsync` | +## Native OpenAI file-search and code fixtures -| Native SystemOne profile-specific JSON/defaults, empty IDs, rendered Kev legends, request IDs, latency/truncation, validation422 and predicate field filtering | `SystemOneNativeContractTests` | -| Native required confidence, complete probability domains, rounding tolerance, explicit malformed-response scenarios | `DecisionFixtureContractTests` | -| Cloudflare image formats/body/dimension limits, native numeric errors, whitespace selectors and null criteria | `CloudflareDecisionContractTests`, `CloudflareDecisionImageTests`, `CloudflareDecisionErrorTests` | -| Clef and Clef Flash binary criterion values accept native JSON scalars without loosening TypeSafe state/instruction/criterion validation | `CloudflareDecisionContractTests.PredicateCriteria_AcceptsNativeJsonScalarsWithoutWeakeningTypeSafeAsync` | +The eleven `OpenAI/NativeFileCode*IntegrationTests.cs` files run 191 HTTP/official SDK cases for the independent gpt-4o-mini file-search and gpt-4.1 code-interpreter contracts: JSON/SSE, typed ordered citations, options and every supported filter operator, exact numeric comparisons, resource matching, include projection, continuation/replay, isolation, invalid-ID normalization, atomic configuration, cache read/miss and request export. Expanded web regression coverage is 105/105, including valid unused include selectors. The independent code-only full server gate passed 1158/1158 with 92.21% line coverage; see [the bounded contract and delivery evidence](../Features/OpenAiNativeFileCode.md). diff --git a/docs/images/admin-dashboard-cache-analytics.jpg b/docs/images/admin-dashboard-cache-analytics.jpg new file mode 100644 index 0000000..a763eca Binary files /dev/null and b/docs/images/admin-dashboard-cache-analytics.jpg differ diff --git a/docs/images/admin-dashboard-conversation.jpg b/docs/images/admin-dashboard-conversation.jpg index cd287d8..c4c5640 100644 Binary files a/docs/images/admin-dashboard-conversation.jpg and b/docs/images/admin-dashboard-conversation.jpg differ diff --git a/docs/images/admin-dashboard-insights.jpg b/docs/images/admin-dashboard-insights.jpg new file mode 100644 index 0000000..3884748 Binary files /dev/null and b/docs/images/admin-dashboard-insights.jpg differ diff --git a/docs/images/admin-dashboard-mobile-request-detail.png b/docs/images/admin-dashboard-mobile-request-detail.png index 617c133..e2b4546 100644 Binary files a/docs/images/admin-dashboard-mobile-request-detail.png and b/docs/images/admin-dashboard-mobile-request-detail.png differ diff --git a/docs/images/admin-dashboard-overview.jpg b/docs/images/admin-dashboard-overview.jpg index 1110911..e2e33cb 100644 Binary files a/docs/images/admin-dashboard-overview.jpg and b/docs/images/admin-dashboard-overview.jpg differ diff --git a/docs/images/admin-dashboard-runtime.jpg b/docs/images/admin-dashboard-runtime.jpg index 1dde163..ecafcd1 100644 Binary files a/docs/images/admin-dashboard-runtime.jpg and b/docs/images/admin-dashboard-runtime.jpg differ diff --git a/docs/images/admin-dashboard-token-usage.png b/docs/images/admin-dashboard-token-usage.png index 0ed63b4..05a3d15 100644 Binary files a/docs/images/admin-dashboard-token-usage.png and b/docs/images/admin-dashboard-token-usage.png differ diff --git a/docs/images/showcase-conversation.jpg b/docs/images/showcase-conversation.jpg new file mode 100644 index 0000000..c295eec Binary files /dev/null and b/docs/images/showcase-conversation.jpg differ diff --git a/docs/images/showcase-insights.jpg b/docs/images/showcase-insights.jpg new file mode 100644 index 0000000..7c17dad Binary files /dev/null and b/docs/images/showcase-insights.jpg differ diff --git a/docs/images/showcase-overview.jpg b/docs/images/showcase-overview.jpg new file mode 100644 index 0000000..e7e4420 Binary files /dev/null and b/docs/images/showcase-overview.jpg differ diff --git a/docs/images/showcase-token-flow.jpg b/docs/images/showcase-token-flow.jpg new file mode 100644 index 0000000..90a9214 Binary files /dev/null and b/docs/images/showcase-token-flow.jpg differ diff --git a/samples/AGENTS.md b/samples/AGENTS.md index cd24094..b970774 100644 --- a/samples/AGENTS.md +++ b/samples/AGENTS.md @@ -2,6 +2,6 @@ Samples must stay runnable and minimal. They are examples of the public API, not hidden test fixtures. -Use the existing `ManagedCode.LlmTck.Hosting` service-defaults methods for Aspire health and liveness endpoints. Do not add a separate ServiceDefaults sample project, ad hoc `app.MapGet("/health", ...)`, or placeholder root status endpoints to sample `Program.cs` files. +Use the existing `ManagedCode.LlmTestServer.Hosting` service-defaults methods for Aspire health and liveness endpoints. Do not add a separate ServiceDefaults sample project, ad hoc `app.MapGet("/health", ...)`, or placeholder root status endpoints to sample `Program.cs` files. -When changing the sample service or AppHost, run the Aspire integration test in `tests/ManagedCode.LlmTck.Tests`. +When changing the sample service or AppHost, run the Aspire integration test in `tests/ManagedCode.LlmTestServer.Tests`. diff --git a/samples/ManagedCode.LlmTck.AppHost/aspire.config.json b/samples/ManagedCode.LlmTck.AppHost/aspire.config.json deleted file mode 100644 index fa80579..0000000 --- a/samples/ManagedCode.LlmTck.AppHost/aspire.config.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "appHost": { - "path": "ManagedCode.LlmTck.AppHost.csproj" - } -} diff --git a/samples/ManagedCode.LlmTck.Service/Dockerfile b/samples/ManagedCode.LlmTck.Service/Dockerfile deleted file mode 100644 index 080052b..0000000 --- a/samples/ManagedCode.LlmTck.Service/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build -WORKDIR /src - -COPY . . -RUN dotnet publish samples/ManagedCode.LlmTck.Service/ManagedCode.LlmTck.Service.csproj \ - --configuration Release \ - --output /app/publish \ - /p:UseAppHost=false - -FROM mcr.microsoft.com/dotnet/aspnet:10.0 -WORKDIR /app - -LABEL org.opencontainers.image.source="https://github.com/managedcode/LlmTck" \ - org.opencontainers.image.description="ManagedCode LLM TCK HTTP service" \ - org.opencontainers.image.licenses="MIT" - -ENV ASPNETCORE_URLS=http://+:8080 -EXPOSE 8080 - -COPY --from=build /app/publish . -ENTRYPOINT ["dotnet", "ManagedCode.LlmTck.Service.dll"] diff --git a/samples/ManagedCode.LlmTck.Service/ManagedCode.LlmTck.Service.csproj b/samples/ManagedCode.LlmTck.Service/ManagedCode.LlmTck.Service.csproj deleted file mode 100644 index 6ab52a0..0000000 --- a/samples/ManagedCode.LlmTck.Service/ManagedCode.LlmTck.Service.csproj +++ /dev/null @@ -1,12 +0,0 @@ - - - false - true - false - - - - - - - diff --git a/samples/ManagedCode.LlmTck.AppHost/AppHost.cs b/samples/ManagedCode.LlmTestServer.AppHost/AppHost.cs similarity index 65% rename from samples/ManagedCode.LlmTck.AppHost/AppHost.cs rename to samples/ManagedCode.LlmTestServer.AppHost/AppHost.cs index 5929143..dd244e0 100644 --- a/samples/ManagedCode.LlmTck.AppHost/AppHost.cs +++ b/samples/ManagedCode.LlmTestServer.AppHost/AppHost.cs @@ -1,9 +1,9 @@ -using ManagedCode.LlmTck.Aspire; +using ManagedCode.LlmTestServer.Aspire; var builder = DistributedApplication.CreateBuilder(args); builder - .AddLlmTck() + .AddLlmTestServer() .WithApiKey("test-key"); builder.Build().Run(); diff --git a/samples/ManagedCode.LlmTck.AppHost/ManagedCode.LlmTck.AppHost.csproj b/samples/ManagedCode.LlmTestServer.AppHost/ManagedCode.LlmTestServer.AppHost.csproj similarity index 74% rename from samples/ManagedCode.LlmTck.AppHost/ManagedCode.LlmTck.AppHost.csproj rename to samples/ManagedCode.LlmTestServer.AppHost/ManagedCode.LlmTestServer.AppHost.csproj index 52bdb53..8ccca26 100644 --- a/samples/ManagedCode.LlmTck.AppHost/ManagedCode.LlmTck.AppHost.csproj +++ b/samples/ManagedCode.LlmTestServer.AppHost/ManagedCode.LlmTestServer.AppHost.csproj @@ -9,6 +9,6 @@ - + diff --git a/samples/ManagedCode.LlmTck.AppHost/Properties/launchSettings.json b/samples/ManagedCode.LlmTestServer.AppHost/Properties/launchSettings.json similarity index 100% rename from samples/ManagedCode.LlmTck.AppHost/Properties/launchSettings.json rename to samples/ManagedCode.LlmTestServer.AppHost/Properties/launchSettings.json diff --git a/samples/ManagedCode.LlmTck.AppHost/appsettings.Development.json b/samples/ManagedCode.LlmTestServer.AppHost/appsettings.Development.json similarity index 100% rename from samples/ManagedCode.LlmTck.AppHost/appsettings.Development.json rename to samples/ManagedCode.LlmTestServer.AppHost/appsettings.Development.json diff --git a/samples/ManagedCode.LlmTck.AppHost/appsettings.json b/samples/ManagedCode.LlmTestServer.AppHost/appsettings.json similarity index 100% rename from samples/ManagedCode.LlmTck.AppHost/appsettings.json rename to samples/ManagedCode.LlmTestServer.AppHost/appsettings.json diff --git a/samples/ManagedCode.LlmTestServer.AppHost/aspire.config.json b/samples/ManagedCode.LlmTestServer.AppHost/aspire.config.json new file mode 100644 index 0000000..14a5dd6 --- /dev/null +++ b/samples/ManagedCode.LlmTestServer.AppHost/aspire.config.json @@ -0,0 +1,5 @@ +{ + "appHost": { + "path": "ManagedCode.LlmTestServer.AppHost.csproj" + } +} diff --git a/samples/ManagedCode.LlmTestServer.Service/ManagedCode.LlmTestServer.Service.csproj b/samples/ManagedCode.LlmTestServer.Service/ManagedCode.LlmTestServer.Service.csproj new file mode 100644 index 0000000..a1ec528 --- /dev/null +++ b/samples/ManagedCode.LlmTestServer.Service/ManagedCode.LlmTestServer.Service.csproj @@ -0,0 +1,31 @@ + + + false + true + false + true + linux-x64;linux-arm64 + linux-x64;linux-arm64 + mcr.microsoft.com/dotnet/aspnet:10.0 + managedcode/llm-test-server + $(Version) + OCI + 1654 + /app/ + false + + + + + + + + + + + + + + + + diff --git a/samples/ManagedCode.LlmTck.Service/Program.cs b/samples/ManagedCode.LlmTestServer.Service/Program.cs similarity index 76% rename from samples/ManagedCode.LlmTck.Service/Program.cs rename to samples/ManagedCode.LlmTestServer.Service/Program.cs index f84e3ff..cd32198 100644 --- a/samples/ManagedCode.LlmTck.Service/Program.cs +++ b/samples/ManagedCode.LlmTestServer.Service/Program.cs @@ -1,5 +1,5 @@ -using ManagedCode.LlmTck.Hosting; -using ManagedCode.LlmTck.Models; +using ManagedCode.LlmTestServer.Hosting; +using ManagedCode.LlmTestServer.Models; var builder = WebApplication.CreateBuilder(args); builder.AddServiceDefaults(); @@ -17,8 +17,8 @@ builder.WebHost.ConfigureKestrel(options => options.ListenAnyIP(port)); } -var requiredBearerToken = builder.Configuration["LlmTck:RequiredBearerToken"]; -builder.Services.AddLlmTck(options => +var requiredBearerToken = LlmTestServerHostingConfiguration.GetRequiredBearerToken(builder.Configuration); +builder.Services.AddLlmTestServer(options => { options.AddDefaultOpenAiModels(); @@ -30,7 +30,7 @@ options.AddChatScenario( "default-blue-whale", scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) + .ForModel(LlmTestServerKnownModelIds.Gpt41Mini) .WhenUserContains("color") .Responds("blue whale", "blue ", "whale") .Responds("blue whale", "blue ", "whale") @@ -40,5 +40,5 @@ var app = builder.Build(); app.MapDefaultEndpoints(); -app.MapLlmTck(); +app.MapLlmTestServer(); app.Run(); diff --git a/samples/ManagedCode.LlmTck.Service/Properties/launchSettings.json b/samples/ManagedCode.LlmTestServer.Service/Properties/launchSettings.json similarity index 100% rename from samples/ManagedCode.LlmTck.Service/Properties/launchSettings.json rename to samples/ManagedCode.LlmTestServer.Service/Properties/launchSettings.json diff --git a/samples/ManagedCode.LlmTck.Service/appsettings.Development.json b/samples/ManagedCode.LlmTestServer.Service/appsettings.Development.json similarity index 100% rename from samples/ManagedCode.LlmTck.Service/appsettings.Development.json rename to samples/ManagedCode.LlmTestServer.Service/appsettings.Development.json diff --git a/samples/ManagedCode.LlmTck.Service/appsettings.json b/samples/ManagedCode.LlmTestServer.Service/appsettings.json similarity index 100% rename from samples/ManagedCode.LlmTck.Service/appsettings.json rename to samples/ManagedCode.LlmTestServer.Service/appsettings.json diff --git a/scripts/coverage.sh b/scripts/coverage.sh index 93f2112..1e09cbb 100644 --- a/scripts/coverage.sh +++ b/scripts/coverage.sh @@ -3,26 +3,26 @@ set -euo pipefail mkdir -p artifacts/coverage/raw # CI normalizes PDB paths. Resolve them back to this checkout without disabling # source filtering or the 90% gate (Coverlet console does not create this map). -dotnet msbuild tests/ManagedCode.LlmTck.Tests/ManagedCode.LlmTck.Tests.csproj \ +dotnet msbuild tests/ManagedCode.LlmTestServer.Tests/ManagedCode.LlmTestServer.Tests.csproj \ -nologo -target:InitializeSourceRootMappedPaths -getItem:SourceRoot \ -p:Configuration=Release -p:ContinuousIntegrationBuild=true > artifacts/coverage/source-roots.json python3 - <<'PYMAP' import json from pathlib import Path roots = json.loads(Path("artifacts/coverage/source-roots.json").read_text())["Items"]["SourceRoot"] -project = Path("tests/ManagedCode.LlmTck.Tests/ManagedCode.LlmTck.Tests.csproj").resolve() +project = Path("tests/ManagedCode.LlmTestServer.Tests/ManagedCode.LlmTestServer.Tests.csproj").resolve() Path("artifacts/coverage/source-roots.map").write_text("".join( f"{project}|{root['Identity']}={root['MappedPath']}\n" for root in roots if root.get("MappedPath") )) PYMAP -dotnet tool run coverlet tests/ManagedCode.LlmTck.Tests/bin/Release/net10.0/ManagedCode.LlmTck.Tests.dll \ +dotnet tool run coverlet tests/ManagedCode.LlmTestServer.Tests/bin/Release/net10.0/ManagedCode.LlmTestServer.Tests.dll \ --target dotnet \ - --targetargs "test --project tests/ManagedCode.LlmTck.Tests/ManagedCode.LlmTck.Tests.csproj --configuration Release --no-build --verbosity normal" \ + --targetargs "test --project tests/ManagedCode.LlmTestServer.Tests/ManagedCode.LlmTestServer.Tests.csproj --configuration Release --no-build --verbosity normal" \ --source-mapping-file artifacts/coverage/source-roots.map \ --format cobertura --format json --output artifacts/coverage/raw/coverage \ - --include '[ManagedCode.LlmTck]*' --include '[ManagedCode.LlmTck.*]*' \ - --exclude '[ManagedCode.LlmTck.Tests]*' --exclude '[ManagedCode.LlmTck.Service]*' --exclude '[ManagedCode.LlmTck.AppHost]*' \ + --include '[ManagedCode.LlmTestServer]*' --include '[ManagedCode.LlmTestServer.*]*' \ + --exclude '[ManagedCode.LlmTestServer.Tests]*' --exclude '[ManagedCode.LlmTestServer.Service]*' --exclude '[ManagedCode.LlmTestServer.AppHost]*' \ --threshold 90 --threshold-type line --threshold-stat Total dotnet tool run reportgenerator -reports:artifacts/coverage/raw/coverage.cobertura.xml \ -targetdir:artifacts/coverage/report '-reporttypes:HtmlSummary;TextSummary' diff --git a/scripts/package-consumer/AnthropicNativePackageTests.cs b/scripts/package-consumer/AnthropicNativePackageTests.cs new file mode 100644 index 0000000..41b233a --- /dev/null +++ b/scripts/package-consumer/AnthropicNativePackageTests.cs @@ -0,0 +1,131 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using global::Anthropic; +using global::Anthropic.Core; +using global::Anthropic.Models.Messages; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Scenarios; +using Aspire.Hosting; +using Aspire.Hosting.Testing; +using ManagedCode.LlmTestServer.Aspire; +using TUnit.Core; + +public sealed class AnthropicNativePackageTests +{ + private static readonly string _model = JsonSerializer.SerializeToElement(Model.ClaudeSonnet5_5).GetString()!; + private const string _key = "anthropic-sdk-key"; + private const string _url = "https://example.com/sdk-source"; + private const string _call = "srvtoolu_sdk"; + + [Test] + public async Task InstalledPackages_OfficialSdkParsesNativeJsonSseErrorsAndHistoryAsync(CancellationToken cancellationToken) + { + var builder = DistributedApplicationTestingBuilder.Create([]); + builder.AddLlmTestServer("anthropic-native-package-server").WithApiKey(_key); + await using var app = await builder.BuildAsync(cancellationToken); + await app.StartAsync(cancellationToken); + await app.ResourceNotifications.WaitForResourceHealthyAsync("anthropic-native-package-server", cancellationToken); + using var http = app.CreateHttpClient("anthropic-native-package-server", LlmTestServerResource.HttpEndpointName); + var control = new LlmTestServerClient(http, _key); + await control.ConfigureAsync(config => config.RequireBearerToken(_key).UseModel(_model, LlmTestServerModelKind.Chat) + .UseChatScenario("anthropic-sdk", scenario => scenario.ForModel(_model) + .RespondsWithItems(Items()).RespondsWithItems(Items()).RespondsWithItems(Items("unavailable")).Responds("SDK history complete.")), cancellationToken); + using var sdk = new AnthropicClient(new ClientOptions + { HttpClient = http, BaseUrl = new Uri(http.BaseAddress!, "/anthropic").ToString(), ApiKey = _key, MaxRetries = 0, ResponseValidation = true }); + var options = Options(); + var first = await sdk.Messages.Create(options, cancellationToken); + AssertContent(first.Content); + Require(first.Usage.ServerToolUse!.WebSearchRequests == 1, "Installed native JSON must report one search."); + + var blocks = new JsonArray(); var input = string.Empty; var stopped = new List(); + var messageStopped = false; var citationDeltas = 0; var searchCount = -1L; + await foreach (var update in sdk.Messages.CreateStreaming(options, cancellationToken)) + { + switch (update.Value) + { + case RawContentBlockStartEvent start: + Require(start.Index == blocks.Count, "Installed SDK stream block indexes must be contiguous."); + blocks.Add(JsonNode.Parse(start.ContentBlock.Json.GetRawText())); + break; + case RawContentBlockDeltaEvent delta: + switch (delta.Delta.Value) + { + case InputJsonDelta json: input += json.PartialJson; break; + case TextDelta text: + var index = checked((int)delta.Index); + blocks[index]!["text"] = (blocks[index]!["text"]?.GetValue() ?? string.Empty) + text.Text; + break; + case CitationsDelta citation: + citationDeltas++; + var citationIndex = checked((int)delta.Index); + blocks[citationIndex]!["citations"] ??= new JsonArray(); + ((JsonArray)blocks[citationIndex]!["citations"]!).Add(JsonNode.Parse(citation.Citation.Json.GetRawText())); + break; + } + break; + case RawContentBlockStopEvent stop: stopped.Add(stop.Index); break; + case RawMessageDeltaEvent delta: + searchCount = delta.Usage.ServerToolUse!.WebSearchRequests; + break; + case RawMessageStopEvent: messageStopped = true; break; + } + } + blocks[0]!["input"] = JsonNode.Parse(input); + Require(messageStopped && stopped.SequenceEqual(new long[] { 0, 1, 2 }) && citationDeltas == 1 && searchCount == 1, "Installed native SDK SSE must complete blocks, citations and usage."); + var streamed = JsonSerializer.Deserialize>(blocks.ToJsonString())!; + AssertContent(streamed); + Require(JsonNode.DeepEquals(blocks, JsonNode.Parse(JsonSerializer.Serialize(first.Content))), "Installed SDK SSE must reconstruct JSON content exactly."); + + var failed = await sdk.Messages.Create(options, cancellationToken); + var errorBlock = (WebSearchToolResultBlock)failed.Content[1].Value!; + Require(errorBlock.Content.TryPickError(out var nativeError) && nativeError.ErrorCode.Json.GetString() == "unavailable", + "Official installed SDK must parse typed native errors inside successful Messages."); + var history = JsonSerializer.Deserialize>(JsonSerializer.Serialize(first.Content))!; + var follow = options with { Messages = [.. options.Messages, new() { Role = Role.Assistant, Content = history }, new() { Role = Role.User, Content = "SDK follow" }] }; + var continued = await sdk.Messages.Create(follow, cancellationToken); + Require(((TextBlock)continued.Content.Single().Value!).Text == "SDK history complete.", "Installed SDK assistant history must roundtrip."); + var traces = (await control.ExportRequestsAsync(cancellationToken: cancellationToken)).Evidence.Requests; + Require(traces.Count == 4, "Installed journal must retain every SDK request."); + var stream = traces.Single(trace => trace.IsStreaming); + Require(stream.TerminalMarkerObserved == true && stream.NativeTools.Count == 1 && stream.ResponseToolCalls.Count == 0, "Installed journal must retain terminal native evidence separately from functions."); + var restored = traces.Single(trace => trace.Messages.Any(message => message.Content == "SDK follow")); + Require(restored.Messages.Single(message => message.Role == "assistant").OutputItems.Count == 3, "Installed typed history must retain ordered native items."); + } + + private static MessageCreateParams Options(long maxUses = 1) + { + return new() + { + Model = Model.ClaudeSonnet5_5, MaxTokens = 256, ToolChoice = new ToolChoiceAuto(), + Messages = [new() { Role = Role.User, Content = "SDK fixture" }], + Tools = [new WebSearchTool20250305 + { MaxUses = maxUses, AllowedDomains = ["example.com/sdk*"], UserLocation = new() { Country = "US", Timezone = "America/New_York" } }], + }; + } + + private static LlmTestServerResponseItem[] Items(string? error = null) + { + return + [new LlmTestServerResponseServerToolUseItem { Id = _call, Input = new LlmTestServerWebSearchInput { Query = "SDK fixture query" } }, + new LlmTestServerResponseServerToolResultItem { ToolUseId = _call, + Result = error is null ? new LlmTestServerWebSearchResults { Results = [new() { Url = _url, Title = "SDK source", EncryptedContent = "opaque-sdk-source" }] } + : new LlmTestServerWebSearchErrorResult { ErrorCode = error } }, + new LlmTestServerResponseMessageItem { Content = "SDK answer.", StreamChunks = ["SDK ", "answer."], + Citations = error is null ? [new LlmTestServerAnthropicWebSearchCitation { Url = _url, EncryptedIndex = "opaque-sdk-index", CitedText = "SDK source excerpt", Title = "SDK source" }] : [] }]; + } + + private static void AssertContent(IReadOnlyList content) + { + if (content.Count != 3 || content[0].Value is not ServerToolUseBlock call || call.ID != _call + || call.Input["query"].GetString() != "SDK fixture query" || content[1].Value is not WebSearchToolResultBlock returned + || returned.ToolUseID != _call || !returned.Content.TryPickWebSearchResultBlocks(out var results) + || results.Single().EncryptedContent != "opaque-sdk-source" || content[2].Value is not TextBlock text || text.Text != "SDK answer." + || text.Citations!.Single().Value is not CitationsWebSearchResultLocation citation || citation.EncryptedIndex != "opaque-sdk-index") + { throw new InvalidOperationException("Official Anthropic SDK must parse typed native calls, results and citations."); } + } + private static void Require(bool condition, string message) + { + if (!condition) { throw new InvalidOperationException(message); } + } +} diff --git a/scripts/package-consumer/AzureOutputPolicyPackageTests.cs b/scripts/package-consumer/AzureOutputPolicyPackageTests.cs new file mode 100644 index 0000000..88fc306 --- /dev/null +++ b/scripts/package-consumer/AzureOutputPolicyPackageTests.cs @@ -0,0 +1,188 @@ +#pragma warning disable OPENAI001 // Exercise the pinned official Responses SDK against installed packages. +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Text; +using Aspire.Hosting; +using Aspire.Hosting.Testing; +using Azure; +using Azure.AI.Inference; +using Azure.AI.OpenAI; +using Azure.Core.Pipeline; +using ManagedCode.LlmTestServer.Aspire; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Scenarios; +using OpenAI; +using OpenAI.Chat; +using OpenAI.Responses; +using TUnit.Core; + +public sealed class AzureOutputPolicyPackageTests +{ + private const string _key = "azure-policy-package-key"; + private const string _model = LlmTestServerKnownModelIds.Gpt41Mini; + private const string _text = "Explicit installed Azure policy text"; + private const string _recovery = "Installed Azure policy recovery"; + + [Test] + [Arguments("deployment")] + [Arguments("inference")] + [Arguments("inference-models")] + [Arguments("azure-chat")] + [Arguments("foundry-chat")] + [Arguments("azure-responses")] + [Arguments("foundry-responses")] + public async Task InstalledPackages_AzurePolicyStreamsThroughOriginalSdkAndExplicitJsonRecoveryAsync(string route, CancellationToken cancellationToken) + { + var resourceName = "azure-policy-" + route; + var builder = DistributedApplicationTestingBuilder.Create([]); + builder.AddLlmTestServer(resourceName).WithApiKey(_key); + await using var app = await builder.BuildAsync(cancellationToken); + await app.StartAsync(cancellationToken); + await app.ResourceNotifications.WaitForResourceHealthyAsync(resourceName, cancellationToken); + using var http = app.CreateHttpClient(resourceName, LlmTestServerResource.HttpEndpointName); + var control = new LlmTestServerClient(http, _key); + var inference = route is "inference" or "inference-models"; + var responses = route.EndsWith("responses", StringComparison.Ordinal); + var kind = inference || responses ? LlmTestServerOutputDispositionKind.Filtered : LlmTestServerOutputDispositionKind.Refused; + var prefix = route.StartsWith("foundry-", StringComparison.Ordinal) ? "/microsoft-foundry" : "/azure-openai"; + var expectedPath = route switch + { + "deployment" => "/azure-openai/openai/deployments/" + _model + "/chat/completions", + "inference" => "/microsoft-foundry/chat/completions", + "inference-models" => "/microsoft-foundry/models/chat/completions", + _ => prefix + "/openai/v1/" + (responses ? "responses" : "chat/completions"), + }; + await control.ConfigureAsync(config => config.RequireBearerToken(_key).UseChatModel(_model) + .UseChatScenario("installed-azure-policy", scenario => scenario.ForModel(_model) + .Responds(_text, "Explicit installed ", "Azure policy text") + .WithOutputDisposition(new() { Kind = kind }) + .Responds(_recovery)), cancellationToken); + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + deadline.CancelAfter(TimeSpan.FromSeconds(60)); + var text = new StringBuilder(); + var refusal = new StringBuilder(); + long inputTokens = 0, outputTokens = 0, totalTokens = 0; + var sawUsage = false; + Func> recover; + if (inference) + { + var options = new AzureAIInferenceClientOptions { Transport = new HttpClientTransport(http) }; + options.Retry.MaxRetries = 0; + var endpoint = route == "inference-models" ? "/microsoft-foundry/models" : "/microsoft-foundry"; + var sdk = new ChatCompletionsClient(new Uri(http.BaseAddress!, endpoint), new AzureKeyCredential(_key), options); + var request = new ChatCompletionsOptions([new ChatRequestUserMessage("policy")]) { Model = _model }; + CompletionsFinishReason? finish = null; + using (var stream = await sdk.CompleteStreamingAsync(request, deadline.Token)) + { + await foreach (var update in stream) + { + text.Append(update.ContentUpdate); + finish = update.FinishReason ?? finish; + if (update.Usage is { } usage) + { + sawUsage = true; inputTokens = usage.PromptTokens; outputTokens = usage.CompletionTokens; totalTokens = usage.TotalTokens; + } + } + } + Require(finish == CompletionsFinishReason.ContentFiltered, "Installed Model Inference SDK must parse typed ContentFiltered."); + recover = async token => + { + var next = (await sdk.CompleteAsync(new ChatCompletionsOptions([new ChatRequestUserMessage("next")]) { Model = _model }, token)).Value; + Require(next.FinishReason == CompletionsFinishReason.Stopped, "Installed inference recovery must have a normal stop."); + return next.Content; + }; + } + else if (responses) + { + var sdk = new ResponsesClient(new ApiKeyCredential(_key), new ResponsesClientOptions + { Endpoint = new Uri(http.BaseAddress!, prefix + "/openai/v1/"), Transport = new HttpClientPipelineTransport(http), RetryPolicy = new ClientRetryPolicy(0) }); + ResponseResult? final = null; + await foreach (var update in sdk.CreateResponseStreamingAsync(new CreateResponseOptions + { Model = _model, StreamingEnabled = true, StoredOutputEnabled = false, InputItems = { ResponseItem.CreateUserMessageItem("policy") } }, deadline.Token)) + { + if (update is StreamingResponseOutputTextDeltaUpdate delta) { text.Append(delta.Delta); } + if (update is StreamingResponseIncompleteUpdate incomplete) { final = incomplete.Response; } + Require(update is not StreamingResponseCompletedUpdate and not StreamingResponseFailedUpdate and not StreamingResponseErrorUpdate, + "Installed filtered Azure Responses must end incomplete, not completed, failed or API error."); + } + Require(final is not null && final.Status == ResponseStatus.Incomplete + && final.IncompleteStatusDetails.Reason == ResponseIncompleteStatusReason.ContentFilter, + "Installed Azure Responses SDK must parse native content_filter incomplete completion."); + var usage = final!.Usage; + sawUsage = usage is not null; + if (usage is not null) { inputTokens = usage.InputTokenCount; outputTokens = usage.OutputTokenCount; totalTokens = usage.TotalTokenCount; } + recover = async token => + { + var next = (await sdk.CreateResponseAsync(new CreateResponseOptions + { Model = _model, StoredOutputEnabled = false, InputItems = { ResponseItem.CreateUserMessageItem("next") } }, token)).Value; + Require(next.Status == ResponseStatus.Completed && next.IncompleteStatusDetails is null, "Installed Azure Responses recovery must complete normally."); + return next.GetOutputText(); + }; + } + else + { + var transport = new HttpClientPipelineTransport(http); + var sdk = route == "deployment" + ? new AzureOpenAIClient(new Uri(http.BaseAddress!, "/azure-openai/"), new ApiKeyCredential(_key), + new AzureOpenAIClientOptions { Transport = transport, RetryPolicy = new ClientRetryPolicy(0) }).GetChatClient(_model) + : new ChatClient(_model, new ApiKeyCredential(_key), new OpenAIClientOptions + { Endpoint = new Uri(http.BaseAddress!, prefix + "/openai/v1/"), Transport = transport, RetryPolicy = new ClientRetryPolicy(0) }); + ChatFinishReason? finish = null; + await foreach (var update in sdk.CompleteChatStreamingAsync([new UserChatMessage("policy")], cancellationToken: deadline.Token)) + { + foreach (var part in update.ContentUpdate) { text.Append(part.Text); } + refusal.Append(update.RefusalUpdate); + finish = update.FinishReason ?? finish; + if (update.Usage is { } usage) + { sawUsage = true; inputTokens = usage.InputTokenCount; outputTokens = usage.OutputTokenCount; totalTokens = usage.TotalTokenCount; } + } + Require(finish == ChatFinishReason.Stop && text.Length == 0 && refusal.ToString() == _text, + "Installed Azure Chat SDK must expose native refusal deltas and normal stop without duplicate ordinary text."); + recover = async token => + { + var next = (await sdk.CompleteChatAsync([new UserChatMessage("next")], cancellationToken: token)).Value; + Require(next.FinishReason == ChatFinishReason.Stop && next.Refusal is null, "Installed Azure Chat recovery must not inherit refusal."); + return next.Content.Single().Text; + }; + } + Require((kind == LlmTestServerOutputDispositionKind.Refused ? refusal.ToString() : text.ToString()) == _text, + "Installed original Azure SDK must retain explicit policy text."); + var first = await CompletedTraceAsync(control, 1, deadline.Token); + Require(first.Path == expectedPath && first.ModelId == _model && first.IsStreaming && first.StatusCode == 200 + && first.TransportState == LlmTestServerProviderHttpTraceTransportState.Completed && first.TerminalMarkerObserved == true + && first.OutputDisposition?.Kind == kind && first.StreamDelivery is null && !first.StreamDeliveryApplied, + "Installed journal must retain the exact Azure route, native200 policy and actual stream terminal."); + Require(sawUsage, $"Installed original {route} SDK stream omitted typed usage; journal input/output/total={first.Usage?.InputTokens}/{first.Usage?.OutputTokens}/{first.Usage?.TotalTokens}."); + Require(first.Usage is { } recorded && recorded.InputTokens == inputTokens && recorded.OutputTokens == outputTokens + && recorded.TotalTokens == totalTokens && inputTokens > 0 && outputTokens > 0, + "Installed SDK stream usage must reconcile with recorded generated/refusal tokens."); + var verified = await control.VerifyAsync(new() + { Requests = [new() { Query = new() { RequestId = first.RequestId }, RequireStreamingTerminalMarker = true }] }, deadline.Token); + Require(verified.Passed, "Installed native Azure policy stream must satisfy protocol-terminal verification."); + Require(await recover(deadline.Token) == _recovery, "Explicit installed SDK JSON retry must consume the next recovery fixture."); + var second = await CompletedTraceAsync(control, 2, deadline.Token); + var summary = await control.GetAssertionsAsync(deadline.Token); + Require(second.Path == expectedPath && !second.IsStreaming && second.StatusCode == 200 && second.OutputDisposition is null + && second.TransportState == LlmTestServerProviderHttpTraceTransportState.Completed + && summary.Matched == 2 && summary.ErrorsReturned == 0 + && summary.Events.Count(item => item.OutputDisposition?.Kind == kind) == 1 + && (await control.GetRequestsAsync(cancellationToken: deadline.Token)).TotalObserved == 2, + "Exactly two original SDK calls must be observed, with one committed policy followed by ordinary JSON recovery."); + } + + private static async Task CompletedTraceAsync(LlmTestServerClient control, int count, CancellationToken cancellationToken) + { + while (true) + { + var evidence = await control.GetRequestsAsync(cancellationToken: cancellationToken); + if (evidence.Requests.Count == count && evidence.Requests.All(trace => trace.IsCompleted)) + return evidence.Requests.OrderBy(trace => trace.Sequence).Last(); + await Task.Delay(20, cancellationToken); + } + } + + private static void Require(bool condition, string message) + { if (!condition) { throw new InvalidOperationException(message); } } +} diff --git a/scripts/package-consumer/Consumer.csproj.template b/scripts/package-consumer/Consumer.csproj.template index 9e146e7..5b2cb02 100644 --- a/scripts/package-consumer/Consumer.csproj.template +++ b/scripts/package-consumer/Consumer.csproj.template @@ -11,7 +11,13 @@ true - + + + + + + + diff --git a/scripts/package-consumer/DeepSeekOutputPolicyPackageTests.cs b/scripts/package-consumer/DeepSeekOutputPolicyPackageTests.cs new file mode 100644 index 0000000..2f531ff --- /dev/null +++ b/scripts/package-consumer/DeepSeekOutputPolicyPackageTests.cs @@ -0,0 +1,106 @@ +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Text; +using Aspire.Hosting; +using Aspire.Hosting.Testing; +using ManagedCode.LlmTestServer.Aspire; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Scenarios; +using OpenAI; +using OpenAI.Chat; +using TUnit.Core; + +public sealed class DeepSeekOutputPolicyPackageTests +{ + private const string _key = "deepseek-policy-package-key"; + private const string _model = "deepseek-flash"; + private const string _path = "/deepseek/v1/chat/completions"; + private const string _text = "Explicit installed DeepSeek prefix"; + private const string _recovery = "Installed DeepSeek recovery"; + + [Test] + public async Task InstalledPackages_DeepSeekFilteredStreamRetainsUsageAndExplicitJsonRecoveryAsync(CancellationToken cancellationToken) + { + const string ResourceName = "deepseek-policy-package"; + var builder = DistributedApplicationTestingBuilder.Create([]); + builder.AddLlmTestServer(ResourceName).WithApiKey(_key); + await using var app = await builder.BuildAsync(cancellationToken); + await app.StartAsync(cancellationToken); + await app.ResourceNotifications.WaitForResourceHealthyAsync(ResourceName, cancellationToken); + using var http = app.CreateHttpClient(ResourceName, LlmTestServerResource.HttpEndpointName); + var control = new LlmTestServerClient(http, _key); + await control.ConfigureAsync(config => config.RequireBearerToken(_key).UseChatModel(_model) + .UseChatScenario("installed-deepseek-policy", scenario => scenario.ForModel(_model) + .Responds(_text, "Explicit installed ", "DeepSeek prefix") + .WithOutputDisposition(new() { Kind = LlmTestServerOutputDispositionKind.Filtered }) + .Responds(_recovery)), cancellationToken); + var sdk = new ChatClient(_model, new ApiKeyCredential(_key), new OpenAIClientOptions + { + Endpoint = new Uri(http.BaseAddress!, "/deepseek/v1/"), + Transport = new HttpClientPipelineTransport(http), + RetryPolicy = new ClientRetryPolicy(0), + }); + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + deadline.CancelAfter(TimeSpan.FromSeconds(60)); + var text = new StringBuilder(); + ChatFinishReason? finish = null; + ChatTokenUsage? usage = null; + var usageUpdates = 0; + await foreach (var update in sdk.CompleteChatStreamingAsync([new UserChatMessage("policy")], cancellationToken: deadline.Token)) + { + foreach (var part in update.ContentUpdate) { text.Append(part.Text); } + finish = update.FinishReason ?? finish; + Require(update.RefusalUpdate is null, "DeepSeek Filtered must not become a model-refusal field."); + if (update.Usage is not null) { usage = update.Usage; usageUpdates++; } + } + Require(text.ToString() == _text && finish == ChatFinishReason.ContentFilter && usage is not null && usageUpdates == 1, + "Installed DeepSeek SDK streaming must retain the configured prefix, native ContentFilter finish and one final usage update."); + var sdkUsage = usage ?? throw new InvalidOperationException("Installed DeepSeek SSE omitted final usage."); + var first = await CompletedTraceAsync(control, 1, deadline.Token); + Require(first.Path == _path && first.ModelId == _model && first.IsStreaming && first.StatusCode == 200 + && first.TransportState == LlmTestServerProviderHttpTraceTransportState.Completed + && first.OutputDisposition?.Kind == LlmTestServerOutputDispositionKind.Filtered + && first.TerminalMarkerObserved == true && first.StreamDelivery is null && !first.StreamDeliveryApplied, + "Installed journal must distinguish a completed native DeepSeek policy stream from HTTP or delivery faults."); + Require(first.Usage is not null && sdkUsage.InputTokenCount == first.Usage.InputTokens + && sdkUsage.OutputTokenCount == first.Usage.OutputTokens && sdkUsage.TotalTokenCount == first.Usage.TotalTokens + && sdkUsage.InputTokenCount > 0 && sdkUsage.OutputTokenCount > 0, + "Installed SDK usage must reconcile with the recorded inference usage, including generated policy-prefix tokens."); + var verified = await control.VerifyAsync(new() + { + Requests = [new() { Query = new() { RequestId = first.RequestId }, RequireStreamingTerminalMarker = true }], + }, deadline.Token); + Require(verified.Passed, "Installed DeepSeek native policy SSE must satisfy actual protocol-terminal verification."); + var summary = await control.GetAssertionsAsync(deadline.Token); + Require(summary.Matched == 1 && summary.ErrorsReturned == 0 + && summary.Events.Single().OutputDisposition?.Kind == LlmTestServerOutputDispositionKind.Filtered, + "The filtered outcome must commit once without becoming a provider HTTP error."); + + var recovered = (await sdk.CompleteChatAsync([new UserChatMessage("next")], cancellationToken: deadline.Token)).Value; + Require(recovered.Content.Single().Text == _recovery && recovered.FinishReason == ChatFinishReason.Stop && recovered.Refusal is null, + "The explicit next installed SDK JSON call must consume the queued recovery with a normal stop."); + var second = await CompletedTraceAsync(control, 2, deadline.Token); + Require(second.Path == _path && !second.IsStreaming && second.StatusCode == 200 && second.OutputDisposition is null + && second.TransportState == LlmTestServerProviderHttpTraceTransportState.Completed, + "Installed JSON recovery must remain on the DeepSeek route and not inherit the prior policy outcome."); + var requests = await control.GetRequestsAsync(cancellationToken: deadline.Token); + summary = await control.GetAssertionsAsync(deadline.Token); + Require(requests.TotalObserved == 2 && summary.Matched == 2 && summary.ErrorsReturned == 0, + "Installed DeepSeek SDK must make exactly two matched requests with retries disabled."); + } + + private static async Task CompletedTraceAsync(LlmTestServerClient control, int count, CancellationToken cancellationToken) + { + while (true) + { + var evidence = await control.GetRequestsAsync(cancellationToken: cancellationToken); + if (evidence.Requests.Count == count && evidence.Requests.All(trace => trace.IsCompleted)) + return evidence.Requests.OrderBy(trace => trace.Sequence).Last(); + await Task.Delay(20, cancellationToken); + } + } + + private static void Require(bool condition, string message) + { if (!condition) { throw new InvalidOperationException(message); } } +} diff --git a/scripts/package-consumer/NativeFileCodePackageTests.cs b/scripts/package-consumer/NativeFileCodePackageTests.cs new file mode 100644 index 0000000..c1fb2da --- /dev/null +++ b/scripts/package-consumer/NativeFileCodePackageTests.cs @@ -0,0 +1,180 @@ +#pragma warning disable OPENAI001 // Exercise the official Responses client against installed packages. +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Net.Http.Headers; +using Aspire.Hosting; +using Aspire.Hosting.Testing; +using ManagedCode.LlmTestServer.Aspire; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Scenarios; +using OpenAI.Responses; +using TUnit.Core; + +public sealed class NativeFileCodePackageTests +{ + private const string _key = "file-code-package-key"; + + [Test] + public async Task InstalledPackages_ServeFileAndCodeJsonStreamAndStoredHistory(CancellationToken cancellationToken) + { + var builder = DistributedApplicationTestingBuilder.Create([]); + builder.AddLlmTestServer("file-code-package-server").WithApiKey(_key); + await using var app = await builder.BuildAsync(cancellationToken); + await app.StartAsync(cancellationToken); + await app.ResourceNotifications.WaitForResourceHealthyAsync("file-code-package-server", cancellationToken); + using var http = app.CreateHttpClient("file-code-package-server", "http"); + http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", _key); + var control = new LlmTestServerClient(http, _key); + var sdk = new ResponsesClient(new ApiKeyCredential(_key), new ResponsesClientOptions + { Endpoint = new Uri(http.BaseAddress!, "/openai/v1/"), Transport = new HttpClientPipelineTransport(http) }); + foreach (var code in new[] { false, true }) + { + await control.ConfigureAsync(config => + { + config.RequireBearerToken(_key); + if (code) { config.UseGpt41(); } else { config.UseGpt4OMini(); } + config.UseChatScenario("installed-file-code", scenario => scenario + .ForModel(code ? LlmTestServerKnownModelIds.Gpt41 : LlmTestServerKnownModelIds.Gpt4OMini) + .RespondsWithItems(Items(code)) + .RespondsWithItems(Items(code)).Responds("installed continuation complete")); + }, cancellationToken); + var first = (await sdk.CreateResponseAsync(Options(code), cancellationToken: cancellationToken)).Value; + AssertOutput(first, code); + var streamOptions = Options(code); + streamOptions.StreamingEnabled = true; + var updates = new List(); + await foreach (var update in sdk.CreateResponseStreamingAsync(streamOptions, cancellationToken: cancellationToken)) + { updates.Add(update); } + Require(updates.Select(update => update.SequenceNumber).SequenceEqual(Enumerable.Range(0, updates.Count)), + "Installed file/code streaming must preserve contiguous event sequence numbers."); + AssertOutput(updates.OfType().Single().Response, code); + if (code) + { + Require(updates.OfType().Count() == 1 + && updates.OfType().Count() == 1 + && updates.OfType().Count() == 1, + "Installed code interpreter must emit native progress events."); + Require(updates.OfType().Count() == 2 + && updates.OfType().Count() == 1, + "Installed code interpreter must emit configured code deltas and the completed code."); + } + else + { + Require(updates.OfType().Count() == 1 + && updates.OfType().Count() == 1 + && updates.OfType().Count() == 1, + "Installed file search must emit native progress events."); + } + Require(updates.OfType().Count() == 1, + "Installed native text must emit its citation exactly once."); + var follow = Options(code); + follow.ToolChoice = ResponseToolChoice.CreateAutoChoice(); + follow.PreviousResponseId = first.Id; + var continued = (await sdk.CreateResponseAsync(follow, cancellationToken: cancellationToken)).Value; + Require(((MessageResponseItem)continued.OutputItems.Single()).Content.Single().Text == "installed continuation complete", + "Installed file/code response history must support previous_response_id."); + var evidence = await control.GetRequestsAsync(cancellationToken: cancellationToken); + Require(evidence.Requests.Count == 3 && evidence.Requests.All(trace => trace.IsSuccess == true), + "Installed native requests must retain successful terminal evidence."); + var streamed = evidence.Requests.Single(trace => trace.IsStreaming); + Require(streamed.TerminalMarkerObserved == true && streamed.ResponseOutputItems.Count == 2 + && streamed.NativeTools.Count == 1 && streamed.ResponseToolCalls.Count == 0, + "Native evidence must expose native declarations and outputs separately from client function calls."); + var restored = evidence.Requests.Single(trace => trace.ResponsePreview?.Contains("installed continuation complete", StringComparison.Ordinal) == true); + Require(restored.Messages.Any(message => message.Role == "assistant" && message.OutputItems.Any(item => + code ? item is LlmTestServerResponseCodeInterpreterItem : item is LlmTestServerResponseFileSearchItem)), + "Installed public history must preserve typed native output items."); + } + } + + private static CreateResponseOptions Options(bool code) + { + var options = new CreateResponseOptions + { + Model = code ? LlmTestServerKnownModelIds.Gpt41 : LlmTestServerKnownModelIds.Gpt4OMini, + InputItems = { ResponseItem.CreateUserMessageItem("installed native fixture") }, + ToolChoice = ResponseToolChoice.CreateRequiredChoice(), + }; + if (code) + { + options.Tools.Add(ResponseTool.CreateCodeInterpreterTool(new CodeInterpreterToolContainer( + CodeInterpreterToolContainerConfiguration.CreateAutomaticContainerConfiguration(["file_input"])))); + options.IncludedProperties.Add(IncludedResponseProperty.CodeInterpreterCallOutputs); + } + else + { + options.Tools.Add(ResponseTool.CreateFileSearchTool(["vs_installed"], 1, null, null)); + options.IncludedProperties.Add(IncludedResponseProperty.FileSearchCallResults); + } + return options; + } + + private static LlmTestServerResponseItem[] Items(bool code) + { + return code ? + [ + new LlmTestServerResponseCodeInterpreterItem + { + Id = "ci_installed", ContainerId = "cntr_installed", InputFileIds = ["file_input"], GeneratedFileIds = ["file_generated"], + Code = "print(42)", CodeChunks = ["print(", "42)"], + Outputs = [new LlmTestServerCodeInterpreterLogsOutput { Logs = "42\n" }, + new LlmTestServerCodeInterpreterImageOutput { Url = "https://example.com/installed-chart.png" }], + }, + new LlmTestServerResponseMessageItem + { + Id = "msg_installed_code", Content = "Chart fixture.", + Citations = [new LlmTestServerContainerFileCitation + { ContainerId = "cntr_installed", FileId = "file_generated", Filename = "chart.png", StartIndex = 0, EndIndex = 0 }], + }, + ] : + [ + new LlmTestServerResponseFileSearchItem + { + Id = "fs_installed", VectorStoreIds = ["vs_installed"], Queries = ["installed fixture"], + Results = [new() { FileId = "file_installed", Filename = "fixture.txt", Text = "Installed fixture.", Score = 0.9 }], + }, + new LlmTestServerResponseMessageItem + { + Id = "msg_installed_file", Content = "File fixture.", + Citations = [new LlmTestServerFileCitation { FileId = "file_installed", Filename = "fixture.txt", Index = 4 }], + }, + ]; + } + + private static void AssertOutput(ResponseResult response, bool code) + { + Require(response.OutputItems.Count == 2 && !response.OutputItems.OfType().Any(), + "File/code output must remain native output items."); + var text = (MessageResponseItem)response.OutputItems[1]; + if (code) + { + var execution = (CodeInterpreterCallResponseItem)response.OutputItems[0]; + Require(execution.Id == "ci_installed" && execution.ContainerId == "cntr_installed" && execution.Code == "print(42)" + && execution.Outputs.OfType().Single().Logs == "42\n" + && execution.Outputs.OfType().Single().ImageUri.AbsoluteUri == "https://example.com/installed-chart.png", + "Official installed SDK must parse code, logs and image output."); + var citation = (ContainerFileCitationMessageAnnotation)text.Content.Single().OutputTextAnnotations.Single(); + Require(citation.ContainerId == "cntr_installed" && citation.FileId == "file_generated" + && citation.StartIndex == 0 && citation.EndIndex == 0, + "Official installed SDK must preserve zero-length container file citations."); + } + else + { + var search = (FileSearchCallResponseItem)response.OutputItems[0]; + var result = search.Results.Single(); + Require(search.Id == "fs_installed" && search.Queries.Single() == "installed fixture" + && result.FileId == "file_installed" && result.Filename == "fixture.txt" && result.Text == "Installed fixture.", + "Official installed SDK must parse included file search results."); + var citation = (FileCitationMessageAnnotation)text.Content.Single().OutputTextAnnotations.Single(); + Require(citation.FileId == "file_installed" && citation.Filename == "fixture.txt" && citation.Index == 4, + "Official installed SDK must preserve file citation insertion indexes."); + } + } + + private static void Require(bool condition, string message) + { + if (!condition) { throw new InvalidOperationException(message); } + } +} +#pragma warning restore OPENAI001 diff --git a/scripts/package-consumer/NativePackageTests.cs b/scripts/package-consumer/NativePackageTests.cs new file mode 100644 index 0000000..5080a36 --- /dev/null +++ b/scripts/package-consumer/NativePackageTests.cs @@ -0,0 +1,174 @@ +#pragma warning disable OPENAI001 // Qualify the official experimental Responses SDK against installed packages. +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Net; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using Aspire.Hosting; +using Aspire.Hosting.Testing; +using ManagedCode.LlmTestServer.Aspire; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Scenarios; +using OpenAI.Responses; +using TUnit.Core; + +public sealed class NativePackageTests +{ + private const string _key = "native-package-key"; + private const string _source = "https://example.com/installed-native"; + private const string _searchId = "ws_installed_native"; + private const string _messageId = "msg_installed_native"; + private const string _functionId = "fc_installed_native"; + private const string _callId = "call_installed_native"; + + [Test] + public async Task InstalledPackages_ServeNativeSearchJsonStreamAndCompleteMixedFunctionRound(CancellationToken cancellationToken) + { + var builder = DistributedApplicationTestingBuilder.Create([]); + builder.AddLlmTestServer("native-package-server").WithApiKey(_key); + await using var app = await builder.BuildAsync(cancellationToken); + await app.StartAsync(cancellationToken); + await app.ResourceNotifications.WaitForResourceHealthyAsync("native-package-server", cancellationToken); + using var http = app.CreateHttpClient("native-package-server", "http"); + http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", _key); + var control = new LlmTestServerClient(http, _key); + await control.ConfigureAsync(config => config.RequireBearerToken(_key).UseGpt41Mini() + .UseConversation("installed-native", graph => graph + .Step("search", step => step.WhenLatestUserEquals("native-start").WhenTurn(1).RespondsWithItems(NativeItems()).Then("follow")) + .Step("follow", step => step.WhenLatestUserEquals("native-follow").WhenTurn(2).Responds("stored native history complete").Complete())) + .UseConversation("installed-mixed", graph => graph + .Step("search", step => step.WhenLatestUserEquals("mixed-start").WhenTurn(1).RespondsWithItems( + new LlmTestServerResponseFunctionCallItem + { Id = _functionId, Call = new() { Id = _callId, Name = "weather", ArgumentsJson = "{}" } }, + NativeItems()[0], NativeItems()[1]).Then("result")) + .Step("result", step => step.WhenToolResult("weather", _callId).WhenTurn(2) + .WhenLatestTextContains("18 degrees").Responds("mixed native round complete").Complete())), cancellationToken); + using (var unauthorizedRequest = new HttpRequestMessage(HttpMethod.Post, "/openai/v1/responses")) + { + unauthorizedRequest.Headers.Authorization = new AuthenticationHeaderValue("Bearer", "wrong-native-key"); + unauthorizedRequest.Content = JsonContent.Create(new + { model = LlmTestServerKnownModelIds.Gpt41Mini, input = "native-start", tools = new[] { new { type = "web_search" } } }); + using var denied = await http.SendAsync(unauthorizedRequest, cancellationToken); + Require(denied.StatusCode == HttpStatusCode.Unauthorized, "The packaged native endpoint must enforce its configured key."); + } + Require((await control.GetConversationsAsync(cancellationToken: cancellationToken)).Count == 0, + "Unauthorized native requests must not create conversation state."); + + using var storedConversation = control.ForConversation("installed-native-stored"); + var storedSdk = Sdk(storedConversation.HttpClient); + var first = (await storedSdk.CreateResponseAsync(Options("native-start"), cancellationToken: cancellationToken)).Value; + AssertNative(first, offset: 0); + Require(first.OutputItems.Count == 2 && !first.OutputItems.OfType().Any(), + "Native web search must be a native item rather than a function call."); + var follow = Options("native-follow"); + follow.PreviousResponseId = first.Id; + follow.ToolChoice = ResponseToolChoice.CreateAutoChoice(); + var storedFinal = (await storedSdk.CreateResponseAsync(follow, cancellationToken: cancellationToken)).Value; + Require(((MessageResponseItem)storedFinal.OutputItems.Single()).Content.Single().Text == "stored native history complete", + "The installed server must restore native previous_response_id history before turn-two matching."); + + using var mixedConversation = control.ForConversation("installed-native-explicit"); + var mixedSdk = Sdk(mixedConversation.HttpClient); + var streamingOptions = Options("mixed-start", mixed: true); + streamingOptions.StoredOutputEnabled = false; + streamingOptions.StreamingEnabled = true; + var updates = new List(); + await foreach (var update in mixedSdk.CreateResponseStreamingAsync(streamingOptions, cancellationToken: cancellationToken)) + { updates.Add(update); } + Require(updates.Select(update => update.SequenceNumber).SequenceEqual(Enumerable.Range(0, updates.Count)), + "Native stream sequence numbers must be contiguous."); + Require(updates.OfType().Single().ItemId == _searchId + && updates.OfType().Single().OutputIndex == 1 + && updates.OfType().Single().ItemId == _searchId, + "The installed package must emit typed native progress updates at the mixed output index."); + var annotation = updates.OfType().Single(); + Require(annotation.OutputIndex == 2 && annotation.ItemId == _messageId + && ((UriCitationMessageAnnotation)annotation.Annotation).Uri.AbsoluteUri == _source, + "The installed stream must retain the native URL citation."); + var mixedFirst = updates.OfType().Single().Response; + Require(mixedFirst.OutputItems.Count == 3, "Mixed output order must include function, native search and text."); + var function = (FunctionCallResponseItem)mixedFirst.OutputItems[0]; + Require(function.Id == _functionId && function.CallId == _callId && function.FunctionName == "weather" + && function.FunctionArguments.ToString() == "{}", "Mixed function output and correlation IDs must remain distinct."); + AssertNative(mixedFirst, offset: 1); + var explicitFollow = Options("unused", mixed: true); + explicitFollow.InputItems.Clear(); + explicitFollow.InputItems.Add(ResponseItem.CreateUserMessageItem("mixed-start")); + foreach (var item in mixedFirst.OutputItems) { explicitFollow.InputItems.Add(item); } + explicitFollow.InputItems.Add(ResponseItem.CreateFunctionCallOutputItem(_callId, "18 degrees")); + var mixedFinal = (await mixedSdk.CreateResponseAsync(explicitFollow, cancellationToken: cancellationToken)).Value; + Require(((MessageResponseItem)mixedFinal.OutputItems.Single()).Content.Single().Text == "mixed native round complete", + "The installed server must complete a real SDK function/native round from explicit native history."); + + var states = await control.GetConversationsAsync(cancellationToken: cancellationToken); + Require(states.Count == 2 && states.All(state => state.ExecutedSteps == 2), + "Both independently correlated packaged conversations must advance exactly two logical turns."); + var evidence = await control.GetRequestsAsync(cancellationToken: cancellationToken); + Require(evidence.Requests.Count == 5 && evidence.Requests.Count(trace => trace.StatusCode == 401) == 1 + && evidence.Requests.Count(trace => trace.IsSuccess == true) == 4, "Public traces must expose all real packaged provider requests."); + var storedTrace = evidence.Requests.Single(trace => trace.ConversationId == storedConversation.Id && trace.StepId == "follow"); + var storedSearch = storedTrace.Messages.Where(message => message.Role == "assistant").SelectMany(message => message.OutputItems) + .OfType().Single(); + Require(storedSearch.Id == _searchId && storedSearch.Sources.Single().Url == _source, + "Public packaged trace serialization must retain typed native stored history."); + var mixedTrace = evidence.Requests.Single(trace => trace.ConversationId == mixedConversation.Id && trace.StepId == "result"); + var history = mixedTrace.Messages.Where(message => message.Role == "assistant").SelectMany(message => message.OutputItems).ToArray(); + Require(history.Length == 3 && ((LlmTestServerResponseFunctionCallItem)history[0]).Id == _functionId + && ((LlmTestServerResponseWebSearchItem)history[1]).Id == _searchId + && ((LlmTestServerResponseMessageItem)history[2]).Id == _messageId, + "Public packaged trace serialization must preserve ordered mixed output identities."); + var streamedTrace = evidence.Requests.Single(trace => trace.ConversationId == mixedConversation.Id && trace.StepId == "search"); + Require(streamedTrace.IsStreaming && streamedTrace.TerminalMarkerObserved == true + && streamedTrace.ResponseOutputItems.Count == 3 && streamedTrace.ResponseToolCalls.Single().Id == _callId + && streamedTrace.NativeTools.Single().Contract == LlmTestServerNativeToolContract.OpenAiResponsesWebSearch, + "Packaged native streaming evidence must include completion, ordered outputs and a separate native declaration."); + } + + private static LlmTestServerResponseItem[] NativeItems() => + [ + new LlmTestServerResponseWebSearchItem { Id = _searchId, Queries = ["installed native fixture"], Sources = [new() { Url = _source }] }, + new LlmTestServerResponseMessageItem + { + Id = _messageId, Content = "Paris fixture.", StreamChunks = ["Paris ", "fixture."], + Citations = [new LlmTestServerUrlCitation() { StartIndex = 0, EndIndex = 5, Url = _source, Title = "Installed fixture source" }], + }, + ]; + + private static ResponsesClient Sdk(HttpClient http) => new(new ApiKeyCredential(_key), new ResponsesClientOptions + { Endpoint = new Uri(http.BaseAddress!, "/openai/v1/"), Transport = new HttpClientPipelineTransport(http) }); + + private static CreateResponseOptions Options(string input, bool mixed = false) + { + var options = new CreateResponseOptions + { + Model = LlmTestServerKnownModelIds.Gpt41Mini, + InputItems = { ResponseItem.CreateUserMessageItem(input) }, + Tools = { ResponseTool.CreateWebSearchTool(null, WebSearchToolContextSize.Low, null) }, + ToolChoice = mixed ? ResponseToolChoice.CreateAutoChoice() : ResponseToolChoice.CreateRequiredChoice(), + IncludedProperties = { IncludedResponseProperty.WebSearchCallActionSources }, + }; + if (mixed) { options.Tools.Add(ResponseTool.CreateFunctionTool("weather", BinaryData.FromString("{\"type\":\"object\"}"), false, "Scripted weather")); } + return options; + } + + private static void AssertNative(ResponseResult response, int offset) + { + var search = (WebSearchCallResponseItem)response.OutputItems[offset]; + var action = (WebSearchSearchAction)search.Action!; + Require(search.Id == _searchId && search.Status == WebSearchCallStatus.Completed && action.Queries.Single() == "installed native fixture" + && ((WebSearchActionUriSource)action.Sources.Single()).Uri.AbsoluteUri == _source, + "The installed official SDK must parse native search identity, action and included sources."); + var message = (MessageResponseItem)response.OutputItems[offset + 1]; + var citation = (UriCitationMessageAnnotation)message.Content.Single().OutputTextAnnotations.Single(); + Require(message.Id == _messageId && message.Content.Single().Text == "Paris fixture." && citation.StartIndex == 0 + && citation.EndIndex == 5 && citation.Uri.AbsoluteUri == _source && citation.Title == "Installed fixture source", + "The installed official SDK must parse the exact configured message and URL citation."); + } + + private static void Require(bool condition, string message) + { + if (!condition) { throw new InvalidOperationException(message); } + } +} +#pragma warning restore OPENAI001 diff --git a/scripts/package-consumer/OutputDispositionPackageTests.cs b/scripts/package-consumer/OutputDispositionPackageTests.cs new file mode 100644 index 0000000..85c70c1 --- /dev/null +++ b/scripts/package-consumer/OutputDispositionPackageTests.cs @@ -0,0 +1,177 @@ +#pragma warning disable OPENAI001 // Exercise the pinned official Responses client using installed NuGet packages. +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Text; +using System.Text.Json; +using Aspire.Hosting; +using Aspire.Hosting.Testing; +using ManagedCode.LlmTestServer.Aspire; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Scenarios; +using Microsoft.Extensions.AI; +using OpenAI; +using OpenAI.Chat; +using OpenAI.Responses; +using TUnit.Core; +using AnthropicMessages = global::Anthropic.Models.Messages; + +public sealed class OutputDispositionPackageTests +{ + private const string _key = "output-policy-package-key"; + private const string _text = "Explicit installed policy text"; + private const string _recovery = "Installed policy recovery"; + + [Test] + [Arguments("chat")] + [Arguments("responses")] + [Arguments("anthropic")] + [Arguments("gemini")] + [Arguments("meai-json")] + [Arguments("meai-stream")] + public async Task InstalledPackages_NativePolicyOutcomeIsTypedAndFollowingInferenceRecoversAsync(string provider, CancellationToken cancellationToken) + { + var resourceName = "output-policy-" + provider; + var builder = DistributedApplicationTestingBuilder.Create([]); + builder.AddLlmTestServer(resourceName).WithApiKey(_key); + await using var app = await builder.BuildAsync(cancellationToken); + await app.StartAsync(cancellationToken); + await app.ResourceNotifications.WaitForResourceHealthyAsync(resourceName, cancellationToken); + using var http = app.CreateHttpClient(resourceName, LlmTestServerResource.HttpEndpointName); + var control = new LlmTestServerClient(http, _key); + var model = provider switch + { + "anthropic" => JsonSerializer.SerializeToElement(AnthropicMessages.Model.ClaudeSonnet5_5).GetString()!, + "gemini" => "gemini-3.8-flash", + _ => LlmTestServerKnownModelIds.Gpt41Mini, + }; + var kind = provider is "responses" or "gemini" ? LlmTestServerOutputDispositionKind.Filtered : LlmTestServerOutputDispositionKind.Refused; + var metadata = kind == LlmTestServerOutputDispositionKind.Refused; + var streaming = provider is "responses" or "gemini" or "meai-stream"; + await control.ConfigureAsync(config => config.RequireBearerToken(_key).UseChatModel(model) + .UseChatScenario("installed-policy", scenario => scenario.ForModel(model) + .Responds(_text, "Explicit installed ", "policy text") + .WithOutputDisposition(new() { Kind = kind, Category = metadata ? "cyber" : null, Explanation = metadata ? "Installed fixture policy" : null }) + .Responds(_recovery)), cancellationToken); + + var chat = provider == "chat" ? new ChatClient(model, new ApiKeyCredential(_key), new OpenAIClientOptions + { Endpoint = new Uri(http.BaseAddress!, "/openai/v1/"), Transport = new HttpClientPipelineTransport(http), RetryPolicy = new ClientRetryPolicy(0) }) : null; + var responses = provider == "responses" ? new ResponsesClient(new ApiKeyCredential(_key), new ResponsesClientOptions + { Endpoint = new Uri(http.BaseAddress!, "/openai/v1/"), Transport = new HttpClientPipelineTransport(http), RetryPolicy = new ClientRetryPolicy(0) }) : null; + using var anthropic = provider == "anthropic" ? new global::Anthropic.AnthropicClient(new global::Anthropic.Core.ClientOptions + { HttpClient = http, BaseUrl = new Uri(http.BaseAddress!, "/anthropic").ToString(), ApiKey = _key, MaxRetries = 0, ResponseValidation = true }) : null; + using var google = provider == "gemini" ? new Google.GenAI.Client(apiKey: _key, enterprise: false, + httpOptions: new() { BaseUrl = new Uri(http.BaseAddress!, "/gemini").ToString(), ApiVersion = "v1beta", RetryOptions = new() { Attempts = 1 } }, + clientOptions: new() { HttpClientFactory = () => http }) : null; + using var abstraction = provider.StartsWith("meai-", StringComparison.Ordinal) ? control.CreateChatClient() : null; + if (chat is not null) + { + var result = (await chat.CompleteChatAsync([new UserChatMessage("policy")], cancellationToken: cancellationToken)).Value; + Require(result.Refusal == _text && result.Content.Count == 0 && result.FinishReason == OpenAI.Chat.ChatFinishReason.Stop, + "Installed OpenAI chat refusal must be the typed refusal field with native stop, without ordinary text duplication."); + } + else if (responses is not null) + { + var text = new StringBuilder(); ResponseResult? final = null; + await foreach (var update in responses.CreateResponseStreamingAsync(new CreateResponseOptions + { Model = model, StreamingEnabled = true, StoredOutputEnabled = false, InputItems = { ResponseItem.CreateUserMessageItem("policy") } }, cancellationToken)) + { + if (update is StreamingResponseOutputTextDeltaUpdate delta) { text.Append(delta.Delta); } + if (update is StreamingResponseIncompleteUpdate incomplete) { final = incomplete.Response; } + Require(update is not StreamingResponseCompletedUpdate and not StreamingResponseFailedUpdate and not StreamingResponseErrorUpdate, + "Installed filtered Responses SSE must end incomplete rather than completed, failed or API error."); + } + Require(text.ToString() == _text && final is not null && final.Status == ResponseStatus.Incomplete + && final.IncompleteStatusDetails.Reason == ResponseIncompleteStatusReason.ContentFilter, + "Installed Responses SDK must retain its explicit prefix and typed content_filter incomplete outcome."); + } + else if (anthropic is not null) + { + var result = await anthropic.Messages.Create(AnthropicOptions(), cancellationToken); + Require(result.StopReason?.Json.GetString() == "refusal" && result.StopDetails is { } refusalDetails + && refusalDetails.Category?.Json.GetString() == "cyber" && refusalDetails.Explanation == "Installed fixture policy" + && string.Concat(result.Content.Select(block => ((AnthropicMessages.TextBlock)block.Value!).Text)) == _text, + "Installed strict Anthropic SDK must parse native refusal stop/details and explicit text."); + } + else if (google is not null) + { + var text = new StringBuilder(); Google.GenAI.Types.GenerateContentResponse? final = null; + await foreach (var update in google.Models.GenerateContentStreamAsync(model, "policy", + new Google.GenAI.Types.GenerateContentConfig { AutomaticContinuation = false }, cancellationToken: cancellationToken)) + { + text.Append(update.Text); + if (update.Candidates?.Any(candidate => candidate.FinishReason is not null) == true) { final = update; } + } + Require(text.ToString() == _text && final is not null && final.Candidates?.Single().FinishReason == Google.GenAI.Types.FinishReason.Safety + && string.IsNullOrEmpty(final.Text) && final.PromptFeedback is null, + "Installed Gemini SDK must observe a prefix followed by candidate SAFETY with no repeated terminal content or input-block envelope."); + } + else + { + Microsoft.Extensions.AI.ChatMessage[] prompt = [new(ChatRole.User, "policy")]; + var result = streaming + ? await abstraction!.GetStreamingResponseAsync(prompt, cancellationToken: cancellationToken).ToChatResponseAsync(cancellationToken) + : await abstraction!.GetResponseAsync(prompt, cancellationToken: cancellationToken); + var refusal = result.Messages.SelectMany(message => message.Contents).OfType().Single(); + Require(refusal.ErrorCode == "Refusal" && refusal.Message == _text && result.Text.Length == 0 + && !result.Messages.SelectMany(message => message.Contents).OfType().Any(), + "Installed IChatClient must represent refusal as one canonical ErrorContent and no duplicate ordinary text."); + } + var first = await CompletedTraceAsync(control, 1, cancellationToken); + Require(first.StatusCode == 200 && first.TransportState == LlmTestServerProviderHttpTraceTransportState.Completed + && first.OutputDisposition is { } policy && policy.Kind == kind && policy.Category == (metadata ? "cyber" : null) + && policy.Explanation == (metadata ? "Installed fixture policy" : null), + "Installed request journal must preserve the native200 policy kind and full bounded metadata."); + var summary = await control.GetAssertionsAsync(cancellationToken); + Require(summary.Matched == 1 && summary.ErrorsReturned == 0 && summary.Events.Single().OutputDisposition?.Kind == kind + && summary.Events.Single().OutputDisposition?.Explanation == (metadata ? "Installed fixture policy" : null), + "Installed runtime evidence must describe committed native policy inference without reporting a generic HTTP error."); + if (streaming) + { + Require(first.TerminalMarkerObserved == true, "Installed native policy SSE must have a real protocol terminal."); + var verified = await control.VerifyAsync(new() + { Requests = [new() { Query = new() { RequestId = first.RequestId }, RequireStreamingTerminalMarker = true }] }, cancellationToken); + Require(verified.Passed, "Installed stream verification must accept actual native policy terminal evidence."); + } + string recovered; + if (chat is not null) { recovered = (await chat.CompleteChatAsync([new UserChatMessage("next")], cancellationToken: cancellationToken)).Value.Content.Single().Text; } + else if (responses is not null) + { + recovered = (await responses.CreateResponseAsync(new CreateResponseOptions + { Model = model, StoredOutputEnabled = false, InputItems = { ResponseItem.CreateUserMessageItem("next") } }, cancellationToken)).Value.GetOutputText(); + } + else if (anthropic is not null) { recovered = ((AnthropicMessages.TextBlock)(await anthropic.Messages.Create(AnthropicOptions(), cancellationToken)).Content.Single().Value!).Text; } + else if (google is not null) + { + recovered = (await google.Models.GenerateContentAsync(model, "next", new Google.GenAI.Types.GenerateContentConfig + { AutomaticContinuation = false }, cancellationToken: cancellationToken)).Text!; + } + else { recovered = (await abstraction!.GetResponseAsync([new Microsoft.Extensions.AI.ChatMessage(ChatRole.User, "next")], cancellationToken: cancellationToken)).Text; } + Require(recovered == _recovery, "The explicit next inference must select the queued recovery after the policy outcome."); + var second = await CompletedTraceAsync(control, 2, cancellationToken); + Require(second.OutputDisposition is null && second.StatusCode == 200, "Installed queued recovery must not inherit prior policy metadata."); + Require((await control.GetRequestsAsync(cancellationToken: cancellationToken)).TotalObserved == 2 + && (await control.GetAssertionsAsync(cancellationToken)).Matched == 2, + "The original SDK must make exactly two observed, matched attempts with no hidden retries."); + } + + private static AnthropicMessages.MessageCreateParams AnthropicOptions() => new() + { Model = AnthropicMessages.Model.ClaudeSonnet5_5, MaxTokens = 256, Messages = [new() { Role = AnthropicMessages.Role.User, Content = "policy" }] }; + + private static async Task CompletedTraceAsync(LlmTestServerClient control, int count, CancellationToken cancellationToken) + { + using var watchdog = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + watchdog.CancelAfter(TimeSpan.FromSeconds(60)); + while (true) + { + var evidence = await control.GetRequestsAsync(cancellationToken: watchdog.Token); + if (evidence.Requests.Count == count && evidence.Requests.All(trace => trace.IsCompleted)) + return evidence.Requests.OrderBy(trace => trace.Sequence).Last(); + await Task.Delay(20, watchdog.Token); + } + } + + private static void Require(bool condition, string message) + { if (!condition) { throw new InvalidOperationException(message); } } +} diff --git a/scripts/package-consumer/PackageTests.cs b/scripts/package-consumer/PackageTests.cs index 6e3b5d8..a1a2645 100644 --- a/scripts/package-consumer/PackageTests.cs +++ b/scripts/package-consumer/PackageTests.cs @@ -1,10 +1,12 @@ +using System.Globalization; using System.Net.Http.Headers; using System.Text; using System.Text.Json; using System.Text.Json.Nodes; using Aspire.Hosting; using Aspire.Hosting.Testing; -using ManagedCode.LlmTck.Aspire; +using ManagedCode.LlmTestServer.Aspire; +using ManagedCode.LlmTestServer.Client; using Microsoft.Playwright; using TUnit.Core; using static Microsoft.Playwright.Assertions; @@ -15,15 +17,11 @@ public sealed class PackageTests public async Task InstalledPackage_ServesAssetsAndInteractiveDashboard(CancellationToken cancellationToken) { var builder = DistributedApplicationTestingBuilder.Create([]); - var resource = builder.AddLlmTck("package-tck").WithApiKey("test-key"); - if (!resource.Resource.WorkingDirectory.StartsWith(AppContext.BaseDirectory, StringComparison.Ordinal)) - throw new InvalidOperationException("The service must come from the installed package output."); - if (Directory.GetFiles(resource.Resource.WorkingDirectory, "*.staticwebassets.runtime.json", SearchOption.AllDirectories).Length != 0) - throw new InvalidOperationException("Package contains a build-machine static assets manifest."); + var resource = builder.AddLlmTestServer("package-test-server").WithApiKey("test-key"); await using var app = await builder.BuildAsync(cancellationToken); await app.StartAsync(cancellationToken); - await app.ResourceNotifications.WaitForResourceHealthyAsync("package-tck", cancellationToken); - using var http = app.CreateHttpClient("package-tck", "http"); + await app.ResourceNotifications.WaitForResourceHealthyAsync("package-test-server", cancellationToken); + using var http = app.CreateHttpClient("package-test-server", "http"); http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", "test-key"); foreach (var path in new[] { "/health", "/admin-api/models", "/_framework/blazor.web.js", "/_content/Microsoft.FluentUI.AspNetCore.Components/Microsoft.FluentUI.AspNetCore.Components.lib.module.js" }) { @@ -77,15 +75,51 @@ public async Task InstalledPackage_ServesAssetsAndInteractiveDashboard(Cancellat if (!(await response.Content.ReadAsStringAsync(cancellationToken)).Contains("0.95", StringComparison.Ordinal)) throw new InvalidOperationException("Installed package lost the native decision fixture: " + model); } + var control = new LlmTestServerClient(http, "test-key"); + var dashboardDemo = await CreateDashboardCacheDemoAsync(control, http, cancellationToken); + var dashboardEvidence = dashboardDemo.NamedConversation; + const int defaultMatchedRequests = 4; + await using (var session = await control.CreateSessionAsync(config => config.RequireBearerToken("test-key") + .UseChatScenario("package-session", scenario => scenario.Responds("isolated package response")), cancellationToken)) + { + using var sessionChat = session.Control.CreateChatClient(); + var sessionReply = await sessionChat.GetResponseAsync( + [new Microsoft.Extensions.AI.ChatMessage(Microsoft.Extensions.AI.ChatRole.User, "session request")], cancellationToken: cancellationToken); + if (sessionReply.Text != "isolated package response" || (await session.Control.GetAssertionsAsync(cancellationToken)).Matched != 1 + || (await control.GetAssertionsAsync(cancellationToken)).Matched != defaultMatchedRequests) + throw new InvalidOperationException("Installed package sessions must isolate requests and assertions from the dashboard runtime."); + } + await VerifyInstalledConversationGraphAsync(control, defaultMatchedRequests, cancellationToken); + await VerifyInstalledToolPresetAsync(control, cancellationToken); using var playwright = await Playwright.CreateAsync(); await using var browser = await playwright.Chromium.LaunchAsync(new() { Headless = true }); - var page = await browser.NewPageAsync(); + var recordingDirectory = Environment.GetEnvironmentVariable("LLM_TEST_SERVER_DEMO_RECORDING_DIRECTORY"); + await using var browserContext = await browser.NewContextAsync(new() + { + ViewportSize = new() { Width = 1440, Height = 1000 }, + RecordVideoDir = recordingDirectory, + RecordVideoSize = recordingDirectory is null ? null : new() { Width = 1440, Height = 1000 }, + }); + var recordedScenes = new List(); + var sceneLabels = new[] { "Authorization", "Request inbox", "Overview", "Cache miss", "Cache hit", "Conversation analytics", "Unlabeled request", "Conversation history", "Runtime", "Mobile detail" }; + var recordingClock = recordingDirectory is null ? null : System.Diagnostics.Stopwatch.StartNew(); + var page = await browserContext.NewPageAsync(); + async Task HoldDemoFrameAsync(int seconds) + { + if (recordingDirectory is not null) + { + var index = recordedScenes.Count + 1; + recordedScenes.Add(new { Index = index, Label = sceneLabels[index - 1], StartSeconds = recordingClock!.Elapsed.TotalSeconds, DurationSeconds = seconds }); + await Task.Delay(TimeSpan.FromSeconds(seconds), cancellationToken); + } + } await page.GotoAsync(http.BaseAddress!.ToString()); + await HoldDemoFrameAsync(7); var runtimeTrigger = page.GetByRole(AriaRole.Button, new() { Name = "Runtime settings", Exact = true }); await runtimeTrigger.ClickAsync(); await Expect(page.GetByLabel("Bearer token", new() { Exact = true })).ToBeVisibleAsync(); await Expect(runtimeTrigger).ToHaveAttributeAsync("aria-expanded", "true"); - await Expect(page.GetByRole(AriaRole.Dialog, new() { Name = "Runtime state" })).ToBeVisibleAsync(); + await Expect(page.GetByRole(AriaRole.Alertdialog, new() { Name = "Runtime state" })).ToBeVisibleAsync(); await page.ScreenshotAsync(new() { Path = "package-dashboard.png", FullPage = true }); await page.GetByLabel("Bearer token", new() { Exact = true }).FillAsync("test-key"); await page.GetByRole(AriaRole.Button, new() { Name = "Close runtime settings", Exact = true }).ClickAsync(); @@ -100,16 +134,390 @@ public async Task InstalledPackage_ServesAssetsAndInteractiveDashboard(Cancellat await Expect(runtimeTrigger).ToBeFocusedAsync(); await page.GetByRole(AriaRole.Button, new() { Name = "Refresh runtime state" }).ClickAsync(); await Expect(page.GetByText("Unauthorized", new() { Exact = true })).ToHaveCountAsync(0); + await Expect(page.GetByTestId("request-row")).ToHaveCountAsync(4); + await page.SetViewportSizeAsync(1440, 1000); + await HoldDemoFrameAsync(7); + await Expect(page.GetByRole(AriaRole.Alertdialog, new() { Name = "Runtime state" })).Not.ToBeVisibleAsync(); + var headerBounds = await page.Locator(".topbar").BoundingBoxAsync(); + var workspaceBounds = await page.Locator(".mail-workspace").BoundingBoxAsync(); + if (headerBounds is null || workspaceBounds is null || Math.Abs(workspaceBounds.Y - headerBounds.Y - headerBounds.Height) > 2) + throw new InvalidOperationException("Closed runtime dialog must not leave empty space above the request workspace."); + var selectedDemoRequest = dashboardEvidence.Requests[^1]; + await page.Locator($"[data-testid='request-row'][data-request-id='{selectedDemoRequest.RequestId}']").ClickAsync(); + await Expect(page.GetByTestId("request-details")).ToBeVisibleAsync(); + await Expect(page.GetByTestId("selected-request-id")).ToHaveTextAsync(selectedDemoRequest.RequestId); + await page.ScreenshotAsync(new() { Path = "package-dashboard-overview.jpg", FullPage = true, Quality = 85 }); + await HoldDemoFrameAsync(6); + var uncachedDemoRequest = dashboardEvidence.Requests[0]; + await page.Locator($"[data-testid='request-row'][data-request-id='{uncachedDemoRequest.RequestId}']").ClickAsync(); + await page.GetByTestId("analytics-tab").ClickAsync(); + await Expect(page.GetByTestId("analytics-panel")).ToBeVisibleAsync(); + await page.GetByTestId("analytics-scope-request").ClickAsync(); + await AssertDashboardAnalyticsAsync(page, [uncachedDemoRequest], dashboardEvidence); + await page.ScreenshotAsync(new() { Path = "package-dashboard-cache-miss.jpg", FullPage = true, Quality = 85 }); + await HoldDemoFrameAsync(10); + await page.Locator($"[data-testid='request-row'][data-request-id='{selectedDemoRequest.RequestId}']").ClickAsync(); + await page.GetByTestId("analytics-tab").ClickAsync(); + await Expect(page.GetByTestId("analytics-panel")).ToBeVisibleAsync(); + await page.GetByTestId("analytics-scope-request").ClickAsync(); + await Expect(page.GetByTestId("analytics-scope-request")).ToHaveAttributeAsync("aria-pressed", "true"); + await AssertDashboardAnalyticsAsync(page, [selectedDemoRequest], dashboardEvidence); + await page.ScreenshotAsync(new() { Path = "package-dashboard-token-usage.png", FullPage = true }); + await page.ScreenshotAsync(new() { Path = "package-dashboard-cache-request.jpg", FullPage = true, Quality = 85 }); + await HoldDemoFrameAsync(10); + await page.GetByTestId("analytics-scope-conversation").ClickAsync(); + await Expect(page.GetByTestId("analytics-scope-conversation")).ToHaveAttributeAsync("aria-pressed", "true"); + await AssertDashboardAnalyticsAsync(page, dashboardEvidence.Requests, dashboardEvidence, conversationScope: true); + var analyticsLayout = new Dictionary(); + foreach (var (label, locator) in new[] + { + ("cache-share", page.GetByTestId("analytics-cache-share")), + ("token-flow", page.GetByTestId("analytics-flow")), + ("weather-row", page.Locator("[data-testid='analytics-tool-row'][data-tool-name='weather']")), + ("analytics-panel", page.GetByTestId("analytics-panel")), + }) + { + var bounds = await locator.BoundingBoxAsync(); + analyticsLayout[label] = bounds; + // The headline figures and the top of the token flow must be on screen; detail cards such as tools sit below the fold. + if ((label is "cache-share" && (bounds is null || bounds.X < 0 || bounds.Y < 0 || bounds.X + bounds.Width > 1440 || bounds.Y + bounds.Height > 1000)) + || (label is "token-flow" && (bounds is null || bounds.X < 0 || bounds.Y < 0 || bounds.X + bounds.Width > 1440 || bounds.Y > 1000 - 160))) + throw new InvalidOperationException($"Dashboard analytics {label} must be visible in the first desktop viewport: {JsonSerializer.Serialize(bounds)}."); + } + await File.WriteAllTextAsync("package-dashboard-analytics-layout.json", + JsonSerializer.Serialize(new { ViewportWidth = 1440, ViewportHeight = 1000, Bounds = analyticsLayout }, + new JsonSerializerOptions { WriteIndented = true }), cancellationToken); + await page.ScreenshotAsync(new() { Path = "package-dashboard-cache-conversation.jpg", FullPage = true, Quality = 85 }); + await HoldDemoFrameAsync(12); + await page.Locator($"[data-testid='request-row'][data-request-id='{dashboardDemo.UnlabeledRequest.RequestId}']").ClickAsync(); + await page.GetByTestId("analytics-tab").ClickAsync(); + await Expect(page.GetByTestId("analytics-scope-conversation")).ToBeDisabledAsync(); + await Expect(page.GetByTestId("analytics-scope-request")).ToHaveAttributeAsync("aria-pressed", "true"); + await AssertDashboardAnalyticsAsync(page, [dashboardDemo.UnlabeledRequest], dashboardEvidence); + await HoldDemoFrameAsync(8); + await page.Locator($"[data-testid='request-row'][data-request-id='{selectedDemoRequest.RequestId}']").ClickAsync(); + await page.GetByRole(AriaRole.Tab, new() { Name = "Conversation", Exact = false }).ClickAsync(); + await page.ScreenshotAsync(new() { Path = "package-dashboard-conversation.jpg", FullPage = true, Quality = 85 }); + await HoldDemoFrameAsync(12); + var insightsTrigger = page.GetByTestId("insights-open"); + await insightsTrigger.ClickAsync(); + await Expect(page.GetByTestId("insights-panel")).ToBeVisibleAsync(); + await Expect(insightsTrigger).ToHaveAttributeAsync("aria-pressed", "true"); + var journalNow = await control.GetRequestsAsync(cancellationToken: cancellationToken); + foreach (var provider in journalNow.Requests.GroupBy(trace => trace.ProviderId)) + await Expect(page.Locator($"[data-testid='insight-providers-value'][data-key='{provider.Key}']")) + .ToHaveAttributeAsync("data-value", provider.Count().ToString(CultureInfo.InvariantCulture)); + await page.ScreenshotAsync(new() { Path = "package-dashboard-insights.jpg", FullPage = true, Quality = 85 }); + await insightsTrigger.ClickAsync(); + await Expect(page.GetByTestId("insights-panel")).ToHaveCountAsync(0); + await runtimeTrigger.ClickAsync(); + await Expect(page.GetByRole(AriaRole.Alertdialog, new() { Name = "Runtime state" })).ToBeVisibleAsync(); + await page.ScreenshotAsync(new() { Path = "package-dashboard-runtime.jpg", FullPage = true, Quality = 85 }); + await HoldDemoFrameAsync(8); + await page.Keyboard.PressAsync("Escape"); + await Expect(page.GetByRole(AriaRole.Alertdialog, new() { Name = "Runtime state" })).Not.ToBeVisibleAsync(); await page.ScreenshotAsync(new() { Path = "package-dashboard-authorized.png", FullPage = true }); await page.SetViewportSizeAsync(390, 844); await runtimeTrigger.ClickAsync(); await Expect(page.GetByLabel("Bearer token", new() { Exact = true })).ToBeVisibleAsync(); - var dialogBounds = await page.GetByRole(AriaRole.Dialog, new() { Name = "Runtime state" }).BoundingBoxAsync(); + var dialogBounds = await page.GetByRole(AriaRole.Alertdialog, new() { Name = "Runtime state" }).BoundingBoxAsync(); if (dialogBounds is null || dialogBounds.X < 0 || dialogBounds.Y < 0 || dialogBounds.X + dialogBounds.Width > 390 || dialogBounds.Y + dialogBounds.Height > 844) throw new InvalidOperationException("Runtime settings must fit the mobile viewport."); await page.ScreenshotAsync(new() { Path = "package-dashboard-runtime-mobile.png", FullPage = true }); await page.GetByRole(AriaRole.Button, new() { Name = "Close runtime settings", Exact = true }).ClickAsync(); await Expect(runtimeTrigger).ToHaveAttributeAsync("aria-expanded", "false"); + await Expect(page.GetByRole(AriaRole.Alertdialog, new() { Name = "Runtime state" })).Not.ToBeVisibleAsync(); await page.ScreenshotAsync(new() { Path = "package-dashboard-authorized-mobile.png", FullPage = true }); + await page.ScreenshotAsync(new() { Path = "package-dashboard-mobile-request-detail.png", FullPage = true }); + await HoldDemoFrameAsync(8); + if (recordingDirectory is not null) + { + await browserContext.CloseAsync(); + var videoPath = page.Video is null ? null : await page.Video.PathAsync(); + var markerPath = Path.Combine(recordingDirectory, "demo-scenes-" + Guid.NewGuid().ToString("N") + ".json"); + await File.WriteAllTextAsync(markerPath, + JsonSerializer.Serialize(new { VideoPath = videoPath, Clock = "Stopwatch started immediately before NewPageAsync", Scenes = recordedScenes }, + new JsonSerializerOptions { WriteIndented = true }), cancellationToken); + } + } + + private static async Task AssertDashboardAnalyticsAsync(IPage page, + IReadOnlyList requests, + ManagedCode.LlmTestServer.Diagnostics.LlmTestServerRequestSnapshot evidence, bool conversationScope = false) + { + var usage = requests.Where(trace => trace.Usage is not null).Select(trace => trace.Usage!).ToArray(); + var toolResults = requests.SelectMany(trace => trace.Messages).Where(message => message.Role == "tool").ToArray(); + var expected = new Dictionary + { + ["requests"] = requests.Count, + ["usage-requests"] = usage.Length, + ["input"] = usage.Sum(item => (long)item.InputTokens), + ["output"] = usage.Sum(item => (long)item.OutputTokens), + ["total"] = usage.Sum(item => (long)item.TotalTokens), + ["reasoning"] = usage.Sum(item => (long)item.ReasoningTokens), + ["cache-read"] = usage.Sum(item => (long)item.CachedInputTokens), + ["cache-write"] = usage.Sum(item => (long)item.CacheCreationInputTokens), + ["cache-hits"] = usage.Count(item => item.CachedInputTokens > 0), + ["tool-calls"] = requests.Sum(trace => (long)trace.ResponseToolCalls.Count), + ["tool-results"] = toolResults.Length, + ["tool-result-ids"] = toolResults.Select(message => message.ToolCallId).Where(id => id is not null).Distinct(StringComparer.Ordinal).Count(), + }; + foreach (var metric in expected) + await Expect(page.GetByTestId("analytics-" + metric.Key)).ToHaveAttributeAsync("data-value", metric.Value.ToString(CultureInfo.InvariantCulture)); + var expectedShare = expected["input"] > 0 ? (double)expected["cache-read"] / expected["input"] : 0; + var expectedHitRate = expected["usage-requests"] > 0 ? (double)expected["cache-hits"] / expected["usage-requests"] : 0; + foreach (var (name, value) in new[] { ("cache-share", expectedShare), ("cache-hit-rate", expectedHitRate) }) + { + var raw = await page.GetByTestId("analytics-" + name).GetAttributeAsync("data-value"); + if (!double.TryParse(raw, NumberStyles.Float, CultureInfo.InvariantCulture, out var actual) || Math.Abs(actual - value) > 0.0000001) + throw new InvalidOperationException($"Dashboard analytics {name} differs from the observed journal: expected {value}, actual {raw}."); + } + await Expect(page.GetByTestId("analytics-retention")).ToBeVisibleAsync(); + await Expect(page.GetByTestId("analytics-retention")).ToHaveAttributeAsync("data-incomplete", + requests.Any(trace => !trace.IsCompleted || trace.Usage is null) || conversationScope && evidence.EvictedCount > 0 ? "true" : "false"); + var payloadShortened = evidence.RetentionTruncated || requests.Any(trace => trace.RequestPreviewTruncated + || trace.ResponsePreviewTruncated || trace.RetentionTruncated || trace.RuntimePayloadTruncated); + await Expect(page.GetByTestId("analytics-payload-retention")).ToHaveCountAsync(payloadShortened ? 1 : 0); + var weather = page.Locator("[data-testid='analytics-tool-row'][data-tool-name='weather']"); + if (!requests.Any(trace => trace.ResponseToolCalls.Any(call => call.Name == "weather") + || trace.Messages.Any(message => message.ToolCalls.Any(call => call.Name == "weather")))) + { + await Expect(weather).ToHaveCountAsync(0); + return; + } + await Expect(weather.GetByTestId("analytics-tool-name")).ToHaveTextAsync("weather"); + await Expect(weather.GetByTestId("analytics-tool-calls-returned")).ToHaveAttributeAsync("data-value", expected["tool-calls"].ToString(CultureInfo.InvariantCulture)); + await Expect(weather.GetByTestId("analytics-tool-results-observed")).ToHaveAttributeAsync("data-value", expected["tool-results"].ToString(CultureInfo.InvariantCulture)); + await Expect(weather.GetByTestId("analytics-result-ids")).ToHaveAttributeAsync("data-value", expected["tool-result-ids"].ToString(CultureInfo.InvariantCulture)); + } + + private sealed record DashboardDemoEvidence( + ManagedCode.LlmTestServer.Diagnostics.LlmTestServerRequestSnapshot NamedConversation, + ManagedCode.LlmTestServer.Diagnostics.LlmTestServerProviderHttpTrace UnlabeledRequest); + + private static async Task CreateDashboardCacheDemoAsync( + LlmTestServerClient control, HttpClient http, CancellationToken cancellationToken) + { + const string model = "gpt-5-mini"; + var weather = ManagedCode.LlmTestServer.Tools.WellKnown.LlmTestServerWellKnownTools.Weather.Success( + new() { Location = "Paris", Unit = "celsius" }, + new() { Location = "Paris", Temperature = 18m, Unit = "celsius", Condition = "sunny" }, + "call_dashboard_weather"); + const string finalAnswer = "Paris is 18 °C and sunny. Take the 14:30 train, walk along the Seine, and leave time for a café near the station."; + await control.ConfigureAsync(config => config.RequireBearerToken("test-key").UseGpt5Mini(reasoningTokens: 23) + .UseConversation("dashboard-cache", graph => graph.ForModel(model) + .Step("weather", step => step.WhenLatestUserContains("Paris").WhenTurn(1).CallsTools(weather.Call).Then("itinerary")) + .Step("itinerary", step => step.WhenToolResult(weather.Call.Name, weather.Call.Id).WhenTurn(2) + .WithToolArgumentsJson(weather.Call.ArgumentsJson).WhenLatestJson("$", weather.Result.Content) + .Responds(finalAnswer, "Paris is 18 °C and sunny. ", "Take the 14:30 train, walk along the Seine, ", + "and leave time for a café near the station.").Complete())) + .UseChatScenario("unlabeled-dashboard", scenario => scenario.ForModel(model).WhenUserContains("unlabeled fixture") + .Responds("This independent request has no conversation selector.")), cancellationToken); + using var conversation = control.ForConversation("dashboard-cache-demo"); + var sdk = new OpenAI.Chat.ChatClient(model, new System.ClientModel.ApiKeyCredential("test-key"), + new OpenAI.OpenAIClientOptions + { + Endpoint = new Uri(conversation.HttpClient.BaseAddress!, "openai/v1"), + Transport = new System.ClientModel.Primitives.HttpClientPipelineTransport(conversation.HttpClient), + }); + var options = new OpenAI.Chat.ChatCompletionOptions + { + Tools = { OpenAI.Chat.ChatTool.CreateFunctionTool(weather.Definition.Name, weather.Definition.Description, + BinaryData.FromString(weather.Definition.ParametersJson)) }, + }; + var reference = "You are a careful travel assistant. Use only the supplied timetable and weather fixtures, explain practical trade-offs, and preserve Unicode place names.\n" + + string.Join("\n", Enumerable.Range(1, 100).Select(index => + $"Travel reference {index:D3}: Reserve a relaxed walking interval, allow twenty minutes before departure, prefer step-free routes, and keep the supplied afternoon timetable unchanged.")); + List messages = + [new OpenAI.Chat.SystemChatMessage(reference), new OpenAI.Chat.UserChatMessage("Plan a relaxed afternoon visit to Paris with weather and the train itinerary.")]; + var first = (await sdk.CompleteChatAsync(messages, options, cancellationToken)).Value; + if (first.ToolCalls.Single().FunctionName != weather.Call.Name || first.ToolCalls.Single().Id != weather.Call.Id + || first.Usage.InputTokenCount < 1024 || first.Usage.InputTokenDetails.CachedTokenCount != 0) + throw new InvalidOperationException("The dashboard demo must start with a real SDK tool call and an uncached eligible prompt."); + messages.Add(new OpenAI.Chat.AssistantChatMessage(first)); + messages.Add(new OpenAI.Chat.ToolChatMessage(weather.Result.ToolCallId, weather.Result.Content)); + var answer = (await sdk.CompleteChatAsync(messages, options, cancellationToken)).Value; + if (answer.Content[0].Text != finalAnswer || answer.Usage.InputTokenDetails.CachedTokenCount <= 0 + || answer.Usage.OutputTokenDetails.ReasoningTokenCount != 23) + throw new InvalidOperationException("The second SDK request must complete the tool round with real cached-input and reasoning usage."); + var streamed = new StringBuilder(); + OpenAI.Chat.ChatTokenUsage? streamingUsage = null; + await foreach (var update in sdk.CompleteChatStreamingAsync(messages, options, cancellationToken)) + { + foreach (var part in update.ContentUpdate) streamed.Append(part.Text); + if (update.Usage is not null) streamingUsage = update.Usage; + } + if (streamed.ToString() != finalAnswer || streamingUsage?.InputTokenDetails.CachedTokenCount is not > 0) + throw new InvalidOperationException("The repeated SDK answer must stream the real cache-hit usage."); + var independentSdk = new OpenAI.Chat.ChatClient(model, new System.ClientModel.ApiKeyCredential("test-key"), + new OpenAI.OpenAIClientOptions { Endpoint = new Uri(http.BaseAddress!, "openai/v1"), + Transport = new System.ClientModel.Primitives.HttpClientPipelineTransport(http) }); + var independent = (await independentSdk.CompleteChatAsync([new OpenAI.Chat.UserChatMessage("An unlabeled fixture request")], + cancellationToken: cancellationToken)).Value; + if (independent.Content[0].Text != "This independent request has no conversation selector.") + throw new InvalidOperationException("The unlabeled dashboard fixture did not reach its separate scenario."); + var evidence = await control.GetRequestsAsync(new() + { Conversation = conversation.Id, Order = ManagedCode.LlmTestServer.Diagnostics.LlmTestServerRequestOrder.Ascending }, cancellationToken); + var traces = evidence.Requests; + if (traces.Count != 3 || evidence.TotalObserved != 4 || evidence.EvictedCount != 0 + || traces.Any(trace => trace.ConversationId != conversation.Id || !trace.ConversationIdWasProvided || !trace.IsCompleted || trace.IsSuccess != true) + || traces[0].Usage?.CachedInputTokens != 0 || traces[1].Usage?.CachedInputTokens is not > 0 || traces[2].Usage?.CachedInputTokens is not > 0 + || traces[0].ResponseToolCalls.Single().Name != weather.Call.Name || traces[2].TerminalMarkerObserved != true) + throw new InvalidOperationException("Dashboard cache analytics must use three complete, explicitly correlated provider requests."); + if (traces[0].Messages.Count != 2 || traces[0].ResponseToolCalls.Single().ArgumentsJson != weather.Call.ArgumentsJson + || traces.Skip(1).Any(trace => trace.Messages.Count != 4 + || trace.Messages.SelectMany(message => message.ToolCalls).Count() != 1 + || !trace.Messages.SelectMany(message => message.ToolCalls).Any(call => call.Id == weather.Call.Id + && call.Name == weather.Call.Name && call.ArgumentsJson == weather.Call.ArgumentsJson) + || !trace.Messages.Any(message => message.Role == "tool" && message.ToolCallId == weather.Result.ToolCallId + && message.Content == weather.Result.Content))) + throw new InvalidOperationException("Preview shortening must retain the real weather call arguments, result content, and correlation IDs."); + if (traces[0].Usage!.InputTokens != first.Usage.InputTokenCount + || traces[1].Usage!.CachedInputTokens != answer.Usage.InputTokenDetails.CachedTokenCount + || traces[2].Usage!.CachedInputTokens != streamingUsage!.InputTokenDetails.CachedTokenCount) + throw new InvalidOperationException("Dashboard trace cache accounting differs from the actual SDK usage."); + var allEvidence = await control.GetRequestsAsync(cancellationToken: cancellationToken); + var unlabeled = allEvidence.Requests.Single(trace => !trace.ConversationIdWasProvided); + if (allEvidence.Requests.Count != 4 || unlabeled.ScenarioId != "unlabeled-dashboard") + throw new InvalidOperationException("Default request grouping must preserve the independent unlabeled request."); + var summary = await control.GetAssertionsAsync(cancellationToken); + if (summary.Matched != 4 || summary.InputTokens != allEvidence.Requests.Sum(trace => trace.Usage!.InputTokens) + || summary.CachedInputTokens != allEvidence.Requests.Sum(trace => trace.Usage!.CachedInputTokens) + || summary.ReasoningTokens != allEvidence.Requests.Sum(trace => trace.Usage!.ReasoningTokens)) + throw new InvalidOperationException("Default runtime assertions must agree with the dashboard demo's observed request usage."); + return new(evidence, unlabeled); + } + + private static async Task VerifyInstalledToolPresetAsync(LlmTestServerClient control, CancellationToken cancellationToken) + { + var fixture = ManagedCode.LlmTestServer.Tools.WellKnown.LlmTestServerWellKnownTools.Calculator.Success( + new() { Expression = "6 * 7" }, new() { Value = 42m }, "call_installed_calculator"); + await using var session = await control.CreateSessionAsync(config => config.RequireBearerToken("test-key").UseGpt41Mini() + .UseConversation("installed-preset", graph => graph + .Step("call", step => step.WhenTurn(1).CallsTools(fixture.Call).Then("result")) + .Step("result", step => step.WhenToolResult(fixture.Call.Name, fixture.Call.Id) + .WithToolArgumentsJson(fixture.Call.ArgumentsJson).WhenLatestJson("$", fixture.Result.Content) + .Responds("42").Complete())), cancellationToken); + var sdk = new OpenAI.Chat.ChatClient("gpt-4.1-mini", new System.ClientModel.ApiKeyCredential("test-key"), + new OpenAI.OpenAIClientOptions + { + Endpoint = new Uri(session.HttpClient.BaseAddress!, "openai/v1"), + Transport = new System.ClientModel.Primitives.HttpClientPipelineTransport(session.HttpClient), + }); + var options = new OpenAI.Chat.ChatCompletionOptions + { + Tools = { OpenAI.Chat.ChatTool.CreateFunctionTool(fixture.Definition.Name, fixture.Definition.Description, + BinaryData.FromString(fixture.Definition.ParametersJson)) }, + }; + List messages = [new OpenAI.Chat.UserChatMessage("calculate")]; + var first = (await sdk.CompleteChatAsync(messages, options, cancellationToken)).Value; + if (first.ToolCalls.Single().FunctionName != fixture.Call.Name || first.ToolCalls.Single().Id != fixture.Call.Id) + throw new InvalidOperationException("Installed well-known preset lost its typed function call."); + messages.Add(new OpenAI.Chat.AssistantChatMessage(first)); + messages.Add(new OpenAI.Chat.ToolChatMessage(fixture.Result.ToolCallId, fixture.Result.Content)); + var answer = (await sdk.CompleteChatAsync(messages, options, cancellationToken)).Value; + var state = (await session.Control.GetConversationsAsync(cancellationToken)).Single(); + if (answer.Content[0].Text != "42" || !state.IsTerminal || state.ExecutedSteps != 2) + throw new InvalidOperationException("Installed well-known preset did not complete its real SDK tool round."); } + + private static async Task VerifyInstalledConversationGraphAsync(LlmTestServerClient control, int defaultMatchedRequests, CancellationToken cancellationToken) + { + await using var session = await control.CreateSessionAsync(config => config.RequireBearerToken("test-key").UseGpt41Mini() + .UseDeterminism("installed-seed", new DateTimeOffset(2001, 2, 3, 4, 5, 6, TimeSpan.Zero)) + .UseConversation("package-graph", graph => graph + .Step("lookup", step => step.WhenLatestUserEquals("weather in Paris").WhenTurn(1) + .CallsTool("weather", """{"city":"Paris"}""", "call_package_graph").Then("answer")) + .Step("answer", step => step.WhenToolResult("weather", "call_package_graph").WhenTurn(2) + .WithToolArgumentsJson("""{"city":"Paris"}""").WhenLatestJsonValue("$.temperature", 18) + .Responds("Paris is 18 °C", "Paris is ", "18 °C").Complete())), cancellationToken); + using var conversation = session.Control.ForConversation("installed-sdk-graph"); + var sdk = new OpenAI.Chat.ChatClient("gpt-4.1-mini", new System.ClientModel.ApiKeyCredential("test-key"), + new OpenAI.OpenAIClientOptions + { + Endpoint = new Uri(conversation.HttpClient.BaseAddress!, "openai/v1"), + Transport = new System.ClientModel.Primitives.HttpClientPipelineTransport(conversation.HttpClient), + }); + var options = new OpenAI.Chat.ChatCompletionOptions + { + Tools = { OpenAI.Chat.ChatTool.CreateFunctionTool("weather", functionParameters: + BinaryData.FromString("""{"type":"object","required":["city"],"properties":{"city":{"const":"Paris"}}}""")) }, + }; + List messages = [new OpenAI.Chat.UserChatMessage("weather in Paris")]; + var first = (await sdk.CompleteChatAsync(messages, options, cancellationToken)).Value; + if (first.CreatedAt != new DateTimeOffset(2001, 2, 3, 4, 5, 6, TimeSpan.Zero)) + throw new InvalidOperationException("Installed fixed-clock SDK fixture time was lost."); + var call = first.ToolCalls.Single(); + if (call.Id != "call_package_graph" || call.FunctionName != "weather" + || call.FunctionArguments.ToString() != """{"city":"Paris"}""") + throw new InvalidOperationException("Installed conversation graph lost its SDK tool-call fixture."); + messages.Add(new OpenAI.Chat.AssistantChatMessage(first)); + messages.Add(new OpenAI.Chat.ToolChatMessage(call.Id, """{"temperature":18}""")); + var explanation = await conversation.Control.ExplainMatchAsync(new() + { + Request = new() + { + Messages = + [ + new() { Role = "user", Content = "weather in Paris" }, + new() { Role = "assistant", ToolCalls = [new() { Id = call.Id, Name = "weather", ArgumentsJson = """{"city":"Paris"}""" }] }, + new() { Role = "tool", ToolCallId = call.Id, Content = """{"temperature":18}""" }, + ], + Tools = [new() { Name = "weather", ParametersJson = """{"type":"object","required":["city"],"properties":{"city":{"const":"Paris"}}}""" }], + }, + }, cancellationToken); + if (explanation.ConversationId != conversation.Id || explanation.SelectedScenarioId != "package-graph" + || explanation.SelectedStepId != "answer" || explanation.TurnIndex != 2 + || (await conversation.Control.GetConversationsAsync(cancellationToken)).Single().ExecutedSteps != 1) + throw new InvalidOperationException("Installed match explanation must identify the active step without consuming it."); + var answer = (await sdk.CompleteChatAsync(messages, options, cancellationToken)).Value; + if (answer.Content[0].Text != "Paris is 18 °C") + throw new InvalidOperationException("Installed SDK conversation did not match its JSON tool result."); + var replay = new StringBuilder(); + await foreach (var update in sdk.CompleteChatStreamingAsync(messages, options, cancellationToken)) + foreach (var part in update.ContentUpdate) replay.Append(part.Text); + var state = (await conversation.Control.GetConversationsAsync(cancellationToken)).Single(); + var events = (await session.Control.GetAssertionsAsync(cancellationToken)).Events; + if (replay.ToString() != "Paris is 18 °C" || state.ConversationId != conversation.Id || state.ExecutedSteps != 2 + || !state.IsTerminal || !events.Last().IsReplay || events.Last().StepId != "answer" + || events.Any(item => item.ConversationId != conversation.Id) + || (await control.GetAssertionsAsync(cancellationToken)).Matched != defaultMatchedRequests) + throw new InvalidOperationException("Installed conversation replay must preserve graph state, header scope and default runtime isolation."); + var evidence = await conversation.Control.GetRequestsAsync(new() { Order = ManagedCode.LlmTestServer.Diagnostics.LlmTestServerRequestOrder.Ascending }, cancellationToken); + var exported = await conversation.Control.ExportRequestsAsync(cancellationToken: cancellationToken); + var ndjson = await conversation.Control.ExportRequestsNdjsonAsync(cancellationToken: cancellationToken); + var verified = await conversation.Control.VerifyAsync(new() + { + NoUnexpectedCalls = true, + Requests = + [ + new() { Name = "all graph calls", MinimumCount = 3, MaximumCount = 3 }, + new() { Name = "stream completion", Query = new() { Streaming = true }, RequireStreamingTerminalMarker = true }, + new() { Name = "tool fixture", Query = new() { Step = "lookup" }, Tools = [new() { Name = "weather", CallId = "call_package_graph", ArgumentsJson = """{"city":"Paris"}""" }] }, + ], + Conversations = [new() { ScenarioId = "package-graph", ConversationId = conversation.Id, VisitedSteps = ["lookup", "answer"], RequireTerminal = true }], + }, cancellationToken); + if (evidence.Requests.Count != 3 || evidence.EvictedCount != 0 || evidence.RetentionTruncated + || evidence.Requests.Any(trace => trace.WireClockMode != "fixed") || evidence.Requests[^1].WireSequence != 2 + || exported.SchemaVersion != 1 || exported.Evidence.Requests.Count != 3 + || ndjson.Split('\n', StringSplitOptions.RemoveEmptyEntries).Length != 4 || !verified.Passed || !verified.EvidenceComplete) + throw new InvalidOperationException("Installed request query/export/verification lost complete fixture evidence."); + await using var retrySession = await control.CreateSessionAsync(config => config.RequireBearerToken("test-key") + .UseChatScenario("sdk-retry", scenario => scenario.Fails(429, "too_many_requests", "try again").RetryAfter(0).Responds("retried")), cancellationToken); + retrySession.HttpClient.DefaultRequestHeaders.Add(ManagedCode.LlmTestServer.Diagnostics.LlmTestServerRequestHeaders.RetryGroup, "installed-sdk-retry"); + var retrySdk = new OpenAI.Chat.ChatClient("gpt-4.1-mini", new System.ClientModel.ApiKeyCredential("test-key"), + new OpenAI.OpenAIClientOptions { Endpoint = new Uri(retrySession.HttpClient.BaseAddress!, "openai/v1"), + Transport = new System.ClientModel.Primitives.HttpClientPipelineTransport(retrySession.HttpClient) }); + var retryAnswer = (await retrySdk.CompleteChatAsync([new OpenAI.Chat.UserChatMessage("retry")], cancellationToken: cancellationToken)).Value; + var retryEvidence = await retrySession.Control.VerifyAsync(new() + { + NoUnexpectedCalls = true, + Retries = [new() { GroupId = "installed-sdk-retry", MinimumAttempts = 2, MaximumAttempts = 2, + StatusSequence = [429, 200], MinimumRetryAfterSeconds = 0, RespectRetryAfter = true }], + }, cancellationToken); + if (retryAnswer.Content[0].Text != "retried" || !retryEvidence.Passed || !retryEvidence.EvidenceComplete) + throw new InvalidOperationException("Installed official SDK retries must be proven by actual provider requests."); + + } + } diff --git a/scripts/package-consumer/ProtocolDeliveryPackageTests.cs b/scripts/package-consumer/ProtocolDeliveryPackageTests.cs new file mode 100644 index 0000000..df99c87 --- /dev/null +++ b/scripts/package-consumer/ProtocolDeliveryPackageTests.cs @@ -0,0 +1,131 @@ +#pragma warning disable OPENAI001 // Qualify the installed package with the pinned official Responses SDK. +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Text; +using System.Text.Json; +using Aspire.Hosting; +using Aspire.Hosting.Testing; +using ManagedCode.LlmTestServer.Aspire; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Scenarios; +using OpenAI.Responses; +using TUnit.Core; +using AnthropicMessages = global::Anthropic.Models.Messages; + +public sealed class ProtocolDeliveryPackageTests +{ + private const string _key = "protocol-delivery-package-key"; + + [Test] + [Arguments("responses")] + [Arguments("anthropic")] + [Arguments("gemini")] + public async Task InstalledPackages_OfficialSdkDetectsMissingStreamTerminalAndExplicitRetryRecoversAsync(string provider, CancellationToken cancellationToken) + { + var resourceName = "protocol-package-" + provider; + var builder = DistributedApplicationTestingBuilder.Create([]); + builder.AddLlmTestServer(resourceName).WithApiKey(_key); + await using var app = await builder.BuildAsync(cancellationToken); + await app.StartAsync(cancellationToken); + await app.ResourceNotifications.WaitForResourceHealthyAsync(resourceName, cancellationToken); + using var http = app.CreateHttpClient(resourceName, LlmTestServerResource.HttpEndpointName); + var control = new LlmTestServerClient(http, _key); + var model = provider switch + { + "anthropic" => JsonSerializer.SerializeToElement(AnthropicMessages.Model.ClaudeSonnet5_5).GetString()!, + "gemini" => "gemini-3.8-flash", + _ => LlmTestServerKnownModelIds.Gpt41Mini, + }; + await control.ConfigureAsync(config => config.RequireBearerToken(_key).UseChatModel(model) + .UseChatScenario("protocol-package-delivery", scenario => scenario.ForModel(model) + .Responds("first second", "", "first ", "", "second") + .WithStreamDelivery(new() { Mode = LlmTestServerStreamDeliveryMode.EarlyCleanEnd, AfterContentFrames = 1 }) + .Responds("installed protocol recovery")), cancellationToken); + + var responses = provider == "responses" ? new ResponsesClient(new ApiKeyCredential(_key), new ResponsesClientOptions + { Endpoint = new Uri(http.BaseAddress!, "/openai/v1/"), Transport = new HttpClientPipelineTransport(http), RetryPolicy = new ClientRetryPolicy(0) }) : null; + using var anthropic = provider == "anthropic" ? new global::Anthropic.AnthropicClient(new global::Anthropic.Core.ClientOptions + { HttpClient = http, BaseUrl = new Uri(http.BaseAddress!, "/anthropic").ToString(), ApiKey = _key, MaxRetries = 0, ResponseValidation = true }) : null; + using var google = provider == "gemini" ? new Google.GenAI.Client(apiKey: _key, enterprise: false, + httpOptions: new() { BaseUrl = new Uri(http.BaseAddress!, "/gemini").ToString(), ApiVersion = "v1beta", RetryOptions = new() { Attempts = 1 } }, + clientOptions: new() { HttpClientFactory = () => http }) : null; + var text = new StringBuilder(); + var terminal = false; + if (responses is not null) + { + await foreach (var update in responses.CreateResponseStreamingAsync(new CreateResponseOptions + { Model = model, StreamingEnabled = true, StoredOutputEnabled = false, InputItems = { ResponseItem.CreateUserMessageItem("stream") } }, cancellationToken)) + { + if (update is StreamingResponseOutputTextDeltaUpdate delta) { text.Append(delta.Delta); } + if (update is StreamingResponseCompletedUpdate) { terminal = true; } + } + } + else if (anthropic is not null) + { + await foreach (var update in anthropic.Messages.CreateStreaming(AnthropicOptions(), cancellationToken)) + { + if (update.Value is AnthropicMessages.RawContentBlockDeltaEvent block && block.Delta.Value is AnthropicMessages.TextDelta delta) { text.Append(delta.Text); } + if (update.Value is AnthropicMessages.RawMessageStopEvent) { terminal = true; } + } + } + else + { + await foreach (var update in google!.Models.GenerateContentStreamAsync(model, "stream", + new Google.GenAI.Types.GenerateContentConfig { AutomaticContinuation = false }, cancellationToken: cancellationToken)) + { + text.Append(update.Text); + if (update.Candidates?.Any(candidate => candidate.FinishReason is not null) == true) { terminal = true; } + } + } + if (text.ToString() != "first " || terminal) + throw new InvalidOperationException($"Installed {provider} SSE must expose the first nonempty text delta without a protocol completion."); + var trace = await CompletedTraceAsync(control, cancellationToken); + var expectedPath = provider switch + { + "responses" => "/openai/v1/responses", + "anthropic" => "/anthropic/v1/messages", + _ => "/gemini/v1beta/models/" + model + ":streamGenerateContent", + }; + if (trace.Path != expectedPath || trace.StatusCode != 200 || trace.TerminalMarkerObserved == true + || trace.TransportState != LlmTestServerProviderHttpTraceTransportState.Completed + || trace.StreamDelivery?.Mode != LlmTestServerStreamDeliveryMode.EarlyCleanEnd + || !trace.StreamDeliveryApplied || trace.EmittedContentFrames != 1) + throw new InvalidOperationException($"Installed {provider} diagnostics lost the applied delivery boundary or reported a false terminal marker."); + var verification = await control.VerifyAsync(new() + { Requests = [new() { Query = new() { RequestId = trace.RequestId }, RequireStreamingTerminalMarker = true }] }, cancellationToken); + if (verification.Passed || !verification.Failures.Any(failure => failure.Code == "stream.terminal" && failure.RequestIds.Contains(trace.RequestId))) + throw new InvalidOperationException($"Installed {provider} verification must reject missing terminal evidence."); + if ((await control.GetAssertionsAsync(cancellationToken)).Matched != 1) + throw new InvalidOperationException($"Installed {provider} SDK must issue exactly one attempt and retain its committed inference."); + + string recovered; + if (responses is not null) + recovered = (await responses.CreateResponseAsync(new CreateResponseOptions + { Model = model, StoredOutputEnabled = false, InputItems = { ResponseItem.CreateUserMessageItem("explicit retry") } }, cancellationToken)).Value.GetOutputText(); + else if (anthropic is not null) + recovered = ((AnthropicMessages.TextBlock)(await anthropic.Messages.Create(AnthropicOptions(), cancellationToken)).Content.Single().Value!).Text; + else + recovered = (await google!.Models.GenerateContentAsync(model, "explicit retry", + new Google.GenAI.Types.GenerateContentConfig { AutomaticContinuation = false }, cancellationToken: cancellationToken)).Text!; + if (recovered != "installed protocol recovery" || (await control.GetAssertionsAsync(cancellationToken)).Matched != 2 + || (await control.GetRequestsAsync(cancellationToken: cancellationToken)).TotalObserved != 2) + throw new InvalidOperationException($"Installed {provider} explicit JSON retry must consume the next fixture exactly once."); + } + + private static AnthropicMessages.MessageCreateParams AnthropicOptions() => new() + { Model = AnthropicMessages.Model.ClaudeSonnet5_5, MaxTokens = 256, Messages = [new() { Role = AnthropicMessages.Role.User, Content = "stream" }] }; + + private static async Task CompletedTraceAsync(LlmTestServerClient control, CancellationToken cancellationToken) + { + using var watchdog = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + watchdog.CancelAfter(TimeSpan.FromSeconds(60)); + while (true) + { + var evidence = await control.GetRequestsAsync(cancellationToken: watchdog.Token); + if (evidence.Requests.Count == 1 && evidence.Requests[0].IsCompleted) { return evidence.Requests[0]; } + watchdog.Token.ThrowIfCancellationRequested(); + } + } +} diff --git a/scripts/package-consumer/ShowcaseTests.cs b/scripts/package-consumer/ShowcaseTests.cs new file mode 100644 index 0000000..c97b3be --- /dev/null +++ b/scripts/package-consumer/ShowcaseTests.cs @@ -0,0 +1,256 @@ +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Text.Json; +using System.Text.Json.Nodes; +using Aspire.Hosting; +using Aspire.Hosting.Testing; +using ManagedCode.LlmTestServer.Aspire; +using Microsoft.Playwright; +using TUnit.Core; +using static Microsoft.Playwright.Assertions; + +// Drives a realistic multi-provider session through the installed package and captures the showcase images used by the +// README, landing page and launch material. The evidence captures in PackageTests stay independent of this traffic. +public sealed class ShowcaseTests +{ + private const string Conversation = "trip-planner"; + + [Test] + public async Task Showcase_CapturesTokenAnalyticsInsightsAndOverviewFromRichTraffic(CancellationToken cancellationToken) + { + var builder = DistributedApplicationTestingBuilder.Create([]); + builder.AddLlmTestServer("showcase-server").WithApiKey("test-key"); + await using var app = await builder.BuildAsync(cancellationToken); + await app.StartAsync(cancellationToken); + await app.ResourceNotifications.WaitForResourceHealthyAsync("showcase-server", cancellationToken); + using var http = app.CreateHttpClient("showcase-server", "http"); + http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", "test-key"); + http.DefaultRequestHeaders.Add("anthropic-version", "2023-06-01"); + + await PostAsync(http, "/admin-api/configure", Configuration(), cancellationToken); + await DriveTripPlannerAsync(http, cancellationToken); + await DriveSideTrafficAsync(http, cancellationToken); + + using var playwright = await Playwright.CreateAsync(); + await using var browser = await playwright.Chromium.LaunchAsync(new() { Headless = true }); + await using var context = await browser.NewContextAsync(new() + { + ViewportSize = new() { Width = 1440, Height = 1040 }, + DeviceScaleFactor = 2, + }); + var page = await context.NewPageAsync(); + await page.GotoAsync(http.BaseAddress!.ToString()); + await page.GetByRole(AriaRole.Button, new() { Name = "Runtime settings", Exact = true }).ClickAsync(); + await page.GetByLabel("Bearer token", new() { Exact = true }).FillAsync("test-key"); + await page.GetByRole(AriaRole.Button, new() { Name = "Close runtime settings", Exact = true }).ClickAsync(); + await page.GetByRole(AriaRole.Button, new() { Name = "Refresh runtime state" }).ClickAsync(); + await Expect(page.GetByTestId("request-row")).ToHaveCountAsync(15); + var tripRows = page.Locator("[data-testid='request-row']", new() { HasTextString = "10 turns" }); + + await tripRows.First.ClickAsync(); + await page.GetByTestId("analytics-tab").ClickAsync(); + await page.GetByTestId("analytics-scope-conversation").ClickAsync(); + await Expect(page.GetByTestId("analytics-flow")).ToBeVisibleAsync(); + await page.Locator(".sankey-band").First.HoverAsync(); + await Expect(page.Locator("#viz-tip")).ToBeVisibleAsync(); + await CaptureAsync(page, "package-showcase-token-flow.jpg"); + + await page.GetByTestId("insights-open").ClickAsync(); + await Expect(page.GetByTestId("insights-panel")).ToBeVisibleAsync(); + var providerArc = page.Locator("[data-testid='insight-providers'] .donut-arc").First; + var arcBox = await providerArc.BoundingBoxAsync() ?? throw new InvalidOperationException("Provider donut is not rendered."); + await providerArc.HoverAsync(new() { Position = new() { X = arcBox.Width / 2 + 4, Y = 4 }, Force = true }); + await Expect(page.Locator("#viz-tip")).ToBeVisibleAsync(); + await CaptureAsync(page, "package-showcase-insights.jpg"); + + var toolCallRow = page.Locator("[data-testid='request-row']", new() { HasTextString = "2 turns" }) + .Filter(new() { HasTextString = "gpt-5-mini" }).First; + await toolCallRow.ClickAsync(); + await Expect(page.GetByTestId("overview-tool-calls")).ToBeVisibleAsync(); + await page.GetByTestId("conversation-strip").First.Locator(".convo-strip-bar > span").First.HoverAsync(); + await Expect(page.Locator("#viz-tip")).ToBeVisibleAsync(); + await CaptureAsync(page, "package-showcase-overview.jpg"); + + await tripRows.First.ClickAsync(); + await page.GetByRole(AriaRole.Tab, new() { Name = "Conversation", Exact = false }).ClickAsync(); + await page.Locator("[data-testid='conversation'] .msg-meter").Nth(1).HoverAsync(); + await Expect(page.Locator("#viz-tip")).ToBeVisibleAsync(); + await CaptureAsync(page, "package-showcase-conversation.jpg"); + } + + private static async Task CaptureAsync(IPage page, string path) + { + await page.WaitForTimeoutAsync(250); + await page.ScreenshotAsync(new() { Path = path, Type = ScreenshotType.Jpeg, Quality = 86 }); + } + + private static async Task DriveTripPlannerAsync(HttpClient http, CancellationToken cancellationToken) + { + var system = "You are a careful travel assistant for a rail and dining concierge. Use only the supplied tools and fixtures, " + + "explain practical trade-offs, keep Unicode place names, and never invent timetables.\n" + + string.Join("\n", Enumerable.Range(1, 60).Select(index => + $"Policy {index:D3}: prefer step-free routes, leave twenty minutes before departure, confirm bookings with a reference, and keep answers under 80 words.")); + var messages = new JsonArray + { + new JsonObject { ["role"] = "system", ["content"] = system }, + new JsonObject { ["role"] = "user", ["content"] = "Plan a relaxed afternoon trip from Paris to Fontainebleau today, with weather and trains." }, + }; + var results = new Dictionary(StringComparer.Ordinal) + { + ["call_weather"] = """{"city":"Paris","temperature":18,"unit":"celsius","condition":"sunny","wind_kph":9}""", + ["call_trains"] = """{"departures":[{"time":"14:30","duration_min":38,"platform":"L"},{"time":"15:30","duration_min":41}],"return":[{"time":"18:12"},{"time":"19:05"}]}""", + ["call_table"] = """{"restaurant":"Le Train Bleu","time":"19:00","guests":2,"reference":"TB-4821"}""", + }; + for (var turn = 0; turn < 5; turn++) + { + if (turn == 3) + { + messages.Add(new JsonObject { ["role"] = "user", ["content"] = "Great. Book a table for two near the station for when we get back." }); + } + + var body = new JsonObject { ["model"] = "gpt-5-mini", ["messages"] = messages.DeepClone(), ["tools"] = Tools() }; + using var request = new HttpRequestMessage(HttpMethod.Post, "/openai/v1/chat/completions") { Content = JsonContent.Create(body) }; + request.Headers.Add("X-Llm-Test-Server-Conversation", Conversation); + using var response = await http.SendAsync(request, cancellationToken); + response.EnsureSuccessStatusCode(); + var message = (await response.Content.ReadFromJsonAsync(cancellationToken))!["choices"]![0]!["message"]!.AsObject(); + var assistant = new JsonObject { ["role"] = "assistant", ["content"] = message["content"]?.DeepClone() }; + if (message["tool_calls"] is JsonArray calls) + { + assistant["tool_calls"] = calls.DeepClone(); + } + + messages.Add(assistant); + foreach (var call in (message["tool_calls"] as JsonArray) ?? []) + { + var id = call!["id"]!.GetValue(); + messages.Add(new JsonObject { ["role"] = "tool", ["tool_call_id"] = id, ["content"] = results[id] }); + } + } + } + + private static async Task DriveSideTrafficAsync(HttpClient http, CancellationToken cancellationToken) + { + await PostAsync(http, "/openai/v1/chat/completions", new JsonObject + { + ["model"] = "gpt-4.1-mini", + ["messages"] = new JsonArray { new JsonObject { ["role"] = "user", ["content"] = "Are the fixtures healthy?" } }, + }, cancellationToken); + using (var stream = await http.PostAsJsonAsync("/openai/v1/chat/completions", new JsonObject + { + ["model"] = "gpt-4.1-mini", + ["stream"] = true, + ["messages"] = new JsonArray { new JsonObject { ["role"] = "user", ["content"] = "Does streaming work?" } }, + }, cancellationToken)) + { + stream.EnsureSuccessStatusCode(); + _ = await stream.Content.ReadAsStringAsync(cancellationToken); + } + + var review = new JsonObject + { + ["model"] = "claude-sonnet-4-5", + ["max_tokens"] = 256, + ["system"] = "You review travel plans for realism. Answer in one sentence.", + ["messages"] = new JsonArray { new JsonObject { ["role"] = "user", ["content"] = "Review: 14:30 train, château, 18:12 back, 19:00 dinner." } }, + }; + await PostAsync(http, "/anthropic/v1/messages", review, cancellationToken); + await PostAsync(http, "/anthropic/v1/messages", review, cancellationToken); + await PostAsync(http, "/gemini/v1beta/models/gemini-2.5-flash:generateContent", new JsonObject + { + ["contents"] = new JsonArray { new JsonObject { ["role"] = "user", ["parts"] = new JsonArray { new JsonObject { ["text"] = "Summarize the Fontainebleau plan in one line." } } } }, + }, cancellationToken); + await PostAsync(http, "/mistral/v1/chat/completions", new JsonObject + { + ["model"] = "mistral-small-latest", + ["messages"] = new JsonArray { new JsonObject { ["role"] = "user", ["content"] = "Translate the plan into French." } }, + }, cancellationToken); + await PostAsync(http, "/groq/openai/v1/chat/completions", new JsonObject + { + ["model"] = "llama-3.3-70b-versatile", + ["messages"] = new JsonArray { new JsonObject { ["role"] = "user", ["content"] = "Give me a packing list." } }, + }, cancellationToken); + await PostAsync(http, "/openai/v1/embeddings", new JsonObject { ["model"] = "text-embedding-3-small", ["input"] = "station café near Gare de Lyon" }, cancellationToken); + (await http.GetAsync("/openai/v1/models", cancellationToken)).EnsureSuccessStatusCode(); + using var missing = await http.PostAsJsonAsync("/openai/v1/chat/completions", new JsonObject + { + ["model"] = "gpt-missing", + ["messages"] = new JsonArray { new JsonObject { ["role"] = "user", ["content"] = "hello" } }, + }, cancellationToken); + if (missing.StatusCode != System.Net.HttpStatusCode.NotFound) + throw new InvalidOperationException("The showcase needs one visible client error."); + } + + private static async Task PostAsync(HttpClient http, string path, JsonNode body, CancellationToken cancellationToken) + { + using var response = await http.PostAsJsonAsync(path, body, cancellationToken); + if (!response.IsSuccessStatusCode) + throw new InvalidOperationException($"{path} returned {(int)response.StatusCode}: {await response.Content.ReadAsStringAsync(cancellationToken)}"); + } + + private static JsonArray Tools() + { + static JsonObject Tool(string name, string description, JsonObject properties, params string[] required) => new() + { + ["type"] = "function", + ["function"] = new JsonObject + { + ["name"] = name, + ["description"] = description, + ["parameters"] = new JsonObject { ["type"] = "object", ["properties"] = properties, ["required"] = new JsonArray(required.Select(item => (JsonNode)item).ToArray()) }, + }, + }; + + return + [ + Tool("get_weather", "Current weather and short forecast for a city.", new JsonObject { ["city"] = new JsonObject { ["type"] = "string" }, ["unit"] = new JsonObject { ["type"] = "string" } }, "city"), + Tool("search_trains", "Regional and intercity departures between two stations after a time.", new JsonObject { ["from"] = new JsonObject { ["type"] = "string" }, ["to"] = new JsonObject { ["type"] = "string" }, ["after"] = new JsonObject { ["type"] = "string" } }, "from", "to"), + Tool("reserve_table", "Reserve a restaurant table near a landmark or station.", new JsonObject { ["area"] = new JsonObject { ["type"] = "string" }, ["time"] = new JsonObject { ["type"] = "string" }, ["guests"] = new JsonObject { ["type"] = "integer" } }, "area", "time", "guests"), + ]; + } + + private static JsonObject Configuration() + { + static JsonObject Call(string id, string name, object arguments) => new() + { + ["toolCalls"] = new JsonArray { new JsonObject { ["id"] = id, ["name"] = name, ["argumentsJson"] = JsonSerializer.Serialize(arguments) } }, + }; + static JsonObject Text(string content) => new() { ["content"] = content }; + + return new JsonObject + { + ["requiredBearerToken"] = "test-key", + ["models"] = new JsonArray + { + new JsonObject { ["id"] = "gpt-5-mini", ["kind"] = "chat", ["reasoningTokens"] = 23 }, + new JsonObject { ["id"] = "gpt-4.1-mini", ["kind"] = "chat" }, + new JsonObject { ["id"] = "claude-sonnet-4-5", ["kind"] = "chat", ["reasoningTokens"] = 12 }, + new JsonObject { ["id"] = "gemini-2.5-flash", ["kind"] = "chat" }, + new JsonObject { ["id"] = "mistral-small-latest", ["kind"] = "chat" }, + new JsonObject { ["id"] = "llama-3.3-70b-versatile", ["kind"] = "chat" }, + new JsonObject { ["id"] = "text-embedding-3-small", ["kind"] = "embedding" }, + }, + ["chatScenarios"] = new JsonArray + { + new JsonObject + { + ["id"] = "trip-planner", + ["modelId"] = "gpt-5-mini", + ["responses"] = new JsonArray + { + Call("call_weather", "get_weather", new { city = "Paris", unit = "celsius" }), + Call("call_trains", "search_trains", new { from = "Paris Gare de Lyon", to = "Fontainebleau", after = "14:00" }), + Text("Paris is 18 °C and sunny. Take the 14:30 regional train from Gare de Lyon (38 min), walk the château gardens, and be back on the 18:12 service."), + Call("call_table", "reserve_table", new { area = "Gare de Lyon", time = "19:00", guests = 2 }), + Text("Booked: a table for two at 19:00 at Le Train Bleu, five minutes from your return platform. Confirmation TB-4821."), + }, + }, + new JsonObject { ["id"] = "quick-check", ["modelId"] = "gpt-4.1-mini", ["responses"] = new JsonArray { Text("All fixtures are healthy."), new JsonObject { ["content"] = "Streaming works.", ["streamChunks"] = new JsonArray { "Streaming ", "works." } } } }, + new JsonObject { ["id"] = "claude-review", ["modelId"] = "claude-sonnet-4-5", ["responses"] = new JsonArray { Text("Realistic; add ten minutes for the château queue."), Text("Looks good.") } }, + new JsonObject { ["id"] = "gemini-summary", ["modelId"] = "gemini-2.5-flash", ["responses"] = new JsonArray { Text("Fontainebleau day trip: 14:30 out, 18:12 back, dinner at 19:00.") } }, + new JsonObject { ["id"] = "mistral-translate", ["modelId"] = "mistral-small-latest", ["responses"] = new JsonArray { Text("Départ 14h30, retour 18h12, dîner à 19h00.") } }, + new JsonObject { ["id"] = "groq-packing", ["modelId"] = "llama-3.3-70b-versatile", ["responses"] = new JsonArray { Text("Comfortable shoes, a light jacket, water and your rail pass.") } }, + }, + }; + } +} diff --git a/scripts/package-consumer/TypeSafeAndDeliveryPackageTests.cs b/scripts/package-consumer/TypeSafeAndDeliveryPackageTests.cs new file mode 100644 index 0000000..0760ac7 --- /dev/null +++ b/scripts/package-consumer/TypeSafeAndDeliveryPackageTests.cs @@ -0,0 +1,132 @@ +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Text; +using System.Text.Json; +using Aspire.Hosting; +using Aspire.Hosting.Testing; +using ManagedCode.LlmTestServer.Aspire; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Decisions; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Scenarios; +using ManagedCode.LlmTestServer.SystemOne; +using OpenAI; +using OpenAI.Chat; +using TUnit.Core; + +public sealed class TypeSafeAndDeliveryPackageTests +{ + private const string _key = "test-key"; + + [Test] + public async Task InstalledPackages_ReadmeTypeSafeJev114ExampleReturnsNativePredicateAsync(CancellationToken cancellationToken) + { + var builder = DistributedApplicationTestingBuilder.Create([]); + builder.AddLlmTestServer("typesafe-readme-package-server").WithApiKey(_key); + await using var app = await builder.BuildAsync(cancellationToken); + await app.StartAsync(cancellationToken); + await app.ResourceNotifications.WaitForResourceHealthyAsync("typesafe-readme-package-server", cancellationToken); + using var http = app.CreateHttpClient("typesafe-readme-package-server", LlmTestServerResource.HttpEndpointName); + var control = new LlmTestServerClient(http, _key); + await control.ConfigureAsync(config => config + .RequireBearerToken("test-key") + .UseDecisionModel(TypeSafeRequestValidation.Jev114Model, LlmTestServerDecisionProvider.TypeSafe) + .UseDecisionScenario(new LlmTestServerDecisionScenario + { + Id = "typesafe-priority", + ModelId = TypeSafeRequestValidation.Jev114Model, + Answers = new() + { + ["urgent"] = new() { Kind = LlmTestServerDecisionKind.Predicate, Probability = 0.95 }, + }, + }), cancellationToken); + + var jev = control.CreateSystemOneClient(); + var decision = await jev.EvaluateAsync(new SystemOneDecisionRequest + { + Model = TypeSafeRequestValidation.Jev114Model, + State = JsonSerializer.SerializeToElement("Please help quickly"), + Questions = new() + { + ["urgent"] = new() + { + Type = SystemOneDecisionTypes.Predicate, + Instructions = JsonSerializer.SerializeToElement("Does this need urgent attention?"), + }, + }, + }, cancellationToken: cancellationToken); + if (decision.Model != TypeSafeRequestValidation.Jev114Model || decision.Answers.Count != 1 + || decision.Answers["urgent"].Type != SystemOneDecisionTypes.Predicate || decision.Answers["urgent"].Probability != 0.95) + throw new InvalidOperationException("Installed packages did not preserve the README TypeSafe/Jev 1.14 native predicate example."); + var summary = await control.GetAssertionsAsync(cancellationToken); + if (summary.Matched != 1 || summary.ErrorsReturned != 0) + throw new InvalidOperationException("The installed TypeSafe/Jev fixture must produce exactly one successful matched decision."); + var trace = (await control.GetRequestsAsync(cancellationToken: cancellationToken)).Requests.Single(); + if (trace.Path != "/systemone/v1/systemone" || trace.OperationId != ManagedCode.LlmTestServer.Providers.LlmTestServerProviderOperationIds.SystemOne.DecisionsCreate + || trace.ModelId != TypeSafeRequestValidation.Jev114Model || trace.StatusCode != 200 || trace.ScenarioId != "typesafe-priority") + throw new InvalidOperationException("The installed README native request is missing its provider/model/scenario trace evidence."); + } + + [Test] + public async Task InstalledPackages_PublicStreamDeliveryPlanProducesPartialSdkStreamThenExplicitRecoveryAsync(CancellationToken cancellationToken) + { + var builder = DistributedApplicationTestingBuilder.Create([]); + builder.AddLlmTestServer("stream-delivery-package-server").WithApiKey(_key); + await using var app = await builder.BuildAsync(cancellationToken); + await app.StartAsync(cancellationToken); + await app.ResourceNotifications.WaitForResourceHealthyAsync("stream-delivery-package-server", cancellationToken); + using var http = app.CreateHttpClient("stream-delivery-package-server", LlmTestServerResource.HttpEndpointName); + var control = new LlmTestServerClient(http, _key); + await control.ConfigureAsync(config => config.RequireBearerToken(_key) + .UseChatScenario("package-stream-delivery", scenario => scenario + .Responds("first second", "first ", "second") + .WithStreamDelivery(new() { Mode = LlmTestServerStreamDeliveryMode.EarlyCleanEnd, AfterContentFrames = 1 }) + .Responds("installed package recovery")), cancellationToken); + var sdk = new ChatClient(LlmTestServerKnownModelIds.Gpt41Mini, new ApiKeyCredential(_key), new OpenAIClientOptions + { + Endpoint = new Uri(http.BaseAddress!, "/openai/v1"), + Transport = new HttpClientPipelineTransport(http), + RetryPolicy = new ClientRetryPolicy(0), + }); + var text = new StringBuilder(); + ChatFinishReason? finish = null; + await foreach (var update in sdk.CompleteChatStreamingAsync([new UserChatMessage("stream")], cancellationToken: cancellationToken)) + { + foreach (var part in update.ContentUpdate) { text.Append(part.Text); } + if (update.FinishReason is { } reason) { finish = reason; } + } + if (text.ToString() != "first " || finish is not null) + throw new InvalidOperationException("The installed delivery plan must end after the first real SDK content update without a finish frame."); + var trace = await CompletedTraceAsync(control, cancellationToken); + if (trace.StatusCode != 200 || trace.TerminalMarkerObserved == true + || trace.StreamDelivery?.Mode != LlmTestServerStreamDeliveryMode.EarlyCleanEnd + || !trace.StreamDeliveryApplied || trace.EmittedContentFrames != 1 + || trace.ResponsePreview?.Contains("[DONE]", StringComparison.Ordinal) != false) + throw new InvalidOperationException("Installed package delivery diagnostics must distinguish the incomplete protocol from HTTP 200."); + var verification = await control.VerifyAsync(new() + { + Requests = [new() { Query = new() { RequestId = trace.RequestId }, RequireStreamingTerminalMarker = true }], + }, cancellationToken); + if (verification.Passed || !verification.Failures.Any(failure => failure.Code == "stream.terminal" && failure.RequestIds.Contains(trace.RequestId))) + throw new InvalidOperationException("Installed package verification must reject a stream without its terminal marker."); + if ((await control.GetAssertionsAsync(cancellationToken)).Matched != 1) + throw new InvalidOperationException("Post-header delivery failure must retain its one committed inference without SDK retries."); + var recovered = (await sdk.CompleteChatAsync([new UserChatMessage("explicit retry")], cancellationToken: cancellationToken)).Value; + if (recovered.Content.Single().Text != "installed package recovery" || (await control.GetAssertionsAsync(cancellationToken)).Matched != 2 + || (await control.GetRequestsAsync(cancellationToken: cancellationToken)).TotalObserved != 2) + throw new InvalidOperationException("The explicit SDK retry must consume the next installed fixture exactly once."); + } + + private static async Task CompletedTraceAsync(LlmTestServerClient control, CancellationToken cancellationToken) + { + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + deadline.CancelAfter(TimeSpan.FromSeconds(10)); + while (true) + { + var evidence = await control.GetRequestsAsync(cancellationToken: deadline.Token); + if (evidence.Requests.Count == 1 && evidence.Requests[0].IsCompleted) { return evidence.Requests[0]; } + deadline.Token.ThrowIfCancellationRequested(); + } + } +} diff --git a/scripts/release.sh b/scripts/release.sh index 6e3e4a7..f55f2ac 100644 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -2,7 +2,7 @@ set -euo pipefail case "${1:?Expected resolve or publish}" in resolve) - version=$(dotnet msbuild src/ManagedCode.LlmTck/ManagedCode.LlmTck.csproj -nologo -v:q -getProperty:Version) + version=$(dotnet msbuild src/ManagedCode.LlmTestServer/ManagedCode.LlmTestServer.csproj -nologo -v:q -getProperty:Version) tag="v$version" echo "version=$version" >> "$GITHUB_OUTPUT" echo "tag=$tag" >> "$GITHUB_OUTPUT" @@ -14,14 +14,35 @@ case "${1:?Expected resolve or publish}" in ;; publish) test -n "${NUGET_API_KEY:-}" || { echo 'NUGET_API_KEY is required.' >&2; exit 1; } - dotnet nuget push 'artifacts/packages/*.nupkg' --source https://api.nuget.org/v3/index.json --api-key "$NUGET_API_KEY" --skip-duplicate + mkdir -p artifacts/packages + python3 - "${TAG_NAME:?TAG_NAME is required}" > artifacts/packages/release-manifest.txt <<'PY' +from pathlib import Path +import sys +import xml.etree.ElementTree as ET +version = sys.argv[1].removeprefix("v") +paths = [Path("artifacts/packages") / f"{ET.parse(project).findtext('.//PackageId')}.{version}.nupkg" + for project in sorted(Path("src").glob("*/*.csproj"))] +missing = [str(path) for path in paths if not path.is_file()] +if missing: + raise SystemExit("Missing release artifacts: " + ", ".join(missing)) +print("\n".join(map(str, paths))) +PY + packages=() + while IFS= read -r package; do packages+=("$package"); done < artifacts/packages/release-manifest.txt + for package in "${packages[@]}"; do + dotnet nuget push "$package" --source https://api.nuget.org/v3/index.json --api-key "$NUGET_API_KEY" --skip-duplicate + done if gh release view "$TAG_NAME" > /dev/null 2>&1; then - gh release upload "$TAG_NAME" artifacts/packages/*.nupkg --clobber + gh release upload "$TAG_NAME" "${packages[@]}" --clobber else - gh release create "$TAG_NAME" artifacts/packages/*.nupkg --title "$TAG_NAME" --generate-notes --target "$GITHUB_SHA" + gh release create "$TAG_NAME" "${packages[@]}" --title "$TAG_NAME" --generate-notes --target "$GITHUB_SHA" fi + ;; + finalize) + # Only the verified public-feed consumer completes delivery. A failed verification + # leaves this marker absent, so a rerun resumes the same version's publication. printf '%s\n' "$TAG_NAME" > artifacts/packages/nuget-published.txt gh release upload "$TAG_NAME" artifacts/packages/nuget-published.txt --clobber ;; - *) echo 'Expected resolve or publish' >&2; exit 2 ;; + *) echo 'Expected resolve, publish or finalize' >&2; exit 2 ;; esac diff --git a/scripts/test-release.py b/scripts/test-release.py deleted file mode 100644 index 44da5b0..0000000 --- a/scripts/test-release.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python3 -"""Exercise release recovery with local fake gh/NuGet CLIs; never publish externally.""" -import os -from pathlib import Path -import subprocess -import tempfile - -script = Path(__file__).with_name("release.sh").resolve() -with tempfile.TemporaryDirectory(prefix="llmtck-release-test-") as directory: - root = Path(directory) - (root / "artifacts/packages").mkdir(parents=True) - (root / "artifacts/packages/example.nupkg").write_text("fixture") - for name, code in { - "dotnet": '''#!/usr/bin/env bash -set -eu -if [ "$1" = msbuild ]; then echo 0.1.1; exit 0; fi -if [ -e fail-push ]; then touch partial-package; exit 1; fi -touch packages-published -''', - "gh": '''#!/usr/bin/env bash -set -eu -case "$2" in - view) test -e release-exists || exit 1; if [ -e delivered ]; then echo nuget-published.txt; fi ;; - create) test -e packages-published; touch release-exists ;; - upload) test -e packages-published; case "$4" in *nuget-published.txt) touch delivered;; esac ;; -esac -''', - }.items(): - file = root / name - file.write_text(code) - file.chmod(0o755) - output = root / "output" - env = dict(os.environ, PATH=str(root) + os.pathsep + os.environ["PATH"], GITHUB_OUTPUT=str(output), - NUGET_API_KEY="fake-test-key", TAG_NAME="v0.1.1", GITHUB_SHA="fixture-sha", FORCE_RELEASE="false") - def run(action, success=True): - result = subprocess.run(["bash", str(script), action], cwd=root, env=env, capture_output=True, text=True) - assert (result.returncode == 0) == success, result.stderr - def should_release(expected): - output.write_text("") - run("resolve") - assert f"should_release={str(expected).lower()}" in output.read_text() - # An older partial GitHub release must not prevent repairing NuGet delivery. - (root / "release-exists").touch() - should_release(True) - (root / "fail-push").touch() - run("publish", False) - assert not (root / "delivered").exists() - should_release(True) - (root / "fail-push").unlink() - run("publish") - should_release(False) - env["FORCE_RELEASE"] = "true" - should_release(True) - env["NUGET_API_KEY"] = "" - run("publish", False) - # New version path: release is created only after NuGet success. - env["NUGET_API_KEY"] = "fake-test-key" - (root / "release-exists").unlink() - (root / "delivered").unlink() - run("publish") - assert (root / "delivered").exists() -print("Release recovery: partial publish, rerun, completion, force and missing-key checks passed.") diff --git a/scripts/verify-container.py b/scripts/verify-container.py new file mode 100644 index 0000000..a9a19ec --- /dev/null +++ b/scripts/verify-container.py @@ -0,0 +1,423 @@ +#!/usr/bin/env python3 +"""Run a real Docker image and verify its standalone HTTP configuration/provider/dashboard flows.""" +import argparse +import json +from pathlib import Path +import secrets +import subprocess +import time +import urllib.error +import urllib.request + + +def command(*arguments, **kwargs): + return subprocess.check_output(arguments, text=True, **kwargs).strip() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--image", required=True) + parser.add_argument("--platform", choices=["linux/amd64", "linux/arm64"]) + parser.add_argument("--anonymous-pull", action="store_true") + parser.add_argument("--require-platform", action="append", default=[]) + args = parser.parse_args() + root = Path(__file__).resolve().parent.parent + evidence = root / "artifacts" / "container-verification" / secrets.token_hex(8) + evidence.mkdir(parents=True) + published_platforms = [] + if args.require_platform and not args.anonymous_pull: + parser.error("--require-platform requires --anonymous-pull") + if args.anonymous_pull: + clean_config = evidence / "anonymous-docker-config" + clean_config.mkdir() + (clean_config / "config.json").write_text('{"auths":{}}\n') + print(f"Pulling {args.image} with an empty Docker configuration", flush=True) + pull_arguments = ["docker", "--config", str(clean_config), "pull"] + if args.platform: + pull_arguments += ["--platform", args.platform] + command(*pull_arguments, args.image) + if args.require_platform: + manifest = json.loads(command("docker", "--config", str(clean_config), "manifest", "inspect", args.image)) + published_platforms = sorted({item["platform"]["os"] + "/" + item["platform"]["architecture"] + for item in manifest.get("manifests", [])}) + missing = set(args.require_platform) - set(published_platforms) + if missing: + raise RuntimeError("Published image is missing platforms: " + ", ".join(sorted(missing))) + inspect_arguments = ["docker", "image", "inspect"] + if args.platform: + inspect_arguments += ["--platform", args.platform] + image = json.loads(command(*inspect_arguments, args.image))[0] + if not image["Config"].get("User") or image["Config"]["User"] in ("root", "0"): + raise RuntimeError("The distribution image must run as a non-root user.") + token = secrets.token_urlsafe(24) + container = command("docker", "run", "--detach", "--platform", image["Os"] + "/" + image["Architecture"], "--publish", "127.0.0.1::8080", + "--env", "LlmTestServer__RequiredBearerToken=" + token, args.image) + results = [] + try: + state = json.loads(command("docker", "inspect", container))[0] + bindings = state["NetworkSettings"].get("Ports", {}).get("8080/tcp") + if not bindings: + raise RuntimeError(f"Container did not allocate its HTTP port; status={state['State']['Status']}, exitCode={state['State']['ExitCode']}") + port = bindings[0]["HostPort"] + base = "http://127.0.0.1:" + port + + def request(path, *, body=None, authorized=True, method=None, extra_headers=None): + headers = dict(extra_headers or {}) + if authorized: + headers["Authorization"] = "Bearer " + token + data = None if body is None else json.dumps(body).encode() + if data is not None: + headers["Content-Type"] = "application/json" + req = urllib.request.Request(base + path, data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(req, timeout=15) as response: + return response.status, response.read(), dict(response.headers) + except urllib.error.HTTPError as error: + return error.code, error.read(), dict(error.headers) + + deadline = time.monotonic() + 120 + while True: + try: + status, _, _ = request("/health", authorized=False) + if status == 200: + break + except (urllib.error.URLError, TimeoutError, ConnectionError): + pass + if time.monotonic() >= deadline: + raise RuntimeError("Container failed to become healthy within 120 seconds.") + if command("docker", "inspect", "--format", "{{.State.Running}}", container) != "true": + raise RuntimeError("Container exited before becoming healthy.") + time.sleep(1) + assert request("/alive", authorized=False)[0] == 200 + results.append("health and liveness") + status, dashboard, _ = request("/", authorized=False) + assert status == 200 and b"LLM Test Server" in dashboard and b"blazor.web.js" in dashboard + assert request("/_framework/blazor.web.js", authorized=False)[0] == 200 + results.append("dashboard HTML and Blazor assets") + assert request("/admin-api/models", authorized=False)[0] == 401 + assert request("/openai/v1/models", authorized=False)[0] == 401 + results.append("management and provider authorization") + configuration = { + "requiredBearerToken": token, + "models": [{"id": "gpt-4.1-mini", "kind": "chat"}], + "chatScenarios": [{"id": "container-smoke", "modelId": "gpt-4.1-mini", + "responses": [{"content": "container reply", "streamChunks": ["container ", "reply"]}, + {"content": "stream reply", "streamChunks": ["stream ", "reply"]}]}], + } + assert request("/admin-api/configure", body=configuration)[0] == 200 + body = {"model": "gpt-4.1-mini", "messages": [{"role": "user", "content": "test"}]} + assert request("/openai/v1/chat/completions", body=body, authorized=False)[0] == 401 + status, response, _ = request("/openai/v1/chat/completions", body=body) + assert status == 200 and json.loads(response)["choices"][0]["message"]["content"] == "container reply" + status, response, _ = request("/openai/v1/chat/completions", body=body | {"stream": True}) + assert status == 200 and b"data: [DONE]" in response and b"stream " in response + results.append("management configure, provider JSON and SSE") + invalid = body | {"model": "not-configured"} + assert request("/openai/v1/chat/completions", body=invalid)[0] == 404 + assertions = json.loads(request("/admin-api/assertions")[1]) + assert assertions["matched"] == 2 and assertions["modelNotFound"] == 1 + results.append("unknown-model validation and real runtime assertions") + source_url = "https://example.com/container-source" + search_id = "ws_container_native" + message_id = "msg_container_native" + citation = {"kind": "url", "startIndex": 0, "endIndex": 5, "url": source_url, "title": "Container fixture"} + native_items = [ + {"kind": "web_search", "id": search_id, "queries": ["container fixture facts"], + "sources": [{"url": source_url}]}, + {"kind": "message", "id": message_id, "content": "Paris fixture.", + "streamChunks": ["Paris ", "fixture."], "citations": [citation]}, + ] + native_configuration = { + "requiredBearerToken": token, + "models": [{"id": "gpt-4.1-mini", "kind": "chat"}], + "chatScenarios": [{"id": "container-native", "modelId": "gpt-4.1-mini", + "responses": [{"outputItems": native_items} for _ in range(3)]}], + } + status, configured, _ = request("/admin-api/configure", body=native_configuration) + assert status == 200, ("Native fixture configuration failed", status, configured) + native_request = { + "model": "gpt-4.1-mini", "input": "run the native fixture", + "tools": [{"type": "web_search", "search_context_size": "low", + "filters": {"allowed_domains": ["example.com"]}, + "user_location": {"type": "approximate", "country": "FR"}}], + "tool_choice": {"type": "web_search"}, + } + + def check_native_output(payload, *, include_sources): + output = payload["output"] + assert [item["type"] for item in output] == ["web_search_call", "message"] + search, message = output + assert search["id"] == search_id and search["status"] == "completed" + assert "role" not in search and "content" not in search and "call_id" not in search + assert search["action"]["type"] == "search" + assert search["action"]["queries"] == ["container fixture facts"] + if include_sources: + assert search["action"]["sources"] == [{"type": "url", "url": source_url}] + else: + assert "sources" not in search["action"] + assert message["id"] == message_id and message["role"] == "assistant" + assert len(message["content"]) == 1 + text = message["content"][0] + assert text["type"] == "output_text" and text["text"] == "Paris fixture." + assert text["annotations"] == [{"type": "url_citation", "start_index": 0, + "end_index": 5, "url": source_url, "title": "Container fixture"}] + + status, response, _ = request("/openai/v1/responses", body=native_request) + assert status == 200, ("Native JSON failed", status, response) + check_native_output(json.loads(response), include_sources=False) + included_request = native_request | {"include": ["web_search_call.action.sources"]} + status, response, _ = request("/openai/v1/responses", body=included_request) + assert status == 200, ("Native JSON with sources failed", status, response) + included = json.loads(response) + check_native_output(included, include_sources=True) + results.append("native Responses configure, ordered JSON, source include gating and URL citation") + status, response, headers = request("/openai/v1/responses", body=included_request | {"stream": True}) + assert status == 200 and headers.get("Content-Type", "").startswith("text/event-stream") + assert response.endswith(b"\n\n"), "Native stream has no complete final SSE frame" + frames = [json.loads(line[6:]) for line in response.splitlines() + if line.startswith(b"data: ") and line != b"data: [DONE]"] + names = [frame["type"] for frame in frames] + assert [frame["sequence_number"] for frame in frames] == list(range(len(frames))) + progress = [frame for frame in frames if frame["type"].startswith("response.web_search_call.")] + assert [frame["type"] for frame in progress] == ["response.web_search_call.in_progress", + "response.web_search_call.searching", "response.web_search_call.completed"] + assert all(frame["item_id"] == search_id and frame["output_index"] == 0 for frame in progress) + assert names.index("response.web_search_call.completed") < names.index("response.output_text.delta") + assert "".join(frame["delta"] for frame in frames if frame["type"] == "response.output_text.delta") == "Paris fixture." + annotations = [frame for frame in frames if frame["type"] == "response.output_text.annotation.added"] + assert len(annotations) == 1 and annotations[0]["item_id"] == message_id + assert annotations[0]["annotation"] == included["output"][1]["content"][0]["annotations"][0] + assert names[-1] == "response.completed" + assert frames[-1]["response"]["output"] == included["output"] + check_native_output(frames[-1]["response"], include_sources=True) + status, response, _ = request("/admin-api/requests?streaming=true&limit=1") + assert status == 200 + trace = json.loads(response)[0] + assert trace["terminalMarkerObserved"] is True and trace["isCompleted"] is True + assert trace["responseToolCalls"] == [] and len(trace["nativeTools"]) == 1 + assert trace["responseOutputItems"][0]["kind"] == "web_search" + assert trace["responseOutputItems"][0]["id"] == search_id + results.append("native SSE progress, deltas, citation, completed output and terminal request evidence") + for code in (False, True): + model = "gpt-4.1" if code else "gpt-4o-mini" + native_kind = "code_interpreter" if code else "file_search" + wire_kind = native_kind + "_call" + output_id = "ci_container" if code else "fs_container" + if code: + native_item = {"kind": native_kind, "id": output_id, "containerId": "cntr_container", + "inputFileIds": ["file_input"], "generatedFileIds": ["file_generated"], + "code": "print(42)", "codeChunks": ["print(", "42)"], + "outputs": [{"kind": "logs", "logs": "42\n"}, + {"kind": "image", "url": "https://example.com/container-chart.png"}]} + native_citation = {"kind": "container_file", "containerId": "cntr_container", "fileId": "file_generated", + "filename": "chart.png", "startIndex": 0, "endIndex": 0} + declaration = {"type": native_kind, "container": {"type": "auto", "file_ids": ["file_input"]}} + include = "code_interpreter_call.outputs" + payload_key = "outputs" + else: + native_item = {"kind": native_kind, "id": output_id, "vectorStoreIds": ["vs_container"], + "queries": ["container file fixture"], "results": [{"fileId": "file_container", "filename": "fixture.txt", + "text": "File fixture.", "score": 0.9, "attributes": {"category": "fixture"}}]} + native_citation = {"kind": "file", "fileId": "file_container", "filename": "fixture.txt", "index": 4} + declaration = {"type": native_kind, "vector_store_ids": ["vs_container"], "max_num_results": 1, + "ranking_options": {"ranker": "auto", "score_threshold": 0.5}, + "filters": {"type": "eq", "key": "category", "value": "fixture"}} + include = "file_search_call.results" + payload_key = "results" + items = [native_item, {"kind": "message", "id": "msg_" + native_kind, "content": "File fixture.", + "citations": [native_citation]}] + configuration = {"requiredBearerToken": token, "models": [{"id": model, "kind": "chat"}], + "chatScenarios": [{"id": "container-file-code", "modelId": model, + "responses": [{"outputItems": items} for _ in range(3)]}]} + status, configured, _ = request("/admin-api/configure", body=configuration) + assert status == 200, (native_kind, status, configured) + body = {"model": model, "input": "container file/code", "tools": [declaration], "tool_choice": "required"} + status, excluded, _ = request("/openai/v1/responses", body=body) + assert status == 200, (native_kind, status, excluded) + assert json.loads(excluded)["output"][0].get(payload_key) is None + body["include"] = [include] + status, included, _ = request("/openai/v1/responses", body=body) + assert status == 200, (native_kind, status, included) + output = json.loads(included)["output"] + assert output[0]["type"] == wire_kind and output[0]["id"] == output_id + annotation = output[1]["content"][0]["annotations"][0] + assert annotation["type"] == ("container_file_citation" if code else "file_citation") + if code: + assert output[0]["code"] == "print(42)" and output[0]["container_id"] == "cntr_container" + assert output[0]["outputs"] == [{"type": "logs", "logs": "42\n"}, + {"type": "image", "url": "https://example.com/container-chart.png"}] + assert annotation["start_index"] == annotation["end_index"] == 0 + else: + assert output[0]["queries"] == ["container file fixture"] + assert output[0]["results"][0]["file_id"] == "file_container" and annotation["index"] == 4 + status, stream, _ = request("/openai/v1/responses", body=body | {"stream": True}) + assert status == 200, (native_kind, status, stream) + frames = [json.loads(line[6:]) for line in stream.decode().splitlines() if line.startswith("data: ")] + assert [frame["sequence_number"] for frame in frames] == list(range(len(frames))) + assert frames[-1]["type"] == "response.completed" and frames[-1]["response"]["output"] == output + progress = [frame["type"] for frame in frames if frame["type"].startswith("response." + wire_kind + ".")] + assert progress == ["response." + wire_kind + ".in_progress", + "response." + wire_kind + (".interpreting" if code else ".searching"), "response." + wire_kind + ".completed"] + if code: + assert "".join(frame["delta"] for frame in frames if frame["type"] == "response.code_interpreter_call_code.delta") == "print(42)" + assert len([frame for frame in frames if frame["type"] == "response.code_interpreter_call_code.done"]) == 1 + status, response, _ = request("/admin-api/requests?streaming=true&limit=1") + trace = json.loads(response)[0] + assert status == 200 and trace["terminalMarkerObserved"] is True and trace["responseToolCalls"] == [] + assert trace["responseOutputItems"][0]["kind"] == native_kind + assert trace["responseOutputItems"][1]["citations"][0]["kind"] == ("container_file" if code else "file") + results.append(native_kind + " configuration, optional includes, typed citations, JSON/SSE and terminal evidence") + anthropic_model = "claude-sonnet-5-5" + anthropic_items = [ + {"kind": "server_tool_use", "id": "srvtoolu_container", "input": {"kind": "web_search", "query": "container facts"}}, + {"kind": "server_tool_result", "toolUseId": "srvtoolu_container", "result": {"kind": "web_search_results", "results": [ + {"url": source_url, "title": "Container source", "encryptedContent": "opaque-container-result"}]}}, + {"kind": "message", "content": "Verified source.", "streamChunks": ["Verified ", "source."], "citations": [ + {"kind": "anthropic_web_search", "url": source_url, "title": "Container source", + "encryptedIndex": "opaque-container-index", "citedText": "Verified source."}]}, + ] + configuration = { + "requiredBearerToken": token, + "models": [{"id": anthropic_model, "kind": "chat"}], + "chatScenarios": [{"id": "container-anthropic-native", "modelId": anthropic_model, + "responses": [{"outputItems": anthropic_items} for _ in range(2)]}], + } + assert request("/admin-api/configure", body=configuration)[0] == 200 + anthropic_request = {"model": anthropic_model, "max_tokens": 256, + "messages": [{"role": "user", "content": "container native search"}], + "tools": [{"type": "web_search_20250305", "name": "web_search", "max_uses": 1, + "allowed_domains": ["example.com"]}], + "tool_choice": {"type": "auto"}} + anthropic_headers = {"anthropic-version": "2023-06-01", "x-api-key": token} + status, response, _ = request("/anthropic/v1/messages", body=anthropic_request, authorized=False, extra_headers=anthropic_headers) + payload = json.loads(response) + assert status == 200, (status, payload) + assert [item["type"] for item in payload["content"]] == ["server_tool_use", "web_search_tool_result", "text"] + assert payload["content"][1]["content"][0]["encrypted_content"] == "opaque-container-result" + assert payload["content"][2]["citations"][0]["encrypted_index"] == "opaque-container-index" + assert payload["usage"]["server_tool_use"]["web_search_requests"] == 1 + status, response, _ = request("/anthropic/v1/messages", body=anthropic_request | {"stream": True}, + authorized=False, extra_headers=anthropic_headers) + assert status == 200, (status, response) + frames = [json.loads(line[6:]) for line in response.decode().splitlines() if line.startswith("data: ")] + starts = [frame for frame in frames if frame["type"] == "content_block_start"] + assert [frame["index"] for frame in starts] == [0, 1, 2] + assert [frame["content_block"]["type"] for frame in starts] == ["server_tool_use", "web_search_tool_result", "text"] + deltas = [frame["delta"] for frame in frames if frame["type"] == "content_block_delta"] + assert json.loads(next(delta["partial_json"] for delta in deltas if delta["type"] == "input_json_delta")) == {"query": "container facts"} + assert "".join(delta["text"] for delta in deltas if delta["type"] == "text_delta") == "Verified source." + assert next(delta["citation"] for delta in deltas if delta["type"] == "citations_delta")["encrypted_index"] == "opaque-container-index" + assert frames[-1]["type"] == "message_stop" + assert next(frame for frame in frames if frame["type"] == "message_delta")["usage"]["server_tool_use"]["web_search_requests"] == 1 + trace = json.loads(request("/admin-api/requests?limit=1")[1])[0] + assert trace["terminalMarkerObserved"] is True and trace["responseToolCalls"] == [] + assert trace["responseOutputItems"][0]["kind"] == "server_tool_use" + assert trace["responseOutputItems"][1]["kind"] == "server_tool_result" + results.append("Anthropic native search call/result, opaque citations, cumulative usage, JSON/SSE and terminal evidence") + deepseek_model = "deepseek-flash" + filtered = {"content": "Visible fixture", "streamChunks": ["Visible ", "fixture"], + "outputDisposition": {"kind": "Filtered"}} + configuration = {"requiredBearerToken": token, + "models": [{"id": deepseek_model, "kind": "chat"}], + "chatScenarios": [{"id": "container-deepseek-policy", "modelId": deepseek_model, + "responses": [filtered, filtered, {"content": "Recovered"}]}]} + assert request("/admin-api/configure", body=configuration)[0] == 200 + deepseek_request = {"model": deepseek_model, "messages": [{"role": "user", "content": "policy"}]} + path = "/deepseek/v1/chat/completions" + assert request(path, body=deepseek_request | {"stream_options": {"include_usage": True}})[0] == 400 + status, response, _ = request(path, body=deepseek_request) + payload = json.loads(response) + assert status == 200 and payload["choices"][0]["finish_reason"] == "content_filter" + assert payload["choices"][0]["message"]["content"] == "Visible fixture" + status, response, _ = request(path, body=deepseek_request | {"stream": True}) + wire = response.decode() + assert status == 200 and wire.endswith("data: [DONE]\n\n") + frames = [json.loads(line[6:]) for line in wire.splitlines() + if line.startswith("data: ") and line != "data: [DONE]"] + assert all(len(frame["choices"]) == 1 for frame in frames) + assert all("usage" not in frame for frame in frames[:-1]) + assert frames[-1]["choices"][0]["finish_reason"] == "content_filter" + assert isinstance(frames[-1]["usage"], dict) + assert "".join(frame["choices"][0]["delta"].get("content", "") for frame in frames) == "Visible fixture" + status, response, _ = request(path, body=deepseek_request) + payload = json.loads(response) + assert status == 200 and payload["choices"][0]["message"]["content"] == "Recovered" + assert payload["choices"][0]["finish_reason"] == "stop" + assert json.loads(request("/admin-api/assertions")[1])["matched"] == 3 + results.append("DeepSeek native filtered JSON/SSE, mandatory final usage, invalid options without fixture consumption and recovery") + azure_routes = [ + ("/azure-openai/openai/deployments/gpt-4.1-mini/chat/completions?api-version=2025-04-01-preview", False, "Refused"), + ("/microsoft-foundry/chat/completions?api-version=2024-05-01-preview", False, "Filtered"), + ("/microsoft-foundry/models/chat/completions?api-version=2024-05-01-preview", False, "Filtered"), + ("/azure-openai/openai/v1/chat/completions", False, "Refused"), + ("/microsoft-foundry/openai/v1/chat/completions", False, "Refused"), + ("/azure-openai/openai/v1/responses", True, "Filtered"), + ("/microsoft-foundry/openai/v1/responses", True, "Refused"), + ] + for path, responses_api, kind in azure_routes: + policy = {"content": "Azure policy text", "streamChunks": ["Azure policy ", "text"], + "outputDisposition": {"kind": kind}} + configuration = {"requiredBearerToken": token, + "models": [{"id": "gpt-4.1-mini", "kind": "chat"}], + "chatScenarios": [{"id": "container-azure-policy", "modelId": "gpt-4.1-mini", + "responses": [policy, policy, {"content": "Azure recovered"}]}]} + status, configured, _ = request("/admin-api/configure", body=configuration) + assert status == 200, (path, status, configured) + body = {"model": "gpt-4.1-mini"} + body.update({"input": "policy", "store": False} if responses_api else + {"messages": [{"role": "user", "content": "policy"}]}) + status, response, _ = request(path, body=body) + payload = json.loads(response) + assert status == 200, (path, status, payload) + if responses_api: + assert payload["status"] == ("incomplete" if kind == "Filtered" else "completed") + if kind == "Filtered": + assert payload["incomplete_details"]["reason"] == "content_filter" + else: + assert payload["output"][0]["content"][0]["type"] == "refusal" + else: + choice = payload["choices"][0] + assert choice["finish_reason"] == ("content_filter" if kind == "Filtered" else "stop") + assert choice["message"]["refusal" if kind == "Refused" else "content"] == "Azure policy text" + streamed = body | {"stream": True} + native_inference = path.split("?")[0] in ( + "/microsoft-foundry/chat/completions", "/microsoft-foundry/models/chat/completions") + if not responses_api and not native_inference: + streamed["stream_options"] = {"include_usage": True} + status, response, _ = request(path, body=streamed) + wire = response.decode() + assert status == 200, (path, status, wire) + frames = [json.loads(line[6:]) for line in wire.splitlines() + if line.startswith("data: ") and line != "data: [DONE]"] + if responses_api: + assert frames[-1]["type"] == ("response.incomplete" if kind == "Filtered" else "response.completed") + else: + assert wire.endswith("data: [DONE]\n\n") + choices = [choice for frame in frames for choice in frame["choices"]] + assert next(choice["finish_reason"] for choice in choices if choice.get("finish_reason")) == ( + "content_filter" if kind == "Filtered" else "stop") + field = "refusal" if kind == "Refused" else "content" + assert "".join(choice["delta"].get(field) or "" for choice in choices) == "Azure policy text" + if native_inference: + assert all(len(frame["choices"]) == 1 for frame in frames) + assert isinstance(frames[-1]["usage"], dict) + status, response, _ = request(path, body=body) + assert status == 200 and "Azure recovered" in response.decode(), (path, status, response) + assert json.loads(request("/admin-api/assertions")[1])["matched"] == 3 + results.append("seven Azure/Foundry text routes: native policy JSON/SSE, valid endings and explicit recovery") + assert request("/admin-api/reset", method="POST")[0] == 200 + assert json.loads(request("/admin-api/assertions")[1])["totalEvents"] == 0 + results.append("management reset") + (evidence / "result.json").write_text(json.dumps({"image": args.image, "imageId": image["Id"], + "repoDigests": image.get("RepoDigests", []), "architecture": image["Architecture"], + "anonymousPull": args.anonymous_pull, "publishedPlatforms": published_platforms, "checks": results}, indent=2) + "\n") + print("Container acceptance passed: " + "; ".join(results), flush=True) + print("Evidence: " + str(evidence), flush=True) + finally: + logs = subprocess.run(["docker", "logs", container], text=True, capture_output=True) + (evidence / "container.log").write_text((logs.stdout + logs.stderr).replace(token, "[redacted]")) + subprocess.run(["docker", "rm", "--force", container], check=True, stdout=subprocess.DEVNULL) + + +if __name__ == "__main__": + main() diff --git a/scripts/verify-package.py b/scripts/verify-package.py index a04ef7d..7755edc 100644 --- a/scripts/verify-package.py +++ b/scripts/verify-package.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Run a real NuGet consumer outside the repository, using an isolated package cache.""" +"""Run a real NuGet consumer with no project references and an isolated package cache.""" import os import base64 import hashlib @@ -10,37 +10,61 @@ import shutil import sys import xml.etree.ElementTree as ET +import argparse + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument("--public-feed", action="store_true", help="Restore exclusively from public NuGet, after publication.") +parser.add_argument("--refresh-screenshots", action="store_true", help="Update checked-in dashboard images after browser verification.") +parser.add_argument("--packages", type=Path, help="Local package directory; defaults to artifacts/packages.") +parser.add_argument("--record-demo", action="store_true", help="Keep a real dashboard browser recording under ignored marketing/raw.") +args = parser.parse_args() root = Path(__file__).resolve().parent.parent -packages = root / "artifacts/packages" +packages = (args.packages or root / "artifacts/packages").resolve() versions = {n.attrib["Include"]: n.attrib["Version"] for n in ET.parse(root / "Directory.Packages.props").iter("PackageVersion")} version = ET.parse(root / "Directory.Build.props").findtext(".//Version") -consumer = Path(tempfile.mkdtemp(prefix="llmtck-package-consumer-")).resolve() -package_cache = Path(tempfile.mkdtemp(prefix="llmtck-package-cache-")).resolve() +evidence = root / "artifacts/package-consumer" +evidence.mkdir(parents=True, exist_ok=True) +consumer = Path(tempfile.mkdtemp(prefix="consumer-", dir=evidence)).resolve() +package_cache = Path(tempfile.mkdtemp(prefix="cache-", dir=evidence)).resolve() print(f"Consumer: {consumer}", flush=True) project = (root / "scripts/package-consumer/Consumer.csproj.template").read_text() -for key, value in {"__VERSION__": version, "__ASPIRE_VERSION__": versions["Aspire.Hosting.Testing"], "__TUNIT_VERSION__": versions["TUnit"], "__PLAYWRIGHT_VERSION__": versions["Microsoft.Playwright"]}.items(): +for key, value in {"__VERSION__": version, "__ASPIRE_VERSION__": versions["Aspire.Hosting.Testing"], "__TUNIT_VERSION__": versions["TUnit"], "__PLAYWRIGHT_VERSION__": versions["Microsoft.Playwright"], "__OPENAI_VERSION__": versions["OpenAI"], "__AZURE_OPENAI_VERSION__": versions["Azure.AI.OpenAI"], "__AZURE_INFERENCE_VERSION__": versions["Azure.AI.Inference"], "__ANTHROPIC_VERSION__": versions["Anthropic"], "__GOOGLE_GENAI_VERSION__": versions["Google.GenAI"]}.items(): project = project.replace(key, value) (consumer / "Consumer.csproj").write_text(project) -(consumer / "PackageTests.cs").write_text((root / "scripts/package-consumer/PackageTests.cs").read_text()) +for source in sorted((root / "scripts/package-consumer").glob("*.cs")): + shutil.copy2(source, consumer / source.name) (consumer / "global.json").write_text((root / "global.json").read_text()) +# Keep evidence repository-owned without inheriting the source repository's CPM, +# analyzers or project references into this otherwise independent consumer. +for name in ["Directory.Build.props", "Directory.Build.targets", "Directory.Packages.props"]: + (consumer / name).write_text("\n") env = dict(os.environ, NUGET_PACKAGES=str(package_cache)) +if args.record_demo: + recording_directory = root / "marketing/raw/dashboard-recordings" + recording_directory.mkdir(parents=True, exist_ok=True) + env["LLM_TEST_SERVER_DEMO_RECORDING_DIRECTORY"] = str(recording_directory) def run(*args): subprocess.run(args, cwd=consumer, env=env, check=True) -run("dotnet", "restore", "Consumer.csproj", "--source", str(packages), "--source", "https://api.nuget.org/v3/index.json") -metadata = json.loads((package_cache / "managedcode.llmtck.aspire" / version / ".nupkg.metadata").read_text()) -expected_hash = base64.b64encode(hashlib.sha512((packages / f"ManagedCode.LlmTck.Aspire.{version}.nupkg").read_bytes()).digest()).decode() +sources = [] if args.public_feed else ["--source", str(packages)] +run("dotnet", "restore", "Consumer.csproj", *sources, "--source", "https://api.nuget.org/v3/index.json") +metadata = json.loads((package_cache / "managedcode.llmtestserver.aspire" / version / ".nupkg.metadata").read_text()) +expected_hash = base64.b64encode(hashlib.sha512((packages / f"ManagedCode.LlmTestServer.Aspire.{version}.nupkg").read_bytes()).digest()).decode() if metadata.get("contentHash") != expected_hash: raise RuntimeError("Restored Aspire package differs from the local artifact being verified.") print("Verified restored package SHA-512 matches the local nupkg.", flush=True) run("dotnet", "build", "Consumer.csproj", "--configuration", "Release", "--no-restore") run("pwsh", "bin/Release/net10.0/playwright.ps1", "install", "chromium", *(["--with-deps"] if sys.platform.startswith("linux") else [])) try: - run("dotnet", "test", "Consumer.csproj", "--configuration", "Release", "--no-build") + run("dotnet", "test", "--project", "Consumer.csproj", "--configuration", "Release", "--no-build") finally: - evidence = root / "artifacts/package-consumer" - evidence.mkdir(parents=True, exist_ok=True) - for image in (consumer / "bin").rglob("package-dashboard*.png"): - shutil.copy2(image, evidence / image.name) + for image in consumer.rglob("package-*.*"): + if image.suffix in {".png", ".jpg"}: + shutil.copy2(image, evidence / image.name) if (consumer / "TestResults").exists(): shutil.copytree(consumer / "TestResults", evidence / "TestResults", dirs_exist_ok=True) +if args.refresh_screenshots: + for name in ["overview.jpg", "conversation.jpg", "runtime.jpg", "mobile-request-detail.png", "token-usage.png", "insights.jpg"]: + shutil.copy2(evidence / f"package-dashboard-{name}", root / "docs/images" / f"admin-dashboard-{name}") + for name in ["token-flow.jpg", "insights.jpg", "overview.jpg", "conversation.jpg"]: + shutil.copy2(evidence / f"package-showcase-{name}", root / "docs/images" / f"showcase-{name}") diff --git a/scripts/verify-release.py b/scripts/verify-release.py new file mode 100644 index 0000000..66d6426 --- /dev/null +++ b/scripts/verify-release.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Verify a real GitHub release and every published NuGet package; never simulate publishing.""" +import hashlib +import json +from pathlib import Path +import subprocess +import time +import xml.etree.ElementTree as ET + +root = Path(__file__).resolve().parent.parent +version = ET.parse(root / "Directory.Build.props").findtext(".//Version") +package_ids = sorted(ET.parse(project).findtext(".//PackageId") for project in (root / "src").glob("*/*.csproj")) +release = json.loads(subprocess.check_output( + ["gh", "release", "view", "v" + version, "--json", "tagName,isDraft,assets,url"], cwd=root, text=True)) +if release["isDraft"] or release["tagName"] != "v" + version: + raise RuntimeError("The matching public GitHub release is not available.") +asset_names = {asset["name"] for asset in release["assets"]} +expected_assets = {f"{package}.{version}.nupkg" for package in package_ids} +if expected_assets - asset_names: + raise RuntimeError("Missing GitHub release assets: " + ", ".join(sorted(expected_assets - asset_names))) +evidence = root / "artifacts/release-verification" / version +evidence.mkdir(parents=True, exist_ok=True) +for package in package_ids: + filename = f"{package}.{version}.nupkg" + url = f"https://api.nuget.org/v3-flatcontainer/{package.lower()}/{version}/{filename.lower()}" + target = evidence / filename + for attempt in range(20): + result = subprocess.run(["curl", "--fail", "--silent", "--show-error", "--max-time", "30", + "--output", str(target), url], capture_output=True, text=True) + if result.returncode == 0: + break + if attempt == 19: + raise RuntimeError(f"Published package remains unavailable: {package} {version}: {result.stderr}") + print(f"Waiting for NuGet: {package} {version} ({attempt + 1}/20)", flush=True) + time.sleep(15) + local = root / "artifacts/packages" / filename + if not local.exists() or hashlib.sha512(local.read_bytes()).digest() != hashlib.sha512(target.read_bytes()).digest(): + raise RuntimeError("Published package bytes differ from the release artifact: " + filename) + print("Verified public package: " + filename, flush=True) +(evidence / "release.json").write_text(json.dumps(release, indent=2) + "\n") +print(f"Verified {len(package_ids)} published packages and GitHub release {release['url']}", flush=True) diff --git a/site/.gitignore b/site/.gitignore new file mode 100644 index 0000000..b947077 --- /dev/null +++ b/site/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +dist/ diff --git a/site/.htmlhintrc b/site/.htmlhintrc new file mode 100644 index 0000000..c4e5e8c --- /dev/null +++ b/site/.htmlhintrc @@ -0,0 +1,14 @@ +{ + "tagname-lowercase": true, + "attr-lowercase": true, + "attr-value-double-quotes": true, + "attr-no-duplication": true, + "doctype-first": true, + "tag-pair": true, + "spec-char-escape": true, + "id-unique": true, + "src-not-empty": true, + "alt-require": true, + "doctype-html5": true, + "title-require": true +} diff --git a/site/AGENTS.md b/site/AGENTS.md index 59ff9e6..1f6d477 100644 --- a/site/AGENTS.md +++ b/site/AGENTS.md @@ -3,3 +3,12 @@ The site is static HTML/CSS. Keep the first viewport focused on the product and include a real raster visual asset. Verify layout at desktop and mobile widths before shipping visual changes. + +## Site commands + +- Install: `npm ci --prefix site` +- Lint: `npm run lint --prefix site` (HTMLHint, Stylelint, JavaScript syntax) +- Build: `npm run build --prefix site` +- Production canonical: `python3 site/build.py --base-url "https://actual-pages-base-url"`; the Pages workflow supplies `configure-pages` output. +- Build output is `site/dist/`; only this directory is published. Keep tooling and local marketing originals out of the Pages artifact. +- `tokens.css` mirrors the dashboard's final warm canvas/ink/lime brand layer. Check the complete CSS cascade when synchronizing it; earlier theme blocks are not the final skin. diff --git a/site/assets/apple-touch-icon.png b/site/assets/apple-touch-icon.png new file mode 100644 index 0000000..286aa85 Binary files /dev/null and b/site/assets/apple-touch-icon.png differ diff --git a/site/assets/favicon-32.png b/site/assets/favicon-32.png new file mode 100644 index 0000000..bc996c7 Binary files /dev/null and b/site/assets/favicon-32.png differ diff --git a/site/assets/llm-tck-hero.png b/site/assets/llm-tck-hero.png deleted file mode 100644 index decd9c7..0000000 Binary files a/site/assets/llm-tck-hero.png and /dev/null differ diff --git a/site/assets/mark.svg b/site/assets/mark.svg new file mode 100644 index 0000000..4e7526b --- /dev/null +++ b/site/assets/mark.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/site/assets/og.png b/site/assets/og.png new file mode 100644 index 0000000..6554ac8 Binary files /dev/null and b/site/assets/og.png differ diff --git a/site/build.py b/site/build.py new file mode 100644 index 0000000..5f43e06 --- /dev/null +++ b/site/build.py @@ -0,0 +1,35 @@ +"""Build the static Pages artifact using the canonical URL supplied by configure-pages.""" +import argparse +import html +import shutil +from pathlib import Path +from urllib.parse import urlparse + +ROOT = Path(__file__).resolve().parent +DEFAULT_URL = "https://managedcode.github.io/LlmTestServer" +parser = argparse.ArgumentParser() +parser.add_argument("--base-url", default=DEFAULT_URL) +args = parser.parse_args() +base_url = args.base_url.rstrip("/") +parsed = urlparse(base_url) +if parsed.scheme != "https" or not parsed.netloc or parsed.query or parsed.fragment or any(c.isspace() for c in base_url): + parser.error("The canonical base URL must be an absolute HTTPS URL without query or fragment.") +output = ROOT / "dist" +if output.exists(): + shutil.rmtree(output) +output.mkdir() +for name in ("index.html", "tokens.css", "styles.css", "site.js"): + content = (ROOT / name).read_text() + (output / name).write_text(content.replace(DEFAULT_URL, html.escape(base_url, quote=True))) +shutil.copytree(ROOT / "assets", output / "assets") +# Reuse the reviewed product captures without duplicating tracked raster assets. +for name in ("showcase-token-flow.jpg", "showcase-overview.jpg", "showcase-insights.jpg", "admin-dashboard-runtime.jpg"): + shutil.copyfile(ROOT.parent / "docs" / "images" / name, output / "assets" / name) +(output / ".nojekyll").touch() +(output / "robots.txt").write_text(f"User-agent: *\nAllow: /\nSitemap: {base_url}/sitemap.xml\n") +(output / "sitemap.xml").write_text( + '\n' + '' + f"{html.escape(base_url)}/\n" +) +print(f"Built {output} for {base_url}/") diff --git a/site/index.html b/site/index.html index 7ae2767..7d85f36 100644 --- a/site/index.html +++ b/site/index.html @@ -1,76 +1,720 @@ - - - LLM TCK - - + + + + LLM Mock Server for .NET and Aspire | LLM Test Server + + + + + + + + + + + + + + + + + + + + + + + +
- LLM TCK + LLM Test Server + Get started
-
-
+
+
-

ManagedCode.LlmTck

-

Deterministic LLM compatibility tests for .NET.

-

- Host a local provider-compatible server with Aspire, script exact model behavior, and call it through HTTP or Microsoft.Extensions.AI. +

.NET 10 · Aspire · Docker

+

Your tests.
Your responses.

+

A deterministic LLM API server for .NET tests.

+

+ Script responses, streaming, tool calls and failures. Call it with your regular provider SDK, then see + exactly what your application sent and where every token went.

- Get started - GitHub + Get started Explore the dashboard +
+
+ + dotnet add package ManagedCode.LlmTestServer.Aspire + +
+
+
+
+ + Tokens tab for a five-request conversation: token flow from system prompt, tool schemas, user, assistant and tool messages into cache read, cache write, uncached input, response and reasoning. +
+ + +
+ A real capture of the dashboard served by the + package. +
+
+
+ +
+
+

13 provider API families, each under its own local namespace

+
    +
  • OpenAI
    • +
  • Azure OpenAI
    • +
  • Microsoft Foundry
    • +
  • Anthropic
    • +
  • Gemini
    • +
  • Amazon Bedrock
    • +
  • Groq
    • +
  • Mistral
    • +
  • Ollama
    • +
  • Cohere
    • +
  • OpenRouter
    • +
  • DeepSeek
    • +
  • Perplexity
    • +
+
+
+ +
+
+

How it works

+

From fixture to a
request you can inspect.

+

Keep your real provider integration in the test. Choose the behavior on the other side.

+
+
    +
  1. + +

    Set the scene.

    +

    Script a model, a reply, a tool call or a failure.

    + 
    .UseChatScenario("greeting", s => s
+  .WhenUserContains("hello")
+  .Responds("Hello from your test!"))
    + +
    2. +
  2. + +

    Make the call.

    +

    Point your provider SDK or IChatClient at the namespaced route.

    + 
    POST /openai/v1/chat/completions
+→ 200 OK
+→ "Hello from your test!"
    + +
    3. +
  3. + +

    Check what happened.

    +

    Assert on the reply and the recorded requests.

    + 
    var state = await control
+  .GetAssertionsAsync();
+// state.Matched == 1
    +
    4. +
+
+ +
+
+
+
+

The built-in request dashboard

+

Less reconstruction.
More context.

+
+

+ Provider, model, status, latency and token usage stay with each request. Select a call to read the + response, the conversation behind it and the wire payloads. Hover any chart for exact values; JSON + payloads and tool-call arguments are highlighted. +

+
+
+ Request overview with the conversation strip, per-message token bars, a tool-call card with highlighted arguments and a tooltip. +
One entry per provider request, with the selected response in the reading pane.
+
+
+
+

Traffic insights

+

All traffic at a glance.

+

+ Providers, models, outcomes, latency, cache and context mix as charts. Click a slice to filter the + request inbox. +

+ Traffic insights view with donut, ring, capsule-bar and dot-matrix charts across providers, models, outcomes and latency. +
+
+

Runtime diagnostics

+

Look inside your test.

+

+ Inspect models, fixtures, provider endpoints, assertions and runtime events from the same debugging + surface. +

+ Runtime state dialog showing configuration and diagnostic information. +
+
+
+
+

Request lifecycle

+

Every call, end to end.

+

+ Your test calls through its usual client. The server answers from your fixtures and records the + exchange for the dashboard and your assertions. +

+
+
+ + +
+ +
+
+
+ +
+
+
+
+

Token analytics

+

See where every
token goes.

+
+
+

+ The Tokens tab splits each request's input by source: system prompt, tool schemas, user, + assistant, tool calls and tool results. The parts add up to the reported input. Then it shows what the + prompt cache read, what it wrote and what stayed fresh. +

+

+ Your tests read the same numbers from GET /admin-api/requests: each request's usage + carries the per-role split as InputComposition. +

+ Read the analytics guide +
+
+ +
+
+
+

Input token flow

+

+ Where 3,140 input tokens came from, and how the prompt cache handled them. Example request. +

+
+ Example request +
+ +
+ +
+
+
+
+

Context mix

+

What the prompt is made of.

+
+
+
+
  • System prompt1,86059.2%
  • Tool schemas42013.4%
  • User2146.8%
  • Assistant34210.9%
  • Tool calls882.8%
  • Tool results2166.9%
+
+
+
+
+

Prompt cache

+

How the input was served.

+
+
+
81.5%from cache
+
  • Cache read2,56081.5%
  • Cache write2568.2%
  • Uncached32410.3%
+
+
+
+
+

Context growth

+

Input per request in one labeled conversation.

+
+
+
    +
  1. 1,204#1
  2. 2,150#2
  3. 2,736#3
  4. 3,140#4
    5. +
+
    +
  • From cache
    • +
  • Fresh input
    • +
+
+

Output55 tokens

+ +
    +
  • Visible text32
    • +
  • Reasoning23
    • +
+
+
+
+

+ Example values. Reasoning is part of output and cache reads/writes are part of input, so they are never + added twice. Requests without reported usage show as missing, not as zero. +

+ +
+
+
+

Traffic insights

+

+ The same journal, sliced by provider, model, outcome, latency, operation and delivery. Open it with + /?view=insights; clicking a slice filters the request inbox. +

+
+ Example values +
+
+
+

Providers

Requests per provider surface.

+
  • OpenAI22
  • Anthropic11
  • Gemini7
  • Azure OpenAI5
  • Other3
+
+
+

Models

One dot per request.

+
+
  • gpt-4.1-mini14
  • claude-sonnet-4-511
  • gpt-5-mini9
  • gemini-2.5-flash7
  • text-embedding-3-small4
  • Other3
+
+
+

Outcomes

Share of requests that returned 2xx.

+
85%success
  • 4xx 4
  • 5xx 1
  • transport 1
  • in flight 1
+
+
+

Latency

Requests per latency band.

+
  1. 30<10 ms
  2. 1210–100 ms
  3. 50.1–1 s
  4. 1≥1 s
+
+
+

Operations

Routed provider operations.

+
  • chat.completions.create44%
  • messages.create23%
  • models.generateContent15%
  • responses.create10%
  • embeddings.create8.3%
+
+
+

Delivery

Streamed chunks or one body.

+
  • Single response31
  • Streaming17
+
+
- LLM TCK compatibility test bench showing model modality nodes connected to a deterministic server.
-
-
-

Coverage

-

Provider behavior you can pin down.

+
+
+

Beyond a single reply

+

Test the conversation.
And the unexpected.

+

Every behavior is a fixture you choose. Nothing is hidden behind retries or randomness.

-
-
01

Chat

Script exact responses, failures, and queued replies.

-
02

Streaming

Emit known SSE chunks and a deterministic done marker.

-
03

Models

Expose a local catalog for chat, embeddings, image, and audio.

-
04

Assertions

Inspect matched, unmatched, auth, exhausted, and error events.

-
05

Embeddings

Return configured vectors for repeatable semantic tests.

-
06

Media

Serve fixed image and audio fixtures without external calls.

+
+
+ +

Tool loops, step by step.

+

Conversation graphs match each call and result, then choose the next reply.

+ Conversation DSL +
+
+ +

Streaming, chunk by chunk.

+

Script exact chunks: one response id, then the terminal marker.

+ Streaming chunks +
+
+ +

Failures on demand.

+

Rate limits and content filters on every provider family.

+ Fault simulation +
+
+ +

More than text.

+

Explicit embeddings, images, audio and video jobs where supported.

+ Modalities +
+
+ +

Parallel tests, separate state.

+

Each session owns its configuration, queue and journal.

+ Test sessions +
+
+ +

Verify, then reproduce.

+

Check order, counts and tool arguments; fix ids with a seed.

+ Verification +
-
-
-

Packages

-

Small surfaces, clear ownership.

-

- Core runtime, OpenAI wire mapping, ASP.NET Core endpoints, Microsoft.Extensions.AI clients, and Aspire AppHost extensions ship as separate packages. +

+
+
+
+

Provider-aware, by design

+

Your provider.
A local endpoint.

+
+
+

+ Every provider family has its own namespace and wire format. Operation coverage varies by provider and + grows incrementally. +

+ Read the operation matrix +
+
+
    +
  • OpenAI/openai/v1Official SDK checked
    • +
  • + Azure OpenAI/azure-openai/openai/deployments/…Official SDK checked +
    • +
  • + Microsoft Foundry/microsoft-foundry/modelsOfficial SDK checked +
    • +
  • Anthropic/anthropic/v1/messages
    • +
  • Gemini/gemini/v1beta/models/…
    • +
  • + Amazon Bedrock/bedrock/model/…/converseOfficial SDK checked +
    • +
  • Groq/groq/openai/v1
    • +
  • Mistral/mistral/v1
    • +
  • Ollama/ollama/api/chat
    • +
  • Cohere/cohere/v2/chat
    • +
  • OpenRouter/openrouter/api/v1
    • +
  • DeepSeek/deepseek/v1
    • +
  • Perplexity/perplexity/v1/sonar
    • +
  • + Native decision APIs/systemone · /cloudflareSystemOne · Workers AI Clef +
    • +
+

+ Each hosted operation is listed in a provider contract with documentation links and behavior tests. A + provider profile does not imply every API of that provider is implemented: check the operation and client + version your application uses.

-
    -
  • ManagedCode.LlmTckScenarios and assertions
    • -
  • ManagedCode.LlmTck.HostingProvider and control endpoints
    • -
  • ManagedCode.LlmTck.ClientControl and AI clients
    • -
  • ManagedCode.LlmTck.AspireAppHost integration
    • -
-
-

Start

-

Add the Aspire resource and call the local endpoint.

- 
builder
-    .AddLlmTck()
-    .WithApiKey("test-key");
+
+
+
+

Get started

+

A local server.
A familiar workflow.

+

Run it from your Aspire AppHost without Docker, or start the standalone container.

+
+
+
+
+ Default +

Aspire AppHost

+

Aspire starts the packaged .NET server and allocates its endpoint. No container runtime needed.

+
+
+
+ Terminal +
+ 
dotnet add package ManagedCode.LlmTestServer.Aspire
+
+
+
+ AppHost.cs +
+ 
using ManagedCode.LlmTestServer.Aspire;
+
+var builder = DistributedApplication.CreateBuilder(args);
+builder.AddLlmTestServer("llm-test-server")
+       .WithApiKey("test-key");
+builder.Build().Run();
+
+ Open the complete integration test +
+
+
+ Standalone +

Docker + management API

+

+ The v1.0.0 image serves provider routes, the dashboard and /admin-api. Load fixtures with + one request. +

+
+
+
+ Terminal +
+ 
docker run --rm -p 8080:8080 \
+  -e LlmTestServer__RequiredBearerToken=local-test-key \
+  ghcr.io/managedcode/llm-test-server:1.0.0
+
+
+
+ Configure +
+ 
curl http://localhost:8080/admin-api/configure \
+  -H 'Authorization: Bearer local-test-key' \
+  -H 'Content-Type: application/json' \
+  --data '{"requiredBearerToken":"local-test-key",
+    "models":[{"id":"gpt-4.1-mini","kind":"chat"}],
+    "chatScenarios":[{"id":"greeting","modelId":"gpt-4.1-mini",
+      "responses":[{"content":"Hello from LLM Test Server"}]}]}'
+
+ Docker distribution guide +
+
+
+

NuGet packages

+ +
+
+
+ +
+
+

A few useful details

+

Before your first test.

+
+
+
+ Does it call a live model? +

+ No. The server returns the fixtures configured for your test. It models API behavior so your application + can make real HTTP or SDK calls against a local server. +

+
+
+ Do I need Docker? +

+ No. The default package-owned Aspire integration starts the packaged .NET server and exposes its allocated + resource endpoint. The container image is an explicit alternative. +

+
+
+ Can I keep my provider SDK? +

+ Use the supported routes with the SDKs listed in the + operation coverage documentation. Compatibility is verified per operation; check your client version and API requirements. +

+
+
+ How are tokens counted? +

+ Usage is deterministic. Text is counted with a tiktoken-compatible tokenizer, so the same request always + reports the same input, output, reasoning and cache values. It does not reproduce a provider's billing. +

+
+
+ Can tests share a server? +

+ Yes. Create separate test sessions for independent configuration, scenario queues, history and + diagnostics. Dispose each session when its test ends. +

+
+
+ Is every provider API implemented? +

+ No. Coverage varies by provider. The operation matrix describes the claimed routes, streaming modes and + SDK evidence. Provider profiles grow incrementally. +

+
+
+ Does this evaluate model quality? +

+ No. It verifies your application's behavior against scripted API fixtures. It does not measure the + quality of a live model's answers. +

+
+
+
+ +
+
+ +

Make the next call predictable

+

Know what you send.
Choose what comes back.

+

Start the server, script a response and inspect your first request.

+
+ Get started View on GitHub +
+
+ + +
+
+

Dashboard preview

+
+ + +
+
+
+
+ +
+
+
+ + +
+ + diff --git a/site/package-lock.json b/site/package-lock.json new file mode 100644 index 0000000..c653462 --- /dev/null +++ b/site/package-lock.json @@ -0,0 +1,1888 @@ +{ + "name": "llm-test-server-site", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "llm-test-server-site", + "devDependencies": { + "htmlhint": "1.9.2", + "stylelint": "17.16.0", + "stylelint-config-standard": "40.0.0" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", + "integrity": "sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.29.7", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", + "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@cacheable/memory": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@cacheable/memory/-/memory-2.2.0.tgz", + "integrity": "sha512-CTLKqLItRCEixEAewD3/j9DB3/o96gpTPD4eJ1v+DGOlxZRZncRQkGYqqnAGCscYd6RNeXfGeiuCphsPtqyIfQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cacheable/utils": "^2.5.0", + "@keyv/bigmap": "^1.3.1", + "hookified": "^1.15.1", + "keyv": "^5.6.0" + } + }, + "node_modules/@cacheable/utils": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@cacheable/utils/-/utils-2.5.0.tgz", + "integrity": "sha512-buipgOVDkkPXNR5+xBpDw7Zk2n1EvU7qBJCNUcL7rhQ//kfpOXPAvQ511Os0vpLYJ1pZnvudNytkQt2hst3wqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "hashery": "^1.5.1", + "keyv": "^5.6.0" + } + }, + "node_modules/@csstools/css-calc": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-3.4.3.tgz", + "integrity": "sha512-iex20d8CHVkyvg6B7UKV7uHnI2Bqo9g+EFfT9E0y+GvTvhZ/DwONJ+9aKb1dlqm0ZiGsL5RXjp0fCoJYnkeDjA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=20.19.0" + }, + "peerDependencies": { + "@csstools/css-parser-algorithms": "^4.0.2", + "@csstools/css-tokenizer": "^4.0.2" + } + }, + "node_modules/@csstools/css-parser-algorithms": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@csstools/css-parser-algorithms/-/css-parser-algorithms-4.0.2.tgz", + "integrity": "sha512-40cSKyMvK+tq4qz6Awrlye2WGuOKt3FwPgtGg6KTfbHOWNw+Rk1rzbAtZnZ6IBhsY491HLRnDXwoyBAijmmILA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=20.19.0" + }, + "peerDependencies": { + "@csstools/css-tokenizer": "^4.0.2" + } + }, + "node_modules/@csstools/css-syntax-patches-for-csstree": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@csstools/css-syntax-patches-for-csstree/-/css-syntax-patches-for-csstree-1.1.15.tgz", + "integrity": "sha512-J0u7HkVl2nzSlhsiTOp4AmwcUQ3D+mGEEKfBy/7To5/y7F2OHwyLrXfrhR0SMgr4p5Lo+eaMVSeai24zUcBIxA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "peerDependencies": { + "css-tree": "^3.2.1" + }, + "peerDependenciesMeta": { + "css-tree": { + "optional": true + } + } + }, + "node_modules/@csstools/css-tokenizer": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@csstools/css-tokenizer/-/css-tokenizer-4.0.2.tgz", + "integrity": "sha512-OoKoR0f76dCY666JlcbhmVTs2drYj1GUXZTYTcbUgJjh9Nv41aFfZ21bPQTERm5+L5cBDo466NltB2lplS5GBw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=20.19.0" + } + }, + "node_modules/@csstools/media-query-list-parser": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@csstools/media-query-list-parser/-/media-query-list-parser-5.0.2.tgz", + "integrity": "sha512-qx19O8AbbLLbDSAcI6YOby/Vq8rV51psgIvlF5OZT/L8fQJDCyQifL6k+kYU4a8ySpzJ4RBzZkIs530bAn8y4Q==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=20.19.0" + }, + "peerDependencies": { + "@csstools/css-parser-algorithms": "^4.0.2", + "@csstools/css-tokenizer": "^4.0.2" + } + }, + "node_modules/@csstools/selector-resolve-nested": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@csstools/selector-resolve-nested/-/selector-resolve-nested-4.0.2.tgz", + "integrity": "sha512-JpjNBJFA+4OnAHKRK4AMp8xrYyLhI9XapOeBlP2PhRU/IylSw+1ep3BFNTksBJnrTS3KT6kqKNVfpzLLwZUW5Q==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "engines": { + "node": ">=20.19.0" + }, + "peerDependencies": { + "postcss-selector-parser": "^7.1.1" + } + }, + "node_modules/@csstools/selector-specificity": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/@csstools/selector-specificity/-/selector-specificity-6.0.1.tgz", + "integrity": "sha512-MBuqFIIvqk4S593MGAskTkPpEUUzZNvGzh8mMxEwl3oHwjQlXmqHEDxNbN3Y0Tsh+xjmem4GZjeg7I0qkwBivg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "engines": { + "node": ">=20.19.0" + }, + "peerDependencies": { + "postcss-selector-parser": "^7.1.1" + } + }, + "node_modules/@keyv/bigmap": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@keyv/bigmap/-/bigmap-1.3.1.tgz", + "integrity": "sha512-WbzE9sdmQtKy8vrNPa9BRnwZh5UF4s1KTmSK0KUVLo3eff5BlQNNWDnFOouNpKfPKDnms9xynJjsMYjMaT/aFQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "hashery": "^1.4.0", + "hookified": "^1.15.0" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "keyv": "^5.6.0" + } + }, + "node_modules/@keyv/serialize": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@keyv/serialize/-/serialize-1.1.1.tgz", + "integrity": "sha512-dXn3FZhPv0US+7dtJsIi2R+c7qWYiReoEh5zUntWCf4oSpMNib8FDhSoed6m3QyZdx5hK7iLFkYk3rNxwt8vTA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@sindresorhus/merge-streams": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/merge-streams/-/merge-streams-4.0.0.tgz", + "integrity": "sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@types/sarif": { + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/@types/sarif/-/sarif-2.1.7.tgz", + "integrity": "sha512-kRz0VEkJqWLf1LLVN4pT1cg1Z9wAuvI6L97V3m2f5B76Tg8d413ddvLBPTEHAZJlnn4XSvu0FkZtViCQGVyrXQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ansi-regex": { + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.4.0.tgz", + "integrity": "sha512-KzTVk2tCWAHtYrvvvaP8bJKJq2pVinhLcGEQdtLIYPbmNGNyYe8QwNaTUYQp2J7/vIsUKt5QCqAfUkYyG9DkOw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true, + "license": "Python-2.0" + }, + "node_modules/astral-regex": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-2.0.0.tgz", + "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/async": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", + "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", + "dev": true, + "license": "MIT" + }, + "node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/brace-expansion": { + "version": "5.0.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.12.tgz", + "integrity": "sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cacheable": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/cacheable/-/cacheable-2.5.0.tgz", + "integrity": "sha512-60cyAOytib/OzBw1JNSoSV/boK1AtHryDIjvVBk7XbN4ugfkM3+Sry7fEjNgPMGgOjuaZPAp8ruZ0Cxafwyq9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cacheable/memory": "^2.2.0", + "@cacheable/utils": "^2.5.0", + "hookified": "^1.15.0", + "keyv": "^5.6.0", + "qified": "^0.10.1" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT" + }, + "node_modules/colord": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/colord/-/colord-2.10.0.tgz", + "integrity": "sha512-AidJptpBJmjTclAp9BkLwJi0T93fo5epJnbaZslpg6QVzpHjAiveF55mE9AcUJiGMqRHgMDY8soMsQtuNYMHfw==", + "dev": true, + "license": "MIT" + }, + "node_modules/commander": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-11.1.0.tgz", + "integrity": "sha512-yPVavfyCcRhmorC7rWlkHn15b4wDVgVmBA7kV4QVBsF7kv/9TKJAbAXVTxvTnwP8HHKjRCJDClKbciiYS7p0DQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=16" + } + }, + "node_modules/cosmiconfig": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-9.0.2.tgz", + "integrity": "sha512-gtTZxTDau1wL7Y7zifc2dd8jHSK/k6BTx/2Xp/BpdlAdnlYWFVt7qhJqgwi7637yRwRQ3qL4ZidbB4I8tA5VOg==", + "dev": true, + "license": "MIT", + "dependencies": { + "env-paths": "^2.2.1", + "import-fresh": "^3.3.0", + "js-yaml": "^4.1.0", + "parse-json": "^5.2.0" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/d-fischer" + }, + "peerDependencies": { + "typescript": ">=4.9.5" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/css-functions-list": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/css-functions-list/-/css-functions-list-3.3.3.tgz", + "integrity": "sha512-8HFEBPKhOpJPEPu70wJJetjKta86Gw9+CCyCnB3sui2qQfOvRyqBy4IKLKKAwdMpWb2lHXWk9Wb4Z6AmaUT1Pg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/css-tree": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-3.2.1.tgz", + "integrity": "sha512-X7sjQzceUhu1u7Y/ylrRZFU2FS6LRiFVp6rKLPg23y3x3c3DOKAwuXGDp+PAGjh6CSnCjYeAul8pcT8bAl+lSA==", + "dev": true, + "license": "MIT", + "dependencies": { + "mdn-data": "2.27.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0" + } + }, + "node_modules/cssesc": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz", + "integrity": "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==", + "dev": true, + "license": "MIT", + "bin": { + "cssesc": "bin/cssesc" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/env-paths": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", + "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/error-ex": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.4.tgz", + "integrity": "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-arrayish": "^0.2.1" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-glob": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", + "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.8" + }, + "engines": { + "node": ">=8.6.0" + } + }, + "node_modules/fast-uri": { + "version": "3.1.9", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.9.tgz", + "integrity": "sha512-hS/0Wt3Ya043oKXs0sIxrMpKQITXhMLIrVAKKXi4gozGC9v7YRWVnXIZzjNPTO02M9BdxLCWqK1yRXgNziVyJQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/fastest-levenshtein": { + "version": "1.0.16", + "resolved": "https://registry.npmjs.org/fastest-levenshtein/-/fastest-levenshtein-1.0.16.tgz", + "integrity": "sha512-eRnCtTTtGZFpQCwhJiUOuxPQWRXVKYDn0b2PeHfXL6/Zi53SLAzAHfVhVWK2AryC/WH05kGfxhFIPvTF0SXQzg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4.9.1" + } + }, + "node_modules/fastq": { + "version": "1.20.3", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.3.tgz", + "integrity": "sha512-XKv5nnLs6nLF71NgiKJLIZFLkPyIEuOselLG7ujZnGrRfQK8HpvY+WqKhAJUAdLomwVHErVS4LfxFlPq0/FTAw==", + "dev": true, + "license": "ISC", + "dependencies": { + "reusify": "^1.0.4" + } + }, + "node_modules/file-entry-cache": { + "version": "11.1.5", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-11.1.5.tgz", + "integrity": "sha512-+PFTHITI08JIGhnNpGNI8T8inUpgZfk3GNEqfT9R2zZV2iFXg3CvqzSl/uEhs7TSGujYRELEANyDvS8Fj7+S7Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "flat-cache": "^6.1.23" + } + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dev": true, + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/flat-cache": { + "version": "6.1.23", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-6.1.23.tgz", + "integrity": "sha512-f++BY9pTk+983xK1FLzlLpmM0i0z+jHmx3QESGkURMXujQZz1k5wzwX6hjnQ8goaD0B+sYnDK1yZ6MTyZfUaqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "cacheable": "^2.5.0", + "flatted": "^3.4.2", + "hookified": "^1.15.0" + } + }, + "node_modules/flatted": { + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.4.tgz", + "integrity": "sha512-5+ybhBZANEJxaH3X5evAFatUxLfEHSr7n6kYJ+1Qd0mUqr4eu9gIf6GDbWHf8RJijHrjjO8G+la14SlL2SeS1Q==", + "dev": true, + "license": "ISC" + }, + "node_modules/fs-extra": { + "version": "11.4.1", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.4.1.tgz", + "integrity": "sha512-KYAb4c9BJQI6QqGKthV68OHe0badztdXJWKo0WtBA9IuCFPTKvE5ZdUBglP833aMjhaSPNO4A5j/EkzZtGlKjA==", + "dev": true, + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=14.14" + } + }, + "node_modules/get-east-asian-width": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.7.0.tgz", + "integrity": "sha512-XjH1AECxf0giL2V1aU8vKyRR2ppRUb5c0EvT7zuJTokQ74bNo52zOtghqdWIqrhUD79fo3x0WfKZdOqxF6LG1Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/glob": { + "version": "13.0.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz", + "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "minimatch": "^10.2.2", + "minipass": "^7.1.3", + "path-scurry": "^2.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/global-modules": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/global-modules/-/global-modules-2.0.0.tgz", + "integrity": "sha512-NGbfmJBp9x8IxyJSd1P+otYK8vonoJactOogrVfFRIAEY1ukil8RSKDz2Yo7wh1oihl51l/r6W4epkeKJHqL8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "global-prefix": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/global-prefix": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/global-prefix/-/global-prefix-3.0.0.tgz", + "integrity": "sha512-awConJSVCHVGND6x3tmMaKcQvwXLhjdkmomy2W+Goaui8YPgYgXJZewhg3fWC+DlfqqQuWg8AwqjGTD2nAPVWg==", + "dev": true, + "license": "MIT", + "dependencies": { + "ini": "^1.3.5", + "kind-of": "^6.0.2", + "which": "^1.3.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/globby": { + "version": "16.2.4", + "resolved": "https://registry.npmjs.org/globby/-/globby-16.2.4.tgz", + "integrity": "sha512-c8B/VNLmxRcmqqenRA9t+9IyOjf9+V6lTxPaUJLqOCONdQkWZ0ETYgX0qbtJqPsgCNusT9MZ5Jeidw8Eb9tn2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sindresorhus/merge-streams": "^4.0.0", + "fast-glob": "^3.3.3", + "ignore": "^7.0.5", + "is-path-inside": "^4.0.0", + "micromatch": "^4.0.8", + "slash": "^5.1.0", + "unicorn-magic": "^0.4.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/globjoin": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/globjoin/-/globjoin-0.1.4.tgz", + "integrity": "sha512-xYfnw62CKG8nLkZBfWbhWwDw02CHty86jfPcc2cr3ZfeuK9ysoVPPEUxf21bAD/rWAgk52SuBrLJlefNy8mvFg==", + "dev": true, + "license": "MIT" + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/hashery": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/hashery/-/hashery-1.5.1.tgz", + "integrity": "sha512-iZyKG96/JwPz1N55vj2Ie2vXbhu440zfUfJvSwEqEbeLluk7NnapfGqa7LH0mOsnDxTF85Mx8/dyR6HfqcbmbQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "hookified": "^1.15.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/hookified": { + "version": "1.15.1", + "resolved": "https://registry.npmjs.org/hookified/-/hookified-1.15.1.tgz", + "integrity": "sha512-MvG/clsADq1GPM2KGo2nyfaWVyn9naPiXrqIe4jYjXNZQt238kWyOGrsyc/DmRAQ+Re6yeo6yX/yoNCG5KAEVg==", + "dev": true, + "license": "MIT" + }, + "node_modules/html-tags": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/html-tags/-/html-tags-5.1.0.tgz", + "integrity": "sha512-n6l5uca7/y5joxZ3LUePhzmBFUJ+U2YWzhMa8XUTecSeSlQiZdF5XAd/Q3/WUl0VsXgUwWi8I7CNIwdI5WN1SQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20.10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/htmlhint": { + "version": "1.9.2", + "resolved": "https://registry.npmjs.org/htmlhint/-/htmlhint-1.9.2.tgz", + "integrity": "sha512-PweWSPA1Pb+AVFIOSpIGu5KhLdmtk/uf/0CpjvrDf6XUWmdTyqUljlylwSxQ0AWLvPGcBxK2n8uISsI4lCOkBQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "async": "3.2.6", + "chalk": "4.1.2", + "commander": "11.1.0", + "glob": "^13.0.6", + "is-glob": "^4.0.3", + "node-sarif-builder": "3.2.0", + "strip-json-comments": "3.1.1", + "xml": "1.0.1" + }, + "bin": { + "htmlhint": "bin/htmlhint" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "Open Collective", + "url": "https://opencollective.com/htmlhint" + } + }, + "node_modules/ignore": { + "version": "7.0.13", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.13.tgz", + "integrity": "sha512-2E8Q6rh+1RRSq/YdaGsNfXsEZy9FVqv6BSSrkmZJP7BnaoPnRfKL+YaofPvP+5rzdk144qSm6i4+hvPgUcbV4Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/import-fresh": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", + "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/import-meta-resolve": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-4.2.0.tgz", + "integrity": "sha512-Iqv2fzaTQN28s/FwZAoFq0ZSs/7hMAHJVX+w8PZl3cY19Pxk6jFFalxQoIfW2826i/fDLXv8IiEZRIT0lDuWcg==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "dev": true, + "license": "ISC" + }, + "node_modules/is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", + "dev": true, + "license": "MIT" + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-path-inside": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-4.0.0.tgz", + "integrity": "sha512-lJJV/5dYS+RcL8uQdBDW9c9uWFLLBNRyFhnAKXw5tVqLlKZ4RMGZKv+YQ/IA3OhD+RpbJa1LLFM1FQPGyIXvOA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/js-yaml": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/json-parse-even-better-errors": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true, + "license": "MIT" + }, + "node_modules/jsonfile": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.1.tgz", + "integrity": "sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/keyv": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-5.6.0.tgz", + "integrity": "sha512-CYDD3SOtsHtyXeEORYRx2qBtpDJFjRTGXUtmNEMGyzYOKj1TE3tycdlho7kA1Ufx9OYWZzg52QFBGALTirzDSw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@keyv/serialize": "^1.1.1" + } + }, + "node_modules/kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/lines-and-columns": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "dev": true, + "license": "MIT" + }, + "node_modules/lodash.truncate": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/lodash.truncate/-/lodash.truncate-4.4.2.tgz", + "integrity": "sha512-jttmRe7bRse52OsWIMDLaXxWqRAmtIUccAQ3garviCqJjafXOfNMO0yMfNpdD6zbGaTU0P5Nz7e7gAT6cKmJRw==", + "dev": true, + "license": "MIT" + }, + "node_modules/lru-cache": { + "version": "11.5.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.3.tgz", + "integrity": "sha512-U4N8FgzmWxc8k1VH8Kr6lQg18U7Fjvby6wXHVRX/ZZ7IwWbRMgrRbP0Wrb5q5NVinryp4SQampHKdvtecItxUg==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/mathml-tag-names": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mathml-tag-names/-/mathml-tag-names-4.0.0.tgz", + "integrity": "sha512-aa6AU2Pcx0VP/XWnh8IGL0SYSgQHDT6Ucror2j2mXeFAlN3ahaNs8EZtG1YiticMkSLj3Gt6VPFfZogt7G5iFQ==", + "dev": true, + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/mdn-data": { + "version": "2.27.1", + "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.27.1.tgz", + "integrity": "sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ==", + "dev": true, + "license": "CC0-1.0" + }, + "node_modules/meow": { + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/meow/-/meow-14.1.0.tgz", + "integrity": "sha512-EDYo6VlmtnumlcBCbh1gLJ//9jvM/ndXHfVXIFrZVr6fGcwTUyCTFNTLCKuY3ffbK8L/+3Mzqnd58RojiZqHVw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/micromatch": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "dev": true, + "license": "MIT", + "dependencies": { + "braces": "^3.0.3", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/minimatch": { + "version": "10.2.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", + "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.8" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "3.3.20", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.20.tgz", + "integrity": "sha512-uKdg2G3GNCKQn9byYOpxbGqrT2fGO5KRt5J/8b3pok8rT6qxGWF6hxMyJiEYtAf+FVyYuD9hRaDqX5uPFYJ4ZQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/node-sarif-builder": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/node-sarif-builder/-/node-sarif-builder-3.2.0.tgz", + "integrity": "sha512-kVIOdynrF2CRodHZeP/97Rh1syTUHBNiw17hUCIVhlhEsWlfJm19MuO56s4MdKbr22xWx6mzMnNAgXzVlIYM9Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/sarif": "^2.1.7", + "fs-extra": "^11.1.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, + "license": "MIT", + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/parse-json": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", + "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-even-better-errors": "^2.3.0", + "lines-and-columns": "^1.1.6" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/path-scurry": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", + "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.31", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.31.tgz", + "integrity": "sha512-YUbWYCgErwnMAVkjUnYa+KlMBzGpHwiIt/DE7f0G33AuX2LH9UcTk6KgWxd3zRU+pN3u/C+Xl3EnMLM4Hea7Sg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.20", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.2" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/postcss-safe-parser": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/postcss-safe-parser/-/postcss-safe-parser-7.1.0.tgz", + "integrity": "sha512-1WzZxRLaAFwEh6Do+zyGpjWV3nGJNxxhuh7Ubu/q1ICImMgZJnLwhoaEpRbG4pJppp2Y1ncL9ffA2f+LhrefQg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss-safe-parser" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "engines": { + "node": ">=18.0" + }, + "peerDependencies": { + "postcss": "^8.4.31" + } + }, + "node_modules/postcss-selector-parser": { + "version": "7.1.7", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.1.7.tgz", + "integrity": "sha512-wWBro/7hJ40ye6TPiXkUQoXKCw23xSsjriUvqd7Uls9Sso3oLiON1flc18cCdnjgCRXw4fBVqUJJeOdDTh3P3g==", + "dev": true, + "license": "MIT", + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss-value-parser": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz", + "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/qified": { + "version": "0.10.1", + "resolved": "https://registry.npmjs.org/qified/-/qified-0.10.1.tgz", + "integrity": "sha512-+Owyggi9IxT1ePKGafcI87ubSmxol6smwJ+RAHDQlx9+9cPwFWDiKFFCPuWhr9ignlGpZ9vDQLw67N4dcTVFEA==", + "dev": true, + "license": "MIT", + "dependencies": { + "hookified": "^2.1.1" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/qified/node_modules/hookified": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/hookified/-/hookified-2.2.0.tgz", + "integrity": "sha512-p/LgFzRN5FeoD3DLS6bkUapeye6E4SI6yJs6KetENd18S+FBthqYq2amJUWpt5z0EQwwHemidjY5OqJGEKm5uA==", + "dev": true, + "license": "MIT" + }, + "node_modules/queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/reusify": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", + "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", + "dev": true, + "license": "MIT", + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "queue-microtask": "^1.2.2" + } + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/slash": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-5.1.0.tgz", + "integrity": "sha512-ZA6oR3T/pEyuqwMgAKT0/hAv8oAXckzbkmR0UkUosQ+Mc4RxGoJkRmwHgHufaenlyAgE1Mxgpdcrf75y6XcnDg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/slice-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-4.0.0.tgz", + "integrity": "sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "astral-regex": "^2.0.0", + "is-fullwidth-code-point": "^3.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/slice-ansi?sponsor=1" + } + }, + "node_modules/source-map-js": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.2.tgz", + "integrity": "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/string-width": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-8.3.0.tgz", + "integrity": "sha512-ZbmZM0JCihQN91dWnxoipT2KOEyHqEyfRXUyjuRhW8b/xnqPDoq4gWEVApTVa9db2wN8mmoikgFBbjh71+cGeQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "get-east-asian-width": "^1.5.0", + "strip-ansi": "^7.1.2" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/stylelint": { + "version": "17.16.0", + "resolved": "https://registry.npmjs.org/stylelint/-/stylelint-17.16.0.tgz", + "integrity": "sha512-ejlWnbJ13x1GU32+zZXa6sYQJsKKkwSdv2B4rAdO3OWdEmvhSJKcXi5pOphQYSzyaj1kXpuGJr7pcMBNfDIuPw==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/stylelint" + }, + { + "type": "github", + "url": "https://github.com/sponsors/stylelint" + } + ], + "license": "MIT", + "dependencies": { + "@csstools/css-calc": "^3.3.0", + "@csstools/css-parser-algorithms": "^4.0.0", + "@csstools/css-syntax-patches-for-csstree": "^1.1.9", + "@csstools/css-tokenizer": "^4.0.0", + "@csstools/media-query-list-parser": "^5.0.0", + "@csstools/selector-resolve-nested": "^4.0.1", + "@csstools/selector-specificity": "^6.0.0", + "colord": "^2.10.0", + "cosmiconfig": "^9.0.2", + "css-functions-list": "^3.3.3", + "css-tree": "^3.2.1", + "debug": "^4.4.3", + "fast-glob": "^3.3.3", + "fastest-levenshtein": "^1.0.16", + "file-entry-cache": "^11.1.5", + "global-modules": "^2.0.0", + "globby": "^16.2.4", + "globjoin": "^0.1.4", + "html-tags": "^5.1.0", + "ignore": "^7.0.6", + "import-meta-resolve": "^4.2.0", + "mathml-tag-names": "^4.0.0", + "meow": "^14.1.0", + "micromatch": "^4.0.8", + "normalize-path": "^3.0.0", + "picocolors": "^1.1.1", + "postcss": "^8.5.26", + "postcss-safe-parser": "^7.0.1", + "postcss-selector-parser": "^7.1.5", + "postcss-value-parser": "^4.2.0", + "string-width": "^8.2.2", + "supports-hyperlinks": "^4.5.0", + "svg-tags": "^1.0.0", + "table": "^6.9.0", + "write-file-atomic": "^7.0.1" + }, + "bin": { + "stylelint": "bin/stylelint.mjs" + }, + "engines": { + "node": ">=20.19.0" + } + }, + "node_modules/stylelint-config-recommended": { + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/stylelint-config-recommended/-/stylelint-config-recommended-18.0.0.tgz", + "integrity": "sha512-mxgT2XY6YZ3HWWe3Di8umG6aBmWmHTblTgu/f10rqFXnyWxjKWwNdjSWkgkwCtxIKnqjSJzvFmPT5yabVIRxZg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/stylelint" + }, + { + "type": "github", + "url": "https://github.com/sponsors/stylelint" + } + ], + "license": "MIT", + "engines": { + "node": ">=20.19.0" + }, + "peerDependencies": { + "stylelint": "^17.0.0" + } + }, + "node_modules/stylelint-config-standard": { + "version": "40.0.0", + "resolved": "https://registry.npmjs.org/stylelint-config-standard/-/stylelint-config-standard-40.0.0.tgz", + "integrity": "sha512-EznGJxOUhtWck2r6dJpbgAdPATIzvpLdK9+i5qPd4Lx70es66TkBPljSg4wN3Qnc6c4h2n+WbUrUynQ3fanjHw==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/stylelint" + }, + { + "type": "github", + "url": "https://github.com/sponsors/stylelint" + } + ], + "license": "MIT", + "dependencies": { + "stylelint-config-recommended": "^18.0.0" + }, + "engines": { + "node": ">=20.19.0" + }, + "peerDependencies": { + "stylelint": "^17.0.0" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/supports-hyperlinks": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/supports-hyperlinks/-/supports-hyperlinks-4.6.0.tgz", + "integrity": "sha512-FL/q/YePS4tFFTNiBOR+zPk2ZrKheqNtmTJ/9OUZ4e1wKHJofDbGQSmSsxS0XNnabSkqexryCaXGvEKz1vf9UQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^5.0.1", + "supports-color": "^10.2.2" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/chalk/supports-hyperlinks?sponsor=1" + } + }, + "node_modules/supports-hyperlinks/node_modules/has-flag": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-5.0.1.tgz", + "integrity": "sha512-CsNUt5x9LUdx6hnk/E2SZLsDyvfqANZSUq4+D3D8RzDJ2M+HDTIkF60ibS1vHaK55vzgiZw1bEPFG9yH7l33wA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/supports-hyperlinks/node_modules/supports-color": { + "version": "10.2.2", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-10.2.2.tgz", + "integrity": "sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "node_modules/svg-tags": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/svg-tags/-/svg-tags-1.0.0.tgz", + "integrity": "sha512-ovssysQTa+luh7A5Weu3Rta6FJlFBBbInjOh722LIt6klpU2/HtdUbszju/G4devcvk8PGt7FCLv5wftu3THUA==", + "dev": true + }, + "node_modules/table": { + "version": "6.9.0", + "resolved": "https://registry.npmjs.org/table/-/table-6.9.0.tgz", + "integrity": "sha512-9kY+CygyYM6j02t5YFHbNz2FN5QmYGv9zAjVp4lCDjlCw7amdckXlEt/bjMhUIfj4ThGRE4gCUH5+yGnNuPo5A==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "ajv": "^8.0.1", + "lodash.truncate": "^4.4.2", + "slice-ansi": "^4.0.0", + "string-width": "^4.2.3", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/table/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/table/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/table/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/unicorn-magic": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/unicorn-magic/-/unicorn-magic-0.4.1.tgz", + "integrity": "sha512-lzlXPoVB0Uy/FvTaUgX39RbixYiSKoc3JSoSf01+0c2B6FR3qdYIPN7Qjo/AV7n6sqLCk0509cT3LRj1oSZ1+A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/universalify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", + "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "dev": true, + "license": "MIT" + }, + "node_modules/which": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", + "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "which": "bin/which" + } + }, + "node_modules/write-file-atomic": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-7.0.1.tgz", + "integrity": "sha512-OTIk8iR8/aCRWBqvxrzxR0hgxWpnYBblY1S5hDWBQfk/VFmJwzmJgQFN3WsoUKHISv2eAwe+PpbUzyL1CKTLXg==", + "dev": true, + "license": "ISC", + "dependencies": { + "signal-exit": "^4.0.1" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/xml": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/xml/-/xml-1.0.1.tgz", + "integrity": "sha512-huCv9IH9Tcf95zuYCsQraZtWnJvBtLVE0QHMOs8bWyZAFZNDcYjsPq1nEx8jKA9y+Beo9v+7OBPRisQTjinQMw==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/site/package.json b/site/package.json new file mode 100644 index 0000000..ae64eb0 --- /dev/null +++ b/site/package.json @@ -0,0 +1,13 @@ +{ + "name": "llm-test-server-site", + "private": true, + "scripts": { + "lint": "htmlhint index.html && stylelint styles.css tokens.css && node --check site.js", + "build": "python3 build.py" + }, + "devDependencies": { + "htmlhint": "1.9.2", + "stylelint": "17.16.0", + "stylelint-config-standard": "40.0.0" + } +} diff --git a/site/site.js b/site/site.js new file mode 100644 index 0000000..d711556 --- /dev/null +++ b/site/site.js @@ -0,0 +1,100 @@ +// Keep all page content and navigation usable when JavaScript is unavailable. +if (navigator.clipboard && window.isSecureContext) { + for (const button of document.querySelectorAll("[data-copy]")) { + button.hidden = false; + button.addEventListener("click", async () => { + const source = document.getElementById(button.dataset.copy); + const status = document.getElementById("copy-status"); + if (!source || !status) return; + try { + await navigator.clipboard.writeText(source.textContent); + status.textContent = "Code copied to clipboard."; + button.textContent = "Copied"; + window.setTimeout(() => { + button.textContent = "Copy"; + }, 1800); + } catch { + status.textContent = "Copy is unavailable. Select and copy the code directly."; + } + }); + } +} + +// Real image links remain the fallback when the browser cannot open a modal. +const imageViewer = document.querySelector(".image-viewer"); +if (imageViewer && typeof imageViewer.showModal === "function") { + const links = [...document.querySelectorAll('a[href^="assets/"]')].filter(link => link.querySelector("img")); + const shots = [...new Map(links.map(link => [link.href, link])).values()]; + const image = imageViewer.querySelector(".image-viewer-image"); + const caption = imageViewer.querySelector(".image-viewer-caption"); + const position = imageViewer.querySelector(".image-viewer-position"); + const zoom = imageViewer.querySelector(".image-viewer-zoom"); + const stage = imageViewer.querySelector(".image-viewer-stage"); + const setZoom = expanded => { + imageViewer.classList.toggle("image-viewer-expanded", expanded); + zoom.setAttribute("aria-pressed", String(expanded)); + zoom.setAttribute("aria-label", expanded ? "Fit screenshot to preview" : "Show screenshot at 100 percent size"); + zoom.textContent = expanded ? "Fit" : "100%"; + stage.scrollTo(0, 0); + }; + let current = 0; + let opener; + let previousOverflow; + + const showShot = index => { + setZoom(false); + current = (index + shots.length) % shots.length; + const source = shots[current].querySelector("img"); + image.src = shots[current].href; + image.alt = source.alt; + caption.textContent = source.alt; + position.textContent = `${current + 1} / ${shots.length}`; + }; + + for (const link of links) { + link.setAttribute("aria-haspopup", "dialog"); + link.addEventListener("click", event => { + if (event.button !== 0 || event.metaKey || event.ctrlKey || event.shiftKey || event.altKey) return; + event.preventDefault(); + opener = link; + showShot(shots.findIndex(shot => shot.href === link.href)); + previousOverflow = document.body.style.overflow; + document.body.style.overflow = "hidden"; + imageViewer.showModal(); + }); + } + + zoom.addEventListener("click", () => setZoom(zoom.getAttribute("aria-pressed") !== "true")); + imageViewer.querySelector(".image-viewer-close").addEventListener("click", () => imageViewer.close()); + imageViewer.querySelector(".image-viewer-previous").addEventListener("click", () => showShot(current - 1)); + imageViewer.querySelector(".image-viewer-next").addEventListener("click", () => showShot(current + 1)); + imageViewer.addEventListener("keydown", event => { + if (event.key === "Tab") { + const buttons = [...imageViewer.querySelectorAll('button, [tabindex="0"]')]; + const first = buttons[0]; + const last = buttons[buttons.length - 1]; + if (event.shiftKey && document.activeElement === first) { + event.preventDefault(); + last.focus(); + } else if (!event.shiftKey && document.activeElement === last) { + event.preventDefault(); + first.focus(); + } + return; + } + if (event.altKey || event.ctrlKey || event.metaKey || event.shiftKey) return; + if (event.target === stage && imageViewer.classList.contains("image-viewer-expanded")) return; + if (event.key === "ArrowLeft" || event.key === "ArrowRight") { + event.preventDefault(); + showShot(current + (event.key === "ArrowRight" ? 1 : -1)); + } + }); + imageViewer.addEventListener("click", event => { + const bounds = imageViewer.getBoundingClientRect(); + if (event.target === imageViewer && (event.clientX < bounds.left || event.clientX > bounds.right || event.clientY < bounds.top || event.clientY > bounds.bottom)) imageViewer.close(); + }); + imageViewer.addEventListener("close", () => { + document.body.style.overflow = previousOverflow; + opener?.focus({ preventScroll: true }); + }); +} diff --git a/site/stylelint.config.mjs b/site/stylelint.config.mjs new file mode 100644 index 0000000..fe3ed9c --- /dev/null +++ b/site/stylelint.config.mjs @@ -0,0 +1,8 @@ +export default { + extends: ['stylelint-config-standard'], + rules: { + // Component rules are scoped by their own class (.legend, .sankey, .capability…) and never + // target the same element; source order follows the page, not global specificity. + 'no-descending-specificity': null, + }, +}; diff --git a/site/styles.css b/site/styles.css index 086d325..3b739a2 100644 --- a/site/styles.css +++ b/site/styles.css @@ -1,26 +1,39 @@ +* { + box-sizing: border-box; +} + :root { - color-scheme: dark; - --bg: #0b1014; - --panel: #111a20; - --panel-strong: #16242d; - --text: #ecf4f5; - --muted: #9fb0b4; - --line: #28424b; - --teal: #34d4c4; - --blue: #7bb6ff; - --amber: #f0ba5a; + --ink: #161616; + --ink-soft: #2a2928; + --surface: #fff; + --radius-card: 18px; + --shadow-card: 0 1px 2px rgb(22 22 22 / 4%), 0 12px 40px rgb(22 22 22 / 6%); + --shadow-float: 0 2px 6px rgb(22 22 22 / 6%), 0 22px 60px rgb(22 22 22 / 14%); + --chart-ink: #161616; + --chart-reasoning: #a39d96; + --graphite: #5b5753; + --stone: #a39d96; + --sand: #d4cbbf; + --cream: #ece6de; + --tint: #f3efea; + --coral: #f2805a; + --lime-deep: #5aa02c; + --lime-line: #8fd14f; } -* { - box-sizing: border-box; +html { + scroll-behavior: smooth; + scroll-padding-top: 96px; } body { margin: 0; - background: var(--bg); color: var(--text); - font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; - line-height: 1.55; + background: var(--canvas); + font-family: -apple-system, BlinkMacSystemFont, "SF Pro Text", "Helvetica Neue", sans-serif; + font-size: 16px; + line-height: 1.6; + -webkit-font-smoothing: antialiased; } a { @@ -28,247 +41,2688 @@ a { text-decoration: none; } -.topbar { - align-items: center; - background: color-mix(in srgb, var(--bg) 86%, transparent); - border-bottom: 1px solid var(--line); - display: flex; - gap: 24px; - justify-content: space-between; - min-height: 64px; - padding: 0 32px; - position: sticky; - top: 0; - z-index: 3; +button { + font: inherit; } -.brand { - font-weight: 800; - letter-spacing: 0; +button, +a { + -webkit-tap-highlight-color: transparent; } -nav { - display: flex; - gap: 18px; - color: var(--muted); - font-size: 14px; +:where(a, button, summary):focus-visible { + outline: 3px solid var(--focus); + outline-offset: 5px; + border-radius: 5px; } -.hero { - display: grid; - gap: 36px; - grid-template-columns: minmax(0, 0.9fr) minmax(360px, 1.1fr); - min-height: calc(100vh - 64px); - padding: 64px 48px 48px; - align-items: center; +a:hover { + color: var(--link); } -.hero-copy { - max-width: 680px; +img { + display: block; + max-width: 100%; + height: auto; } -.eyebrow { - color: var(--teal); - font-size: 13px; - font-weight: 700; - letter-spacing: 0.12em; - margin: 0 0 12px; - text-transform: uppercase; +figure, +h1, +h2, +h3, +p, +ul, +ol { + margin: 0; +} + +ul, +ol { + padding: 0; + list-style: none; +} + +code, +pre { + font-family: var(--code-font); +} + +pre { + margin: 0; } h1, h2, -h3, -p { - margin-top: 0; +h3 { + text-wrap: balance; } h1 { - font-size: clamp(42px, 6vw, 74px); - line-height: 0.98; - letter-spacing: 0; - margin-bottom: 22px; + font-size: clamp(60px, 6.6vw, 98px); + line-height: 0.99; + letter-spacing: -0.062em; + font-weight: 650; } h2 { - font-size: clamp(30px, 4vw, 48px); - line-height: 1.05; - letter-spacing: 0; - margin-bottom: 18px; + font-size: clamp(38px, 4.1vw, 60px); + line-height: 1.06; + letter-spacing: -0.048em; + font-weight: 620; } h3 { - font-size: 20px; - letter-spacing: 0; - margin-bottom: 8px; + font-size: 21px; + line-height: 1.25; + letter-spacing: -0.025em; + font-weight: 620; +} + +.section-wrap { + width: min(1240px, calc(100% - 96px)); + margin-inline: auto; } -.lede { +.eyebrow { + margin-bottom: 20px; color: var(--muted); - font-size: 20px; - max-width: 620px; + font-size: 11px; + font-weight: 650; + letter-spacing: 0.1em; + text-transform: uppercase; } -.actions { +.section-intro > :where(p:not(.eyebrow)) { + max-width: 440px; + margin-top: 20px; + color: var(--muted); +} + +.section-heading { + display: grid; + grid-template-columns: 1.2fr 1fr; + gap: 90px; + align-items: end; + margin-bottom: 46px; +} + +.section-heading > :where(p), +.section-heading > div > :where(p:not(.eyebrow)) { + max-width: 420px; + color: var(--muted); + font-size: 15px; +} + +/* Header */ + +.topbar { + position: sticky; + z-index: 10; + top: 0; display: flex; - flex-wrap: wrap; - gap: 12px; - margin-top: 30px; + gap: 28px; + align-items: center; + justify-content: space-between; + height: 72px; + padding-inline: max(32px, calc((100% - 1240px) / 2)); + border-bottom: 1px solid var(--line-soft); + background: rgb(249 247 245 / 86%); + backdrop-filter: saturate(160%) blur(18px); } -.primary, -.secondary { +.topbar::after { + content: ""; + position: absolute; + right: 0; + bottom: -1px; + left: 0; + height: 1px; + background: linear-gradient( + 90deg, + transparent 4%, + rgb(164 242 88 / 70%) 30%, + rgb(219 50 118 / 45%) 66%, + rgb(255 126 93 / 50%) 84%, + transparent 98% + ); + pointer-events: none; +} + +.brand { + display: inline-flex; + gap: 11px; align-items: center; - border: 1px solid var(--line); + font-size: 16px; + font-weight: 620; + letter-spacing: -0.025em; + white-space: nowrap; +} + +.brand-mark { + width: 32px; + height: 32px; border-radius: 8px; + box-shadow: 0 4px 12px rgb(22 22 22 / 18%); +} + +nav { + display: flex; + gap: 28px; + align-items: center; + margin-left: auto; + color: var(--text-soft); + font-size: 13px; +} + +/* Buttons and links */ + +.button { display: inline-flex; - font-weight: 700; - min-height: 44px; - padding: 0 18px; + gap: 18px; + align-items: center; + justify-content: center; + min-height: 50px; + padding: 12px 22px; + border-radius: 12px; + color: var(--ink); + background: var(--accent); + box-shadow: + 0 6px 18px rgb(105 156 53 / 22%), + inset 0 1px rgb(255 255 255 / 45%); + font-size: 14px; + font-weight: 620; + line-height: 1.4; + transition: + background 0.15s ease, + transform 0.15s ease; } -.primary { - background: var(--teal); - border-color: var(--teal); - color: #06110f; +.button:hover { + color: var(--ink); + background: #b2fa68; + transform: translateY(-1px); } -.secondary { - background: #132029; - color: var(--text); +.button-small { + min-height: 38px; + padding: 8px 14px; + border-radius: 9px; + font-size: 12px; } -.hero-art { - aspect-ratio: 16 / 10; - border: 1px solid var(--line); - border-radius: 8px; - box-shadow: 0 30px 80px rgb(0 0 0 / 42%); - display: block; - height: auto; - object-fit: cover; - width: 100%; +.actions { + display: flex; + flex-wrap: wrap; + gap: 26px; + align-items: center; } -.band, -.split, -.start { - border-top: 1px solid var(--line); - padding: 72px 48px; +.text-link, +.inline-link { + display: inline-flex; + gap: 10px; + align-items: center; + color: var(--link); + font-size: 14px; + font-weight: 550; } -.section-heading, -.split > div, -.start { - max-width: 1040px; +.inline-link { + margin-top: 18px; } -.grid { +/* Hero */ + +.hero { display: grid; - gap: 14px; - grid-template-columns: repeat(3, minmax(0, 1fr)); - max-width: 1120px; + grid-template-columns: 0.92fr 1.08fr; + gap: 56px; + align-items: center; + min-height: 720px; + padding-block: 72px 88px; } -article { - background: var(--panel); +.hero-pill { + display: inline-flex; + gap: 8px; + align-items: center; + margin-bottom: 26px; + padding: 6px 12px; border: 1px solid var(--line); - border-radius: 8px; - min-height: 168px; - padding: 22px; + border-radius: 999px; + color: var(--text-soft); + background: rgb(255 255 255 / 70%); + font-size: 12px; + font-weight: 550; } -article span { - color: var(--amber); - display: block; - font-size: 13px; - font-weight: 800; - margin-bottom: 22px; +.status-dot, +.caption-dot { + display: inline-block; + flex-shrink: 0; + width: 7px; + height: 7px; + border-radius: 50%; + background: #5bc97a; + box-shadow: 0 0 0 3px rgb(91 201 122 / 18%); +} + +.accent-text { + color: var(--text); + background: linear-gradient(transparent 80%, var(--accent) 80%, var(--accent) 94%, transparent 94%); +} + +.hero-definition { + margin-top: 28px; + font-size: 21px; + font-weight: 550; + line-height: 1.4; + letter-spacing: -0.025em; } -article p, -.split p { +.hero-lede { + max-width: 460px; + margin-top: 14px; color: var(--muted); + font-size: 16px; + line-height: 1.7; } -.split { - display: grid; - gap: 40px; - grid-template-columns: minmax(0, 0.9fr) minmax(320px, 1fr); +.hero .actions { + margin-top: 30px; + gap: 24px; } -.package-list { - display: grid; +.install-chip { + display: inline-flex; gap: 10px; - list-style: none; - margin: 0; - padding: 0; + align-items: center; + max-width: 100%; + margin-top: 30px; + padding: 6px 6px 6px 14px; + border-radius: 11px; + color: #e9e6e2; + background: var(--ink); + box-shadow: 0 8px 24px rgb(22 22 22 / 14%); } -.package-list li { - align-items: center; - background: var(--panel-strong); +.install-chip code { + min-width: 0; + overflow: hidden; + font-size: 12px; + text-overflow: ellipsis; + white-space: nowrap; +} + +.install-prompt { + color: var(--accent); + font-family: var(--code-font); + font-size: 12px; +} + +.install-chip .copy-button { + color: var(--ink); + background: var(--accent); +} + +.hero-stage { + position: relative; + min-width: 0; + padding: 26px 0 30px 26px; +} + +.hero-stage::before { + content: ""; + position: absolute; + z-index: -1; + inset: -40px -60px -20px 0; + background: + radial-gradient(closest-side at 70% 40%, rgb(164 242 88 / 22%), transparent), + radial-gradient(closest-side at 30% 80%, rgb(255 126 93 / 14%), transparent); + filter: blur(10px); +} + +.hero-window { + overflow: hidden; border: 1px solid var(--line); - border-radius: 8px; + border-radius: 16px; + background: var(--surface); + box-shadow: var(--shadow-float); +} + +.window-bar { display: flex; - gap: 16px; + gap: 8px; + align-items: center; justify-content: space-between; - min-height: 68px; - padding: 16px 18px; + height: 36px; + padding-inline: 13px; + border-bottom: 1px solid var(--line-soft); + color: var(--muted); + background: #fbfaf8; + font-size: 10px; +} + +.window-dots { + display: flex; + gap: 5px; +} + +.window-dots i { + width: 8px; + height: 8px; + border-radius: 50%; + background: #dad6d1; +} + +.window-dots i:first-child { + background: #ff7e5d; +} + +.window-dots i:nth-child(2) { + background: #e8b445; +} + +.window-dots i:nth-child(3) { + background: #a4f258; +} + +.window-label { + font-family: var(--code-font); + font-size: 9px; + letter-spacing: 0.1em; +} + +.screenshot-link { + display: block; +} + +.float-card { + position: absolute; + display: grid; + gap: 4px; + padding: 12px 14px; + border: 1px solid rgb(255 255 255 / 80%); + border-radius: 14px; + background: rgb(255 255 255 / 86%); + box-shadow: var(--shadow-float); + backdrop-filter: blur(14px) saturate(160%); +} + +.float-request { + top: 0; + left: 0; + display: flex; + gap: 8px; + align-items: center; + padding: 9px 12px; + font-size: 11px; +} + +.float-request code { + font-size: 11px; +} + +.float-method { + padding: 2px 6px; + border-radius: 5px; + color: #71398c; + background: rgb(137 68 171 / 10%); + font-size: 10px; + font-weight: 650; +} + +.float-status { + padding: 2px 7px; + border-radius: 999px; + color: #337c49; + background: rgb(91 201 122 / 16%); + font-size: 10px; + font-weight: 650; +} + +.float-cache { + right: -18px; + bottom: 0; + display: flex; + gap: 14px; + align-items: center; + width: 272px; + padding: 12px 16px 12px 12px; +} + +.float-cache .ring-center strong { + font-size: 19px; +} + +.float-cache .ring-center span { + font-size: 9.5px; +} + +.float-copy { + display: grid; + gap: 3px; } -.package-list span { +.float-kicker { + color: var(--text); + font-size: 12px; + font-weight: 650; +} + +.float-note { color: var(--muted); + font-size: 11px; + line-height: 1.4; } -pre { - background: #071015; - border: 1px solid var(--line); - border-radius: 8px; - color: var(--blue); - overflow-x: auto; - padding: 22px; +.hero-stage figcaption { + display: flex; + gap: 8px; + align-items: center; + margin-top: 16px; + color: var(--muted); + font-size: 11px; +} + +.hero-stage .caption-dot { + width: 5px; + height: 5px; + box-shadow: none; +} + +/* Provider strip */ + +.provider-strip { + padding-block: 26px; + border-block: 1px solid var(--line-soft); + background: rgb(255 255 255 / 55%); +} + +.provider-strip p { + margin-bottom: 14px; + color: var(--muted); + font-size: 12px; + text-align: center; +} + +.provider-strip ul { + display: flex; + flex-wrap: wrap; + gap: 8px 10px; + justify-content: center; +} + +.provider-strip li { + padding: 5px 12px; + border: 1px solid var(--line-soft); + border-radius: 999px; + color: var(--text-soft); + background: var(--surface); + font-size: 13px; + font-weight: 550; + letter-spacing: -0.01em; +} + +/* Workflow */ + +.workflow { + padding-block: 96px 100px; +} + +.workflow-flow { + --flow-gap: 64px; + + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: var(--flow-gap); + margin-top: 52px; +} + +.flow-node { + position: relative; + display: flex; + flex-direction: column; + min-width: 0; + padding: 24px; + border: 1px solid var(--line-soft); + border-radius: var(--radius-card); + background: var(--surface); + box-shadow: var(--shadow-card); +} + +.flow-glyph { + display: grid; + place-items: center; + width: 40px; + height: 40px; + margin-bottom: 18px; + border-radius: 50%; + color: var(--accent); + background: var(--ink); } -code { - font-family: "SFMono-Regular", Consolas, monospace; +.flow-glyph svg { + width: 20px; + height: 20px; + fill: none; + stroke: currentcolor; + stroke-width: 1.8; + stroke-linecap: round; + stroke-linejoin: round; +} + +.flow-node p { + margin-top: 10px; + margin-bottom: 20px; + color: var(--muted); font-size: 14px; } -@media (max-width: 860px) { - .topbar { - align-items: flex-start; - flex-direction: column; - gap: 8px; - padding: 14px 20px; +.flow-node p code { + color: var(--text-soft); + font-size: 12px; +} + +.flow-link { + position: absolute; + top: 43px; + left: 100%; + width: var(--flow-gap); + height: 2px; + background: repeating-linear-gradient(90deg, var(--graphite) 0 5px, transparent 5px 10px); +} + +.flow-link::after { + content: ""; + position: absolute; + top: 50%; + right: 3px; + width: 8px; + height: 8px; + border-top: 2px solid var(--graphite); + border-right: 2px solid var(--graphite); + transform: translateY(-50%) rotate(45deg); +} + +.flow-link i { + position: absolute; + top: 50%; + left: 0; + width: 8px; + height: 8px; + margin-top: -4px; + border-radius: 50%; + background: var(--accent); + box-shadow: 0 0 0 3px rgb(164 242 88 / 28%); + animation: flow-x 2.6s ease-in-out infinite; +} + +@keyframes flow-x { + 0% { + opacity: 0; + transform: translateX(0); } - nav { - flex-wrap: wrap; + 15%, + 80% { + opacity: 1; } - .hero, - .split { - grid-template-columns: 1fr; + 100% { + opacity: 0; + transform: translateX(calc(var(--flow-gap) - 16px)); } +} - .hero { - min-height: auto; - padding: 46px 20px 32px; +@keyframes flow-y { + 0% { + opacity: 0; + transform: translateY(0); } - .band, - .split, - .start { - padding: 56px 20px; + 15%, + 80% { + opacity: 1; } - .grid { - grid-template-columns: 1fr; + 100% { + opacity: 0; + transform: translateY(calc(var(--flow-gap) - 16px)); } +} - .package-list li { - align-items: flex-start; - flex-direction: column; - gap: 4px; +.step-code { + margin-top: auto; + padding: 14px 16px; + overflow-x: auto; + border-radius: 12px; + color: #e9e6e2; + background: var(--ink); + font-size: 11.5px; + line-height: 1.75; +} + +.code-keyword { + color: #c39bff; +} + +.code-string { + color: #b9f27a; +} + +.code-comment { + color: #8f8a84; +} + +/* Dashboard */ + +.dashboard-section { + padding-block: 96px 80px; + border-block: 1px solid var(--line-soft); + background: linear-gradient(180deg, #f3efea, var(--canvas) 40%); +} + +.dashboard-figure { + overflow: hidden; + border: 1px solid var(--line); + border-radius: var(--radius-card); + background: var(--surface); + box-shadow: var(--shadow-card); +} + +.dashboard-figure > a { + display: block; +} + +.dashboard-figure figcaption { + padding: 12px 20px; + border-top: 1px solid var(--line-soft); + color: var(--muted); + font-size: 12px; +} + +.detail-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 42px; + margin-top: 64px; +} + +.detail-grid .eyebrow { + margin-bottom: 12px; +} + +.detail-grid :where(p:not(.eyebrow)) { + max-width: 480px; + margin-top: 12px; + color: var(--muted); + font-size: 14px; +} + +.detail-image { + display: block; + margin-top: 24px; + overflow: hidden; + border: 1px solid var(--line); + border-radius: 14px; + background: var(--surface); + box-shadow: var(--shadow-card); +} + +.lifecycle { + margin-top: 84px; +} + +.lifecycle-heading { + max-width: 640px; + margin-bottom: 26px; +} + +.lifecycle-heading .eyebrow { + margin-bottom: 12px; +} + +.lifecycle-heading h3 { + font-size: 30px; + letter-spacing: -0.03em; +} + +.lifecycle-heading p { + margin-top: 10px; + color: var(--muted); + font-size: 15px; +} + +.lifecycle-canvas { + padding: 30px; + border: 1px solid var(--line-soft); + border-radius: var(--radius-card); + background-color: var(--surface); + background-image: radial-gradient(circle, rgb(22 22 22 / 8%) 1px, transparent 1.3px); + background-size: 18px 18px; + box-shadow: var(--shadow-card); +} + +.lifecycle-svg { + display: block; + width: 100%; + height: auto; + overflow: visible; +} + +.lifecycle-tall { + display: none; +} + +.lc-link { + color: var(--graphite); + fill: none; + stroke: var(--graphite); + stroke-width: 1.6; + stroke-dasharray: 5 5; +} + +.lc-link-return { + color: var(--lime-deep); + stroke: var(--lime-deep); +} + +.lc-node rect { + fill: #fff; + stroke: rgb(22 22 22 / 14%); +} + +.lc-node-core rect { + fill: var(--ink); + stroke: var(--ink); +} + +.lc-node-core .lc-chip { + fill: rgb(164 242 88 / 12%); + stroke: rgb(164 242 88 / 55%); +} + +.lc-title { + fill: var(--text); + font-size: 15px; + font-weight: 650; +} + +.lc-node-core .lc-title { + fill: #fff; +} + +.lc-sub { + fill: var(--muted); + font-size: 12.5px; +} + +.lc-mono { + fill: var(--text-soft); + font-family: var(--code-font); + font-size: 12px; +} + +.lc-chip { + fill: var(--tint); +} + +.lc-chip-text { + fill: var(--text-soft); + font-size: 11.5px; + font-weight: 550; +} + +.lc-node-core .lc-chip-text { + fill: var(--accent); +} + +.lc-pill rect { + fill: #fff; + stroke: rgb(22 22 22 / 12%); +} + +.lc-pill text, +.lc-side { + fill: var(--text-soft); + font-size: 11.5px; + font-weight: 550; +} + +.lc-packet { + opacity: 0; + fill: none; + stroke: #76c043; + stroke-width: 8; + stroke-linecap: round; + stroke-dasharray: 2 200; + stroke-dashoffset: 2; + animation: lc-packet-1 7s linear infinite; +} + +.lc-packet-2 { + animation-name: lc-packet-2; +} + +.lc-packet-3 { + animation-name: lc-packet-3; +} + +@keyframes lc-packet-1 { + 0% { + opacity: 1; + stroke-dashoffset: 2; + } + + 30% { + opacity: 1; + stroke-dashoffset: -100; + } + + 31%, + 100% { + opacity: 0; + stroke-dashoffset: -100; + } +} + +@keyframes lc-packet-2 { + 0%, + 32% { + opacity: 0; + stroke-dashoffset: 2; + } + + 33% { + opacity: 1; + stroke-dashoffset: 2; + } + + 64% { + opacity: 1; + stroke-dashoffset: -100; + } + + 65%, + 100% { + opacity: 0; + stroke-dashoffset: -100; + } +} + +@keyframes lc-packet-3 { + 0%, + 36% { + opacity: 0; + stroke-dashoffset: 2; + } + + 37% { + opacity: 1; + stroke-dashoffset: 2; + } + + 70% { + opacity: 1; + stroke-dashoffset: -100; + } + + 71%, + 100% { + opacity: 0; + stroke-dashoffset: -100; + } +} + +/* Token analytics */ + +.analytics-section { + position: relative; + padding-block: 104px; + overflow: hidden; + color: #f3f1ee; + background: + radial-gradient(900px 500px at 8% 0%, rgb(164 242 88 / 12%), transparent 65%), + radial-gradient(800px 480px at 100% 10%, rgb(219 50 118 / 14%), transparent 60%), + var(--ink); +} + +.analytics-section .eyebrow { + color: var(--accent); +} + +.section-heading-dark > :where(p), +.section-heading-dark > div > :where(p:not(.eyebrow)) { + color: rgb(243 241 238 / 72%); +} + +.analytics-section .inline-link { + color: var(--accent); +} + +.chart-card { + min-width: 0; + padding: 26px; + border-radius: 22px; + color: var(--text); + background: var(--surface); + box-shadow: 0 30px 80px rgb(0 0 0 / 35%); +} + +.chart-card-header { + display: flex; + gap: 16px; + align-items: flex-start; + justify-content: space-between; + margin-bottom: 22px; +} + +.chart-card-header h3 { + font-size: 18px; +} + +.chart-card-header p { + margin-top: 4px; + color: var(--muted); + font-size: 13px; + line-height: 1.5; +} + +.sankey { + display: grid; + grid-template-columns: minmax(104px, 1fr) minmax(0, 2.8fr) minmax(104px, 1fr); + height: 390px; +} + +.sankey-labels { + position: relative; +} + +.sankey-labels li { + position: absolute; + display: grid; + line-height: 1.2; + transform: translateY(-50%); +} + +.sankey-labels span { + color: var(--text-soft); + font-size: 13px; +} + +.sankey-labels strong { + font-size: 13px; + font-weight: 650; + font-variant-numeric: tabular-nums; +} + +.sankey-labels-left li { + right: 14px; + text-align: right; +} + +.sankey-labels-right li { + left: 14px; +} + +.sankey-plot { + position: relative; +} + +.sankey-plot svg { + position: absolute; + inset: 0; + width: 100%; + height: 100%; +} + +.sankey-node { + position: absolute; + width: 10px; + border-radius: 3px; +} + +.sankey-node-left { + left: 0; +} + +.sankey-node-right { + right: 0; +} + +.sankey-node-center { + left: 50%; + width: 14px; + background: var(--chart-ink); + transform: translateX(-50%); +} + +.sankey-center-label { + position: absolute; + left: 50%; + display: grid; + padding: 2px 8px; + border-radius: 8px; + background: rgb(255 255 255 / 88%); + line-height: 1.2; + text-align: center; + transform: translate(-50%, calc(-100% - 10px)); +} + +.sankey-center-label span { + color: var(--muted); + font-size: 12px; +} + +.sankey-center-label strong { + font-size: 15px; + font-weight: 650; + font-variant-numeric: tabular-nums; +} + +.chart-grid { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 20px; + margin-top: 20px; +} + +.donut-wrap { + display: grid; + gap: 22px; +} + +.donut { + position: relative; + width: 168px; + height: 168px; + margin-inline: auto; +} + +.donut svg { + display: block; + width: 100%; + height: 100%; +} + +.donut-center { + position: absolute; + inset: 0; + display: grid; + place-content: center; + line-height: 1.15; + text-align: center; +} + +.donut-center strong { + font-size: 24px; + font-weight: 650; + letter-spacing: -0.03em; + font-variant-numeric: tabular-nums; +} + +.donut-center span { + color: var(--muted); + font-size: 11px; +} + +.legend { + display: grid; + gap: 2px; +} + +.legend li { + display: grid; + grid-template-columns: 10px minmax(0, 1fr) auto 50px; + gap: 10px; + align-items: center; + padding-block: 4px; + font-size: 13px; +} + +.legend span:not(.swatch) { + overflow: hidden; + color: var(--text-soft); + text-overflow: ellipsis; + white-space: nowrap; +} + +.legend strong { + font-weight: 650; + font-variant-numeric: tabular-nums; + text-align: right; +} + +.legend em { + color: var(--muted); + font-size: 12px; + font-style: normal; + font-variant-numeric: tabular-nums; + text-align: right; +} + +.swatch { + width: 10px; + height: 10px; + border-radius: 3px; +} + +.swatch-ink { + background: var(--chart-ink); +} + +.swatch-reasoning { + background: var(--chart-reasoning); +} + +.legend-inline { + display: flex; + flex-wrap: wrap; + gap: 4px 16px; + margin-top: 12px; +} + +.legend-inline li { + display: inline-flex; + gap: 7px; + padding: 0; + font-size: 12px; +} + +.growth { + display: grid; + grid-template-columns: repeat(4, minmax(0, 1fr)); + gap: 14px; + align-items: end; + height: 168px; + padding-bottom: 2px; + border-bottom: 1px solid var(--line); +} + +.growth li { + display: flex; + flex-direction: column; + align-items: center; + justify-content: flex-end; + height: 100%; +} + +.growth-total { + margin-bottom: 6px; + font-size: 12px; + font-weight: 650; + font-variant-numeric: tabular-nums; +} + +.growth-bar { + position: relative; + display: flex; + flex-direction: column; + justify-content: flex-end; + width: 100%; + max-width: 44px; + max-height: calc(100% - 46px); + overflow: hidden; + border-radius: 6px 6px 2px 2px; + background: #d6d1cb; +} + +.growth-cached { + display: block; + width: 100%; + border-top: 2px solid var(--surface); + background: #5aa02c; +} + +.growth-index { + order: 3; + margin-top: 6px; + color: var(--muted); + font-size: 11px; +} + +.output-split { + margin-top: 22px; + padding-top: 18px; + border-top: 1px solid var(--line-soft); +} + +.output-split p { + display: flex; + justify-content: space-between; + margin-bottom: 10px; + font-size: 13px; +} + +.output-split p span { + color: var(--text-soft); +} + +.output-split p strong { + font-weight: 650; +} + +.output-bar { + display: flex; + gap: 2px; + height: 10px; +} + +.output-bar > span { + height: 100%; +} + +.output-visible { + border-radius: 999px 2px 2px 999px; + background: var(--chart-ink); +} + +.output-reasoning { + border-radius: 2px 999px 999px 2px; + background: var(--chart-reasoning); +} + +.output-split .legend-inline strong { + margin-left: 2px; +} + +.chart-note { + max-width: 760px; + margin-top: 22px; + color: rgb(243 241 238 / 62%); + font-size: 13px; +} + +.example-chip { + flex: none; + padding: 4px 10px; + border: 1px solid rgb(255 255 255 / 22%); + border-radius: 999px; + color: rgb(243 241 238 / 80%); + font-size: 10.5px; + font-weight: 650; + letter-spacing: 0.06em; + text-transform: uppercase; +} + +.example-chip-light { + border-color: var(--line); + color: var(--muted); +} + +.ring { + position: relative; + flex: none; + margin-inline: auto; +} + +.ring svg { + display: block; + width: 100%; + height: 100%; + transform: rotate(-90deg); +} + +.ring-track, +.ring-arc { + fill: none; +} + +.ring-track { + stroke: var(--cream); +} + +.ring-center { + position: absolute; + inset: 0; + display: grid; + place-content: center; + line-height: 1.1; + text-align: center; +} + +.ring-center strong { + font-size: 30px; + font-weight: 650; + letter-spacing: -0.035em; + font-variant-numeric: tabular-nums; +} + +.ring-center span { + color: var(--muted); + font-size: 11px; +} + +.insights-block { + margin-top: 72px; +} + +.insights-heading { + display: flex; + gap: 20px; + align-items: flex-end; + justify-content: space-between; + margin-bottom: 22px; +} + +.insights-heading h3 { + color: #fff; + font-size: 26px; + letter-spacing: -0.03em; +} + +.insights-heading p { + margin-top: 6px; + color: rgb(243 241 238 / 72%); + font-size: 14px; +} + +.insight-grid { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 20px; +} + +.insight-card { + display: flex; + flex-direction: column; + min-height: 372px; +} + +.insight-card .chart-card-header { + margin-bottom: 14px; +} + +.insight-visual { + display: flex; + flex: 1; + flex-direction: column; + gap: 16px; + justify-content: center; + min-height: 0; +} + +.capsules { + display: grid; + gap: 8px; +} + +.capsule { + position: relative; + display: flex; + align-items: center; + justify-content: space-between; + height: 36px; + padding-inline: 14px; + overflow: hidden; + border-radius: 999px; + background: var(--tint); + font-size: 13px; +} + +.capsule-fill { + position: absolute; + inset: 0 auto 0 0; + border-radius: 999px; + background: var(--sand); +} + +.capsule-accent .capsule-fill { + background: var(--accent); +} + +.capsule-label, +.capsule strong { + position: relative; +} + +.capsule-label { + color: var(--text); + font-weight: 550; +} + +.capsule strong { + font-weight: 650; + font-variant-numeric: tabular-nums; +} + +.dots { + display: block; + width: 100%; + max-width: 300px; + height: auto; + margin-inline: auto; +} + +.legend-compact { + grid-template-columns: repeat(2, minmax(0, 1fr)); + column-gap: 16px; + margin-top: 16px; +} + +.legend-single { + grid-template-columns: minmax(0, 1fr); +} + +.legend-compact li, +.legend-ops li { + grid-template-columns: 10px minmax(0, 1fr) auto; + padding-block: 3px; + font-size: 12px; +} + +.swatch-round { + border-radius: 50%; +} + +.status-chips { + display: flex; + flex-wrap: wrap; + gap: 6px; + justify-content: center; +} + +.status-chips li { + display: inline-flex; + gap: 6px; + align-items: center; + padding: 4px 10px; + border-radius: 999px; + color: var(--text-soft); + background: var(--tint); + font-size: 12px; +} + +.status-chips strong { + font-weight: 650; +} + +.status-dot-sm { + width: 7px; + height: 7px; + border-radius: 50%; +} + +.pillcols { + display: grid; + grid-template-columns: repeat(4, minmax(0, 1fr)); + gap: 12px; + align-items: end; + height: 220px; +} + +.pillcol { + display: grid; + grid-template-rows: auto minmax(0, 1fr) auto; + gap: 7px; + justify-items: center; + height: 100%; +} + +.pillcol strong { + font-size: 13px; + font-weight: 650; + font-variant-numeric: tabular-nums; +} + +.pillcol-track { + display: flex; + align-items: flex-end; + width: 40px; + height: 100%; + overflow: hidden; + border-radius: 999px; + background: var(--tint); +} + +.pillcol-fill { + width: 100%; + border-radius: 999px; + background: var(--graphite); +} + +.pillcol-accent .pillcol-fill { + background: var(--accent); +} + +.pillcol-label { + color: var(--muted); + font-size: 11.5px; + white-space: nowrap; +} + +.segbar { + display: flex; + gap: 3px; + height: 16px; +} + +.segbar span { + flex-basis: 0; + min-width: 4px; + border-radius: 999px; +} + +.legend-ops code { + overflow: hidden; + color: var(--text-soft); + font-size: 12px; + text-overflow: ellipsis; + white-space: nowrap; +} + +.bubbles { + display: block; + width: 100%; + max-width: 250px; + height: auto; + margin-inline: auto; +} + +.bubble-sand { + fill: #e3dbd0; +} + +.bubble-ink { + fill: var(--ink); +} + +.bubble-text { + fill: var(--text); + font-size: 18px; + font-weight: 650; +} + +.bubble-text-light { + fill: var(--accent); +} + +.legend-center { + justify-content: center; +} + +/* Capabilities */ + +.capabilities { + padding-block: 104px; +} + +.capability-grid { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 24px; + margin-top: 52px; +} + +.capability { + display: flex; + flex-direction: column; + min-width: 0; + padding-bottom: 24px; + overflow: hidden; + border: 1px solid var(--line-soft); + border-radius: var(--radius-card); + background: var(--surface); + box-shadow: var(--shadow-card); +} + +.capability h3 { + margin-top: 22px; + padding-inline: 24px; + font-size: 19px; +} + +.capability p { + margin-top: 8px; + padding-inline: 24px; + color: var(--muted); + font-size: 14px; +} + +.capability .inline-link { + margin-top: auto; + padding: 16px 24px 0; +} + +.capability-visual { + display: grid; + place-items: center; + min-height: 204px; + padding: 16px 12px; + border-bottom: 1px solid var(--line-soft); +} + +.cap-svg { + display: block; + width: 100%; + max-width: 360px; + height: auto; + overflow: visible; + color: var(--graphite); +} + +.cap-text { + fill: var(--text-soft); + font-size: 12.5px; + font-weight: 550; +} + +.cap-small { + font-size: 11px; +} + +.cap-muted { + fill: var(--muted); +} + +.cap-mono { + fill: var(--text); + font-family: var(--code-font); + font-size: 11.5px; +} + +.cap-on-ink { + fill: var(--accent); +} + +.cap-link { + fill: none; + stroke: var(--graphite); + stroke-width: 1.6; + stroke-dasharray: 4 4; +} + +.cap-node { + fill: var(--ink); + stroke: #fff; + stroke-width: 3; +} + +.cap-node-accent { + fill: var(--accent); + stroke: var(--ink); + stroke-width: 2; +} + +.cap-axis { + stroke: var(--line); + stroke-width: 1.5; +} + +.cap-bracket { + fill: none; + stroke: var(--stone); + stroke-width: 1.4; +} + +.cap-chunk { + fill: var(--tint); + stroke: rgb(22 22 22 / 10%); +} + +.cap-done, +.cap-seal { + fill: var(--ink); +} + +.cap-tickdot { + fill: var(--graphite); +} + +.cap-tickdot-accent { + fill: var(--accent); + stroke: var(--ink); + stroke-width: 1.5; +} + +.cap-status { + fill: #fff; + stroke: var(--ink); + stroke-width: 1.6; +} + +.cap-status.cap-status-done { + fill: var(--ink); +} + +.cap-status-fault { + fill: var(--coral); + stroke: var(--coral); +} + +.cap-status-retry { + stroke: var(--graphite); + stroke-dasharray: 3 3; +} + +.cap-code { + fill: var(--text); + font-family: var(--code-font); + font-size: 13px; + font-weight: 650; +} + +.cap-code-fault, +.cap-code-done { + fill: #fff; +} + +.cap-retry { + fill: none; + stroke: var(--graphite); + stroke-width: 1.8; + stroke-linecap: round; +} + +.cap-cell { + fill: #fff; + stroke: rgb(22 22 22 / 12%); +} + +.cap-glyph { + fill: none; + stroke: var(--ink); + stroke-width: 1.8; +} + +.cap-glyph-fill { + fill: var(--ink); +} + +.cap-glyph-accent { + fill: var(--accent); + stroke: var(--ink); + stroke-width: 1.2; +} + +.cap-divider { + stroke: var(--line); + stroke-dasharray: 4 4; +} + +.cap-lane { + stroke-width: 2.5; + stroke-linecap: round; +} + +.cap-lane-accent, +.cap-station-accent { + stroke: var(--lime-line); +} + +.cap-lane-ink, +.cap-station-ink { + stroke: var(--ink); +} + +.cap-badge-accent { + fill: var(--accent); +} + +.cap-badge-ink { + fill: var(--ink); +} + +.cap-badge-text { + font-size: 13px; + font-weight: 700; +} + +.cap-badge-text-accent { + fill: var(--ink); +} + +.cap-badge-text-ink { + fill: #fff; +} + +.cap-station { + fill: #fff; + stroke-width: 2.5; +} + +.cap-row { + fill: var(--tint); +} + +.cap-check-ink, +.cap-seal-check { + fill: none; + stroke-linecap: round; + stroke-linejoin: round; +} + +.cap-check-ink { + stroke: var(--ink); + stroke-width: 2; +} + +.cap-seal-ring { + fill: none; + stroke: rgb(164 242 88 / 50%); + stroke-width: 1.2; + stroke-dasharray: 3 3; +} + +.cap-seal-check { + stroke: var(--accent); + stroke-width: 4; +} + +/* Providers */ + +.coverage-section { + padding-block: 96px; + border-top: 1px solid var(--line-soft); + background: linear-gradient(180deg, #f3efea, var(--canvas) 50%); +} + +.provider-grid { + display: grid; + grid-template-columns: repeat(4, minmax(0, 1fr)); + gap: 12px; +} + +.provider-grid li { + display: grid; + gap: 6px; + align-content: start; + min-width: 0; + padding: 18px; + border: 1px solid var(--line-soft); + border-radius: 14px; + background: var(--surface); + transition: + border-color 0.15s ease, + transform 0.15s ease; +} + +.provider-grid li:hover { + border-color: rgb(131 204 60 / 60%); + transform: translateY(-2px); +} + +.provider-grid strong { + font-size: 15px; + font-weight: 620; + letter-spacing: -0.015em; +} + +.provider-grid code { + overflow: hidden; + color: var(--muted); + font-size: 11px; + text-overflow: ellipsis; + white-space: nowrap; +} + +.sdk-tag { + width: fit-content; + margin-top: 4px; + padding: 2px 8px; + border-radius: 999px; + color: #365b16; + background: rgb(164 242 88 / 26%); + font-size: 10.5px; + font-weight: 600; +} + +.provider-grid .provider-native { + grid-column: span 3; + border-style: dashed; + background: transparent; +} + +.sdk-tag-muted { + color: var(--text-soft); + background: rgb(22 22 22 / 6%); +} + +.coverage-note { + max-width: 760px; + margin-top: 24px; + color: var(--muted); + font-size: 13px; +} + +/* Get started */ + +.start-section { + padding-block: 104px; + border-block: 1px solid var(--line-soft); +} + +.start-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 24px; + margin-top: 48px; +} + +.start-card { + display: flex; + flex-direction: column; + min-width: 0; + padding: 28px; + border: 1px solid var(--line-soft); + border-radius: 22px; + background: var(--surface); + box-shadow: var(--shadow-card); +} + +.start-card header { + margin-bottom: 22px; +} + +.start-card header p { + margin-top: 8px; + color: var(--muted); + font-size: 14px; +} + +.start-card header code { + color: var(--text-soft); + font-size: 12px; +} + +.start-badge { + display: inline-block; + margin-bottom: 12px; + padding: 3px 10px; + border-radius: 999px; + color: #365b16; + background: rgb(164 242 88 / 30%); + font-size: 11px; + font-weight: 650; +} + +.start-badge-muted { + color: var(--text-soft); + background: rgb(22 22 22 / 7%); +} + +.start-card .inline-link { + margin-top: auto; + padding-top: 8px; +} + +.code-panel { + margin-bottom: 14px; + overflow: hidden; + border-radius: 13px; + color: #e9e6e2; + background: var(--ink); +} + +.code-heading { + display: flex; + gap: 10px; + align-items: center; + justify-content: space-between; + min-height: 40px; + padding: 8px 10px 8px 16px; + border-bottom: 1px solid rgb(255 255 255 / 8%); + color: #a39d96; + font-size: 11px; +} + +.code-panel pre { + padding: 16px; + overflow-x: auto; + font-size: 12px; + line-height: 1.8; +} + +.copy-button { + min-height: 28px; + padding: 4px 10px; + border: 0; + border-radius: 7px; + color: #e9e6e2; + background: rgb(255 255 255 / 10%); + font-size: 11px; + cursor: pointer; +} + +.copy-button:hover { + color: var(--ink); + background: var(--accent); +} + +.package-row { + display: flex; + flex-wrap: wrap; + gap: 12px 18px; + align-items: center; + margin-top: 32px; +} + +.package-row p { + color: var(--muted); + font-size: 13px; +} + +.package-row ul { + display: flex; + flex-wrap: wrap; + gap: 8px; +} + +.package-row a { + display: inline-block; + padding: 5px 12px; + border: 1px solid var(--line); + border-radius: 999px; + color: var(--text-soft); + background: var(--surface); + font-family: var(--code-font); + font-size: 12px; +} + +.package-row a:hover { + border-color: rgb(131 204 60 / 70%); + color: var(--link); +} + +/* FAQ */ + +.faq-section { + display: grid; + grid-template-columns: 0.85fr 1.15fr; + gap: 100px; + padding-block: 96px; +} + +.faq-list details { + padding: 20px 0; + border-bottom: 1px solid var(--line); +} + +.faq-list details:first-child { + border-top: 1px solid var(--line); +} + +summary { + padding-right: 15px; + font-size: 15px; + font-weight: 550; + cursor: pointer; +} + +.faq-list p { + margin-top: 14px; + padding-right: 20px; + color: var(--muted); + font-size: 14px; +} + +.faq-list a { + color: var(--link); + text-decoration: underline; + text-underline-offset: 3px; +} + +/* Closing and footer */ + +.closing { + padding-block: 96px 110px; + border-top: 1px solid var(--line-soft); + text-align: center; + background: + radial-gradient(600px 300px at 50% 0%, rgb(164 242 88 / 16%), transparent 70%), + var(--canvas); +} + +.closing-mark { + width: 88px; + height: 88px; + margin: 0 auto 28px; + border-radius: 20px; + box-shadow: 0 18px 50px rgb(22 22 22 / 22%); +} + +.closing h2 { + font-size: clamp(42px, 5vw, 72px); +} + +.closing .section-wrap > :where(p:not(.eyebrow)) { + margin-top: 22px; + color: var(--muted); + font-size: 15px; +} + +.closing .actions { + justify-content: center; + margin-top: 30px; +} + +.footer { + display: flex; + gap: 30px; + align-items: center; + padding-block: 28px; + border-top: 1px solid var(--line); +} + +.footer .brand { + font-size: 13px; +} + +.footer .brand-mark { + width: 26px; + height: 26px; + border-radius: 7px; +} + +.footer p, +.footer > div { + color: var(--muted); + font-size: 12px; +} + +.footer > div { + display: flex; + gap: 24px; + margin-left: auto; +} + +.skip-link { + position: fixed; + z-index: 20; + top: -100px; + left: 16px; + padding: 10px 18px; + border: 1px solid var(--line); + border-radius: 8px; + background: var(--surface); +} + +.skip-link:focus { + top: 12px; +} + +.visually-hidden { + position: absolute; + width: 1px; + height: 1px; + margin: -1px; + padding: 0; + overflow: hidden; + border: 0; + white-space: nowrap; + clip-path: inset(50%); +} + +/* Screenshot viewer */ + +.image-viewer { + width: min(1200px, 96vw); + max-width: 96vw; + max-height: 94dvh; + padding: 0; + border: 1px solid var(--line); + border-radius: 16px; + color: var(--text); + background: var(--canvas); + box-shadow: 0 24px 100px rgb(0 0 0 / 25%); +} + +.image-viewer::backdrop { + background: rgb(15 15 15 / 72%); + backdrop-filter: blur(6px); +} + +.image-viewer-content { + padding: 16px; +} + +.image-viewer-toolbar, +.image-viewer-navigation { + display: flex; + gap: 12px; + align-items: center; + justify-content: space-between; +} + +.image-viewer-toolbar { + margin-bottom: 12px; +} + +.image-viewer-toolbar h2 { + margin: 0; + font-size: 15px; + letter-spacing: normal; +} + +.image-viewer-actions { + display: flex; + flex-shrink: 0; + gap: 6px; +} + +.image-viewer button { + min-height: 40px; + padding: 8px 12px; + border: 1px solid var(--line); + border-radius: 8px; + color: inherit; + background: var(--canvas); + font: inherit; + font-size: 12px; + cursor: pointer; +} + +.image-viewer button:hover { + background: var(--accent); +} + +.image-viewer-stage { + max-height: calc(94dvh - 180px); + overflow: auto; + border-radius: 6px; + overscroll-behavior: contain; +} + +.image-viewer-image { + display: block; + width: 100%; + max-height: calc(94dvh - 180px); + border-radius: 6px; + object-fit: contain; +} + +.image-viewer-expanded .image-viewer-stage { + height: calc(94dvh - 180px); +} + +.image-viewer-expanded .image-viewer-image { + width: auto; + max-width: none; + max-height: none; +} + +.image-viewer-caption { + margin-top: 10px; + color: var(--muted); + font-size: 12px; + line-height: 1.5; +} + +.image-viewer-navigation { + margin-top: 12px; + font-size: 12px; +} + +/* Responsive */ + +@media (width <= 1180px) { + .section-wrap { + width: calc(100% - 64px); + } + + .hero { + gap: 36px; + } + + .section-heading, + .faq-section { + gap: 50px; + } + + .capability-grid { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + + .chart-grid { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + + .chart-grid > :last-child { + grid-column: 1 / -1; + } + + .insight-grid { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + + .provider-grid { + grid-template-columns: repeat(3, minmax(0, 1fr)); + } + + .provider-grid .provider-native { + grid-column: span 2; + } + + .float-cache { + right: -8px; + } +} + +@media (width <= 900px) { + .topbar { + gap: 16px; + height: 64px; + padding-inline: 24px; + } + + nav { + gap: 20px; + } + + nav a:nth-child(3) { + display: none; + } + + .section-wrap { + width: calc(100% - 48px); + } + + .hero { + grid-template-columns: minmax(0, 1fr); + gap: 36px; + min-height: auto; + padding-block: 56px 72px; + } + + h1 { + font-size: clamp(60px, 12vw, 88px); + } + + .hero-lede { + max-width: 560px; + } + + .hero-stage { + max-width: 720px; + } + + .section-heading { + grid-template-columns: minmax(0, 1fr); + gap: 20px; + } + + .section-heading > :where(p), + .section-heading > div > :where(p:not(.eyebrow)) { + max-width: 560px; + } + + .workflow-flow { + --flow-gap: 48px; + + grid-template-columns: minmax(0, 1fr); + } + + .flow-link { + top: 100%; + left: 43px; + width: 2px; + height: var(--flow-gap); + background: repeating-linear-gradient(180deg, var(--graphite) 0 5px, transparent 5px 10px); + } + + .flow-link::after { + inset: auto auto 3px 50%; + transform: translateX(-50%) rotate(135deg); + } + + .flow-link i { + top: 0; + left: 50%; + margin-top: 0; + margin-left: -4px; + animation-name: flow-y; + } + + .lifecycle-wide { + display: none; + } + + .lifecycle-tall { + display: block; + max-width: 420px; + margin-inline: auto; + } + + .lifecycle-canvas { + padding: 18px 10px; + } + + .start-grid, + .detail-grid { + grid-template-columns: minmax(0, 1fr); + } + + .faq-section { + grid-template-columns: minmax(0, 1fr); + gap: 30px; + } + + .workflow, + .capabilities, + .coverage-section, + .start-section, + .faq-section, + .analytics-section { + padding-block: 72px; + } + + .dashboard-section { + padding-block: 72px 60px; + } + + .footer { + flex-wrap: wrap; + gap: 16px; + } + + .footer > div { + width: 100%; + margin-left: 0; + } +} + +@media (width <= 640px) { + .topbar { + height: 60px; + padding-inline: 18px; + } + + .topbar nav { + display: none; + } + + .topbar .button-small { + margin-left: auto; + } + + .brand { + gap: 9px; + font-size: 14px; + } + + .brand-mark { + width: 28px; + height: 28px; + } + + .section-wrap { + width: calc(100% - 36px); + } + + .hero { + padding-top: 40px; + } + + h1 { + font-size: clamp(54px, 15vw, 72px); + } + + h2 { + font-size: 36px; + } + + .hero-definition { + font-size: 18px; + } + + .hero-lede { + font-size: 15px; + } + + .install-chip { + display: flex; + } + + .install-chip code { + font-size: 11px; + } + + .hero-stage { + padding: 0; + } + + .hero-stage::before { + inset: -30px 0 -10px; + } + + .float-request { + display: none; + } + + .float-cache { + position: relative; + right: auto; + bottom: auto; + width: auto; + margin: -28px 12px 0; + } + + .hero-window { + border-radius: 12px; + } + + .window-bar { + height: 30px; + font-size: 9px; + } + + .flow-node { + padding: 20px; + } + + .chart-card { + padding: 18px 16px; + border-radius: 18px; + } + + .chart-card-header { + flex-direction: column; + gap: 10px; + } + + .sankey { + grid-template-columns: 82px minmax(0, 1fr) 82px; + height: 400px; + } + + .sankey-labels span, + .sankey-labels strong { + font-size: 11.5px; + } + + .sankey-labels-left li { + right: 8px; + } + + .sankey-labels-right li { + left: 8px; + } + + .sankey-node { + width: 8px; + } + + .sankey-node-center { + width: 10px; + } + + .chart-grid, + .insight-grid, + .capability-grid { + grid-template-columns: minmax(0, 1fr); + } + + .insights-heading { + flex-direction: column; + align-items: flex-start; + } + + .insight-card { + min-height: 0; + } + + .chart-grid > :last-child { + grid-column: auto; + } + + .provider-grid { + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 8px; + } + + .provider-grid li { + padding: 14px; + } + + .provider-grid .provider-native { + grid-column: 1 / -1; + } + + .start-card { + padding: 18px; + } + + .code-panel pre { + font-size: 11px; + } + + .closing h2 { + font-size: 40px; + } +} + +@media (width <= 480px) { + .image-viewer-content { + padding: 12px; + } + + .image-viewer-stage, + .image-viewer-image { + max-height: calc(94dvh - 230px); + } + + .image-viewer-expanded .image-viewer-stage { + height: calc(94dvh - 230px); + } +} + +@media (prefers-reduced-motion: reduce) { + html { + scroll-behavior: auto; + } + + .button, + .provider-grid li { + transition: none; + } + + .flow-link i, + .lc-packet { + animation: none; + } + + .flow-link i { + opacity: 0; } } diff --git a/site/tokens.css b/site/tokens.css new file mode 100644 index 0000000..7b07dff --- /dev/null +++ b/site/tokens.css @@ -0,0 +1,21 @@ +/* Mirrors LlmTestServerAdminPage.razor's final brand layer, not earlier skins. + The static site owns this small semantic subset; review alongside dashboard changes. */ +:root { + color-scheme: light; + + --canvas: #f9f7f5; + --pane: #fff; + --pane-selected: rgb(164 242 88 / 14%); + --text: #161616; + --text-soft: #363432; + --muted: rgb(18 18 18 / 60%); + --accent: #a4f258; + --accent-strong: #83cc3c; + --line: rgb(22 22 22 / 13%); + --line-soft: rgb(22 22 22 / 8%); + --brand-coral: #ff7e5d; + --brand-magenta: #db3276; + --link: #396623; + --focus: #396623; + --code-font: "SFMono-Regular", "SF Mono", ui-monospace, menlo, monaco, consolas, monospace; +} diff --git a/src/ManagedCode.LlmTck.Anthropic/AGENTS.md b/src/ManagedCode.LlmTck.Anthropic/AGENTS.md deleted file mode 100644 index d7a9b96..0000000 --- a/src/ManagedCode.LlmTck.Anthropic/AGENTS.md +++ /dev/null @@ -1,5 +0,0 @@ -# ManagedCode.LlmTck.Anthropic - -This package owns Anthropic compatibility constants, provider profiles, and future Messages API wire contracts. - -Keep deterministic runtime behavior in `ManagedCode.LlmTck`; keep HTTP hosting in `ManagedCode.LlmTck.Hosting`. diff --git a/src/ManagedCode.LlmTck.Anthropic/ManagedCode.LlmTck.Anthropic.csproj b/src/ManagedCode.LlmTck.Anthropic/ManagedCode.LlmTck.Anthropic.csproj deleted file mode 100644 index 5dec6b3..0000000 --- a/src/ManagedCode.LlmTck.Anthropic/ManagedCode.LlmTck.Anthropic.csproj +++ /dev/null @@ -1,13 +0,0 @@ - - - ManagedCode.LlmTck.Anthropic - Anthropic Messages compatibility profile and adapters for ManagedCode.LlmTck. - - - - - - - - - diff --git a/src/ManagedCode.LlmTck.Aspire/AGENTS.md b/src/ManagedCode.LlmTck.Aspire/AGENTS.md deleted file mode 100644 index 19fbca8..0000000 --- a/src/ManagedCode.LlmTck.Aspire/AGENTS.md +++ /dev/null @@ -1,9 +0,0 @@ -# ManagedCode.LlmTck.Aspire - -This package owns Aspire AppHost extensions. - -Extension methods should be declarative and should configure the package-owned `LlmTckResource` through Aspire primitives. Consumer examples must use `builder.AddLlmTck()` without service project paths, generated `Projects.*` metadata types, Docker, or a container runtime. Keep container hosting as an explicit opt-in API such as `builder.AddLlmTckContainer()`, not the default local AppHost path. Every behavior change needs an Aspire integration test that starts the AppHost. - -The resource's `http` endpoint is the provider endpoint for Aspire consumers. Do not add a separate provider-endpoint setter or hardcoded loopback URL; callers must use `GetEndpoint("http")` or `CreateHttpClient(...)` from the running Aspire resource. - -Do not add `LLM_TCK_*_COMPATIBILITY` environment flags or fluent compatibility methods unless they drive real route/runtime behavior; Aspire compatibility proof belongs in provider-client tests, not config echo endpoints. diff --git a/src/ManagedCode.LlmTck.Aspire/ManagedCode.LlmTck.Aspire.csproj b/src/ManagedCode.LlmTck.Aspire/ManagedCode.LlmTck.Aspire.csproj deleted file mode 100644 index 080efde..0000000 --- a/src/ManagedCode.LlmTck.Aspire/ManagedCode.LlmTck.Aspire.csproj +++ /dev/null @@ -1,40 +0,0 @@ - - - ManagedCode.LlmTck.Aspire - Aspire AppHost resource extensions for ManagedCode.LlmTck provider emulation. - - - - - - - - - - - - - - $(MSBuildThisFileDirectory)obj/$(Configuration)/service-publish/ - - - - - - <_PublishedLlmTckService Include="$(LlmTckServicePublishDir)**/*" /> - - - - diff --git a/src/ManagedCode.LlmTck.Aspire/buildTransitive/ManagedCode.LlmTck.Aspire.targets b/src/ManagedCode.LlmTck.Aspire/buildTransitive/ManagedCode.LlmTck.Aspire.targets deleted file mode 100644 index 651fb3a..0000000 --- a/src/ManagedCode.LlmTck.Aspire/buildTransitive/ManagedCode.LlmTck.Aspire.targets +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - diff --git a/src/ManagedCode.LlmTck.AzureOpenAI/AGENTS.md b/src/ManagedCode.LlmTck.AzureOpenAI/AGENTS.md deleted file mode 100644 index 3a4fa7f..0000000 --- a/src/ManagedCode.LlmTck.AzureOpenAI/AGENTS.md +++ /dev/null @@ -1,5 +0,0 @@ -# ManagedCode.LlmTck.AzureOpenAI - -This package owns Azure OpenAI compatibility constants, provider profiles, and future Azure OpenAI wire contracts. - -Keep deterministic runtime behavior in `ManagedCode.LlmTck`; keep HTTP hosting in `ManagedCode.LlmTck.Hosting`. diff --git a/src/ManagedCode.LlmTck.AzureOpenAI/ManagedCode.LlmTck.AzureOpenAI.csproj b/src/ManagedCode.LlmTck.AzureOpenAI/ManagedCode.LlmTck.AzureOpenAI.csproj deleted file mode 100644 index b23d89d..0000000 --- a/src/ManagedCode.LlmTck.AzureOpenAI/ManagedCode.LlmTck.AzureOpenAI.csproj +++ /dev/null @@ -1,10 +0,0 @@ - - - ManagedCode.LlmTck.AzureOpenAI - Azure OpenAI compatibility profile and adapters for ManagedCode.LlmTck. - - - - - - diff --git a/src/ManagedCode.LlmTck.Bedrock/AGENTS.md b/src/ManagedCode.LlmTck.Bedrock/AGENTS.md deleted file mode 100644 index 6698494..0000000 --- a/src/ManagedCode.LlmTck.Bedrock/AGENTS.md +++ /dev/null @@ -1,5 +0,0 @@ -# ManagedCode.LlmTck.Bedrock - -This package owns Amazon Bedrock compatibility constants, provider profiles, and future Bedrock wire contracts. - -Keep deterministic runtime behavior in `ManagedCode.LlmTck`; keep HTTP hosting in `ManagedCode.LlmTck.Hosting`. diff --git a/src/ManagedCode.LlmTck.Bedrock/ManagedCode.LlmTck.Bedrock.csproj b/src/ManagedCode.LlmTck.Bedrock/ManagedCode.LlmTck.Bedrock.csproj deleted file mode 100644 index c386879..0000000 --- a/src/ManagedCode.LlmTck.Bedrock/ManagedCode.LlmTck.Bedrock.csproj +++ /dev/null @@ -1,14 +0,0 @@ - - - ManagedCode.LlmTck.Bedrock - Amazon Bedrock compatibility profile and adapters for ManagedCode.LlmTck. - - - - - - - - - - diff --git a/src/ManagedCode.LlmTck.Client/LlmTckAudioContent.cs b/src/ManagedCode.LlmTck.Client/LlmTckAudioContent.cs deleted file mode 100644 index 30581f3..0000000 --- a/src/ManagedCode.LlmTck.Client/LlmTckAudioContent.cs +++ /dev/null @@ -1,3 +0,0 @@ -namespace ManagedCode.LlmTck.Client; - -public sealed record LlmTckAudioContent(byte[] Bytes, string MediaType); diff --git a/src/ManagedCode.LlmTck.Client/LlmTckClientConfigurationBuilder.cs b/src/ManagedCode.LlmTck.Client/LlmTckClientConfigurationBuilder.cs deleted file mode 100644 index c9d8f7a..0000000 --- a/src/ManagedCode.LlmTck.Client/LlmTckClientConfigurationBuilder.cs +++ /dev/null @@ -1,329 +0,0 @@ -using ManagedCode.LlmTck.Configuration; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Scenarios; - -namespace ManagedCode.LlmTck.Client; - -public sealed class LlmTckClientConfigurationBuilder -{ - private readonly LlmTckConfigurationBuilder _builder = new(); - - public LlmTckClientConfigurationBuilder RequireBearerToken(string token) - { - _builder.RequireBearerToken(token); - return this; - } - - public LlmTckClientConfigurationBuilder SimulateRateLimitAfter(int allowedRequests) - { - _builder.SimulateRateLimitAfter(allowedRequests); - return this; - } - - public LlmTckClientConfigurationBuilder SimulateContentFilter(params string[] blockedTerms) - { - _builder.SimulateContentFilter(blockedTerms); - return this; - } - - public LlmTckClientConfigurationBuilder UseModel( - string id, - LlmTckModelKind kind, - int reasoningTokens = 0 - ) - { - _builder.AddModel(id, kind, reasoningTokens); - return this; - } - - public LlmTckClientConfigurationBuilder UseReasoningChatModel( - string id, - int reasoningTokens - ) - { - _builder.AddReasoningChatModel(id, reasoningTokens); - return this; - } - - public LlmTckClientConfigurationBuilder UseGpt55(int reasoningTokens = 0) - { - return UseModel(LlmTckKnownModelIds.Gpt55, LlmTckModelKind.Chat, reasoningTokens); - } - - public LlmTckClientConfigurationBuilder UseGpt55Pro(int reasoningTokens = 0) - { - return UseModel(LlmTckKnownModelIds.Gpt55Pro, LlmTckModelKind.Chat, reasoningTokens); - } - - public LlmTckClientConfigurationBuilder UseGpt54(int reasoningTokens = 0) - { - return UseModel(LlmTckKnownModelIds.Gpt54, LlmTckModelKind.Chat, reasoningTokens); - } - - public LlmTckClientConfigurationBuilder UseGpt54Pro(int reasoningTokens = 0) - { - return UseModel(LlmTckKnownModelIds.Gpt54Pro, LlmTckModelKind.Chat, reasoningTokens); - } - - public LlmTckClientConfigurationBuilder UseGpt54Mini(int reasoningTokens = 0) - { - return UseModel(LlmTckKnownModelIds.Gpt54Mini, LlmTckModelKind.Chat, reasoningTokens); - } - - public LlmTckClientConfigurationBuilder UseGpt54Nano(int reasoningTokens = 0) - { - return UseModel(LlmTckKnownModelIds.Gpt54Nano, LlmTckModelKind.Chat, reasoningTokens); - } - - public LlmTckClientConfigurationBuilder UseGpt5(int reasoningTokens = 0) - { - return UseModel(LlmTckKnownModelIds.Gpt5, LlmTckModelKind.Chat, reasoningTokens); - } - - public LlmTckClientConfigurationBuilder UseGpt5Pro(int reasoningTokens = 0) - { - return UseModel(LlmTckKnownModelIds.Gpt5Pro, LlmTckModelKind.Chat, reasoningTokens); - } - - public LlmTckClientConfigurationBuilder UseGpt5Mini(int reasoningTokens = 0) - { - return UseModel(LlmTckKnownModelIds.Gpt5Mini, LlmTckModelKind.Chat, reasoningTokens); - } - - public LlmTckClientConfigurationBuilder UseGpt5Nano(int reasoningTokens = 0) - { - return UseModel(LlmTckKnownModelIds.Gpt5Nano, LlmTckModelKind.Chat, reasoningTokens); - } - - public LlmTckClientConfigurationBuilder UseGpt41() - { - return UseChatModel(LlmTckKnownModelIds.Gpt41); - } - - public LlmTckClientConfigurationBuilder UseGpt41Mini() - { - return UseChatModel(LlmTckKnownModelIds.Gpt41Mini); - } - - public LlmTckClientConfigurationBuilder UseGpt4OMini() - { - return UseChatModel(LlmTckKnownModelIds.Gpt4OMini); - } - - public LlmTckClientConfigurationBuilder UseTextEmbedding3Small() - { - return UseEmbeddingModel(LlmTckKnownModelIds.TextEmbedding3Small); - } - - public LlmTckClientConfigurationBuilder UseTextEmbedding3Large() - { - return UseEmbeddingModel(LlmTckKnownModelIds.TextEmbedding3Large); - } - - public LlmTckClientConfigurationBuilder UseTextEmbeddingAda002() - { - return UseEmbeddingModel(LlmTckKnownModelIds.TextEmbeddingAda002); - } - - public LlmTckClientConfigurationBuilder UseGptImage2() - { - return UseImageModel(LlmTckKnownModelIds.GptImage2); - } - - public LlmTckClientConfigurationBuilder UseGptImage15() - { - return UseImageModel(LlmTckKnownModelIds.GptImage15); - } - - public LlmTckClientConfigurationBuilder UseGptImage1() - { - return UseImageModel(LlmTckKnownModelIds.GptImage1); - } - - public LlmTckClientConfigurationBuilder UseGptImage1Mini() - { - return UseImageModel(LlmTckKnownModelIds.GptImage1Mini); - } - - public LlmTckClientConfigurationBuilder UseGpt4OMiniTts() - { - return UseAudioModel(LlmTckKnownModelIds.Gpt4OMiniTts); - } - - public LlmTckClientConfigurationBuilder UseTts1() - { - return UseAudioModel(LlmTckKnownModelIds.Tts1); - } - - public LlmTckClientConfigurationBuilder UseTts1Hd() - { - return UseAudioModel(LlmTckKnownModelIds.Tts1Hd); - } - - public LlmTckClientConfigurationBuilder UseSora2() - { - return UseVideoModel(LlmTckKnownModelIds.Sora2); - } - - public LlmTckClientConfigurationBuilder UseSora2Pro() - { - return UseVideoModel(LlmTckKnownModelIds.Sora2Pro); - } - - public LlmTckClientConfigurationBuilder UseDefaultOpenAiModels() - { - _builder.AddDefaultOpenAiModels(); - return this; - } - - public LlmTckClientConfigurationBuilder UseCurrentOpenAiChatModels(int reasoningTokens = 0) - { - _builder.AddCurrentOpenAiChatModels(reasoningTokens); - return this; - } - - public LlmTckClientConfigurationBuilder UseOpenAiEmbeddingModels() - { - _builder.AddOpenAiEmbeddingModels(); - return this; - } - - public LlmTckClientConfigurationBuilder UseOpenAiImageModels() - { - _builder.AddOpenAiImageModels(); - return this; - } - - public LlmTckClientConfigurationBuilder UseOpenAiAudioModels() - { - _builder.AddOpenAiAudioModels(); - return this; - } - - public LlmTckClientConfigurationBuilder UseOpenAiVideoModels() - { - _builder.AddOpenAiVideoModels(); - return this; - } - - public LlmTckClientConfigurationBuilder UseKnownOpenAiModels(int reasoningTokens = 0) - { - _builder.AddKnownOpenAiModels(reasoningTokens); - return this; - } - - public LlmTckClientConfigurationBuilder UseChatModel(string id) - { - return UseModel(id, LlmTckModelKind.Chat); - } - - public LlmTckClientConfigurationBuilder UseEmbeddingModel(string id) - { - return UseModel(id, LlmTckModelKind.Embedding); - } - - public LlmTckClientConfigurationBuilder UseImageModel(string id) - { - return UseModel(id, LlmTckModelKind.Image); - } - - public LlmTckClientConfigurationBuilder UseAudioModel(string id) - { - return UseModel(id, LlmTckModelKind.Audio); - } - - public LlmTckClientConfigurationBuilder UseVideoModel(string id) - { - return UseModel(id, LlmTckModelKind.Video); - } - - public LlmTckClientConfigurationBuilder UseChatScenario( - string id, - Action configure - ) - { - _builder.AddChatScenario(id, configure); - return this; - } - - public LlmTckClientConfigurationBuilder UseDecisionModel(string id, ManagedCode.LlmTck.Decisions.LlmTckDecisionProvider provider) - { - _builder.AddDecisionModel(id, provider); - return this; - } - - public LlmTckClientConfigurationBuilder UseDecisionScenario(ManagedCode.LlmTck.Decisions.LlmTckDecisionScenario scenario) - { - _builder.AddDecisionScenario(scenario); - return this; - } - - public LlmTckClientConfigurationBuilder UseDataset( - string id, - Action configure - ) - { - _builder.AddDataset(id, configure); - return this; - } - - public LlmTckClientConfigurationBuilder UseEmbeddingVector(params float[] values) - { - _builder.WithDefaultEmbeddingVector(values); - return this; - } - - public LlmTckClientConfigurationBuilder UseImageDataUri(string dataUri) - { - _builder.WithDefaultImageDataUri(dataUri); - return this; - } - - public LlmTckClientConfigurationBuilder UseAudio(byte[] bytes, string mediaType) - { - _builder.WithDefaultAudio(bytes, mediaType); - return this; - } - - public LlmTckClientConfigurationBuilder UseVideo(byte[] bytes, string mediaType) - { - _builder.WithDefaultVideo(bytes, mediaType); - return this; - } - - public LlmTckClientConfigurationBuilder UseTranscriptionText(string text) - { - _builder.WithDefaultTranscriptionText(text); - return this; - } - - public LlmTckClientConfigurationBuilder UseTranscriptionUsage(LlmTckTranscriptionUsage usage) - { - _builder.WithDefaultTranscriptionUsage(usage); - return this; - } - - public LlmTckClientConfigurationBuilder WithoutTranscriptionUsage() - { - _builder.WithoutDefaultTranscriptionUsage(); - return this; - } - - public LlmTckClientConfigurationBuilder UseTranslationText(string text) - { - _builder.WithDefaultTranslationText(text); - return this; - } - - public LlmTckClientConfigurationBuilder UseChatResponseCapacity(int maxResponses, long maxBytes) - { - _builder.WithChatResponseCapacity(maxResponses, maxBytes); - return this; - } - - public LlmTckConfiguration Build() - { - return _builder.Build(); - } -} diff --git a/src/ManagedCode.LlmTck.Client/ManagedCode.LlmTck.Client.csproj b/src/ManagedCode.LlmTck.Client/ManagedCode.LlmTck.Client.csproj deleted file mode 100644 index 7092d9f..0000000 --- a/src/ManagedCode.LlmTck.Client/ManagedCode.LlmTck.Client.csproj +++ /dev/null @@ -1,18 +0,0 @@ - - - ManagedCode.LlmTck.Client - Typed control client and assertion helpers for ManagedCode.LlmTck test hosts. - - - - - - - - - - - - - - diff --git a/src/ManagedCode.LlmTck.Cloudflare/AGENTS.md b/src/ManagedCode.LlmTck.Cloudflare/AGENTS.md deleted file mode 100644 index f88d026..0000000 --- a/src/ManagedCode.LlmTck.Cloudflare/AGENTS.md +++ /dev/null @@ -1,3 +0,0 @@ -# ManagedCode.LlmTck.Cloudflare - -Owns native Cloudflare decision wire contracts and compatibility profiles. Keep deterministic fixtures in `ManagedCode.LlmTck` and routes in `ManagedCode.LlmTck.Hosting`. DTOs use explicit JSON names; preserve opaque choice IDs and native scores. diff --git a/src/ManagedCode.LlmTck.Cloudflare/CloudflareCompatibility.cs b/src/ManagedCode.LlmTck.Cloudflare/CloudflareCompatibility.cs deleted file mode 100644 index fdeedf0..0000000 --- a/src/ManagedCode.LlmTck.Cloudflare/CloudflareCompatibility.cs +++ /dev/null @@ -1,24 +0,0 @@ -using ManagedCode.LlmTck.Providers; -namespace ManagedCode.LlmTck.Cloudflare; - -public static class CloudflareCompatibility -{ - public const string ProviderId = LlmTckCompatibilityTags.Cloudflare; - public static LlmTckProviderProfile Profile { get; } = new() - { - Id = ProviderId, - DisplayName = "Cloudflare Workers AI (Clef)", - Protocol = LlmTckProtocolFamily.Cloudflare, - DefaultEndpointPath = "/cloudflare/client/v4/accounts/{accountId}/ai/run/@cf/cloudflare/clef", - Capabilities = [LlmTckProviderCapability.Decisions], - CompatibilityTags = [ProviderId], - ApiContract = new() - { - DocumentationUrl = "https://developers.cloudflare.com/workers-ai/models/clef/", - DocumentationRetrievedOn = "2026-10-20", - DocumentationVersion = "native Clef/Clef-flash decisions and embedded PNG/JPEG/WebP images", - Operations = [ new() { Id = LlmTckProviderOperationIds.Cloudflare.ClefDecisionsCreate, Method = "POST", Path = "/cloudflare/client/v4/accounts/{accountId}/ai/run/@cf/cloudflare/clef", DocumentationUrl = "https://developers.cloudflare.com/workers-ai/models/clef/", ImplementedByHosting = true, Capabilities = [LlmTckProviderCapability.Decisions] }, -new() { Id = LlmTckProviderOperationIds.Cloudflare.ClefFlashDecisionsCreate, Method = "POST", Path = "/cloudflare/client/v4/accounts/{accountId}/ai/run/@cf/cloudflare/clef-flash", DocumentationUrl = "https://developers.cloudflare.com/workers-ai/models/clef/", ImplementedByHosting = true, Capabilities = [LlmTckProviderCapability.Decisions] } ] - }, - }; -} diff --git a/src/ManagedCode.LlmTck.Cloudflare/ManagedCode.LlmTck.Cloudflare.csproj b/src/ManagedCode.LlmTck.Cloudflare/ManagedCode.LlmTck.Cloudflare.csproj deleted file mode 100644 index 5336d17..0000000 --- a/src/ManagedCode.LlmTck.Cloudflare/ManagedCode.LlmTck.Cloudflare.csproj +++ /dev/null @@ -1,5 +0,0 @@ - - ManagedCode.LlmTck.CloudflareNative Cloudflare decision API compatibility for ManagedCode.LlmTck. - - - diff --git a/src/ManagedCode.LlmTck.Cohere/AGENTS.md b/src/ManagedCode.LlmTck.Cohere/AGENTS.md deleted file mode 100644 index 0b19af4..0000000 --- a/src/ManagedCode.LlmTck.Cohere/AGENTS.md +++ /dev/null @@ -1,5 +0,0 @@ -# ManagedCode.LlmTck.Cohere - -This package owns Cohere compatibility constants, provider profiles, and future Cohere wire contracts. - -Keep deterministic runtime behavior in `ManagedCode.LlmTck`; keep HTTP hosting in `ManagedCode.LlmTck.Hosting`. diff --git a/src/ManagedCode.LlmTck.Cohere/ManagedCode.LlmTck.Cohere.csproj b/src/ManagedCode.LlmTck.Cohere/ManagedCode.LlmTck.Cohere.csproj deleted file mode 100644 index 87081c9..0000000 --- a/src/ManagedCode.LlmTck.Cohere/ManagedCode.LlmTck.Cohere.csproj +++ /dev/null @@ -1,13 +0,0 @@ - - - ManagedCode.LlmTck.Cohere - Cohere compatibility profile and adapters for ManagedCode.LlmTck. - - - - - - - - - diff --git a/src/ManagedCode.LlmTck.DeepSeek/AGENTS.md b/src/ManagedCode.LlmTck.DeepSeek/AGENTS.md deleted file mode 100644 index 1ac4213..0000000 --- a/src/ManagedCode.LlmTck.DeepSeek/AGENTS.md +++ /dev/null @@ -1,5 +0,0 @@ -# ManagedCode.LlmTck.DeepSeek - -This package owns DeepSeek compatibility constants, provider profiles, and future DeepSeek wire contracts. - -Keep deterministic runtime behavior in `ManagedCode.LlmTck`; keep HTTP hosting in `ManagedCode.LlmTck.Hosting`. diff --git a/src/ManagedCode.LlmTck.DeepSeek/ManagedCode.LlmTck.DeepSeek.csproj b/src/ManagedCode.LlmTck.DeepSeek/ManagedCode.LlmTck.DeepSeek.csproj deleted file mode 100644 index 2bae547..0000000 --- a/src/ManagedCode.LlmTck.DeepSeek/ManagedCode.LlmTck.DeepSeek.csproj +++ /dev/null @@ -1,10 +0,0 @@ - - - ManagedCode.LlmTck.DeepSeek - DeepSeek compatibility profile and adapters for ManagedCode.LlmTck. - - - - - - diff --git a/src/ManagedCode.LlmTck.Foundry/AGENTS.md b/src/ManagedCode.LlmTck.Foundry/AGENTS.md deleted file mode 100644 index 36cec37..0000000 --- a/src/ManagedCode.LlmTck.Foundry/AGENTS.md +++ /dev/null @@ -1,5 +0,0 @@ -# ManagedCode.LlmTck.Foundry - -This package owns Microsoft Foundry compatibility constants, provider profiles, and future Foundry wire contracts. - -Keep deterministic runtime behavior in `ManagedCode.LlmTck`; keep HTTP hosting in `ManagedCode.LlmTck.Hosting`. diff --git a/src/ManagedCode.LlmTck.Foundry/ManagedCode.LlmTck.Foundry.csproj b/src/ManagedCode.LlmTck.Foundry/ManagedCode.LlmTck.Foundry.csproj deleted file mode 100644 index c0252e0..0000000 --- a/src/ManagedCode.LlmTck.Foundry/ManagedCode.LlmTck.Foundry.csproj +++ /dev/null @@ -1,10 +0,0 @@ - - - ManagedCode.LlmTck.Foundry - Microsoft Foundry compatibility profile and adapters for ManagedCode.LlmTck. - - - - - - diff --git a/src/ManagedCode.LlmTck.Gemini/AGENTS.md b/src/ManagedCode.LlmTck.Gemini/AGENTS.md deleted file mode 100644 index 1bd01bd..0000000 --- a/src/ManagedCode.LlmTck.Gemini/AGENTS.md +++ /dev/null @@ -1,5 +0,0 @@ -# ManagedCode.LlmTck.Gemini - -This package owns Gemini compatibility constants, provider profiles, and future Gemini wire contracts. - -Keep deterministic runtime behavior in `ManagedCode.LlmTck`; keep HTTP hosting in `ManagedCode.LlmTck.Hosting`. diff --git a/src/ManagedCode.LlmTck.Gemini/GeminiCompatibility.cs b/src/ManagedCode.LlmTck.Gemini/GeminiCompatibility.cs deleted file mode 100644 index efc290e..0000000 --- a/src/ManagedCode.LlmTck.Gemini/GeminiCompatibility.cs +++ /dev/null @@ -1,105 +0,0 @@ -using ManagedCode.LlmTck.Providers; - -namespace ManagedCode.LlmTck.Gemini; - -public static class GeminiCompatibility -{ - public const string ProviderId = LlmTckCompatibilityTags.Gemini; - - public static LlmTckProviderProfile Profile { get; } = - new() - { - Id = ProviderId, - DisplayName = "Gemini", - Protocol = LlmTckProtocolFamily.Gemini, - DefaultEndpointPath = "/gemini/v1beta/models/{model}:generateContent", - Capabilities = - [ - LlmTckProviderCapability.Chat, - LlmTckProviderCapability.StreamingChat, - LlmTckProviderCapability.Tools, - LlmTckProviderCapability.StructuredOutput, - LlmTckProviderCapability.Embeddings, - LlmTckProviderCapability.Images, - LlmTckProviderCapability.Video, - LlmTckProviderCapability.Audio, - ], - CompatibilityTags = - [ - ProviderId, - LlmTckCompatibilityTags.GoogleAI, - LlmTckCompatibilityTags.OpenAICompatible, - ], - ApiContract = new() - { - DocumentationUrl = "https://ai.google.dev/api", - DocumentationRetrievedOn = "2026-09-07", - DocumentationVersion = "v1beta Generative Language API", - Operations = - [ - new() - { - Id = LlmTckProviderOperationIds.Gemini.ModelsGenerateContent, - Method = "POST", - Path = "/gemini/v1beta/models/{model}:generateContent", - DocumentationUrl = "https://ai.google.dev/api/generate-content", - ImplementedByHosting = true, - Capabilities = - [ - LlmTckProviderCapability.Chat, - LlmTckProviderCapability.Tools, - LlmTckProviderCapability.StructuredOutput, - LlmTckProviderCapability.Images, - LlmTckProviderCapability.Audio, - ], - }, - new() - { - Id = LlmTckProviderOperationIds.Gemini.ModelsStreamGenerateContent, - Method = "POST", - Path = "/gemini/v1beta/models/{model}:streamGenerateContent", - DocumentationUrl = "https://ai.google.dev/api/generate-content", - SupportsStreaming = true, - ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.StreamingChat, LlmTckProviderCapability.Tools, LlmTckProviderCapability.StructuredOutput], - }, - new() - { - Id = LlmTckProviderOperationIds.Gemini.ModelsEmbedContent, - Method = "POST", - Path = "/gemini/v1beta/models/{model}:embedContent", - DocumentationUrl = "https://ai.google.dev/api/embeddings", - ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Embeddings], - }, - new() - { - Id = LlmTckProviderOperationIds.Gemini.ModelsPredictLongRunningVideo, - Method = "POST", - Path = "/gemini/v1beta/models/{model}:predictLongRunning", - DocumentationUrl = "https://ai.google.dev/gemini-api/docs/video", - ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Video], - }, - new() - { - Id = LlmTckProviderOperationIds.Gemini.ModelsOperationsGetVideo, - Method = "GET", - Path = "/gemini/v1beta/models/{model}/operations/{operationId}", - DocumentationUrl = "https://ai.google.dev/gemini-api/docs/video", - ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Video], - }, - new() - { - Id = LlmTckProviderOperationIds.Gemini.FilesGetGeneratedVideo, - Method = "GET", - Path = "/gemini/v1beta/files/{fileId}", - DocumentationUrl = "https://ai.google.dev/api/files", - ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Video], - }, - ], - }, - }; -} diff --git a/src/ManagedCode.LlmTck.Gemini/ManagedCode.LlmTck.Gemini.csproj b/src/ManagedCode.LlmTck.Gemini/ManagedCode.LlmTck.Gemini.csproj deleted file mode 100644 index 9570653..0000000 --- a/src/ManagedCode.LlmTck.Gemini/ManagedCode.LlmTck.Gemini.csproj +++ /dev/null @@ -1,13 +0,0 @@ - - - ManagedCode.LlmTck.Gemini - Gemini compatibility profile and adapters for ManagedCode.LlmTck. - - - - - - - - - diff --git a/src/ManagedCode.LlmTck.Groq/AGENTS.md b/src/ManagedCode.LlmTck.Groq/AGENTS.md deleted file mode 100644 index debe9b0..0000000 --- a/src/ManagedCode.LlmTck.Groq/AGENTS.md +++ /dev/null @@ -1,5 +0,0 @@ -# ManagedCode.LlmTck.Groq - -This package owns Groq compatibility constants, provider profiles, and future Groq wire contracts. - -Keep deterministic runtime behavior in `ManagedCode.LlmTck`; keep HTTP hosting in `ManagedCode.LlmTck.Hosting`. diff --git a/src/ManagedCode.LlmTck.Groq/ManagedCode.LlmTck.Groq.csproj b/src/ManagedCode.LlmTck.Groq/ManagedCode.LlmTck.Groq.csproj deleted file mode 100644 index f0c4936..0000000 --- a/src/ManagedCode.LlmTck.Groq/ManagedCode.LlmTck.Groq.csproj +++ /dev/null @@ -1,10 +0,0 @@ - - - ManagedCode.LlmTck.Groq - Groq compatibility profile and adapters for ManagedCode.LlmTck. - - - - - - diff --git a/src/ManagedCode.LlmTck.Hosting/LlmTckAdminPanel.razor b/src/ManagedCode.LlmTck.Hosting/LlmTckAdminPanel.razor deleted file mode 100644 index 6369049..0000000 --- a/src/ManagedCode.LlmTck.Hosting/LlmTckAdminPanel.razor +++ /dev/null @@ -1,1305 +0,0 @@ -@namespace ManagedCode.LlmTck.Hosting -@using System.Globalization -@using System.Text -@using System.Text.Encodings.Web -@using System.Text.Json -@using Microsoft.AspNetCore.Components.Web -@using ManagedCode.LlmTck.Control -@using ManagedCode.LlmTck.Models -@using ManagedCode.LlmTck.Providers -@using ManagedCode.LlmTck.Runtime -@using ManagedCode.LlmTck.Scenarios -@using Microsoft.Extensions.Options -@using Microsoft.FluentUI.AspNetCore.Components -@implements IAsyncDisposable -@inject ILlmTckRuntime Runtime -@inject ILlmTckProviderHttpTraceStore TraceStore -@inject IOptions TraceOptions - -@{ - var visibleTraces = FilterTraces(); - var providerOptions = GetProviderOptions(); - var modelOptions = GetModelOptions(); -} - -
-
-
- -
-
Deterministic provider lab
-

LLM TCK request dashboardLLM TCK

-
-
- -
-
- - @_statusText -
- - - - -
-
- - -
-
-
-
Runtime setup & diagnostics
-

Runtime state

-
-
- @FormatBytes(TraceStore.RetainedByteCount) retained - -
-
-
- - -
-
-
-

Fixtures / models@_models.Count

-
- @if (_models.Count == 0) - { -
No models loaded.
- } - else - { -
    - @foreach (var model in _models) - { -
  • @model.Id@model.Kind · @model.OwnedBy
    • - } -
- } -
-
- -
-

Provider APIs@_providerRoutes.Count

-
-
    - @foreach (var route in _providerRoutes) - { -
  • @route.Name@route.Route
    • - } -
-
-
- -
-

Runtime actions4

-
-
    -
  • Refresh stateGET @LlmTckControlRoutes.Assertions
    • -
  • List fixturesGET @LlmTckControlRoutes.Models
    • -
  • Configure runtimePOST @LlmTckControlRoutes.Configure
    • -
  • Reset runtimePOST @LlmTckControlRoutes.Reset
    • -
-
-
- -
-

Assertions@_summary.TotalEvents

-
-
    -
  • Matched@_summary.Matched
    • -
  • Unmatched@_summary.Unmatched
    • -
  • Model not found@_summary.ModelNotFound
    • -
  • Auth failed@_summary.AuthFailed
    • -
  • Scenario exhausted@_summary.ScenarioExhausted
    • -
  • Errors returned@_summary.ErrorsReturned
    • -
-
-
- -
-

Runtime events@_events.Count

-
- @if (_events.Count == 0) - { -
No runtime events yet.
- } - else - { -
    - @foreach (var runtimeEvent in _events.AsEnumerable().Reverse()) - { -
  • - @runtimeEvent.Kind@runtimeEvent.Message - @runtimeEvent.ModelId · @runtimeEvent.Timestamp.ToLocalTime().ToString("HH:mm:ss", CultureInfo.InvariantCulture) -
    • - } -
- } -
-
-
-
- - -
@_bannerText
- -
- - -
-
-
-

Provider requests

-

@visibleTraces.Count of @_traces.Count · newest first

-
- @visibleTraces.Count -
- -
- -
- - - - -
-
- - - - @if (visibleTraces.Count == 0) - { -
-
- @(_traces.Count == 0 ? "No provider requests yet" : "No requests match these filters") - @(_traces.Count == 0 ? "Call a namespaced provider endpoint to start the inbox." : "Clear or change filters to see more calls.") -
-
- } - else - { -
- @foreach (var trace in visibleTraces) - { - var usage = UsageFor(trace); - var messages = MessagesFor(trace); - var messageCount = messages.Count; - - } -
- } -
- - -
- -
Served by MapLlmTck(); the request journal is bounded and never records authorization headers.
-
-@code { - private const string _overviewTab = "overview"; - private const string _conversationTab = "conversation"; - private const string _wireTab = "wire"; - private const string _streamTab = "stream"; - private const string _runtimeTab = "runtime"; - private const int _collapsedTurnThreshold = 420; - private const int _turnPreviewLimit = 132; - private static readonly TimeSpan _autoRefreshInterval = TimeSpan.FromSeconds(15); - private static readonly JsonSerializerOptions _prettyJsonOptions = new() - { - Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, - WriteIndented = true, - }; - private static readonly IReadOnlyList _providerRoutes = - [ - new("OpenAI", LlmTckProviderRouteNamespaces.OpenAI), - new("Azure OpenAI", LlmTckProviderRouteNamespaces.AzureOpenAI), - new("Microsoft Foundry", LlmTckProviderRouteNamespaces.MicrosoftFoundry), - new("Anthropic", LlmTckProviderRouteNamespaces.Anthropic), - new("Gemini", LlmTckProviderRouteNamespaces.Gemini), - new("Groq", LlmTckProviderRouteNamespaces.Groq), - new("Mistral", LlmTckProviderRouteNamespaces.Mistral), - new("Ollama", LlmTckProviderRouteNamespaces.Ollama), - new("Cohere", LlmTckProviderRouteNamespaces.Cohere), - new("Amazon Bedrock", LlmTckProviderRouteNamespaces.Bedrock), - new("OpenRouter", LlmTckProviderRouteNamespaces.OpenRouter), - new("DeepSeek", LlmTckProviderRouteNamespaces.DeepSeek), - new("Perplexity", LlmTckProviderRouteNamespaces.Perplexity), - ]; - - private IReadOnlyList _models = []; - private IReadOnlyList _traces = []; - private IReadOnlyList _events = []; - private IReadOnlyDictionary _eventsByRequestId = - new Dictionary(StringComparer.Ordinal); - private LlmTckAssertionSummary _summary = new(); - private string _bearerToken = string.Empty; - private string _search = string.Empty; - private string _providerFilter = string.Empty; - private string _modelFilter = string.Empty; - private string _outcomeFilter = string.Empty; - private string _activeDetailTab = _overviewTab; - private string? _selectedRequestId; - private bool _mobileDetailsOpen; - private bool _autoRefresh = true; - private bool _resetting; - private bool _runtimeDialogHidden = true; - private bool _interactive; - private bool _restoreRuntimeTriggerFocus; - private ElementReference _runtimeDialogTrigger; - private FluentDialog _runtimeDialog = default!; - private const string ClosedDialogState = "closed"; - private string _statusText = "Connecting…"; - private string _statusClass = string.Empty; - private string? _bannerKind; - private string? _bannerText; - private CancellationTokenSource? _autoRefreshCancellation; - - private string WorkspaceCss => _mobileDetailsOpen ? "shell mobile-reading" : "shell mobile-list"; - private string StatusCss => string.IsNullOrWhiteSpace(_statusClass) - ? "connection" - : $"connection {_statusClass}"; - private string BannerCss => string.IsNullOrWhiteSpace(_bannerText) - ? "notice" - : $"notice show {_bannerKind}"; - private string BannerRole => string.Equals(_bannerKind, "err", StringComparison.Ordinal) - ? "alert" - : "status"; - private string? AccessToken => string.IsNullOrWhiteSpace(_bearerToken) - ? null - : _bearerToken.Trim(); - private string PreviewLimitLabel => $"{FormatBytes(TraceOptions.Value.MaxPreviewBytes)} preview"; - private int SuccessfulTraceCount => _traces.Count(trace => trace.IsSuccess is true); - private int FailedTraceCount => _traces.Count(trace => trace.IsSuccess is false); - private int StreamingTraceCount => _traces.Count(trace => trace.IsStreaming); - private int CacheTraceCount => _traces.Count(HasCacheActivity); - private bool HasActiveFilters => !string.IsNullOrWhiteSpace(_search) - || !string.IsNullOrWhiteSpace(_providerFilter) - || !string.IsNullOrWhiteSpace(_modelFilter) - || !string.IsNullOrWhiteSpace(_outcomeFilter); - private LlmTckProviderHttpTrace? SelectedTrace => _selectedRequestId is null - ? null - : _traces.FirstOrDefault(trace => - string.Equals(trace.RequestId, _selectedRequestId, StringComparison.Ordinal) - ); - private IReadOnlyList GetProviderOptions() => _traces - .Select(trace => trace.ProviderId) - .Where(provider => !string.IsNullOrWhiteSpace(provider)) - .Distinct(StringComparer.OrdinalIgnoreCase) - .Order(StringComparer.OrdinalIgnoreCase) - .ToList(); - private IReadOnlyList GetModelOptions() => _traces - .Select(ModelName) - .Where(model => model != "—") - .Distinct(StringComparer.OrdinalIgnoreCase) - .Order(StringComparer.OrdinalIgnoreCase) - .ToList(); - - protected override void OnInitialized() - { - RefreshState(); - } - - protected override async Task OnAfterRenderAsync(bool firstRender) - { - if (firstRender) - { - _interactive = true; - StateHasChanged(); - StartAutoRefresh(); - } - - if (_restoreRuntimeTriggerFocus) - { - _restoreRuntimeTriggerFocus = false; - await _runtimeDialogTrigger.FocusAsync(); - } - } - - private Task RefreshAsync() - { - RefreshState(); - return Task.CompletedTask; - } - - private async Task ResetAsync() - { - if (!Runtime.IsBearerTokenAccepted(AccessToken)) - { - SetUnauthorizedState(); - return; - } - - _resetting = true; - try - { - TraceStore.Reset(); - await Runtime.ResetAsync(); - RefreshState(); - await CloseRuntimeDialogAsync(); - SetBanner("warn", "Runtime reset · scenarios, events and provider request traces were cleared."); - } - catch (Exception exception) - { - _statusClass = "err"; - _statusText = "Server error"; - SetBanner("err", $"Reset failed: {exception.Message}"); - } - finally - { - _resetting = false; - } - } - - private Task SetAutoRefreshAsync(ChangeEventArgs args) - { - _autoRefresh = args.Value is true - || string.Equals(args.Value?.ToString(), "true", StringComparison.OrdinalIgnoreCase); - if (_autoRefresh) - { - StartAutoRefresh(); - } - else - { - StopAutoRefresh(); - } - - return Task.CompletedTask; - } - - private async Task OpenRuntimeDialogAsync() - { - await _runtimeDialog.ShowAsync(); - _runtimeDialogHidden = false; - } - - private async Task CloseRuntimeDialogAsync() - { - await _runtimeDialog.HideAsync(); - _runtimeDialogHidden = true; - _restoreRuntimeTriggerFocus = true; - } - - private void RuntimeDialogToggled(DialogToggleEventArgs args) - { - if (args.Id == _runtimeDialog.Id && string.Equals(args.NewState, ClosedDialogState, StringComparison.Ordinal)) - { - _runtimeDialogHidden = true; - _restoreRuntimeTriggerFocus = true; - } - } - - private void RefreshState() - { - _statusText = "Loading…"; - if (!Runtime.IsBearerTokenAccepted(AccessToken)) - { - SetUnauthorizedState(); - return; - } - - _models = Runtime.GetModels(); - _summary = Runtime.GetAssertionSummary(); - _events = _summary.Events; - var eventsByRequestId = new Dictionary(StringComparer.Ordinal); - foreach (var runtimeEvent in _events) - { - if (!string.IsNullOrWhiteSpace(runtimeEvent.RequestId)) - { - eventsByRequestId[runtimeEvent.RequestId] = runtimeEvent; - } - } - - _eventsByRequestId = eventsByRequestId; - _traces = TraceStore.GetSnapshot(); - if (_selectedRequestId is null - || !_traces.Any(trace => - string.Equals(trace.RequestId, _selectedRequestId, StringComparison.Ordinal) - )) - { - _selectedRequestId = _traces.FirstOrDefault()?.RequestId; - _activeDetailTab = _overviewTab; - if (_selectedRequestId is null) - { - _mobileDetailsOpen = false; - } - } - - _statusClass = "ok"; - _statusText = _traces.Any(trace => !trace.IsCompleted) ? "Requests in flight" : "Ready"; - ClearBanner(); - } - - private void SetUnauthorizedState() - { - _models = []; - _traces = []; - _events = []; - _eventsByRequestId = new Dictionary(StringComparer.Ordinal); - _summary = new LlmTckAssertionSummary(); - _selectedRequestId = null; - _mobileDetailsOpen = false; - _statusClass = "err"; - _statusText = "Unauthorized"; - SetBanner("err", "Enter the configured bearer token to load runtime state and request traces."); - } - - private void StartAutoRefresh() - { - StopAutoRefresh(); - if (!_autoRefresh) - { - return; - } - - var cancellation = new CancellationTokenSource(); - _autoRefreshCancellation = cancellation; - _ = RunAutoRefreshAsync(cancellation.Token); - } - - private void StopAutoRefresh() - { - var cancellation = _autoRefreshCancellation; - _autoRefreshCancellation = null; - if (cancellation is null) - { - return; - } - - cancellation.Cancel(); - cancellation.Dispose(); - } - - private async Task RunAutoRefreshAsync(CancellationToken cancellationToken) - { - try - { - using var timer = new PeriodicTimer(_autoRefreshInterval); - while (await timer.WaitForNextTickAsync(cancellationToken).ConfigureAwait(false)) - { - await InvokeAsync(() => - { - RefreshState(); - StateHasChanged(); - }); - } - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - } - catch (ObjectDisposedException) when (cancellationToken.IsCancellationRequested) - { - } - } - - private IReadOnlyList FilterTraces() - { - IEnumerable query = _traces; - if (!string.IsNullOrWhiteSpace(_search)) - { - var search = _search.Trim(); - query = query.Where(trace => TraceContains(trace, search)); - } - - if (!string.IsNullOrWhiteSpace(_providerFilter)) - { - query = query.Where(trace => - string.Equals(trace.ProviderId, _providerFilter, StringComparison.OrdinalIgnoreCase) - ); - } - - if (!string.IsNullOrWhiteSpace(_modelFilter)) - { - query = query.Where(trace => - string.Equals(ModelName(trace), _modelFilter, StringComparison.OrdinalIgnoreCase) - ); - } - - query = _outcomeFilter switch - { - "success" => query.Where(trace => trace.IsSuccess is true), - "error" => query.Where(trace => trace.IsSuccess is false), - "stream" => query.Where(trace => trace.IsStreaming), - "cache" => query.Where(HasCacheActivity), - _ => query, - }; - return query.ToList(); - } - - private static bool TraceContains(LlmTckProviderHttpTrace trace, string search) - { - return Contains(trace.ProviderId, search) - || Contains(trace.Path, search) - || Contains(trace.OperationId, search) - || Contains(trace.ModelId, search) - || Contains(trace.RequestPreview, search) - || Contains(trace.ResponsePreview, search) - || Contains(trace.RuntimeResponse, search) - || Contains(trace.RuntimeMessage, search); - } - - private static bool Contains(string? value, string search) - { - return value?.Contains(search, StringComparison.OrdinalIgnoreCase) == true; - } - - private void ClearFilters() - { - _search = string.Empty; - _providerFilter = string.Empty; - _modelFilter = string.Empty; - _outcomeFilter = string.Empty; - } - - private void SetOutcomeFilter(string filter) - { - _outcomeFilter = filter; - } - - private bool IsOutcomeFilter(string filter) - { - return string.Equals(_outcomeFilter, filter, StringComparison.Ordinal); - } - - private string RailFilterCss(string filter) - { - return IsOutcomeFilter(filter) ? "rail-item selected" : "rail-item"; - } - - private void SetProviderFilter(string provider) - { - _providerFilter = provider; - } - - private bool IsProviderFilter(string provider) - { - return string.Equals(_providerFilter, provider, StringComparison.OrdinalIgnoreCase); - } - - private string ProviderFilterCss(string provider) - { - return IsProviderFilter(provider) ? "rail-item provider selected" : "rail-item provider"; - } - - private int ProviderTraceCount(string provider) - { - return _traces.Count(trace => - string.Equals(trace.ProviderId, provider, StringComparison.OrdinalIgnoreCase) - ); - } - - private void SelectTrace(string requestId) - { - _selectedRequestId = requestId; - _activeDetailTab = _overviewTab; - _mobileDetailsOpen = true; - } - - private void BackToRequests() - { - _mobileDetailsOpen = false; - } - - private bool IsSelected(LlmTckProviderHttpTrace trace) - { - return string.Equals(trace.RequestId, _selectedRequestId, StringComparison.Ordinal); - } - - private string InboxItemCss(LlmTckProviderHttpTrace trace) - { - return IsSelected(trace) ? "inbox-item selected" : "inbox-item"; - } - - private static bool HasCacheActivity(LlmTckProviderHttpTrace trace) - { - return trace.Usage is { } usage - && (usage.CachedInputTokens > 0 || usage.CacheCreationInputTokens > 0); - } - - private bool IsDetailTab(string tab) - { - return string.Equals(_activeDetailTab, tab, StringComparison.Ordinal); - } - - private string DetailTabCss(string tab) - { - return IsDetailTab(tab) ? "detail-tab selected" : "detail-tab"; - } - - private string DetailPanelCss(string tab) - { - return IsDetailTab(tab) ? "detail-panel active" : "detail-panel"; - } - - private void SetDetailTab(string tab) - { - _activeDetailTab = tab; - } - - private LlmTckRuntimeEvent? RuntimeEventFor(LlmTckProviderHttpTrace trace) - { - return _eventsByRequestId.GetValueOrDefault(trace.RequestId); - } - - private LlmTckTokenUsage? UsageFor(LlmTckProviderHttpTrace trace) - { - return trace.Usage ?? RuntimeEventFor(trace)?.Usage; - } - - private IReadOnlyList MessagesFor(LlmTckProviderHttpTrace trace) - { - return trace.Messages.Count > 0 - ? trace.Messages - : RuntimeEventFor(trace)?.Messages ?? []; - } - - private IReadOnlyList ChunksFor(LlmTckProviderHttpTrace trace) - { - return trace.StreamChunks.Count > 0 - ? trace.StreamChunks - : RuntimeEventFor(trace)?.StreamChunks ?? []; - } - - private string ModelName(LlmTckProviderHttpTrace trace) - { - return string.IsNullOrWhiteSpace(trace.ModelId) - ? RuntimeEventFor(trace)?.ModelId ?? "—" - : trace.ModelId; - } - - private string RequestPreview(LlmTckProviderHttpTrace trace) - { - var messages = MessagesFor(trace); - var message = messages.LastOrDefault(candidate => - string.Equals(candidate.Role, "user", StringComparison.OrdinalIgnoreCase) - ) - ?? messages.LastOrDefault(); - if (message is not null && !string.IsNullOrWhiteSpace(message.Content)) - { - return TurnPreview(message.Content); - } - - return !string.IsNullOrWhiteSpace(trace.OperationId) - ? trace.OperationId - : trace.Path; - } - - private static string ProviderInitial(string providerId) - { - return ProviderName(providerId) switch - { - "Azure OpenAI" => "Az", - "Microsoft Foundry" => "M", - "Amazon Bedrock" => "B", - "OpenRouter" => "Or", - "DeepSeek" => "D", - "Perplexity" => "P", - { Length: > 0 } name => name[..1], - _ => "?", - }; - } - - private static bool ShouldCollapseTurn(string role, string content) - { - return content.Length > _collapsedTurnThreshold - && (string.Equals(role, "system", StringComparison.OrdinalIgnoreCase) - || string.Equals(role, "tool", StringComparison.OrdinalIgnoreCase)); - } - - private static string TurnPreview(string content) - { - var builder = new StringBuilder(Math.Min(content.Length, _turnPreviewLimit + 1)); - var pendingSpace = false; - var truncated = false; - foreach (var character in content) - { - if (char.IsWhiteSpace(character)) - { - pendingSpace = builder.Length > 0; - continue; - } - - if (pendingSpace) - { - if (builder.Length >= _turnPreviewLimit) - { - truncated = true; - break; - } - - builder.Append(' '); - pendingSpace = false; - } - - if (builder.Length >= _turnPreviewLimit) - { - truncated = true; - break; - } - - builder.Append(character); - } - - return truncated ? builder.Append('…').ToString() : builder.ToString(); - } - - private static string FormatCharacterCount(string content) - { - return $"{content.Length.ToString("N0", CultureInfo.InvariantCulture)} chars"; - } - - private static string ProviderName(string providerId) - { - return providerId switch - { - LlmTckCompatibilityTags.OpenAI => "OpenAI", - LlmTckCompatibilityTags.AzureOpenAI => "Azure OpenAI", - LlmTckCompatibilityTags.MicrosoftFoundry => "Microsoft Foundry", - LlmTckCompatibilityTags.Anthropic => "Anthropic", - LlmTckCompatibilityTags.Gemini => "Gemini", - LlmTckCompatibilityTags.Groq => "Groq", - LlmTckCompatibilityTags.Mistral => "Mistral", - LlmTckCompatibilityTags.Ollama => "Ollama", - LlmTckCompatibilityTags.Cohere => "Cohere", - LlmTckCompatibilityTags.Bedrock => "Amazon Bedrock", - LlmTckCompatibilityTags.OpenRouter => "OpenRouter", - LlmTckCompatibilityTags.DeepSeek => "DeepSeek", - LlmTckCompatibilityTags.Perplexity => "Perplexity", - _ => string.IsNullOrWhiteSpace(providerId) ? "Unknown" : providerId, - }; - } - - private static string StatusCssFor(LlmTckProviderHttpTrace trace) - { - if (!trace.IsCompleted) - { - return "status-chip warning"; - } - - if (trace.TransportState != LlmTckProviderHttpTraceTransportState.Completed) - { - return "status-chip error"; - } - - return trace.StatusCode switch - { - null => "status-chip error", - < 400 => "status-chip success", - >= 400 => "status-chip error", - }; - } - - private static string StatusText(LlmTckProviderHttpTrace trace) - { - if (!trace.IsCompleted) - { - return "Running"; - } - - if (trace.TransportState != LlmTckProviderHttpTraceTransportState.Completed) - { - return trace.TransportState.ToString(); - } - - return trace.StatusCode switch - { - null => "No status", - 200 => "200 OK", - 201 => "201 Created", - 202 => "202 Accepted", - 204 => "204 Empty", - 400 => "400 Invalid", - 401 => "401 Auth", - 404 => "404 Missing", - 405 => "405 Method", - 409 => "409 Conflict", - 429 => "429 Limited", - >= 500 => $"{trace.StatusCode} Server", - _ => trace.StatusCode.Value.ToString(CultureInfo.InvariantCulture), - }; - } - - private static string InboxStatusText(LlmTckProviderHttpTrace trace) - { - if (!trace.IsCompleted) - { - return "Live"; - } - - return trace.StatusCode?.ToString(CultureInfo.InvariantCulture) - ?? trace.TransportState.ToString(); - } - - private static string FormatDuration(double? milliseconds) - { - if (milliseconds is null) - { - return "running"; - } - - return milliseconds < 1 - ? "<1 ms" - : $"{milliseconds.Value:N0} ms"; - } - - private static string FormatBytes(long? bytes) - { - if (bytes is null) - { - return "unknown"; - } - - if (bytes < 1024) - { - return $"{bytes.Value:N0} B"; - } - - return $"{bytes.Value / 1024d:N1} KiB"; - } - - private static string TurnCss(string role) - { - if (string.Equals(role, "system", StringComparison.OrdinalIgnoreCase)) - { - return "turn system"; - } - - if (string.Equals(role, "user", StringComparison.OrdinalIgnoreCase)) - { - return "turn user"; - } - - if (string.Equals(role, "assistant", StringComparison.OrdinalIgnoreCase)) - { - return "turn assistant"; - } - - return string.Equals(role, "tool", StringComparison.OrdinalIgnoreCase) - ? "turn tool" - : "turn"; - } - - private static string RuntimeResponseTurnCss(LlmTckProviderHttpTrace trace) - { - return trace.RuntimeEventKind == LlmTckEventKind.ErrorReturned - ? "turn error" - : "turn assistant"; - } - - private static string RuntimeResponseRole(LlmTckProviderHttpTrace trace) - { - if (trace.IsSuccess is false || trace.RuntimeEventKind == LlmTckEventKind.ErrorReturned) - { - return "provider error"; - } - - return string.IsNullOrWhiteSpace(trace.RuntimeResponse) - ? "provider response" - : "assistant"; - } - - private static string AriaBoolean(bool value) - { - return value ? "true" : "false"; - } - - private static string EventTone(LlmTckEventKind kind) - { - return kind switch - { - LlmTckEventKind.Matched => "success", - LlmTckEventKind.Unmatched - or LlmTckEventKind.ScenarioExhausted => "warning", - _ => "error", - }; - } - - private static string PrettyPayload(string? payload, bool truncated) - { - if (string.IsNullOrWhiteSpace(payload)) - { - return "No body captured."; - } - - var suffix = truncated ? "\n\n[preview truncated]" : string.Empty; - try - { - using var document = JsonDocument.Parse(payload); - return JsonSerializer.Serialize(document.RootElement, _prettyJsonOptions) + suffix; - } - catch (JsonException) - { - return payload + suffix; - } - } - - private static string Empty(string? value) - { - return string.IsNullOrWhiteSpace(value) ? "—" : value; - } - - private void SetBanner(string kind, string text) - { - _bannerKind = kind; - _bannerText = text; - } - - private void ClearBanner() - { - _bannerKind = null; - _bannerText = null; - } - - public ValueTask DisposeAsync() - { - StopAutoRefresh(); - return ValueTask.CompletedTask; - } - - private sealed record ProviderRoute(string Name, string Route); -} diff --git a/src/ManagedCode.LlmTck.Hosting/LlmTckProviderHttpTrace.cs b/src/ManagedCode.LlmTck.Hosting/LlmTckProviderHttpTrace.cs deleted file mode 100644 index de1006b..0000000 --- a/src/ManagedCode.LlmTck.Hosting/LlmTckProviderHttpTrace.cs +++ /dev/null @@ -1,196 +0,0 @@ -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Scenarios; - -namespace ManagedCode.LlmTck.Hosting; - -public sealed record LlmTckProviderHttpTrace -{ - public string RequestId { get; init; } = string.Empty; - - public string ProviderId { get; init; } = string.Empty; - - public string ProviderNamespace { get; init; } = string.Empty; - - public string Method { get; init; } = string.Empty; - - public string Path { get; init; } = string.Empty; - - public string? Route { get; init; } - - public string? OperationId { get; init; } - - public DateTimeOffset StartedAt { get; init; } - - public DateTimeOffset? CompletedAt { get; init; } - - public double? DurationMilliseconds { get; init; } - - public int? StatusCode { get; init; } - - public bool IsCompleted { get; init; } - - public LlmTckProviderHttpTraceTransportState TransportState { get; init; } - - public bool? IsSuccess => !IsCompleted - ? null - : TransportState != LlmTckProviderHttpTraceTransportState.Completed - ? false - : StatusCode is null - ? null - : StatusCode < 400; - - public bool IsStreaming { get; init; } - - public string? RequestContentType { get; init; } - - public long? RequestByteCount { get; init; } - - public string? RequestPreview { get; init; } - - public bool RequestPreviewTruncated { get; init; } - - public string? ResponseContentType { get; init; } - - public long ResponseByteCount { get; init; } - - public string? ResponsePreview { get; init; } - - public bool ResponsePreviewTruncated { get; init; } - - public string? ModelId { get; init; } - - public string? ScenarioId { get; init; } - - public LlmTckEventKind? RuntimeEventKind { get; init; } - - public string? RuntimeMessage { get; init; } - - public string? RuntimeResponse { get; init; } - - public string? ErrorCode { get; init; } - - public string? ErrorMessage { get; init; } - - public string? TransportError { get; init; } - - public LlmTckTokenUsage? Usage { get; init; } - - public IReadOnlyList Messages { get; init; } = []; - - public IReadOnlyList StreamChunks { get; init; } = []; - - public bool RuntimePayloadTruncated { get; init; } - - public bool RetentionTruncated { get; init; } - - public int RetainedByteCount { get; init; } -} - -public sealed record LlmTckProviderHttpTraceCompletion -{ - public DateTimeOffset CompletedAt { get; init; } - - public double DurationMilliseconds { get; init; } - - public int? StatusCode { get; init; } - - public LlmTckProviderHttpTraceTransportState TransportState { get; init; } - - public string? Route { get; init; } - - public string? OperationId { get; init; } - - public bool IsStreaming { get; init; } - - public string? ResponseContentType { get; init; } - - public long ResponseByteCount { get; init; } - - public string? ResponsePreview { get; init; } - - public bool ResponsePreviewTruncated { get; init; } - - public string? TransportError { get; init; } -} - -public sealed record LlmTckProviderHttpTraceRequestCapture -{ - public long? ByteCount { get; init; } - - public string? Preview { get; init; } - - public bool PreviewTruncated { get; init; } - - public string? ModelId { get; init; } -} - -public sealed record LlmTckProviderHttpTraceEnrichment -{ - public string? OperationId { get; init; } - - public string? ModelId { get; init; } - - public string? ScenarioId { get; init; } - - public LlmTckEventKind? RuntimeEventKind { get; init; } - - public string? RuntimeMessage { get; init; } - - public string? RuntimeResponse { get; init; } - - public string? ErrorCode { get; init; } - - public string? ErrorMessage { get; init; } - - public LlmTckTokenUsage? Usage { get; init; } - - public IReadOnlyList? Messages { get; init; } - - public IReadOnlyList? StreamChunks { get; init; } - - public bool? IsStreaming { get; init; } - - public bool? RuntimePayloadTruncated { get; init; } -} - -public sealed class LlmTckProviderHttpTraceOptions -{ - public const int DefaultMaxEntries = 500; - public const int DefaultMaxPreviewBytes = 64 * 1024; - public const int DefaultMaxRetainedBytes = 16 * 1024 * 1024; - public const string DefaultRequestIdHeaderName = "x-llm-tck-request-id"; - - public int MaxEntries { get; set; } = DefaultMaxEntries; - - public int MaxPreviewBytes { get; set; } = DefaultMaxPreviewBytes; - - public int MaxRetainedBytes { get; set; } = DefaultMaxRetainedBytes; - - public string RequestIdHeaderName { get; set; } = DefaultRequestIdHeaderName; -} - -public interface ILlmTckProviderHttpTraceStore -{ - long RetainedByteCount { get; } - - IReadOnlyList GetSnapshot(); - - void Start(LlmTckProviderHttpTrace trace); - - bool TryCaptureRequest(string requestId, LlmTckProviderHttpTraceRequestCapture capture); - - bool TryComplete(string requestId, LlmTckProviderHttpTraceCompletion completion); - - bool TryEnrich(string requestId, LlmTckProviderHttpTraceEnrichment enrichment); - - void Reset(); -} - -public enum LlmTckProviderHttpTraceTransportState -{ - InProgress, - Completed, - Cancelled, - Aborted, - Faulted, -} diff --git a/src/ManagedCode.LlmTck.Hosting/ManagedCode.LlmTck.Hosting.csproj b/src/ManagedCode.LlmTck.Hosting/ManagedCode.LlmTck.Hosting.csproj deleted file mode 100644 index 27a0f42..0000000 --- a/src/ManagedCode.LlmTck.Hosting/ManagedCode.LlmTck.Hosting.csproj +++ /dev/null @@ -1,25 +0,0 @@ - - - ManagedCode.LlmTck.Hosting - ASP.NET Core endpoint mapping for ManagedCode.LlmTck provider emulation and control APIs. - - - - - - - - - - - - - - - - - - - - - diff --git a/src/ManagedCode.LlmTck.Mistral/AGENTS.md b/src/ManagedCode.LlmTck.Mistral/AGENTS.md deleted file mode 100644 index 5a84cce..0000000 --- a/src/ManagedCode.LlmTck.Mistral/AGENTS.md +++ /dev/null @@ -1,5 +0,0 @@ -# ManagedCode.LlmTck.Mistral - -This package owns Mistral compatibility constants, provider profiles, and future Mistral wire contracts. - -Keep deterministic runtime behavior in `ManagedCode.LlmTck`; keep HTTP hosting in `ManagedCode.LlmTck.Hosting`. diff --git a/src/ManagedCode.LlmTck.Mistral/ManagedCode.LlmTck.Mistral.csproj b/src/ManagedCode.LlmTck.Mistral/ManagedCode.LlmTck.Mistral.csproj deleted file mode 100644 index 32b69c7..0000000 --- a/src/ManagedCode.LlmTck.Mistral/ManagedCode.LlmTck.Mistral.csproj +++ /dev/null @@ -1,10 +0,0 @@ - - - ManagedCode.LlmTck.Mistral - Mistral compatibility profile and adapters for ManagedCode.LlmTck. - - - - - - diff --git a/src/ManagedCode.LlmTck.Ollama/AGENTS.md b/src/ManagedCode.LlmTck.Ollama/AGENTS.md deleted file mode 100644 index 46c5855..0000000 --- a/src/ManagedCode.LlmTck.Ollama/AGENTS.md +++ /dev/null @@ -1,5 +0,0 @@ -# ManagedCode.LlmTck.Ollama - -This package owns Ollama compatibility constants, provider profiles, and future Ollama wire contracts. - -Keep deterministic runtime behavior in `ManagedCode.LlmTck`; keep HTTP hosting in `ManagedCode.LlmTck.Hosting`. diff --git a/src/ManagedCode.LlmTck.Ollama/ManagedCode.LlmTck.Ollama.csproj b/src/ManagedCode.LlmTck.Ollama/ManagedCode.LlmTck.Ollama.csproj deleted file mode 100644 index be6c1d2..0000000 --- a/src/ManagedCode.LlmTck.Ollama/ManagedCode.LlmTck.Ollama.csproj +++ /dev/null @@ -1,13 +0,0 @@ - - - ManagedCode.LlmTck.Ollama - Ollama compatibility profile and adapters for ManagedCode.LlmTck. - - - - - - - - - diff --git a/src/ManagedCode.LlmTck.OpenAI/ManagedCode.LlmTck.OpenAI.csproj b/src/ManagedCode.LlmTck.OpenAI/ManagedCode.LlmTck.OpenAI.csproj deleted file mode 100644 index 2f01239..0000000 --- a/src/ManagedCode.LlmTck.OpenAI/ManagedCode.LlmTck.OpenAI.csproj +++ /dev/null @@ -1,13 +0,0 @@ - - - ManagedCode.LlmTck.OpenAI - OpenAI-compatible wire models and adapters for ManagedCode.LlmTck. - - - - - - - - - diff --git a/src/ManagedCode.LlmTck.OpenAI/OpenAiRequestValidation.cs b/src/ManagedCode.LlmTck.OpenAI/OpenAiRequestValidation.cs deleted file mode 100644 index 4005dd3..0000000 --- a/src/ManagedCode.LlmTck.OpenAI/OpenAiRequestValidation.cs +++ /dev/null @@ -1,75 +0,0 @@ -using System.Text.Json; -using ManagedCode.LlmTck.Providers; -using static ManagedCode.LlmTck.ProviderValidation.LlmTckJsonValidation; - -namespace ManagedCode.LlmTck.OpenAI; - -public static class OpenAiRequestValidation -{ - public static LlmTckRequestValidationResult ValidateChat(JsonElement body) - { - return Check(body, value => Validate(value, false)); - } - - public static LlmTckRequestValidationResult ValidateResponse(JsonElement body) - { - return Check(body, value => Validate(value, true)); - } - - private static void Validate(JsonElement body, bool responses) - { - if (body.TryGetProperty("functions", out _) || body.TryGetProperty("function_call", out _)) - { - throw new JsonException("Use tools and tool_choice."); - } - foreach (var tool in OptionalArray(body, "tools")) - { - Choice(tool.GetProperty("type"), "function"); - var function = responses ? tool : tool.GetProperty("function"); - Name(function); OptionalSchema(function, "parameters"); - } - if (body.TryGetProperty("tool_choice", out var choice)) - { - if (choice.ValueKind == JsonValueKind.Object) - { - Choice(choice.GetProperty("type"), "function"); - Name(responses ? choice : choice.GetProperty("function")); - } - else { Choice(choice, "auto", "none", "required"); } - } - if (body.TryGetProperty("parallel_tool_calls", out var parallel)) { _ = parallel.GetBoolean(); } - if (body.TryGetProperty("response_format", out var format) && format.ValueKind != JsonValueKind.Null) - { - ValidateFormat(format, false); - } - if (responses && body.TryGetProperty("text", out var text) && text.ValueKind != JsonValueKind.Null && text.TryGetProperty("format", out format)) - { - ValidateFormat(format, true); - } - var field = responses ? "input" : "messages"; - if (!body.TryGetProperty(field, out var messages) || messages.ValueKind != JsonValueKind.Array) { return; } - foreach (var message in messages.EnumerateArray()) - { - if (message.ValueKind != JsonValueKind.Object) { continue; } - foreach (var call in OptionalArray(message, "tool_calls", allowNull: true)) - { - Name(call.GetProperty("function")); NonEmpty(call.GetProperty("id")); String(call.GetProperty("function").GetProperty("arguments")); - } - if (responses && message.TryGetProperty("type", out var type)) - { - if (type.GetString() == "function_call") { Name(message); NonEmpty(message.GetProperty("call_id")); String(message.GetProperty("arguments")); } - if (type.GetString() == "function_call_output") { NonEmpty(message.GetProperty("call_id")); _ = message.GetProperty("output"); } - } - } - } - - private static void ValidateFormat(JsonElement format, bool responses) - { - Choice(format.GetProperty("type"), "text", "json_object", "json_schema"); - if (format.GetProperty("type").GetString() == "json_schema") - { - Schema(responses ? format.GetProperty("schema") : format.GetProperty("json_schema").GetProperty("schema")); - } - OptionalSchema(format, "schema"); - } -} diff --git a/src/ManagedCode.LlmTck.OpenAI/OpenAiResponseModels.cs b/src/ManagedCode.LlmTck.OpenAI/OpenAiResponseModels.cs deleted file mode 100644 index a21cd4d..0000000 --- a/src/ManagedCode.LlmTck.OpenAI/OpenAiResponseModels.cs +++ /dev/null @@ -1,151 +0,0 @@ -using System.Text.Json; -using System.Text.Json.Serialization; - -namespace ManagedCode.LlmTck.OpenAI; - -public sealed record OpenAiResponseRequest -{ - [JsonPropertyName("parallel_tool_calls")] - public bool ParallelToolCalls { get; init; } = true; - - [JsonPropertyName("tools")] public List Tools { get; init; } = []; - [JsonPropertyName("tool_choice")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public JsonElement ToolChoice { get; init; } - [JsonPropertyName("text")] public OpenAiResponseTextOptions? Text { get; init; } - - [JsonPropertyName("model")] - public string Model { get; init; } = string.Empty; - - [JsonPropertyName("input")] - public JsonElement Input { get; init; } - - [JsonPropertyName("instructions")] - public string? Instructions { get; init; } - - [JsonPropertyName("stream")] - public bool Stream { get; init; } - - [JsonPropertyName("store")] - public bool? Store { get; init; } - - [JsonPropertyName("previous_response_id")] - public string? PreviousResponseId { get; init; } - - [JsonPropertyName("prompt_cache_key")] - public string? PromptCacheKey { get; init; } - - [JsonPropertyName("session_id")] - public string? SessionId { get; init; } -} - -public sealed record OpenAiResponse -{ - [JsonPropertyName("id")] - public string Id { get; init; } = string.Empty; - - [JsonPropertyName("object")] - public string ObjectType { get; init; } = "response"; - - [JsonPropertyName("created_at")] - public long CreatedAt { get; init; } - - [JsonPropertyName("model")] - public string Model { get; init; } = string.Empty; - - [JsonPropertyName("output")] - public List Output { get; init; } = []; - - [JsonPropertyName("usage")] - public OpenAiResponseUsage Usage { get; init; } = new(); - - [JsonPropertyName("status")] - public string Status { get; init; } = "completed"; -} - -public sealed record OpenAiResponseOutputMessage -{ - [JsonPropertyName("call_id")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? CallId { get; init; } - [JsonPropertyName("name")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? Name { get; init; } - [JsonPropertyName("arguments")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? Arguments { get; init; } - - [JsonPropertyName("type")] - public string Type { get; init; } = "message"; - - [JsonPropertyName("id")] - public string Id { get; init; } = string.Empty; - - [JsonPropertyName("status")] - public string Status { get; init; } = "completed"; - - [JsonPropertyName("role")] - public string Role { get; init; } = "assistant"; - - [JsonPropertyName("content")] - public List Content { get; init; } = []; -} - -public sealed record OpenAiResponseOutputText -{ - [JsonPropertyName("type")] - public string Type { get; init; } = "output_text"; - - [JsonPropertyName("text")] - public string Text { get; init; } = string.Empty; - - [JsonPropertyName("annotations")] - public List Annotations { get; init; } = []; -} - -public sealed record OpenAiResponseUsage -{ - [JsonPropertyName("input_tokens")] - public int InputTokens { get; init; } - - [JsonPropertyName("output_tokens")] - public int OutputTokens { get; init; } - - [JsonPropertyName("total_tokens")] - public int TotalTokens { get; init; } - - [JsonPropertyName("input_tokens_details")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public OpenAiInputTokensDetails? InputTokensDetails { get; init; } - - [JsonPropertyName("output_tokens_details")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public OpenAiOutputTokensDetails? OutputTokensDetails { get; init; } -} - -public sealed record OpenAiInputTokensDetails -{ - [JsonPropertyName("cached_tokens")] - public int CachedTokens { get; init; } - - [JsonPropertyName("cache_write_tokens")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public int? CacheWriteTokens { get; init; } -} - -public sealed record OpenAiOutputTokensDetails -{ - [JsonPropertyName("reasoning_tokens")] - public int ReasoningTokens { get; init; } -} - -public sealed record OpenAiResponseTool -{ - [JsonPropertyName("type")] public string Type { get; init; } = "function"; - [JsonPropertyName("name")] public string Name { get; init; } = string.Empty; - [JsonPropertyName("description")] public string? Description { get; init; } - [JsonPropertyName("parameters")] public JsonElement Parameters { get; init; } = JsonSerializer.SerializeToElement(new { type = "object" }); -} -public sealed record OpenAiResponseTextOptions -{ - [JsonPropertyName("format")] public OpenAiResponseTextFormat? Format { get; init; } -} -public sealed record OpenAiResponseTextFormat -{ - [JsonPropertyName("type")] public string Type { get; init; } = "text"; - [JsonPropertyName("name")] public string? Name { get; init; } - [JsonPropertyName("schema")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public JsonElement Schema { get; init; } - [JsonPropertyName("strict")] public bool? Strict { get; init; } -} diff --git a/src/ManagedCode.LlmTck.OpenRouter/AGENTS.md b/src/ManagedCode.LlmTck.OpenRouter/AGENTS.md deleted file mode 100644 index de29888..0000000 --- a/src/ManagedCode.LlmTck.OpenRouter/AGENTS.md +++ /dev/null @@ -1,5 +0,0 @@ -# ManagedCode.LlmTck.OpenRouter - -This package owns OpenRouter compatibility constants, provider profiles, and future OpenRouter wire contracts. - -Keep deterministic runtime behavior in `ManagedCode.LlmTck`; keep HTTP hosting in `ManagedCode.LlmTck.Hosting`. diff --git a/src/ManagedCode.LlmTck.OpenRouter/ManagedCode.LlmTck.OpenRouter.csproj b/src/ManagedCode.LlmTck.OpenRouter/ManagedCode.LlmTck.OpenRouter.csproj deleted file mode 100644 index 48286b3..0000000 --- a/src/ManagedCode.LlmTck.OpenRouter/ManagedCode.LlmTck.OpenRouter.csproj +++ /dev/null @@ -1,10 +0,0 @@ - - - ManagedCode.LlmTck.OpenRouter - OpenRouter compatibility profile and adapters for ManagedCode.LlmTck. - - - - - - diff --git a/src/ManagedCode.LlmTck.Perplexity/AGENTS.md b/src/ManagedCode.LlmTck.Perplexity/AGENTS.md deleted file mode 100644 index fd6f863..0000000 --- a/src/ManagedCode.LlmTck.Perplexity/AGENTS.md +++ /dev/null @@ -1,5 +0,0 @@ -# ManagedCode.LlmTck.Perplexity - -This package owns Perplexity compatibility constants, provider profiles, and future Perplexity wire contracts. - -Keep deterministic runtime behavior in `ManagedCode.LlmTck`; keep HTTP hosting in `ManagedCode.LlmTck.Hosting`. diff --git a/src/ManagedCode.LlmTck.Perplexity/ManagedCode.LlmTck.Perplexity.csproj b/src/ManagedCode.LlmTck.Perplexity/ManagedCode.LlmTck.Perplexity.csproj deleted file mode 100644 index 69f81ad..0000000 --- a/src/ManagedCode.LlmTck.Perplexity/ManagedCode.LlmTck.Perplexity.csproj +++ /dev/null @@ -1,10 +0,0 @@ - - - ManagedCode.LlmTck.Perplexity - Perplexity compatibility profile and adapters for ManagedCode.LlmTck. - - - - - - diff --git a/src/ManagedCode.LlmTck.SystemOne/AGENTS.md b/src/ManagedCode.LlmTck.SystemOne/AGENTS.md deleted file mode 100644 index af5b78a..0000000 --- a/src/ManagedCode.LlmTck.SystemOne/AGENTS.md +++ /dev/null @@ -1,3 +0,0 @@ -# ManagedCode.LlmTck.SystemOne - -Owns native SystemOne decision wire contracts and compatibility profiles. Keep deterministic fixtures in `ManagedCode.LlmTck` and routes in `ManagedCode.LlmTck.Hosting`. DTOs use explicit JSON names; preserve opaque choice IDs and native scores. diff --git a/src/ManagedCode.LlmTck.SystemOne/ManagedCode.LlmTck.SystemOne.csproj b/src/ManagedCode.LlmTck.SystemOne/ManagedCode.LlmTck.SystemOne.csproj deleted file mode 100644 index 8bf7283..0000000 --- a/src/ManagedCode.LlmTck.SystemOne/ManagedCode.LlmTck.SystemOne.csproj +++ /dev/null @@ -1,5 +0,0 @@ - - ManagedCode.LlmTck.SystemOneNative SystemOne decision API compatibility for ManagedCode.LlmTck. - - - diff --git a/src/ManagedCode.LlmTck.SystemOne/SystemOneCompatibility.cs b/src/ManagedCode.LlmTck.SystemOne/SystemOneCompatibility.cs deleted file mode 100644 index ba62d6d..0000000 --- a/src/ManagedCode.LlmTck.SystemOne/SystemOneCompatibility.cs +++ /dev/null @@ -1,23 +0,0 @@ -using ManagedCode.LlmTck.Providers; -namespace ManagedCode.LlmTck.SystemOne; - -public static class SystemOneCompatibility -{ - public const string ProviderId = LlmTckCompatibilityTags.SystemOne; - public static LlmTckProviderProfile Profile { get; } = new() - { - Id = ProviderId, - DisplayName = "SystemOne (Jev and Kev)", - Protocol = LlmTckProtocolFamily.SystemOne, - DefaultEndpointPath = "/systemone/v1/systemone", - Capabilities = [LlmTckProviderCapability.Decisions], - CompatibilityTags = [ProviderId], - ApiContract = new() - { - DocumentationUrl = "https://docs.typesafe.ai/api", - DocumentationRetrievedOn = "2026-10-20", - DocumentationVersion = "Jev 1.14 JSON/multipart decisions, images and box; Kev native decisions", - Operations = [new() { Id = LlmTckProviderOperationIds.SystemOne.DecisionsCreate, Method = "POST", Path = "/systemone/v1/systemone", DocumentationUrl = "https://docs.typesafe.ai/api", ImplementedByHosting = true, Capabilities = [LlmTckProviderCapability.Decisions] }] - }, - }; -} diff --git a/src/ManagedCode.LlmTck/Control/LlmTckControlRoutes.cs b/src/ManagedCode.LlmTck/Control/LlmTckControlRoutes.cs deleted file mode 100644 index b192516..0000000 --- a/src/ManagedCode.LlmTck/Control/LlmTckControlRoutes.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace ManagedCode.LlmTck.Control; - -public static class LlmTckControlRoutes -{ - public const string Admin = "/"; - public const string Models = "/admin-api/models"; - public const string Assertions = "/admin-api/assertions"; - public const string Configure = "/admin-api/configure"; - public const string Reset = "/admin-api/reset"; -} diff --git a/src/ManagedCode.LlmTck/Decisions/LlmTckDecisionFixtureValidation.cs b/src/ManagedCode.LlmTck/Decisions/LlmTckDecisionFixtureValidation.cs deleted file mode 100644 index ac584b6..0000000 --- a/src/ManagedCode.LlmTck/Decisions/LlmTckDecisionFixtureValidation.cs +++ /dev/null @@ -1,27 +0,0 @@ -using System.Globalization; - -namespace ManagedCode.LlmTck.Decisions; - -internal static class LlmTckDecisionFixtureValidation -{ - private const double _probabilityRoundingError = 0.00005; - private const double _floatingPointTolerance = 0.000001; - - public static bool IsConformant(LlmTckDecisionProvider provider, LlmTckDecisionQuestion question, LlmTckDecisionAnswer answer) - { - if (provider == LlmTckDecisionProvider.Unspecified || answer.Kind == LlmTckDecisionKind.Predicate) { return true; } - if (answer.Kind == LlmTckDecisionKind.Box) - { return provider == LlmTckDecisionProvider.TypeSafe && answer.Box is { IsValid: true } && answer.Probabilities.Count == 0; } - var domain = question.Kind == LlmTckDecisionKind.Score - ? Enumerable.Range(0, question.Options.Count).Select(index => index.ToString(CultureInfo.InvariantCulture)).ToArray() - : question.Options.ToArray(); - if (!answer.Confidence.HasValue || answer.Probabilities.Count != domain.Length - || !domain.All(answer.Probabilities.ContainsKey)) { return false; } - var tolerance = provider is LlmTckDecisionProvider.Kev or LlmTckDecisionProvider.Cloudflare - ? _probabilityRoundingError * domain.Length + _floatingPointTolerance : _floatingPointTolerance; - if (Math.Abs(answer.Probabilities.Values.Sum() - 1) > tolerance) { return false; } - if (provider != LlmTckDecisionProvider.OpenAI && answer.Kind == LlmTckDecisionKind.Choice && answer.Probabilities[answer.Choice!] + _floatingPointTolerance < answer.Probabilities.Values.Max()) { return false; } - return provider != LlmTckDecisionProvider.Cloudflare - || Math.Abs(answer.Confidence.Value - answer.Probabilities.Values.Max()) <= _probabilityRoundingError + _floatingPointTolerance; - } -} diff --git a/src/ManagedCode.LlmTck/Models/LlmTckModel.cs b/src/ManagedCode.LlmTck/Models/LlmTckModel.cs deleted file mode 100644 index df17678..0000000 --- a/src/ManagedCode.LlmTck/Models/LlmTckModel.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace ManagedCode.LlmTck.Models; - -public sealed record LlmTckModel -{ - public string Id { get; init; } = LlmTckKnownModelIds.Gpt41Mini; - - public LlmTckModelKind Kind { get; init; } = LlmTckModelKind.Chat; - - public string OwnedBy { get; init; } = "llm-tck"; - - public ManagedCode.LlmTck.Decisions.LlmTckDecisionProvider DecisionProvider { get; init; } - - public int ReasoningTokens { get; init; } -} diff --git a/src/ManagedCode.LlmTck/Models/LlmTckModelKind.cs b/src/ManagedCode.LlmTck/Models/LlmTckModelKind.cs deleted file mode 100644 index c4861eb..0000000 --- a/src/ManagedCode.LlmTck/Models/LlmTckModelKind.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace ManagedCode.LlmTck.Models; - -public enum LlmTckModelKind -{ - Chat, - Embedding, - Image, - Audio, - Video, - Decision, -} diff --git a/src/ManagedCode.LlmTck/Providers/LlmTckProviderApiContract.cs b/src/ManagedCode.LlmTck/Providers/LlmTckProviderApiContract.cs deleted file mode 100644 index cf14026..0000000 --- a/src/ManagedCode.LlmTck/Providers/LlmTckProviderApiContract.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace ManagedCode.LlmTck.Providers; - -public sealed record LlmTckProviderApiContract -{ - public required string DocumentationUrl { get; init; } - - public required string DocumentationRetrievedOn { get; init; } - - public string? DocumentationVersion { get; init; } - - public IReadOnlyList Operations { get; init; } = []; -} diff --git a/src/ManagedCode.LlmTck/Providers/LlmTckProviderProfile.cs b/src/ManagedCode.LlmTck/Providers/LlmTckProviderProfile.cs deleted file mode 100644 index aa1573f..0000000 --- a/src/ManagedCode.LlmTck/Providers/LlmTckProviderProfile.cs +++ /dev/null @@ -1,18 +0,0 @@ -namespace ManagedCode.LlmTck.Providers; - -public sealed record LlmTckProviderProfile -{ - public required string Id { get; init; } - - public required string DisplayName { get; init; } - - public required LlmTckProtocolFamily Protocol { get; init; } - - public string DefaultEndpointPath { get; init; } = "/"; - - public IReadOnlyList Capabilities { get; init; } = []; - - public IReadOnlyList CompatibilityTags { get; init; } = []; - - public required LlmTckProviderApiContract ApiContract { get; init; } -} diff --git a/src/ManagedCode.LlmTck/Runtime/ILlmTckRuntimeRequestScope.cs b/src/ManagedCode.LlmTck/Runtime/ILlmTckRuntimeRequestScope.cs deleted file mode 100644 index a87c5f6..0000000 --- a/src/ManagedCode.LlmTck/Runtime/ILlmTckRuntimeRequestScope.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace ManagedCode.LlmTck.Runtime; - -/// -/// Optional provider-neutral correlation scope used by hosts to associate runtime events with an -/// external request without changing the operation methods on . -/// -public interface ILlmTckRuntimeRequestScope -{ - IDisposable BeginRequest(string requestId); -} diff --git a/src/ManagedCode.LlmTck/Runtime/ILlmTckVideoStore.cs b/src/ManagedCode.LlmTck/Runtime/ILlmTckVideoStore.cs deleted file mode 100644 index da25b6a..0000000 --- a/src/ManagedCode.LlmTck/Runtime/ILlmTckVideoStore.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace ManagedCode.LlmTck.Runtime; - -public interface ILlmTckVideoStore -{ - LlmTckVideoResult StoreVideo(string provider, LlmTckVideoResult video); - IReadOnlyList ListVideos(string provider); - LlmTckVideoResult? FindVideo(string provider, string id, bool generation = false); - bool DeleteVideo(string provider, string id); -} diff --git a/src/ManagedCode.LlmTck/Runtime/LlmTckRuntime.Tools.cs b/src/ManagedCode.LlmTck/Runtime/LlmTckRuntime.Tools.cs deleted file mode 100644 index 1f20472..0000000 --- a/src/ManagedCode.LlmTck/Runtime/LlmTckRuntime.Tools.cs +++ /dev/null @@ -1,91 +0,0 @@ -using System.Text.Json; -using Json.Schema; -using ManagedCode.LlmTck.Scenarios; - -namespace ManagedCode.LlmTck.Runtime; - -public sealed partial class LlmTckRuntime -{ - private static string? ValidateFixture(LlmTckChatRequest request, LlmTckScenarioResponse response) - { - if (response.Error is not null) - { - return null; - } - - try - { - if (request.Tools is null || request.Tools.Any(tool => tool is null || string.IsNullOrWhiteSpace(tool.Name))) - { - return "Tool declarations require names."; - } - - if (request.Tools.Select(tool => tool.Name).Distinct(StringComparer.Ordinal).Count() != request.Tools.Count) - { - return "Tool names must be unique."; - } - - foreach (var tool in request.Tools) - { - _ = LlmTckJsonSchema.Matches("{}", tool.ParametersJson); - } - - if (request.ResponseSchemaJson is { } declaredSchema) - { - _ = LlmTckJsonSchema.Matches("{}", declaredSchema); - } - - if (!request.AllowParallelToolCalls && response.ToolCalls.Count > 1) - { - return "The selected fixture contains parallel tool calls, which the request disabled."; - } - - if (request.ToolChoice == LlmTckToolChoice.Required && response.ToolCalls.Count == 0) - { - return "The selected fixture must contain a tool call."; - } - - if (request.ToolChoice == LlmTckToolChoice.None && response.ToolCalls.Count > 0) - { - return "The selected fixture contains a tool call but tool choice is none."; - } - - foreach (var call in response.ToolCalls) - { - var definition = request.Tools.FirstOrDefault(tool => tool.Name == call.Name); - if (definition is null || request.RequiredToolName is { } required && required != call.Name) - { - return "The fixture calls a tool that was not selected or declared in the request."; - } - - if (!LlmTckJsonSchema.Matches(call.ArgumentsJson, definition.ParametersJson)) - { - return "Fixture tool arguments do not match the declared parameter schema."; - } - } - if (response.ToolCalls.Select(call => call.Id).Distinct(StringComparer.Ordinal).Count() != response.ToolCalls.Count) - { - return "Fixture tool call IDs must be unique."; - } - - if (response.ToolCalls.Count == 0 && (request.RequireJson || request.ResponseSchemaJson is not null)) - { - LlmTckJsonSchema.ValidateJson(response.Content); - if (request.ResponseSchemaJson is { } schema && !LlmTckJsonSchema.Matches(response.Content, schema)) - { - return "The selected fixture does not match the requested JSON schema."; - } - - if (response.StreamChunks.Count > 0 && string.Concat(response.StreamChunks) != response.Content) - { - return "Structured-output stream chunks must concatenate to the validated JSON fixture."; - } - } - return null; - } - catch (Exception exception) when (exception is JsonException or JsonSchemaException or ArgumentException) - { - return $"Invalid JSON or schema in request/fixture: {exception.Message}"; - } - } -} diff --git a/src/ManagedCode.LlmTck/Scenarios/LlmTckChatRequest.cs b/src/ManagedCode.LlmTck/Scenarios/LlmTckChatRequest.cs deleted file mode 100644 index c23f9f0..0000000 --- a/src/ManagedCode.LlmTck/Scenarios/LlmTckChatRequest.cs +++ /dev/null @@ -1,49 +0,0 @@ -using ManagedCode.LlmTck.Models; - -namespace ManagedCode.LlmTck.Scenarios; - -public sealed record LlmTckChatRequest -{ - /// - /// Optional caller-provided identifier copied to runtime trace events for stable correlation. - /// - public string? RequestId { get; init; } - - public string ModelId { get; init; } = LlmTckKnownModelIds.Gpt41Mini; - - public List Messages { get; init; } = []; - - /// Exact provider/transport namespace for optional stored-response continuation. - public string? HistoryNamespace { get; init; } - public string? PreviousResponseId { get; init; } - public bool StoreResponse { get; init; } - - public bool Stream { get; init; } - public List Tools { get; init; } = []; - public LlmTckToolChoice ToolChoice { get; init; } - /// Whether a fixture may contain more than one tool call in a response. - public bool AllowParallelToolCalls { get; init; } = true; - public string? RequiredToolName { get; init; } - public bool RequireJson { get; init; } - public string? ResponseSchemaJson { get; init; } - - /// Warms the prompt cache without consuming a configured response. - public bool PopulateCacheOnly { get; init; } - - public LlmTckPromptCachePolicy PromptCachePolicy { get; init; } - - public string? PromptCacheKey { get; init; } -} - -public enum LlmTckPromptCachePolicy -{ - None, - OpenAiCompatible, - Mistral, - Anthropic, - Gemini, - Bedrock, - DeepSeek, - OpenRouter, - Ollama, -} diff --git a/src/ManagedCode.LlmTck/Scenarios/LlmTckMatchMode.cs b/src/ManagedCode.LlmTck/Scenarios/LlmTckMatchMode.cs deleted file mode 100644 index cca7f93..0000000 --- a/src/ManagedCode.LlmTck/Scenarios/LlmTckMatchMode.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace ManagedCode.LlmTck.Scenarios; - -public enum LlmTckMatchMode -{ - Exact, - Contains, -} diff --git a/src/ManagedCode.LlmTck/Scenarios/LlmTckMessage.cs b/src/ManagedCode.LlmTck/Scenarios/LlmTckMessage.cs deleted file mode 100644 index 36c7e09..0000000 --- a/src/ManagedCode.LlmTck/Scenarios/LlmTckMessage.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace ManagedCode.LlmTck.Scenarios; - -public sealed record LlmTckMessage -{ - public string? ToolCallId { get; init; } - - public string Role { get; init; } = "user"; - - public List ToolCalls { get; init; } = []; - - public string Content { get; init; } = string.Empty; -} diff --git a/src/ManagedCode.LlmTck/Scenarios/LlmTckRequestMatch.cs b/src/ManagedCode.LlmTck/Scenarios/LlmTckRequestMatch.cs deleted file mode 100644 index 57c3258..0000000 --- a/src/ManagedCode.LlmTck/Scenarios/LlmTckRequestMatch.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace ManagedCode.LlmTck.Scenarios; - -public sealed record LlmTckRequestMatch -{ - public LlmTckMatchMode Mode { get; init; } = LlmTckMatchMode.Contains; - - public List Messages { get; init; } = []; -} diff --git a/src/ManagedCode.LlmTck/Scenarios/LlmTckScenario.cs b/src/ManagedCode.LlmTck/Scenarios/LlmTckScenario.cs deleted file mode 100644 index 4db2179..0000000 --- a/src/ManagedCode.LlmTck/Scenarios/LlmTckScenario.cs +++ /dev/null @@ -1,16 +0,0 @@ -using ManagedCode.LlmTck.Models; - -namespace ManagedCode.LlmTck.Scenarios; - -public sealed record LlmTckScenario -{ - public string Id { get; init; } = Guid.NewGuid().ToString("N"); - - public string ModelId { get; init; } = LlmTckKnownModelIds.Gpt41Mini; - - public string? RequiredBearerToken { get; init; } - - public LlmTckRequestMatch Match { get; init; } = new(); - - public List Responses { get; init; } = []; -} diff --git a/src/ManagedCode.LlmTck/Scenarios/LlmTckScenarioBuilder.cs b/src/ManagedCode.LlmTck/Scenarios/LlmTckScenarioBuilder.cs deleted file mode 100644 index 6cdb2ef..0000000 --- a/src/ManagedCode.LlmTck/Scenarios/LlmTckScenarioBuilder.cs +++ /dev/null @@ -1,141 +0,0 @@ -namespace ManagedCode.LlmTck.Scenarios; - -public sealed class LlmTckScenarioBuilder(string id) -{ - private LlmTckScenario _scenario = new() { Id = id }; - - public LlmTckScenarioBuilder ForModel(string modelId) - { - ArgumentException.ThrowIfNullOrWhiteSpace(modelId); - _scenario = _scenario with { ModelId = modelId }; - return this; - } - - public LlmTckScenarioBuilder RequireBearerToken(string token) - { - ArgumentException.ThrowIfNullOrWhiteSpace(token); - _scenario = _scenario with { RequiredBearerToken = token }; - return this; - } - - public LlmTckScenarioBuilder WhenUserContains(string content) - { - ArgumentException.ThrowIfNullOrWhiteSpace(content); - _scenario.Match.Messages.Add(new LlmTckMessage { Role = "user", Content = content }); - return this; - } - - public LlmTckScenarioBuilder WithExactMatch(params LlmTckMessage[] messages) - { - ArgumentNullException.ThrowIfNull(messages); - _scenario.Match.Messages.Clear(); - _scenario.Match.Messages.AddRange(messages); - _scenario = _scenario with - { - Match = _scenario.Match with { Mode = LlmTckMatchMode.Exact }, - }; - return this; - } - - public LlmTckScenarioBuilder Responds(string content, params string[] streamChunks) - { - ArgumentNullException.ThrowIfNull(content); - _scenario.Responses.Add( - new LlmTckScenarioResponse - { - Content = content, - StreamChunks = streamChunks.Length > 0 ? [.. streamChunks] : [content], - } - ); - return this; - } - - public LlmTckScenarioBuilder CallsTool(string name, string argumentsJson, string callId = "call_fixture") - { - return CallsTools(new LlmTckToolCall { Id = callId, Name = name, ArgumentsJson = argumentsJson }); - } - - public LlmTckScenarioBuilder CallsTools(params LlmTckToolCall[] calls) - { - ArgumentNullException.ThrowIfNull(calls); - if (calls.Length == 0) - { - throw new ArgumentException("At least one tool call is required.", nameof(calls)); - } - - foreach (var call in calls) - { - ArgumentNullException.ThrowIfNull(call); - ArgumentException.ThrowIfNullOrWhiteSpace(call.Name); - ArgumentException.ThrowIfNullOrWhiteSpace(call.Id); - LlmTckJsonSchema.ValidateJson(call.ArgumentsJson); - } - _scenario.Responses.Add(new() { ToolCalls = [.. calls] }); - return this; - } - - public LlmTckScenarioBuilder RespondsJson(string json) - { - LlmTckJsonSchema.ValidateJson(json); - return Responds(json); - } - - public LlmTckScenarioBuilder Fails(int statusCode, string code, string message) - { - ArgumentException.ThrowIfNullOrWhiteSpace(code); - ArgumentException.ThrowIfNullOrWhiteSpace(message); - _scenario.Responses.Add( - new LlmTckScenarioResponse - { - Error = new LlmTckScenarioError - { - StatusCode = statusCode, - Code = code, - Message = message, - }, - } - ); - return this; - } - - public LlmTckScenarioBuilder DelaysBy(int milliseconds) - { - if (milliseconds < 0) - { - throw new ArgumentOutOfRangeException(nameof(milliseconds), "Delay cannot be negative."); - } - - if (_scenario.Responses.Count == 0) - { - _scenario.Responses.Add(new LlmTckScenarioResponse()); - } - - var index = _scenario.Responses.Count - 1; - _scenario.Responses[index] = _scenario.Responses[index] with - { - DelayMilliseconds = milliseconds, - }; - return this; - } - - public LlmTckScenario Build() - { - if (_scenario.Responses.Count == 0) - { - throw new InvalidOperationException("A scenario requires at least one response."); - } - - return _scenario with - { - Match = _scenario.Match with { Messages = _scenario.Match.Messages.Select(message => message with { ToolCalls = [.. message.ToolCalls] }).ToList() }, - Responses = - [ - .. _scenario.Responses.Select(response => response with - { - StreamChunks = [.. response.StreamChunks], - ToolCalls = [.. response.ToolCalls], - }), - ], - }; - } -} diff --git a/src/ManagedCode.LlmTck/Scenarios/LlmTckScenarioDataset.cs b/src/ManagedCode.LlmTck/Scenarios/LlmTckScenarioDataset.cs deleted file mode 100644 index b5342a4..0000000 --- a/src/ManagedCode.LlmTck/Scenarios/LlmTckScenarioDataset.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace ManagedCode.LlmTck.Scenarios; - -public sealed record LlmTckScenarioDataset -{ - public string Id { get; init; } = Guid.NewGuid().ToString("N"); - - public List ChatScenarios { get; init; } = []; -} diff --git a/src/ManagedCode.LlmTck/Scenarios/LlmTckScenarioDatasetBuilder.cs b/src/ManagedCode.LlmTck/Scenarios/LlmTckScenarioDatasetBuilder.cs deleted file mode 100644 index 82ba683..0000000 --- a/src/ManagedCode.LlmTck/Scenarios/LlmTckScenarioDatasetBuilder.cs +++ /dev/null @@ -1,39 +0,0 @@ -using ManagedCode.LlmTck.Configuration; - -namespace ManagedCode.LlmTck.Scenarios; - -public sealed class LlmTckScenarioDatasetBuilder(string id) -{ - private readonly LlmTckScenarioDataset _dataset = new() { Id = id }; - - public LlmTckScenarioDatasetBuilder AddChatScenario( - string scenarioId, - Action configure - ) - { - ArgumentException.ThrowIfNullOrWhiteSpace(scenarioId); - ArgumentNullException.ThrowIfNull(configure); - - var builder = new LlmTckScenarioBuilder(scenarioId); - configure(builder); - var scenario = builder.Build(); - - _dataset.ChatScenarios.RemoveAll(existing => - string.Equals(existing.Id, scenarioId, StringComparison.OrdinalIgnoreCase) - ); - _dataset.ChatScenarios.Add(scenario); - - return this; - } - - public LlmTckScenarioDataset Build() - { - return _dataset with - { - ChatScenarios = - [ - .. _dataset.ChatScenarios.Select(LlmTckConfigurationBuilder.SnapshotScenario), - ], - }; - } -} diff --git a/src/ManagedCode.LlmTck/Scenarios/LlmTckScenarioError.cs b/src/ManagedCode.LlmTck/Scenarios/LlmTckScenarioError.cs deleted file mode 100644 index 2459575..0000000 --- a/src/ManagedCode.LlmTck/Scenarios/LlmTckScenarioError.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace ManagedCode.LlmTck.Scenarios; - -public sealed record LlmTckScenarioError -{ - public int StatusCode { get; init; } = 500; - - public string Code { get; init; } = "llm_tck_error"; - - public string Message { get; init; } = "Scripted LLM TCK error."; -} diff --git a/src/ManagedCode.LlmTck/Scenarios/LlmTckScenarioResponse.cs b/src/ManagedCode.LlmTck/Scenarios/LlmTckScenarioResponse.cs deleted file mode 100644 index cc9b174..0000000 --- a/src/ManagedCode.LlmTck/Scenarios/LlmTckScenarioResponse.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace ManagedCode.LlmTck.Scenarios; - -public sealed record LlmTckScenarioResponse -{ - public List ToolCalls { get; init; } = []; - - public string Content { get; init; } = string.Empty; - - public List StreamChunks { get; init; } = []; - - public LlmTckScenarioError? Error { get; init; } - - public int DelayMilliseconds { get; init; } -} diff --git a/src/ManagedCode.LlmTck/Scenarios/LlmTckTools.cs b/src/ManagedCode.LlmTck/Scenarios/LlmTckTools.cs deleted file mode 100644 index 429dab7..0000000 --- a/src/ManagedCode.LlmTck/Scenarios/LlmTckTools.cs +++ /dev/null @@ -1,17 +0,0 @@ -namespace ManagedCode.LlmTck.Scenarios; - -public sealed record LlmTckToolDefinition -{ - public string Name { get; init; } = string.Empty; - public string? Description { get; init; } - public string ParametersJson { get; init; } = "{\"type\":\"object\"}"; -} - -public sealed record LlmTckToolCall -{ - public string Id { get; init; } = string.Empty; - public string Name { get; init; } = string.Empty; - public string ArgumentsJson { get; init; } = "{}"; -} - -public enum LlmTckToolChoice { Auto, None, Required } diff --git a/src/ManagedCode.LlmTestServer.Anthropic/AGENTS.md b/src/ManagedCode.LlmTestServer.Anthropic/AGENTS.md new file mode 100644 index 0000000..e059945 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Anthropic/AGENTS.md @@ -0,0 +1,5 @@ +# ManagedCode.LlmTestServer.Anthropic + +This package owns Anthropic compatibility constants, provider profiles, and Messages API wire contracts. + +Keep deterministic runtime behavior in `ManagedCode.LlmTestServer`; keep HTTP hosting in `ManagedCode.LlmTestServer.Hosting`. diff --git a/src/ManagedCode.LlmTck.Anthropic/AnthropicCompatibility.cs b/src/ManagedCode.LlmTestServer.Anthropic/AnthropicCompatibility.cs similarity index 56% rename from src/ManagedCode.LlmTck.Anthropic/AnthropicCompatibility.cs rename to src/ManagedCode.LlmTestServer.Anthropic/AnthropicCompatibility.cs index a3a2b40..b694272 100644 --- a/src/ManagedCode.LlmTck.Anthropic/AnthropicCompatibility.cs +++ b/src/ManagedCode.LlmTestServer.Anthropic/AnthropicCompatibility.cs @@ -1,30 +1,30 @@ -using ManagedCode.LlmTck.Providers; +using ManagedCode.LlmTestServer.Providers; -namespace ManagedCode.LlmTck.Anthropic; +namespace ManagedCode.LlmTestServer.Anthropic; public static class AnthropicCompatibility { - public const string ProviderId = LlmTckCompatibilityTags.Anthropic; + public const string ProviderId = LlmTestServerCompatibilityTags.Anthropic; - public static LlmTckProviderProfile Profile { get; } = + public static LlmTestServerProviderProfile Profile { get; } = new() { Id = ProviderId, DisplayName = "Anthropic", - Protocol = LlmTckProtocolFamily.AnthropicMessages, + Protocol = LlmTestServerProtocolFamily.AnthropicMessages, DefaultEndpointPath = "/anthropic/v1/messages", Capabilities = [ - LlmTckProviderCapability.Chat, - LlmTckProviderCapability.StreamingChat, - LlmTckProviderCapability.Tools, - LlmTckProviderCapability.StructuredOutput, + LlmTestServerProviderCapability.Chat, + LlmTestServerProviderCapability.StreamingChat, + LlmTestServerProviderCapability.Tools, + LlmTestServerProviderCapability.StructuredOutput, ], CompatibilityTags = [ ProviderId, - LlmTckCompatibilityTags.Claude, - LlmTckCompatibilityTags.MessagesApi, + LlmTestServerCompatibilityTags.Claude, + LlmTestServerCompatibilityTags.MessagesApi, ], ApiContract = new() { @@ -35,7 +35,7 @@ public static class AnthropicCompatibility [ new() { - Id = LlmTckProviderOperationIds.Anthropic.MessagesCreate, + Id = LlmTestServerProviderOperationIds.Anthropic.MessagesCreate, Method = "POST", Path = "/anthropic/v1/messages", DocumentationUrl = "https://platform.claude.com/docs/en/api/messages/create", @@ -45,10 +45,10 @@ public static class AnthropicCompatibility ImplementedByHosting = true, Capabilities = [ - LlmTckProviderCapability.Chat, - LlmTckProviderCapability.StreamingChat, - LlmTckProviderCapability.Tools, - LlmTckProviderCapability.StructuredOutput, + LlmTestServerProviderCapability.Chat, + LlmTestServerProviderCapability.StreamingChat, + LlmTestServerProviderCapability.Tools, + LlmTestServerProviderCapability.StructuredOutput, ], }, ], diff --git a/src/ManagedCode.LlmTck.Anthropic/AnthropicMessagesModels.cs b/src/ManagedCode.LlmTestServer.Anthropic/AnthropicMessagesModels.cs similarity index 71% rename from src/ManagedCode.LlmTck.Anthropic/AnthropicMessagesModels.cs rename to src/ManagedCode.LlmTestServer.Anthropic/AnthropicMessagesModels.cs index 1f1046f..7c76df7 100644 --- a/src/ManagedCode.LlmTck.Anthropic/AnthropicMessagesModels.cs +++ b/src/ManagedCode.LlmTestServer.Anthropic/AnthropicMessagesModels.cs @@ -1,7 +1,7 @@ using System.Text.Json; using System.Text.Json.Serialization; -namespace ManagedCode.LlmTck.Anthropic; +namespace ManagedCode.LlmTestServer.Anthropic; public sealed record AnthropicMessagesRequest { @@ -60,6 +60,9 @@ public sealed record AnthropicMessageResponse [JsonPropertyName("stop_reason")] public string? StopReason { get; init; } = "end_turn"; + [JsonPropertyName("stop_details")] + public AnthropicRefusalStopDetails? StopDetails { get; init; } + [JsonPropertyName("stop_sequence")] public string? StopSequence { get; init; } @@ -73,6 +76,11 @@ public sealed record AnthropicContentBlock [JsonPropertyName("name")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? Name { get; init; } [JsonPropertyName("input")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public JsonElement? Input { get; init; } + [JsonPropertyName("caller")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public AnthropicDirectCaller? Caller { get; init; } + [JsonPropertyName("tool_use_id")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? ToolUseId { get; init; } + [JsonPropertyName("content")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public JsonElement? Content { get; init; } + [JsonPropertyName("citations")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public List? Citations { get; init; } + [JsonPropertyName("type")] public string Type { get; init; } = "text"; @@ -83,6 +91,10 @@ public sealed record AnthropicContentBlock public sealed record AnthropicUsage { + [JsonPropertyName("server_tool_use")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public AnthropicServerToolUsage? ServerToolUse { get; init; } + [JsonPropertyName("input_tokens")] public int InputTokens { get; init; } @@ -166,9 +178,22 @@ private static string ReadContentBlock(JsonElement block) public sealed record AnthropicTool { + private static readonly JsonElement _defaultInputSchema = JsonSerializer.SerializeToElement(new { type = "object" }); + [JsonPropertyName("name")] public string Name { get; init; } = string.Empty; - [JsonPropertyName("description")] public string? Description { get; init; } - [JsonPropertyName("input_schema")] public JsonElement InputSchema { get; init; } = JsonSerializer.SerializeToElement(new { type = "object" }); + [JsonPropertyName("description")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? Description { get; init; } + [JsonPropertyName("input_schema")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public JsonElement InputSchema + { + get => field.ValueKind == JsonValueKind.Undefined && Type is null or "custom" ? _defaultInputSchema : field; + init; + } + [JsonPropertyName("type")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? Type { get; init; } + [JsonPropertyName("max_uses")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public int? MaxUses { get; init; } + [JsonPropertyName("allowed_domains")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public List? AllowedDomains { get; init; } + [JsonPropertyName("blocked_domains")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public List? BlockedDomains { get; init; } + [JsonPropertyName("user_location")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public AnthropicWebSearchLocation? UserLocation { get; init; } } public sealed record AnthropicToolChoice { @@ -185,3 +210,10 @@ public sealed record AnthropicOutputFormat [JsonPropertyName("type")] public string Type { get; init; } = "json_schema"; [JsonPropertyName("schema")] public JsonElement Schema { get; init; } } + +public sealed record AnthropicRefusalStopDetails +{ + [JsonPropertyName("type")] public string Type { get; init; } = "refusal"; + [JsonPropertyName("category")] public string? Category { get; init; } + [JsonPropertyName("explanation")] public string? Explanation { get; init; } +} diff --git a/src/ManagedCode.LlmTestServer.Anthropic/AnthropicNativeWebSearchModels.cs b/src/ManagedCode.LlmTestServer.Anthropic/AnthropicNativeWebSearchModels.cs new file mode 100644 index 0000000..82948de --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Anthropic/AnthropicNativeWebSearchModels.cs @@ -0,0 +1,56 @@ +using System.Text.Json.Serialization; +using ManagedCode.LlmTestServer.Models; + +namespace ManagedCode.LlmTestServer.Anthropic; + +public static class AnthropicNativeWebSearchContract +{ + public const string ToolType = "web_search_20250305"; + public const string ToolName = "web_search"; + public const string ModelId = LlmTestServerKnownModelIds.ClaudeSonnet55; +} + +public sealed record AnthropicWebSearchLocation +{ + [JsonPropertyName("type")] public string Type { get; init; } = "approximate"; + [JsonPropertyName("city")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? City { get; init; } + [JsonPropertyName("region")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? Region { get; init; } + [JsonPropertyName("country")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? Country { get; init; } + [JsonPropertyName("timezone")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? Timezone { get; init; } +} + +public sealed record AnthropicWebSearchResult +{ + [JsonPropertyName("type")] public string Type { get; init; } = "web_search_result"; + [JsonPropertyName("url")] public string Url { get; init; } = string.Empty; + [JsonPropertyName("title")] public string Title { get; init; } = string.Empty; + [JsonPropertyName("encrypted_content")] public string EncryptedContent { get; init; } = string.Empty; + [JsonPropertyName("page_age")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? PageAge { get; init; } +} + +public sealed record AnthropicWebSearchToolResultError +{ + [JsonPropertyName("type")] public string Type { get; init; } = "web_search_tool_result_error"; + [JsonPropertyName("error_code")] public string ErrorCode { get; init; } = string.Empty; +} + +public sealed record AnthropicWebSearchCitation +{ + [JsonPropertyName("type")] public string Type { get; init; } = "web_search_result_location"; + [JsonPropertyName("url")] public string Url { get; init; } = string.Empty; + [JsonPropertyName("encrypted_index")] public string EncryptedIndex { get; init; } = string.Empty; + [JsonPropertyName("cited_text")] public string CitedText { get; init; } = string.Empty; + [JsonPropertyName("title")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? Title { get; init; } +} + +public sealed record AnthropicDirectCaller +{ + [JsonPropertyName("type")] public string Type { get; init; } = "direct"; +} + +public sealed record AnthropicServerToolUsage +{ + [JsonPropertyName("web_fetch_requests")] public int WebFetchRequests { get; init; } + + [JsonPropertyName("web_search_requests")] public int WebSearchRequests { get; init; } +} diff --git a/src/ManagedCode.LlmTestServer.Anthropic/AnthropicNativeWebSearchValidation.cs b/src/ManagedCode.LlmTestServer.Anthropic/AnthropicNativeWebSearchValidation.cs new file mode 100644 index 0000000..6ce246a --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Anthropic/AnthropicNativeWebSearchValidation.cs @@ -0,0 +1,230 @@ +using System.Globalization; +using System.Text.Json; +using static ManagedCode.LlmTestServer.ProviderValidation.LlmTestServerJsonValidation; + +namespace ManagedCode.LlmTestServer.Anthropic; + +internal static class AnthropicNativeWebSearchValidation +{ + private const int _maxItems = 128; + + internal static bool ValidateTool(JsonElement tool) + { + Object(tool); + if (!tool.TryGetProperty("type", out var type) || type.ValueKind == JsonValueKind.Null) { return false; } + String(type); + if (type.GetString() == "custom") { return false; } + Fields(tool, "type", "name", "max_uses", "allowed_domains", "blocked_domains", "user_location"); + Choice(type, AnthropicNativeWebSearchContract.ToolType); + Choice(tool.GetProperty("name"), AnthropicNativeWebSearchContract.ToolName); + if (tool.TryGetProperty("max_uses", out var max) && max.ValueKind != JsonValueKind.Null + && (!max.TryGetInt32(out var value) || value is < 0 or > _maxItems)) + { throw new JsonException("Native max_uses must be zero to 128 in this fixture contract."); } + if (tool.TryGetProperty("allowed_domains", out var allowed) && allowed.ValueKind != JsonValueKind.Null + && tool.TryGetProperty("blocked_domains", out var blocked) && blocked.ValueKind != JsonValueKind.Null) + { throw new JsonException("Allowed and blocked domains are mutually exclusive."); } + foreach (var name in new[] { "allowed_domains", "blocked_domains" }) + { + if (!tool.TryGetProperty(name, out var domains) || domains.ValueKind == JsonValueKind.Null) { continue; } + BoundedArray(domains); + foreach (var domain in domains.EnumerateArray()) { ValidateDomain(domain); } + } + if (tool.TryGetProperty("user_location", out var location) && location.ValueKind != JsonValueKind.Null) + { ValidateLocation(location); } + return true; + } + + internal static void ValidateBody(JsonElement body, bool hasNative) + { + if (body.TryGetProperty("model", out var model) && model.ValueKind == JsonValueKind.String + && model.GetString() == AnthropicNativeWebSearchContract.ModelId + && body.TryGetProperty("tool_choice", out var currentChoice) && currentChoice.ValueKind == JsonValueKind.Object + && currentChoice.TryGetProperty("type", out var currentType) && currentType.ValueKind == JsonValueKind.String + && currentType.GetString() is "any" or "tool") + { throw new JsonException("Claude Sonnet 5.5 does not support forced tool choice."); } + if (hasNative && body.GetProperty("model").GetString() != AnthropicNativeWebSearchContract.ModelId) + { throw new JsonException("Native web search requires the qualified fixture model."); } + if (body.TryGetProperty("tool_choice", out var choice) && choice.ValueKind == JsonValueKind.Object + && choice.TryGetProperty("name", out var choiceName) && choiceName.ValueKind == JsonValueKind.String + && choiceName.GetString() == AnthropicNativeWebSearchContract.ToolName && hasNative) + { + Fields(choice, "type", "name", "disable_parallel_tool_use"); + Choice(choice.GetProperty("type"), "tool"); + } + foreach (var message in OptionalArray(body, "messages")) + { + if (!message.TryGetProperty("content", out var content)) { continue; } + if (content.ValueKind == JsonValueKind.Object && content.TryGetProperty("type", out var objectType) + && objectType.ValueKind == JsonValueKind.String + && objectType.GetString() is "server_tool_use" or "web_search_tool_result") + { throw new JsonException("Native history must be a content array."); } + if (content.ValueKind != JsonValueKind.Array) { continue; } + var native = false; + foreach (var part in content.EnumerateArray()) + { + if (part.ValueKind != JsonValueKind.Object || !part.TryGetProperty("type", out var type)) { continue; } + String(type); + switch (type.GetString()) + { + case "server_tool_use": + Fields(part, "type", "id", "name", "input", "caller"); + ValidateCaller(part); + Id(part.GetProperty("id")); + Choice(part.GetProperty("name"), AnthropicNativeWebSearchContract.ToolName); + var input = part.GetProperty("input"); + Fields(input, "query"); + NonEmpty(input.GetProperty("query")); + native = true; + break; + case "web_search_tool_result": + Fields(part, "type", "tool_use_id", "content", "caller"); + ValidateCaller(part); + Id(part.GetProperty("tool_use_id")); + ValidateResults(part.GetProperty("content")); + native = true; + break; + case "text" when part.TryGetProperty("citations", out var citations): + if (citations.ValueKind == JsonValueKind.Null) { break; } + Array(citations); + if (citations.GetArrayLength() > 0) + { + Fields(part, "type", "text", "citations", "cache_control"); + String(part.GetProperty("text")); + BoundedArray(citations); + foreach (var citation in citations.EnumerateArray()) { ValidateCitation(citation); } + native = true; + } + break; + default: + var name = type.GetString()!; + if (name.EndsWith("_tool_result", StringComparison.Ordinal) + || name.StartsWith("web_search", StringComparison.Ordinal) + || name.StartsWith("code_execution", StringComparison.Ordinal)) + { throw new JsonException("Unsupported native history block."); } + break; + } + } + if (!native) { continue; } + Choice(message.GetProperty("role"), "assistant"); + BoundedArray(content); + foreach (var part in content.EnumerateArray()) + { + Object(part); + Choice(part.GetProperty("type"), "text", "tool_use", "server_tool_use", "web_search_tool_result"); + if (part.GetProperty("type").GetString() == "tool_use") { Id(part.GetProperty("id")); ValidateCaller(part); } + if (part.GetProperty("type").GetString() == "text") + { + Fields(part, "type", "text", "citations", "cache_control"); + String(part.GetProperty("text")); + } + } + } + } + + internal static void ValidateCaller(JsonElement block) + { + if (!block.TryGetProperty("caller", out var caller) || caller.ValueKind == JsonValueKind.Null) { return; } + Fields(caller, "type"); + Choice(caller.GetProperty("type"), "direct"); + } + + private static void ValidateResults(JsonElement content) + { + if (content.ValueKind == JsonValueKind.Object) + { + Fields(content, "type", "error_code"); + Choice(content.GetProperty("type"), "web_search_tool_result_error"); + Choice(content.GetProperty("error_code"), "invalid_tool_input", "unavailable", "max_uses_exceeded", "too_many_requests", "query_too_long", "request_too_large"); + return; + } + BoundedArray(content); + foreach (var result in content.EnumerateArray()) + { + Fields(result, "type", "url", "title", "encrypted_content", "page_age"); + Choice(result.GetProperty("type"), "web_search_result"); + Url(result.GetProperty("url")); + String(result.GetProperty("title")); + NonEmpty(result.GetProperty("encrypted_content")); + OptionalString(result, "page_age"); + } + } + + private static void ValidateCitation(JsonElement citation) + { + Fields(citation, "type", "url", "encrypted_index", "cited_text", "title"); + Choice(citation.GetProperty("type"), "web_search_result_location"); + Url(citation.GetProperty("url")); + NonEmpty(citation.GetProperty("encrypted_index")); + String(citation.GetProperty("cited_text")); + OptionalString(citation, "title"); + } + + private static void ValidateLocation(JsonElement location) + { + Fields(location, "type", "city", "region", "country", "timezone"); + Choice(location.GetProperty("type"), "approximate"); + var populated = false; + foreach (var name in new[] { "city", "region", "country", "timezone" }) + { + if (!location.TryGetProperty(name, out var value) || value.ValueKind == JsonValueKind.Null) { continue; } + NonEmpty(value); + populated = true; + if (name == "country") + { + var country = value.GetString()!; + if (country.Length != 2 || !country.All(char.IsAsciiLetter) + || !CultureInfo.GetCultures(CultureTypes.SpecificCultures).Any(culture => + string.Equals(new RegionInfo(culture.Name).TwoLetterISORegionName, country, StringComparison.OrdinalIgnoreCase))) + { throw new JsonException("An ISO country code is required."); } + } + if (name == "timezone" && (!TimeZoneInfo.TryFindSystemTimeZoneById(value.GetString()!, out var zone) + || !zone.HasIanaId)) + { throw new JsonException("An IANA timezone is required."); } + } + if (!populated) { throw new JsonException("At least one location field is required."); } + } + + private static void ValidateDomain(JsonElement value) + { + NonEmpty(value); + var domain = value.GetString()!; + var host = domain.Split('/', 2)[0]; + if (domain.Contains("://", StringComparison.Ordinal) || domain.Contains('\\') + || domain.Contains('?') || domain.Contains('#') || domain.Any(char.IsWhiteSpace) + || host.Length > 253 || Uri.CheckHostName(host) != UriHostNameType.Dns + || host.Split('.').Any(label => label.Length is < 1 or > 63 || label[0] == '-' || label[^1] == '-' + || label.Any(character => !char.IsAsciiLetterOrDigit(character) && character != '-'))) + { throw new JsonException("A scheme-less domain with optional path is required."); } + } + + private static void Url(JsonElement value) + { + NonEmpty(value); + if (!Uri.TryCreate(value.GetString(), UriKind.Absolute, out var uri) || uri.Scheme is not ("http" or "https")) + { throw new JsonException("An HTTP URL is required."); } + } + + private static void Id(JsonElement value) + { + NonEmpty(value); + if (value.GetString()!.Length > _maxItems) { throw new JsonException("Native IDs are bounded to 128 characters."); } + } + + private static void OptionalString(JsonElement value, string name) + { + if (value.TryGetProperty(name, out var member) && member.ValueKind != JsonValueKind.Null) { String(member); } + } + + private static void BoundedArray(JsonElement value) + { + Array(value); + if (value.GetArrayLength() > _maxItems) { throw new JsonException("Native arrays are bounded to 128 items."); } + } + + private static void Fields(JsonElement value, params string[] names) + { + Object(value); + if (value.EnumerateObject().Any(property => !names.Contains(property.Name, StringComparer.Ordinal))) + { throw new JsonException("Unsupported native field."); } + } +} diff --git a/src/ManagedCode.LlmTck.Anthropic/AnthropicRequestValidation.cs b/src/ManagedCode.LlmTestServer.Anthropic/AnthropicRequestValidation.cs similarity index 66% rename from src/ManagedCode.LlmTck.Anthropic/AnthropicRequestValidation.cs rename to src/ManagedCode.LlmTestServer.Anthropic/AnthropicRequestValidation.cs index 5f36796..6e8441c 100644 --- a/src/ManagedCode.LlmTck.Anthropic/AnthropicRequestValidation.cs +++ b/src/ManagedCode.LlmTestServer.Anthropic/AnthropicRequestValidation.cs @@ -1,12 +1,12 @@ using System.Text.Json; -using ManagedCode.LlmTck.Providers; -using static ManagedCode.LlmTck.ProviderValidation.LlmTckJsonValidation; +using ManagedCode.LlmTestServer.Providers; +using static ManagedCode.LlmTestServer.ProviderValidation.LlmTestServerJsonValidation; -namespace ManagedCode.LlmTck.Anthropic; +namespace ManagedCode.LlmTestServer.Anthropic; public static class AnthropicRequestValidation { - public static LlmTckRequestValidationResult Validate(JsonElement body) + public static LlmTestServerRequestValidationResult Validate(JsonElement body) { return Check(body, ValidateBody); } @@ -17,7 +17,13 @@ private static void ValidateBody(JsonElement body) { throw new JsonException("Legacy function fields are unsupported."); } - foreach (var tool in OptionalArray(body, "tools")) { Name(tool); Schema(tool.GetProperty("input_schema")); } + var hasNative = false; + foreach (var tool in OptionalArray(body, "tools")) + { + if (AnthropicNativeWebSearchValidation.ValidateTool(tool)) { hasNative = true; } + else { Name(tool); Schema(tool.GetProperty("input_schema")); } + } + AnthropicNativeWebSearchValidation.ValidateBody(body, hasNative); if (body.TryGetProperty("tool_choice", out var choice)) { Choice(choice.GetProperty("type"), "auto", "none", "any", "tool"); @@ -34,7 +40,7 @@ private static void ValidateBody(JsonElement body) foreach (var part in content.EnumerateArray()) { if (part.ValueKind != JsonValueKind.Object || !part.TryGetProperty("type", out var type)) { continue; } - if (type.GetString() == "tool_use") { Name(part); NonEmpty(part.GetProperty("id")); Object(part.GetProperty("input")); } + if (type.GetString() == "tool_use") { AnthropicNativeWebSearchValidation.ValidateCaller(part); Name(part); NonEmpty(part.GetProperty("id")); Object(part.GetProperty("input")); } if (type.GetString() == "tool_result") { NonEmpty(part.GetProperty("tool_use_id")); _ = part.GetProperty("content"); } } } diff --git a/src/ManagedCode.LlmTestServer.Anthropic/AnthropicWireMapper.NativeWebSearch.cs b/src/ManagedCode.LlmTestServer.Anthropic/AnthropicWireMapper.NativeWebSearch.cs new file mode 100644 index 0000000..94ca74d --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Anthropic/AnthropicWireMapper.NativeWebSearch.cs @@ -0,0 +1,215 @@ +using System.Text.Json; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Scenarios; + +namespace ManagedCode.LlmTestServer.Anthropic; + +public static partial class AnthropicWireMapper +{ + private static bool IsNativeWebTool(AnthropicTool tool) + { + return tool.Type == AnthropicNativeWebSearchContract.ToolType; + } + + private static bool IsNamedNativeChoice(AnthropicMessagesRequest request) + { + return request.ToolChoice is { Type: "tool", Name: AnthropicNativeWebSearchContract.ToolName } + && request.Tools.Any(IsNativeWebTool); + } + + private static LlmTestServerNativeToolDeclaration ToNativeDeclaration(AnthropicTool tool) + { + return new() + { + Contract = LlmTestServerNativeToolContract.AnthropicMessagesWebSearch20250305, + AnthropicWebSearchOptions = new() + { + MaxUses = tool.MaxUses, + AllowedDomains = tool.AllowedDomains is { } allowed ? [.. allowed] : null, + BlockedDomains = tool.BlockedDomains is { } blocked ? [.. blocked] : null, + UserLocation = tool.UserLocation is { } location ? new() + { + Type = location.Type, + City = location.City, + Region = location.Region, + Country = location.Country, + Timezone = location.Timezone, + } : null, + }, + }; + } + + private static List ToOrderedContent(LlmTestServerChatResult result) + { + return result.OutputItems.Select(item => item switch + { + LlmTestServerResponseMessageItem message => new AnthropicContentBlock + { + Text = message.Content, + Citations = message.Citations.Count > 0 ? message.Citations.Select(ToNativeCitation).ToList() : null, + }, + LlmTestServerResponseFunctionCallItem function => ToToolBlock(function.Call), + LlmTestServerResponseServerToolUseItem call => new AnthropicContentBlock + { + Type = "server_tool_use", + Caller = new(), + Text = null, + Id = call.Id, + Name = AnthropicNativeWebSearchContract.ToolName, + Input = JsonSerializer.SerializeToElement(new { query = ((LlmTestServerWebSearchInput)call.Input).Query }), + }, + LlmTestServerResponseServerToolResultItem returned => new AnthropicContentBlock + { + Type = "web_search_tool_result", + Caller = new(), + Text = null, + ToolUseId = returned.ToolUseId, + Content = ToNativeResultContent(returned.Result), + }, + _ => throw new InvalidOperationException("Unsupported Anthropic output item."), + }).ToList(); + } + + private static JsonElement ToNativeResultContent(LlmTestServerServerToolResult result) + { + return result switch + { + LlmTestServerWebSearchResults found => JsonSerializer.SerializeToElement(found.Results.Select(item => new AnthropicWebSearchResult + { Url = item.Url, Title = item.Title, EncryptedContent = item.EncryptedContent, PageAge = item.PageAge }).ToList()), + LlmTestServerWebSearchErrorResult error => JsonSerializer.SerializeToElement(new AnthropicWebSearchToolResultError { ErrorCode = error.ErrorCode }), + _ => throw new InvalidOperationException("Unsupported Anthropic server tool result."), + }; + } + + private static AnthropicWebSearchCitation ToNativeCitation(LlmTestServerResponseCitation citation) + { + return citation is LlmTestServerAnthropicWebSearchCitation web ? new() + { + Url = web.Url, + Title = web.Title, + EncryptedIndex = web.EncryptedIndex, + CitedText = web.CitedText, + } : throw new InvalidOperationException("Unsupported Anthropic citation."); + } + + private static AnthropicServerToolUsage? NativeServerToolUsage(LlmTestServerChatResult result) + { + if (!result.OutputItems.Any(item => item is LlmTestServerResponseServerToolUseItem)) { return null; } + return new() + { + WebSearchRequests = result.OutputItems.OfType() + .Count(item => item.Result is not LlmTestServerWebSearchErrorResult { ErrorCode: "max_uses_exceeded" }), + }; + } + + public static IEnumerable ToOrderedContentStreamEvents(LlmTestServerChatResult result) + { + var content = ToOrderedContent(result); + for (var index = 0; index < content.Count; index++) + { + var block = content[index]; + if (block.Type is "server_tool_use" or "tool_use") + { + yield return new { type = "content_block_start", index, content_block = block with { Input = JsonSerializer.SerializeToElement(new { }) } }; + yield return new { type = "content_block_delta", index, delta = new { type = "input_json_delta", partial_json = block.Input!.Value.GetRawText() } }; + } + else if (block.Type == "text") + { + yield return new { type = "content_block_start", index, content_block = block with { Text = string.Empty, Citations = block.Citations is null ? null : [] } }; + var plan = (LlmTestServerResponseMessageItem)result.OutputItems[index]; + IReadOnlyList chunks = plan.StreamChunks.Count > 0 ? plan.StreamChunks : [plan.Content]; + foreach (var chunk in chunks) + { + yield return new { type = "content_block_delta", index, delta = new { type = "text_delta", text = chunk } }; + } + foreach (var citation in block.Citations ?? []) + { + yield return new { type = "content_block_delta", index, delta = new { type = "citations_delta", citation } }; + } + } + else + { + yield return new { type = "content_block_start", index, content_block = block }; + } + yield return new { type = "content_block_stop", index }; + } + } + + private static bool HasNativeHistory(IEnumerable blocks) + { + return blocks.Any(block => block.ValueKind == JsonValueKind.Object + && (block.TryGetProperty("type", out var type) && type.GetString() is "server_tool_use" or "web_search_tool_result" + || block.TryGetProperty("citations", out var citations) && citations.ValueKind == JsonValueKind.Array && citations.GetArrayLength() > 0)); + } + + private static LlmTestServerMessage ReadNativeAssistant(IEnumerable blocks) + { + var items = new List(); + var calls = new List(); + foreach (var block in blocks) + { + switch (block.GetProperty("type").GetString()) + { + case "text": + items.Add(new LlmTestServerResponseMessageItem + { + Content = block.GetProperty("text").GetString()!, + Citations = block.TryGetProperty("citations", out var citations) && citations.ValueKind == JsonValueKind.Array + ? citations.EnumerateArray().Select(ReadNativeCitation).ToList() : [], + }); + break; + case "tool_use": + var function = new LlmTestServerToolCall + { Id = block.GetProperty("id").GetString()!, Name = block.GetProperty("name").GetString()!, ArgumentsJson = block.GetProperty("input").GetRawText() }; + calls.Add(function); + items.Add(new LlmTestServerResponseFunctionCallItem { Id = function.Id, Call = function }); + break; + case "server_tool_use": + items.Add(new LlmTestServerResponseServerToolUseItem + { + Id = block.GetProperty("id").GetString()!, + Input = new LlmTestServerWebSearchInput { Query = block.GetProperty("input").GetProperty("query").GetString()! }, + }); + break; + case "web_search_tool_result": + items.Add(new LlmTestServerResponseServerToolResultItem + { ToolUseId = block.GetProperty("tool_use_id").GetString()!, Result = ReadNativeResult(block.GetProperty("content")) }); + break; + } + } + return new() + { + Role = "assistant", + OutputItems = items, + ToolCalls = calls, + Content = string.Concat(items.OfType().Select(item => item.Content)), + }; + } + + private static LlmTestServerResponseCitation ReadNativeCitation(JsonElement citation) + { + return new LlmTestServerAnthropicWebSearchCitation + { + Url = citation.GetProperty("url").GetString()!, + EncryptedIndex = citation.GetProperty("encrypted_index").GetString()!, + CitedText = citation.GetProperty("cited_text").GetString()!, + Title = citation.TryGetProperty("title", out var title) && title.ValueKind == JsonValueKind.String ? title.GetString() : null, + }; + } + + private static LlmTestServerServerToolResult ReadNativeResult(JsonElement content) + { + return content.ValueKind == JsonValueKind.Object + ? new LlmTestServerWebSearchErrorResult { ErrorCode = content.GetProperty("error_code").GetString()! } + : new LlmTestServerWebSearchResults + { + Results = content.EnumerateArray().Select(result => new LlmTestServerAnthropicWebSearchResult + { + Url = result.GetProperty("url").GetString()!, + Title = result.GetProperty("title").GetString()!, + EncryptedContent = result.GetProperty("encrypted_content").GetString()!, + PageAge = result.TryGetProperty("page_age", out var pageAge) && pageAge.ValueKind == JsonValueKind.String ? pageAge.GetString() : null, + }).ToList(), + }; + } +} diff --git a/src/ManagedCode.LlmTck.Anthropic/AnthropicWireMapper.cs b/src/ManagedCode.LlmTestServer.Anthropic/AnthropicWireMapper.cs similarity index 54% rename from src/ManagedCode.LlmTck.Anthropic/AnthropicWireMapper.cs rename to src/ManagedCode.LlmTestServer.Anthropic/AnthropicWireMapper.cs index 6738e1e..d94b887 100644 --- a/src/ManagedCode.LlmTck.Anthropic/AnthropicWireMapper.cs +++ b/src/ManagedCode.LlmTestServer.Anthropic/AnthropicWireMapper.cs @@ -1,30 +1,35 @@ using System.Text.Json; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Scenarios; +using ManagedCode.LlmTestServer.Determinism; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Scenarios; -namespace ManagedCode.LlmTck.Anthropic; +namespace ManagedCode.LlmTestServer.Anthropic; -public static class AnthropicWireMapper +public static partial class AnthropicWireMapper { public const string VersionHeaderName = "anthropic-version"; public const string SupportedVersion = "2023-06-01"; - public static LlmTckChatRequest ToRuntimeRequest(AnthropicMessagesRequest request) + public static LlmTestServerChatRequest ToRuntimeRequest(AnthropicMessagesRequest request) { - var messages = new List(); + var messages = new List(); var system = AnthropicContentReader.ReadTextContent(request.System); if (!string.IsNullOrWhiteSpace(system)) { - messages.Add(new LlmTckMessage { Role = "system", Content = system }); + messages.Add(new LlmTestServerMessage { Role = "system", Content = system }); } messages.AddRange(request.Messages.SelectMany(ReadMessages)); return new() { - Tools = request.Tools.Select(tool => new LlmTckToolDefinition { Name = tool.Name, Description = tool.Description, ParametersJson = tool.InputSchema.GetRawText() }).ToList(), - ToolChoice = request.ToolChoice?.Type switch { "any" or "tool" => LlmTckToolChoice.Required, "none" => LlmTckToolChoice.None, _ => LlmTckToolChoice.Auto }, - RequiredToolName = request.ToolChoice?.Name, + Tools = request.Tools.Where(tool => !IsNativeWebTool(tool)).Select(tool => new LlmTestServerToolDefinition + { Name = tool.Name, Description = tool.Description, ParametersJson = tool.InputSchema.ValueKind == JsonValueKind.Undefined ? "{}" : tool.InputSchema.GetRawText() }).ToList(), + NativeToolSurface = LlmTestServerNativeToolSurface.AnthropicMessages, + NativeTools = request.Tools.Where(IsNativeWebTool).Select(ToNativeDeclaration).ToList(), + RequiredNativeTool = IsNamedNativeChoice(request) ? LlmTestServerNativeToolContract.AnthropicMessagesWebSearch20250305 : null, + ToolChoice = request.ToolChoice?.Type switch { "any" or "tool" => LlmTestServerToolChoice.Required, "none" => LlmTestServerToolChoice.None, _ => LlmTestServerToolChoice.Auto }, + RequiredToolName = IsNamedNativeChoice(request) ? null : request.ToolChoice?.Name, AllowParallelToolCalls = request.ToolChoice?.DisableParallelToolUse != true, RequireJson = request.OutputConfig?.Format is not null, ResponseSchemaJson = request.OutputConfig?.Format?.Schema.GetRawText(), @@ -32,25 +37,27 @@ public static LlmTckChatRequest ToRuntimeRequest(AnthropicMessagesRequest reques Stream = request.Stream, PopulateCacheOnly = request.MaxTokens == 0, PromptCachePolicy = HasPromptCacheControl(request) - ? LlmTckPromptCachePolicy.Anthropic - : LlmTckPromptCachePolicy.None, + ? LlmTestServerPromptCachePolicy.Anthropic + : LlmTestServerPromptCachePolicy.None, Messages = messages, }; } - public static AnthropicMessageResponse ToMessageResponse(LlmTckChatResult result, bool cacheOnly = false) + public static AnthropicMessageResponse ToMessageResponse(LlmTestServerChatResult result, bool cacheOnly = false) { return new() { - Id = CreateMessageId(), + Id = CreateMessageId(result.WireMetadata), Model = result.ModelId, - Content = cacheOnly ? [] : result.ToolCalls.Count > 0 ? (result.Content.Length > 0 ? new[] { new AnthropicContentBlock { Text = result.Content } } : []).Concat(result.ToolCalls.Select(ToToolBlock)).ToList() : [new AnthropicContentBlock { Text = result.Content }], - StopReason = cacheOnly ? "max_tokens" : result.ToolCalls.Count > 0 ? "tool_use" : "end_turn", - Usage = CreateUsage(result.Usage), + Content = cacheOnly ? [] : result.OutputItems.Count > 0 ? ToOrderedContent(result) : result.ToolCalls.Count > 0 ? (result.Content.Length > 0 ? new[] { new AnthropicContentBlock { Text = result.Content } } : []).Concat(result.ToolCalls.Select(ToToolBlock)).ToList() : [new AnthropicContentBlock { Text = result.Content }], + StopReason = cacheOnly ? "max_tokens" : result.OutputDisposition?.Kind == LlmTestServerOutputDispositionKind.Refused ? "refusal" : result.ToolCalls.Count > 0 ? "tool_use" : "end_turn", + StopDetails = !cacheOnly && result.OutputDisposition?.Kind == LlmTestServerOutputDispositionKind.Refused + ? new() { Category = result.OutputDisposition.Category, Explanation = result.OutputDisposition.Explanation } : null, + Usage = CreateUsage(result.Usage) with { ServerToolUse = NativeServerToolUsage(result) }, }; } - public static AnthropicErrorResponse ToError(string type, string message) + public static AnthropicErrorResponse ToError(string type, string message, LlmTestServerResponseMetadata? metadata = null) { return new() { @@ -59,25 +66,26 @@ public static AnthropicErrorResponse ToError(string type, string message) Type = type, Message = message, }, - RequestId = CreateRequestId(), + RequestId = CreateRequestId(metadata), }; } - public static object ToMessageStartEvent(LlmTckChatResult result) + public static object ToMessageStartEvent(LlmTestServerChatResult result) { return new { type = "message_start", message = new AnthropicMessageResponse { - Id = CreateMessageId(), + Id = CreateMessageId(result.WireMetadata), Model = result.ModelId, StopReason = null, Content = [], Usage = CreateUsage( result.Usage, result.Usage.OutputTokens > 0 ? 1 : 0 - ), + ) with + { ServerToolUse = NativeServerToolUsage(result) is null ? null : new AnthropicServerToolUsage() }, }, }; } @@ -115,15 +123,26 @@ public static object ToContentBlockStopEvent() }; } - public static object ToMessageDeltaEvent(LlmTckChatResult result, bool cacheOnly = false) + public static object ToMessageDeltaEvent(LlmTestServerChatResult result, bool cacheOnly = false) { + if (!cacheOnly && NativeServerToolUsage(result) is { } nativeUsage) + { + return new + { + type = "message_delta", + delta = new { stop_reason = result.ToolCalls.Count > 0 ? "tool_use" : "end_turn", stop_sequence = (string?)null }, + usage = new { output_tokens = result.Usage.OutputTokens, server_tool_use = nativeUsage }, + }; + } return new { type = "message_delta", delta = new { - stop_reason = cacheOnly ? "max_tokens" : result.ToolCalls.Count > 0 ? "tool_use" : "end_turn", + stop_reason = cacheOnly ? "max_tokens" : result.OutputDisposition?.Kind == LlmTestServerOutputDispositionKind.Refused ? "refusal" : result.ToolCalls.Count > 0 ? "tool_use" : "end_turn", stop_sequence = (string?)null, + stop_details = !cacheOnly && result.OutputDisposition?.Kind == LlmTestServerOutputDispositionKind.Refused + ? new AnthropicRefusalStopDetails { Category = result.OutputDisposition.Category, Explanation = result.OutputDisposition.Explanation } : null, }, usage = new { output_tokens = result.Usage.OutputTokens }, }; @@ -137,13 +156,13 @@ public static object ToMessageStopEvent() }; } - public static AnthropicContentBlock ToToolBlock(LlmTckToolCall call) + public static AnthropicContentBlock ToToolBlock(LlmTestServerToolCall call) { return new() - { Type = "tool_use", Id = call.Id, Name = call.Name, Input = JsonSerializer.Deserialize(call.ArgumentsJson), Text = null }; + { Type = "tool_use", Caller = new(), Id = call.Id, Name = call.Name, Input = JsonSerializer.Deserialize(call.ArgumentsJson), Text = null }; } - public static IEnumerable ToToolStreamEvents(LlmTckChatResult result, int offset = 0) + public static IEnumerable ToToolStreamEvents(LlmTestServerChatResult result, int offset = 0) { for (var position = 0; position < result.ToolCalls.Count; position++) { @@ -155,13 +174,18 @@ public static IEnumerable ToToolStreamEvents(LlmTckChatResult result, in } } - private static IEnumerable ReadMessages(AnthropicInputMessage message) + private static IEnumerable ReadMessages(AnthropicInputMessage message) { if (message.Content.ValueKind != JsonValueKind.Array) { yield return new() { Role = message.Role, Content = message.TextContent }; yield break; } var blocks = message.Content.EnumerateArray().ToArray(); + if (message.Role == "assistant" && HasNativeHistory(blocks)) + { + yield return ReadNativeAssistant(blocks); + yield break; + } var calls = blocks.Where(block => block.TryGetProperty("type", out var type) && type.GetString() == "tool_use") - .Select(block => new LlmTckToolCall { Id = block.GetProperty("id").GetString()!, Name = block.GetProperty("name").GetString()!, ArgumentsJson = block.GetProperty("input").GetRawText() }).ToList(); + .Select(block => new LlmTestServerToolCall { Id = block.GetProperty("id").GetString()!, Name = block.GetProperty("name").GetString()!, ArgumentsJson = block.GetProperty("input").GetRawText() }).ToList(); var text = string.Concat(blocks.Where(block => block.TryGetProperty("type", out var type) && type.GetString() == "text").Select(block => block.GetProperty("text").GetString())); if (calls.Count > 0 || text.Length > 0) { @@ -174,13 +198,13 @@ private static IEnumerable ReadMessages(AnthropicInputMessage mes } } - private static string CreateMessageId() + private static string CreateMessageId(LlmTestServerResponseMetadata? metadata) { - return $"msg_llmtck_{Guid.NewGuid():N}"; + return "msg_llmtestserver_" + (metadata ?? LlmTestServerResponseMetadata.Live()).IdFor("message"); } private static AnthropicUsage CreateUsage( - LlmTckTokenUsage usage, + LlmTestServerTokenUsage usage, int? outputTokens = null ) { @@ -218,9 +242,9 @@ private static bool HasCacheControl(JsonElement? cacheControl) return cacheControl is { ValueKind: not JsonValueKind.Undefined and not JsonValueKind.Null }; } - private static string CreateRequestId() + private static string CreateRequestId(LlmTestServerResponseMetadata? metadata) { - return $"req_llmtck_{Guid.NewGuid():N}"; + return "req_llmtestserver_" + (metadata ?? LlmTestServerResponseMetadata.Live()).IdFor("request"); } } diff --git a/src/ManagedCode.LlmTestServer.Anthropic/ManagedCode.LlmTestServer.Anthropic.csproj b/src/ManagedCode.LlmTestServer.Anthropic/ManagedCode.LlmTestServer.Anthropic.csproj new file mode 100644 index 0000000..9e73602 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Anthropic/ManagedCode.LlmTestServer.Anthropic.csproj @@ -0,0 +1,13 @@ + + + ManagedCode.LlmTestServer.Anthropic + Anthropic Messages compatibility profile and adapters for ManagedCode.LlmTestServer. + + + + + + + + + diff --git a/src/ManagedCode.LlmTestServer.Aspire/AGENTS.md b/src/ManagedCode.LlmTestServer.Aspire/AGENTS.md new file mode 100644 index 0000000..67469c0 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Aspire/AGENTS.md @@ -0,0 +1,9 @@ +# ManagedCode.LlmTestServer.Aspire + +This package owns Aspire AppHost extensions. + +Extension methods should be declarative and should configure the package-owned `LlmTestServerResource` through Aspire primitives. Consumer examples must use `builder.AddLlmTestServer()` without service project paths, generated `Projects.*` metadata types, Docker, or a container runtime. Keep container hosting as an explicit opt-in API such as `builder.AddLlmTestServerContainer()`, not the default local AppHost path. Every behavior change needs an Aspire integration test that starts the AppHost. + +The resource's `http` endpoint is the provider endpoint for Aspire consumers. Do not add a separate provider-endpoint setter or hardcoded loopback URL; callers must use `GetEndpoint("http")` or `CreateHttpClient(...)` from the running Aspire resource. + +Do not add `LLM_TEST_SERVER_*_COMPATIBILITY` environment flags or fluent compatibility methods unless they drive real route/runtime behavior; Aspire compatibility proof belongs in provider-client tests, not config echo endpoints. diff --git a/src/ManagedCode.LlmTck.Aspire/LlmTckAspireExtensions.cs b/src/ManagedCode.LlmTestServer.Aspire/LlmTestServerAspireExtensions.cs similarity index 51% rename from src/ManagedCode.LlmTck.Aspire/LlmTckAspireExtensions.cs rename to src/ManagedCode.LlmTestServer.Aspire/LlmTestServerAspireExtensions.cs index 0ebf4e6..5b35fe6 100644 --- a/src/ManagedCode.LlmTck.Aspire/LlmTckAspireExtensions.cs +++ b/src/ManagedCode.LlmTestServer.Aspire/LlmTestServerAspireExtensions.cs @@ -1,58 +1,58 @@ using Aspire.Hosting; using Aspire.Hosting.ApplicationModel; -namespace ManagedCode.LlmTck.Aspire; +namespace ManagedCode.LlmTestServer.Aspire; -public static class LlmTckAspireExtensions +public static class LlmTestServerAspireExtensions { - public static IResourceBuilder AddLlmTck( + public static IResourceBuilder AddLlmTestServer( this IDistributedApplicationBuilder builder, - [ResourceName] string name = LlmTckResource.DefaultName + [ResourceName] string name = LlmTestServerResource.DefaultName ) { ArgumentNullException.ThrowIfNull(builder); ArgumentException.ThrowIfNullOrWhiteSpace(name); - var serviceDirectory = LlmTckServiceLocator.GetServiceDirectory(); - var resource = new LlmTckResource(name, "dotnet", serviceDirectory); + var serviceDirectory = LlmTestServerServiceLocator.GetServiceDirectory(); + var resource = new LlmTestServerResource(name, "dotnet", serviceDirectory); return builder.AddResource(resource) - .WithArgs("ManagedCode.LlmTck.Service.dll") - .WithHttpEndpoint(name: LlmTckResource.HttpEndpointName, env: "PORT") + .WithArgs("ManagedCode.LlmTestServer.Service.dll") + .WithHttpEndpoint(name: LlmTestServerResource.HttpEndpointName, env: "PORT") .WithHttpHealthCheck("/health"); } - public static IResourceBuilder AddLlmTckContainer( + public static IResourceBuilder AddLlmTestServerContainer( this IDistributedApplicationBuilder builder, - [ResourceName] string name = LlmTckResource.DefaultName + [ResourceName] string name = LlmTestServerResource.DefaultName ) { ArgumentNullException.ThrowIfNull(builder); ArgumentException.ThrowIfNullOrWhiteSpace(name); - var resource = new LlmTckContainerResource(name); + var resource = new LlmTestServerContainerResource(name); return builder.AddResource(resource) - .WithImage(LlmTckContainerImageTags.Image, LlmTckContainerImageTags.Tag) - .WithImageRegistry(LlmTckContainerImageTags.Registry) + .WithImage(LlmTestServerContainerImageTags.Image, LlmTestServerContainerImageTags.Tag) + .WithImageRegistry(LlmTestServerContainerImageTags.Registry) .WithHttpEndpoint( - targetPort: LlmTckContainerResource.HttpPort, - name: LlmTckResource.HttpEndpointName + targetPort: LlmTestServerContainerResource.HttpPort, + name: LlmTestServerResource.HttpEndpointName ) .WithHttpHealthCheck("/health"); } public static EndpointReference GetHttpEndpoint( - this IResourceBuilder builder + this IResourceBuilder builder ) { ArgumentNullException.ThrowIfNull(builder); - return builder.GetEndpoint(LlmTckResource.HttpEndpointName); + return builder.GetEndpoint(LlmTestServerResource.HttpEndpointName); } public static ReferenceExpression GetOpenAiEndpoint( - this IResourceBuilder builder + this IResourceBuilder builder ) { ArgumentNullException.ThrowIfNull(builder); @@ -61,16 +61,16 @@ this IResourceBuilder builder } public static EndpointReference GetHttpEndpoint( - this IResourceBuilder builder + this IResourceBuilder builder ) { ArgumentNullException.ThrowIfNull(builder); - return builder.GetEndpoint(LlmTckResource.HttpEndpointName); + return builder.GetEndpoint(LlmTestServerResource.HttpEndpointName); } public static ReferenceExpression GetOpenAiEndpoint( - this IResourceBuilder builder + this IResourceBuilder builder ) { ArgumentNullException.ThrowIfNull(builder); @@ -87,7 +87,7 @@ string apiKey ArgumentNullException.ThrowIfNull(builder); ArgumentException.ThrowIfNullOrWhiteSpace(apiKey); - return builder.WithEnvironment("LlmTck__RequiredBearerToken", apiKey); + return builder.WithEnvironment("LlmTestServer__RequiredBearerToken", apiKey); } } diff --git a/src/ManagedCode.LlmTck.Aspire/LlmTckContainerImageTags.cs b/src/ManagedCode.LlmTestServer.Aspire/LlmTestServerContainerImageTags.cs similarity index 73% rename from src/ManagedCode.LlmTck.Aspire/LlmTckContainerImageTags.cs rename to src/ManagedCode.LlmTestServer.Aspire/LlmTestServerContainerImageTags.cs index a10fbb4..f6ed0df 100644 --- a/src/ManagedCode.LlmTck.Aspire/LlmTckContainerImageTags.cs +++ b/src/ManagedCode.LlmTestServer.Aspire/LlmTestServerContainerImageTags.cs @@ -1,20 +1,20 @@ using System.Reflection; -namespace ManagedCode.LlmTck.Aspire; +namespace ManagedCode.LlmTestServer.Aspire; -public static class LlmTckContainerImageTags +public static class LlmTestServerContainerImageTags { private const string _fallbackTag = "0.1.1"; public const string Registry = "ghcr.io"; - public const string Image = "managedcode/llm-tck"; + public const string Image = "managedcode/llm-test-server"; public static string Tag { get; } = GetPackageVersion(); private static string GetPackageVersion() { - var version = typeof(LlmTckContainerImageTags) + var version = typeof(LlmTestServerContainerImageTags) .Assembly.GetCustomAttribute() ?.InformationalVersion; if (string.IsNullOrWhiteSpace(version)) diff --git a/src/ManagedCode.LlmTck.Aspire/LlmTckResource.cs b/src/ManagedCode.LlmTestServer.Aspire/LlmTestServerResource.cs similarity index 55% rename from src/ManagedCode.LlmTck.Aspire/LlmTckResource.cs rename to src/ManagedCode.LlmTestServer.Aspire/LlmTestServerResource.cs index 5af82db..7e8d76a 100644 --- a/src/ManagedCode.LlmTck.Aspire/LlmTckResource.cs +++ b/src/ManagedCode.LlmTestServer.Aspire/LlmTestServerResource.cs @@ -1,18 +1,18 @@ using Aspire.Hosting; using Aspire.Hosting.ApplicationModel; -namespace ManagedCode.LlmTck.Aspire; +namespace ManagedCode.LlmTestServer.Aspire; -public sealed class LlmTckResource(string name, string command, string workingDirectory) +public sealed class LlmTestServerResource(string name, string command, string workingDirectory) : ExecutableResource(name, command, workingDirectory), IResourceWithServiceDiscovery { - public const string DefaultName = "llm-tck"; + public const string DefaultName = "llm-test-server"; public const string HttpEndpointName = "http"; } -public sealed class LlmTckContainerResource(string name) +public sealed class LlmTestServerContainerResource(string name) : ContainerResource(name), IResourceWithServiceDiscovery { diff --git a/src/ManagedCode.LlmTck.Aspire/LlmTckServiceLocator.cs b/src/ManagedCode.LlmTestServer.Aspire/LlmTestServerServiceLocator.cs similarity index 76% rename from src/ManagedCode.LlmTck.Aspire/LlmTckServiceLocator.cs rename to src/ManagedCode.LlmTestServer.Aspire/LlmTestServerServiceLocator.cs index 7c8857e..ffdf2ef 100644 --- a/src/ManagedCode.LlmTck.Aspire/LlmTckServiceLocator.cs +++ b/src/ManagedCode.LlmTestServer.Aspire/LlmTestServerServiceLocator.cs @@ -1,9 +1,9 @@ -namespace ManagedCode.LlmTck.Aspire; +namespace ManagedCode.LlmTestServer.Aspire; -internal static class LlmTckServiceLocator +internal static class LlmTestServerServiceLocator { - private const string _serviceDirectoryName = "llm-tck-service"; - private const string _serviceAssemblyName = "ManagedCode.LlmTck.Service.dll"; + private const string _serviceDirectoryName = "llm-test-server-service"; + private const string _serviceAssemblyName = "ManagedCode.LlmTestServer.Service.dll"; public static string GetServiceDirectory() { @@ -21,8 +21,8 @@ public static string GetServiceDirectory() } throw new InvalidOperationException( - $"The packaged LLM TCK service executable was not found at '{packageServiceDirectory}'. " - + "Restore the ManagedCode.LlmTck.Aspire package or use AddLlmTckContainer() for the container-backed resource." + $"The packaged LLM Test Server service executable was not found at '{packageServiceDirectory}'. " + + "Restore the ManagedCode.LlmTestServer.Aspire package or use AddLlmTestServerContainer() for the container-backed resource." ); } @@ -36,7 +36,7 @@ private static bool ContainsServiceAssembly(string directory) var current = new DirectoryInfo(startDirectory); while (current is not null) { - var solutionPath = Path.Combine(current.FullName, "ManagedCode.LlmTck.slnx"); + var solutionPath = Path.Combine(current.FullName, "ManagedCode.LlmTestServer.slnx"); if (File.Exists(solutionPath)) { foreach (var configuration in new[] { "Release", "Debug" }) @@ -44,7 +44,7 @@ private static bool ContainsServiceAssembly(string directory) var candidate = Path.Combine( current.FullName, "samples", - "ManagedCode.LlmTck.Service", + "ManagedCode.LlmTestServer.Service", "bin", configuration, "net10.0" diff --git a/src/ManagedCode.LlmTestServer.Aspire/ManagedCode.LlmTestServer.Aspire.csproj b/src/ManagedCode.LlmTestServer.Aspire/ManagedCode.LlmTestServer.Aspire.csproj new file mode 100644 index 0000000..b14cbc7 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Aspire/ManagedCode.LlmTestServer.Aspire.csproj @@ -0,0 +1,40 @@ + + + ManagedCode.LlmTestServer.Aspire + Aspire AppHost resource extensions for ManagedCode.LlmTestServer provider emulation. + + + + + + + + + + + + + + $(MSBuildThisFileDirectory)obj/$(Configuration)/service-publish/ + + + + + + <_PublishedLlmTestServerService Include="$(LlmTestServerServicePublishDir)**/*" /> + + + + diff --git a/src/ManagedCode.LlmTestServer.Aspire/buildTransitive/ManagedCode.LlmTestServer.Aspire.targets b/src/ManagedCode.LlmTestServer.Aspire/buildTransitive/ManagedCode.LlmTestServer.Aspire.targets new file mode 100644 index 0000000..ad9049d --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Aspire/buildTransitive/ManagedCode.LlmTestServer.Aspire.targets @@ -0,0 +1,10 @@ + + + + + + diff --git a/src/ManagedCode.LlmTestServer.AzureOpenAI/AGENTS.md b/src/ManagedCode.LlmTestServer.AzureOpenAI/AGENTS.md new file mode 100644 index 0000000..59df5d3 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.AzureOpenAI/AGENTS.md @@ -0,0 +1,5 @@ +# ManagedCode.LlmTestServer.AzureOpenAI + +This package owns Azure OpenAI compatibility constants, provider profiles, and Azure OpenAI wire contracts. + +Keep deterministic runtime behavior in `ManagedCode.LlmTestServer`; keep HTTP hosting in `ManagedCode.LlmTestServer.Hosting`. diff --git a/src/ManagedCode.LlmTck.AzureOpenAI/AzureOpenAiCompatibility.cs b/src/ManagedCode.LlmTestServer.AzureOpenAI/AzureOpenAiCompatibility.cs similarity index 67% rename from src/ManagedCode.LlmTck.AzureOpenAI/AzureOpenAiCompatibility.cs rename to src/ManagedCode.LlmTestServer.AzureOpenAI/AzureOpenAiCompatibility.cs index 07fb40f..4ec568a 100644 --- a/src/ManagedCode.LlmTck.AzureOpenAI/AzureOpenAiCompatibility.cs +++ b/src/ManagedCode.LlmTestServer.AzureOpenAI/AzureOpenAiCompatibility.cs @@ -1,35 +1,35 @@ -using ManagedCode.LlmTck.Providers; +using ManagedCode.LlmTestServer.Providers; -namespace ManagedCode.LlmTck.AzureOpenAI; +namespace ManagedCode.LlmTestServer.AzureOpenAI; public static class AzureOpenAiCompatibility { - public const string ProviderId = LlmTckCompatibilityTags.AzureOpenAI; + public const string ProviderId = LlmTestServerCompatibilityTags.AzureOpenAI; - public static LlmTckProviderProfile Profile { get; } = + public static LlmTestServerProviderProfile Profile { get; } = new() { Id = ProviderId, DisplayName = "Azure OpenAI", - Protocol = LlmTckProtocolFamily.AzureOpenAI, + Protocol = LlmTestServerProtocolFamily.AzureOpenAI, DefaultEndpointPath = "/azure-openai/openai/deployments/{deployment}/chat/completions", Capabilities = [ - LlmTckProviderCapability.Chat, - LlmTckProviderCapability.StreamingChat, - LlmTckProviderCapability.Tools, - LlmTckProviderCapability.StructuredOutput, - LlmTckProviderCapability.Embeddings, - LlmTckProviderCapability.Images, - LlmTckProviderCapability.Video, - LlmTckProviderCapability.Audio, - LlmTckProviderCapability.StreamingAudio, + LlmTestServerProviderCapability.Chat, + LlmTestServerProviderCapability.StreamingChat, + LlmTestServerProviderCapability.Tools, + LlmTestServerProviderCapability.StructuredOutput, + LlmTestServerProviderCapability.Embeddings, + LlmTestServerProviderCapability.Images, + LlmTestServerProviderCapability.Video, + LlmTestServerProviderCapability.Audio, + LlmTestServerProviderCapability.StreamingAudio, ], CompatibilityTags = [ ProviderId, - LlmTckCompatibilityTags.OpenAICompatible, - LlmTckCompatibilityTags.Foundry, + LlmTestServerCompatibilityTags.OpenAICompatible, + LlmTestServerCompatibilityTags.Foundry, ], ApiContract = new() { @@ -40,61 +40,61 @@ public static class AzureOpenAiCompatibility [ new() { - Id = LlmTckProviderOperationIds.AzureOpenAI.V1AudioTranscriptionsCreate, + Id = LlmTestServerProviderOperationIds.AzureOpenAI.V1AudioTranscriptionsCreate, Method = "POST", Path = "/azure-openai/openai/v1/audio/transcriptions", DocumentationUrl = "https://learn.microsoft.com/en-us/azure/foundry/openai/reference-preview-latest", ApiVersion = "v1 (default) or preview", SupportsStreaming = true, ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Audio, LlmTckProviderCapability.StreamingAudio], + Capabilities = [LlmTestServerProviderCapability.Audio, LlmTestServerProviderCapability.StreamingAudio], }, new() { - Id = LlmTckProviderOperationIds.AzureOpenAI.V1ChatCompletionsCreate, + Id = LlmTestServerProviderOperationIds.AzureOpenAI.V1ChatCompletionsCreate, Method = "POST", Path = "/azure-openai/openai/v1/chat/completions", DocumentationUrl = "https://learn.microsoft.com/en-us/azure/foundry/openai/api-version-lifecycle", ApiVersion = "v1", SupportsStreaming = true, ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Chat, LlmTckProviderCapability.StreamingChat, LlmTckProviderCapability.Tools, LlmTckProviderCapability.StructuredOutput], + Capabilities = [LlmTestServerProviderCapability.Chat, LlmTestServerProviderCapability.StreamingChat, LlmTestServerProviderCapability.Tools, LlmTestServerProviderCapability.StructuredOutput], }, new() { - Id = LlmTckProviderOperationIds.AzureOpenAI.V1ResponsesCreate, + Id = LlmTestServerProviderOperationIds.AzureOpenAI.V1ResponsesCreate, Method = "POST", Path = "/azure-openai/openai/v1/responses", DocumentationUrl = "https://learn.microsoft.com/en-us/azure/foundry/openai/api-version-lifecycle", ApiVersion = "v1", SupportsStreaming = true, ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Chat, LlmTckProviderCapability.StreamingChat, LlmTckProviderCapability.Tools, LlmTckProviderCapability.StructuredOutput], + Capabilities = [LlmTestServerProviderCapability.Chat, LlmTestServerProviderCapability.StreamingChat, LlmTestServerProviderCapability.Tools, LlmTestServerProviderCapability.StructuredOutput], }, new() { - Id = LlmTckProviderOperationIds.AzureOpenAI.V1EmbeddingsCreate, + Id = LlmTestServerProviderOperationIds.AzureOpenAI.V1EmbeddingsCreate, Method = "POST", Path = "/azure-openai/openai/v1/embeddings", DocumentationUrl = "https://learn.microsoft.com/en-us/azure/foundry/openai/api-version-lifecycle", ApiVersion = "v1", SupportsStreaming = false, ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Embeddings], + Capabilities = [LlmTestServerProviderCapability.Embeddings], }, new() { - Id = LlmTckProviderOperationIds.AzureOpenAI.V1ImagesCreate, + Id = LlmTestServerProviderOperationIds.AzureOpenAI.V1ImagesCreate, Method = "POST", Path = "/azure-openai/openai/v1/images/generations", DocumentationUrl = "https://learn.microsoft.com/en-us/azure/foundry/openai/reference-preview-latest", ApiVersion = "v1 (default) or preview", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Images], + Capabilities = [LlmTestServerProviderCapability.Images], }, new() { - Id = LlmTckProviderOperationIds.AzureOpenAI.ChatCompletionsCreate, + Id = LlmTestServerProviderOperationIds.AzureOpenAI.ChatCompletionsCreate, Method = "POST", Path = "/azure-openai/openai/deployments/{deployment}/chat/completions", DocumentationUrl = @@ -104,68 +104,68 @@ public static class AzureOpenAiCompatibility ImplementedByHosting = true, Capabilities = [ - LlmTckProviderCapability.Chat, - LlmTckProviderCapability.StreamingChat, - LlmTckProviderCapability.Tools, - LlmTckProviderCapability.StructuredOutput, + LlmTestServerProviderCapability.Chat, + LlmTestServerProviderCapability.StreamingChat, + LlmTestServerProviderCapability.Tools, + LlmTestServerProviderCapability.StructuredOutput, ], }, new() { - Id = LlmTckProviderOperationIds.AzureOpenAI.EmbeddingsCreate, + Id = LlmTestServerProviderOperationIds.AzureOpenAI.EmbeddingsCreate, Method = "POST", Path = "/azure-openai/openai/deployments/{deployment}/embeddings", DocumentationUrl = "https://learn.microsoft.com/en-us/azure/foundry/openai/reference", ApiVersion = "2024-10-21", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Embeddings], + Capabilities = [LlmTestServerProviderCapability.Embeddings], }, new() { - Id = LlmTckProviderOperationIds.AzureOpenAI.ImagesCreate, + Id = LlmTestServerProviderOperationIds.AzureOpenAI.ImagesCreate, Method = "POST", Path = "/azure-openai/openai/deployments/{deployment}/images/generations", DocumentationUrl = "https://learn.microsoft.com/en-us/azure/foundry/openai/reference", ApiVersion = "2024-10-21", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Images], + Capabilities = [LlmTestServerProviderCapability.Images], }, new() { - Id = LlmTckProviderOperationIds.AzureOpenAI.AudioSpeechCreate, + Id = LlmTestServerProviderOperationIds.AzureOpenAI.AudioSpeechCreate, Method = "POST", Path = "/azure-openai/openai/deployments/{deployment}/audio/speech", DocumentationUrl = "https://learn.microsoft.com/en-us/azure/foundry/openai/reference", ApiVersion = "2024-10-21", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Audio], + Capabilities = [LlmTestServerProviderCapability.Audio], }, new() { - Id = LlmTckProviderOperationIds.AzureOpenAI.AudioTranscriptionsCreate, + Id = LlmTestServerProviderOperationIds.AzureOpenAI.AudioTranscriptionsCreate, Method = "POST", Path = "/azure-openai/openai/deployments/{deployment}/audio/transcriptions", DocumentationUrl = "https://learn.microsoft.com/en-us/azure/foundry/openai/reference", ApiVersion = "2024-10-21", RequiredHeader = "api-key", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Audio], + Capabilities = [LlmTestServerProviderCapability.Audio], }, new() { - Id = LlmTckProviderOperationIds.AzureOpenAI.AudioTranslationsCreate, + Id = LlmTestServerProviderOperationIds.AzureOpenAI.AudioTranslationsCreate, Method = "POST", Path = "/azure-openai/openai/deployments/{deployment}/audio/translations", DocumentationUrl = "https://learn.microsoft.com/en-us/azure/foundry/openai/reference", ApiVersion = "2024-10-21", RequiredHeader = "api-key", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Audio], + Capabilities = [LlmTestServerProviderCapability.Audio], }, new() { - Id = LlmTckProviderOperationIds.AzureOpenAI.VideoGenerationJobsCreate, + Id = LlmTestServerProviderOperationIds.AzureOpenAI.VideoGenerationJobsCreate, Method = "POST", Path = "/azure-openai/openai/v1/video/generations/jobs", DocumentationUrl = @@ -173,11 +173,11 @@ public static class AzureOpenAiCompatibility ApiVersion = "v1 preview", RequiredHeader = "api-key", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Video], + Capabilities = [LlmTestServerProviderCapability.Video], }, new() { - Id = LlmTckProviderOperationIds.AzureOpenAI.VideoGenerationJobsList, + Id = LlmTestServerProviderOperationIds.AzureOpenAI.VideoGenerationJobsList, Method = "GET", Path = "/azure-openai/openai/v1/video/generations/jobs", DocumentationUrl = @@ -185,11 +185,11 @@ public static class AzureOpenAiCompatibility ApiVersion = "v1 preview", RequiredHeader = "api-key", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Video], + Capabilities = [LlmTestServerProviderCapability.Video], }, new() { - Id = LlmTckProviderOperationIds.AzureOpenAI.VideoGenerationJobsRetrieve, + Id = LlmTestServerProviderOperationIds.AzureOpenAI.VideoGenerationJobsRetrieve, Method = "GET", Path = "/azure-openai/openai/v1/video/generations/jobs/{jobId}", DocumentationUrl = @@ -197,11 +197,11 @@ public static class AzureOpenAiCompatibility ApiVersion = "v1 preview", RequiredHeader = "api-key", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Video], + Capabilities = [LlmTestServerProviderCapability.Video], }, new() { - Id = LlmTckProviderOperationIds.AzureOpenAI.VideoGenerationJobsDelete, + Id = LlmTestServerProviderOperationIds.AzureOpenAI.VideoGenerationJobsDelete, Method = "DELETE", Path = "/azure-openai/openai/v1/video/generations/jobs/{jobId}", DocumentationUrl = @@ -209,11 +209,11 @@ public static class AzureOpenAiCompatibility ApiVersion = "v1 preview", RequiredHeader = "api-key", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Video], + Capabilities = [LlmTestServerProviderCapability.Video], }, new() { - Id = LlmTckProviderOperationIds.AzureOpenAI.VideoGenerationsRetrieve, + Id = LlmTestServerProviderOperationIds.AzureOpenAI.VideoGenerationsRetrieve, Method = "GET", Path = "/azure-openai/openai/v1/video/generations/{generationId}", DocumentationUrl = @@ -221,11 +221,11 @@ public static class AzureOpenAiCompatibility ApiVersion = "v1 preview", RequiredHeader = "api-key", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Video], + Capabilities = [LlmTestServerProviderCapability.Video], }, new() { - Id = LlmTckProviderOperationIds.AzureOpenAI.VideoGenerationsThumbnailRetrieve, + Id = LlmTestServerProviderOperationIds.AzureOpenAI.VideoGenerationsThumbnailRetrieve, Method = "GET", Path = "/azure-openai/openai/v1/video/generations/{generationId}/content/thumbnail", DocumentationUrl = @@ -233,11 +233,11 @@ public static class AzureOpenAiCompatibility ApiVersion = "v1 preview", RequiredHeader = "api-key", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Video], + Capabilities = [LlmTestServerProviderCapability.Video], }, new() { - Id = LlmTckProviderOperationIds.AzureOpenAI.VideoGenerationsContentRetrieve, + Id = LlmTestServerProviderOperationIds.AzureOpenAI.VideoGenerationsContentRetrieve, Method = "GET", Path = "/azure-openai/openai/v1/video/generations/{generationId}/content/video", DocumentationUrl = @@ -245,11 +245,11 @@ public static class AzureOpenAiCompatibility ApiVersion = "v1 preview", RequiredHeader = "api-key", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Video], + Capabilities = [LlmTestServerProviderCapability.Video], }, new() { - Id = LlmTckProviderOperationIds.AzureOpenAI.VideoGenerationsContentHead, + Id = LlmTestServerProviderOperationIds.AzureOpenAI.VideoGenerationsContentHead, Method = "HEAD", Path = "/azure-openai/openai/v1/video/generations/{generationId}/content/video", DocumentationUrl = @@ -257,7 +257,7 @@ public static class AzureOpenAiCompatibility ApiVersion = "v1 preview", RequiredHeader = "api-key", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Video], + Capabilities = [LlmTestServerProviderCapability.Video], }, ], }, diff --git a/src/ManagedCode.LlmTestServer.AzureOpenAI/ManagedCode.LlmTestServer.AzureOpenAI.csproj b/src/ManagedCode.LlmTestServer.AzureOpenAI/ManagedCode.LlmTestServer.AzureOpenAI.csproj new file mode 100644 index 0000000..78cec33 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.AzureOpenAI/ManagedCode.LlmTestServer.AzureOpenAI.csproj @@ -0,0 +1,10 @@ + + + ManagedCode.LlmTestServer.AzureOpenAI + Azure OpenAI compatibility profile and adapters for ManagedCode.LlmTestServer. + + + + + + diff --git a/src/ManagedCode.LlmTestServer.Bedrock/AGENTS.md b/src/ManagedCode.LlmTestServer.Bedrock/AGENTS.md new file mode 100644 index 0000000..66b7561 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Bedrock/AGENTS.md @@ -0,0 +1,5 @@ +# ManagedCode.LlmTestServer.Bedrock + +This package owns Amazon Bedrock compatibility constants, provider profiles, and Bedrock wire contracts. + +Keep deterministic runtime behavior in `ManagedCode.LlmTestServer`; keep HTTP hosting in `ManagedCode.LlmTestServer.Hosting`. diff --git a/src/ManagedCode.LlmTck.Bedrock/BedrockCompatibility.cs b/src/ManagedCode.LlmTestServer.Bedrock/BedrockCompatibility.cs similarity index 60% rename from src/ManagedCode.LlmTck.Bedrock/BedrockCompatibility.cs rename to src/ManagedCode.LlmTestServer.Bedrock/BedrockCompatibility.cs index 80907b7..1e7ad5f 100644 --- a/src/ManagedCode.LlmTck.Bedrock/BedrockCompatibility.cs +++ b/src/ManagedCode.LlmTestServer.Bedrock/BedrockCompatibility.cs @@ -1,31 +1,31 @@ -using ManagedCode.LlmTck.Providers; +using ManagedCode.LlmTestServer.Providers; -namespace ManagedCode.LlmTck.Bedrock; +namespace ManagedCode.LlmTestServer.Bedrock; public static class BedrockCompatibility { - public const string ProviderId = LlmTckCompatibilityTags.Bedrock; + public const string ProviderId = LlmTestServerCompatibilityTags.Bedrock; - public static LlmTckProviderProfile Profile { get; } = + public static LlmTestServerProviderProfile Profile { get; } = new() { Id = ProviderId, DisplayName = "Amazon Bedrock", - Protocol = LlmTckProtocolFamily.Bedrock, + Protocol = LlmTestServerProtocolFamily.Bedrock, DefaultEndpointPath = "/bedrock/model/{modelId}/converse", Capabilities = [ - LlmTckProviderCapability.Chat, - LlmTckProviderCapability.StreamingChat, - LlmTckProviderCapability.Tools, - LlmTckProviderCapability.StructuredOutput, - LlmTckProviderCapability.Embeddings, - LlmTckProviderCapability.Images, + LlmTestServerProviderCapability.Chat, + LlmTestServerProviderCapability.StreamingChat, + LlmTestServerProviderCapability.Tools, + LlmTestServerProviderCapability.StructuredOutput, + LlmTestServerProviderCapability.Embeddings, + LlmTestServerProviderCapability.Images, ], CompatibilityTags = [ - LlmTckCompatibilityTags.AmazonBedrock, - LlmTckCompatibilityTags.AwsBedrock, + LlmTestServerCompatibilityTags.AmazonBedrock, + LlmTestServerCompatibilityTags.AwsBedrock, ProviderId, ], ApiContract = new() @@ -38,7 +38,7 @@ public static class BedrockCompatibility [ new() { - Id = LlmTckProviderOperationIds.Bedrock.Converse, + Id = LlmTestServerProviderOperationIds.Bedrock.Converse, Method = "POST", Path = "/bedrock/model/{modelId}/converse", DocumentationUrl = @@ -46,25 +46,25 @@ public static class BedrockCompatibility ImplementedByHosting = true, Capabilities = [ - LlmTckProviderCapability.Chat, - LlmTckProviderCapability.Tools, - LlmTckProviderCapability.StructuredOutput, + LlmTestServerProviderCapability.Chat, + LlmTestServerProviderCapability.Tools, + LlmTestServerProviderCapability.StructuredOutput, ], }, new() { - Id = LlmTckProviderOperationIds.Bedrock.ConverseStream, + Id = LlmTestServerProviderOperationIds.Bedrock.ConverseStream, Method = "POST", Path = "/bedrock/model/{modelId}/converse-stream", DocumentationUrl = "https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ConverseStream.html", SupportsStreaming = true, ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.StreamingChat, LlmTckProviderCapability.Tools, LlmTckProviderCapability.StructuredOutput], + Capabilities = [LlmTestServerProviderCapability.StreamingChat, LlmTestServerProviderCapability.Tools, LlmTestServerProviderCapability.StructuredOutput], }, new() { - Id = LlmTckProviderOperationIds.Bedrock.InvokeModel, + Id = LlmTestServerProviderOperationIds.Bedrock.InvokeModel, Method = "POST", Path = "/bedrock/model/{modelId}/invoke", DocumentationUrl = @@ -72,20 +72,20 @@ public static class BedrockCompatibility ImplementedByHosting = true, Capabilities = [ - LlmTckProviderCapability.Embeddings, - LlmTckProviderCapability.Images, + LlmTestServerProviderCapability.Embeddings, + LlmTestServerProviderCapability.Images, ], }, new() { - Id = LlmTckProviderOperationIds.Bedrock.InvokeModelWithResponseStream, + Id = LlmTestServerProviderOperationIds.Bedrock.InvokeModelWithResponseStream, Method = "POST", Path = "/bedrock/model/{modelId}/invoke-with-response-stream", DocumentationUrl = "https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_InvokeModelWithResponseStream.html", SupportsStreaming = true, ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.StreamingChat], + Capabilities = [LlmTestServerProviderCapability.StreamingChat], }, ], }, diff --git a/src/ManagedCode.LlmTck.Bedrock/BedrockEventStreamEncoder.cs b/src/ManagedCode.LlmTestServer.Bedrock/BedrockEventStreamEncoder.cs similarity index 96% rename from src/ManagedCode.LlmTck.Bedrock/BedrockEventStreamEncoder.cs rename to src/ManagedCode.LlmTestServer.Bedrock/BedrockEventStreamEncoder.cs index 624d6e3..de6b84a 100644 --- a/src/ManagedCode.LlmTck.Bedrock/BedrockEventStreamEncoder.cs +++ b/src/ManagedCode.LlmTestServer.Bedrock/BedrockEventStreamEncoder.cs @@ -1,7 +1,7 @@ using System.Text.Json; using Amazon.Runtime.EventStreams; -namespace ManagedCode.LlmTck.Bedrock; +namespace ManagedCode.LlmTestServer.Bedrock; /// Encodes a Bedrock event using the official AWS event-stream framing implementation. public static class BedrockEventStreamEncoder diff --git a/src/ManagedCode.LlmTck.Bedrock/BedrockModels.cs b/src/ManagedCode.LlmTestServer.Bedrock/BedrockModels.cs similarity index 99% rename from src/ManagedCode.LlmTck.Bedrock/BedrockModels.cs rename to src/ManagedCode.LlmTestServer.Bedrock/BedrockModels.cs index d5b51e4..3886481 100644 --- a/src/ManagedCode.LlmTck.Bedrock/BedrockModels.cs +++ b/src/ManagedCode.LlmTestServer.Bedrock/BedrockModels.cs @@ -1,7 +1,7 @@ using System.Text.Json; using System.Text.Json.Serialization; -namespace ManagedCode.LlmTck.Bedrock; +namespace ManagedCode.LlmTestServer.Bedrock; public sealed record BedrockConverseRequest { diff --git a/src/ManagedCode.LlmTck.Bedrock/BedrockRequestValidation.cs b/src/ManagedCode.LlmTestServer.Bedrock/BedrockRequestValidation.cs similarity index 88% rename from src/ManagedCode.LlmTck.Bedrock/BedrockRequestValidation.cs rename to src/ManagedCode.LlmTestServer.Bedrock/BedrockRequestValidation.cs index 9e20483..2c4b87d 100644 --- a/src/ManagedCode.LlmTck.Bedrock/BedrockRequestValidation.cs +++ b/src/ManagedCode.LlmTestServer.Bedrock/BedrockRequestValidation.cs @@ -1,12 +1,12 @@ using System.Text.Json; -using ManagedCode.LlmTck.Providers; -using static ManagedCode.LlmTck.ProviderValidation.LlmTckJsonValidation; +using ManagedCode.LlmTestServer.Providers; +using static ManagedCode.LlmTestServer.ProviderValidation.LlmTestServerJsonValidation; -namespace ManagedCode.LlmTck.Bedrock; +namespace ManagedCode.LlmTestServer.Bedrock; public static class BedrockRequestValidation { - public static LlmTckRequestValidationResult Validate(JsonElement body) + public static LlmTestServerRequestValidationResult Validate(JsonElement body) { return Check(body, ValidateBody); } diff --git a/src/ManagedCode.LlmTck.Bedrock/BedrockWireMapper.cs b/src/ManagedCode.LlmTestServer.Bedrock/BedrockWireMapper.cs similarity index 86% rename from src/ManagedCode.LlmTck.Bedrock/BedrockWireMapper.cs rename to src/ManagedCode.LlmTestServer.Bedrock/BedrockWireMapper.cs index 17e1093..8c3ad45 100644 --- a/src/ManagedCode.LlmTck.Bedrock/BedrockWireMapper.cs +++ b/src/ManagedCode.LlmTestServer.Bedrock/BedrockWireMapper.cs @@ -1,13 +1,13 @@ using System.Text; using System.Text.Json; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Scenarios; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Scenarios; -namespace ManagedCode.LlmTck.Bedrock; +namespace ManagedCode.LlmTestServer.Bedrock; public static class BedrockWireMapper { - public static LlmTckChatRequest ToRuntimeRequest( + public static LlmTestServerChatRequest ToRuntimeRequest( string modelId, BedrockConverseRequest request, bool stream @@ -21,26 +21,26 @@ bool stream var system = ReadText(request.System); if (!string.IsNullOrWhiteSpace(system)) { - messages.Insert(0, new LlmTckMessage { Role = "system", Content = system }); + messages.Insert(0, new LlmTestServerMessage { Role = "system", Content = system }); } return new() { - Tools = request.ToolConfig?.Tools.Select(tool => new LlmTckToolDefinition { Name = tool.ToolSpec.Name, Description = tool.ToolSpec.Description, ParametersJson = tool.ToolSpec.InputSchema.Json.GetRawText() }).ToList() ?? [], - ToolChoice = request.ToolConfig?.ToolChoice is { } choice && (choice.Any is not null || choice.Tool is not null) ? LlmTckToolChoice.Required : LlmTckToolChoice.Auto, + Tools = request.ToolConfig?.Tools.Select(tool => new LlmTestServerToolDefinition { Name = tool.ToolSpec.Name, Description = tool.ToolSpec.Description, ParametersJson = tool.ToolSpec.InputSchema.Json.GetRawText() }).ToList() ?? [], + ToolChoice = request.ToolConfig?.ToolChoice is { } choice && (choice.Any is not null || choice.Tool is not null) ? LlmTestServerToolChoice.Required : LlmTestServerToolChoice.Auto, RequiredToolName = request.ToolConfig?.ToolChoice?.Tool?.Name, RequireJson = request.OutputConfig?.TextFormat is not null, ResponseSchemaJson = request.OutputConfig?.TextFormat?.Structure?.JsonSchema?.Schema, ModelId = modelId, Stream = stream, PromptCachePolicy = HasPromptCacheMarker(request) - ? LlmTckPromptCachePolicy.Bedrock - : LlmTckPromptCachePolicy.None, + ? LlmTestServerPromptCachePolicy.Bedrock + : LlmTestServerPromptCachePolicy.None, Messages = messages, }; } - public static LlmTckChatRequest ToRuntimeRequest( + public static LlmTestServerChatRequest ToRuntimeRequest( string modelId, JsonElement request, bool stream = false @@ -51,11 +51,11 @@ public static LlmTckChatRequest ToRuntimeRequest( ModelId = modelId, Stream = stream, PromptCachePolicy = HasPromptCacheMarker(request) - ? LlmTckPromptCachePolicy.Bedrock - : LlmTckPromptCachePolicy.None, + ? LlmTestServerPromptCachePolicy.Bedrock + : LlmTestServerPromptCachePolicy.None, Messages = [ - new LlmTckMessage + new LlmTestServerMessage { Role = "user", Content = ReadInvokeInput(request), @@ -64,7 +64,7 @@ public static LlmTckChatRequest ToRuntimeRequest( }; } - public static BedrockConverseResponse ToConverseResponse(LlmTckChatResult result) + public static BedrockConverseResponse ToConverseResponse(LlmTestServerChatResult result) { return new() { @@ -88,9 +88,9 @@ public static BedrockConverseResponse ToConverseResponse(LlmTckChatResult result }; } - private static IEnumerable ReadMessages(BedrockMessage message) + private static IEnumerable ReadMessages(BedrockMessage message) { - var calls = message.Content.Where(block => block.ToolUse is not null).Select(block => new LlmTckToolCall { Id = block.ToolUse!.ToolUseId, Name = block.ToolUse.Name, ArgumentsJson = block.ToolUse.Input.GetRawText() }).ToList(); + var calls = message.Content.Where(block => block.ToolUse is not null).Select(block => new LlmTestServerToolCall { Id = block.ToolUse!.ToolUseId, Name = block.ToolUse.Name, ArgumentsJson = block.ToolUse.Input.GetRawText() }).ToList(); if (calls.Count > 0 || message.Content.Any(block => block.ToolResult is null)) { yield return new() { Role = message.Role, Content = ReadText(message.Content), ToolCalls = calls }; @@ -101,7 +101,7 @@ private static IEnumerable ReadMessages(BedrockMessage message) yield return new() { Role = "tool", ToolCallId = block.ToolResult!.ToolUseId, Content = string.Concat(block.ToolResult.Content.Select(value => value.TryGetProperty("text", out var text) ? text.GetString() : value.GetRawText())) }; } } - public static IEnumerable ToToolStreamEvents(LlmTckChatResult result, int offset = 0) + public static IEnumerable ToToolStreamEvents(LlmTestServerChatResult result, int offset = 0) { for (var position = 0; position < result.ToolCalls.Count; position++) { @@ -114,13 +114,13 @@ public static IEnumerable ToToolStreamEvents(LlmTckChatResult result, in } public static BedrockTitanTextResponse ToTitanTextResponse( - LlmTckChatResult result, + LlmTestServerChatResult result, string inputText ) { return new() { - InputTextTokenCount = LlmTckTokenCounter.CountTextTokens(inputText), + InputTextTokenCount = LlmTestServerTokenCounter.CountTextTokens(inputText), Results = [ new BedrockTitanTextResult @@ -140,7 +140,7 @@ string inputText return new() { Embedding = vector, - InputTextTokenCount = LlmTckTokenCounter.CountTextTokens(inputText), + InputTextTokenCount = LlmTestServerTokenCounter.CountTextTokens(inputText), EmbeddingsByType = new Dictionary>(StringComparer.Ordinal) { ["float"] = vector, @@ -192,7 +192,7 @@ public static object ToConverseMessageStopEvent(bool toolUse = false) return new { messageStop = new { stopReason = toolUse ? "tool_use" : "end_turn" } }; } - public static object ToConverseMetadataEvent(LlmTckChatResult result) + public static object ToConverseMetadataEvent(LlmTestServerChatResult result) { return new { diff --git a/src/ManagedCode.LlmTestServer.Bedrock/ManagedCode.LlmTestServer.Bedrock.csproj b/src/ManagedCode.LlmTestServer.Bedrock/ManagedCode.LlmTestServer.Bedrock.csproj new file mode 100644 index 0000000..4d12576 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Bedrock/ManagedCode.LlmTestServer.Bedrock.csproj @@ -0,0 +1,14 @@ + + + ManagedCode.LlmTestServer.Bedrock + Amazon Bedrock compatibility profile and adapters for ManagedCode.LlmTestServer. + + + + + + + + + + diff --git a/src/ManagedCode.LlmTck.Client/AGENTS.md b/src/ManagedCode.LlmTestServer.Client/AGENTS.md similarity index 87% rename from src/ManagedCode.LlmTck.Client/AGENTS.md rename to src/ManagedCode.LlmTestServer.Client/AGENTS.md index 9b19995..997f539 100644 --- a/src/ManagedCode.LlmTck.Client/AGENTS.md +++ b/src/ManagedCode.LlmTestServer.Client/AGENTS.md @@ -1,4 +1,4 @@ -# ManagedCode.LlmTck.Client +# ManagedCode.LlmTestServer.Client This package contains the control client and `Microsoft.Extensions.AI` implementations. diff --git a/src/ManagedCode.LlmTestServer.Client/LlmTestServerAudioContent.cs b/src/ManagedCode.LlmTestServer.Client/LlmTestServerAudioContent.cs new file mode 100644 index 0000000..fe5dd18 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Client/LlmTestServerAudioContent.cs @@ -0,0 +1,3 @@ +namespace ManagedCode.LlmTestServer.Client; + +public sealed record LlmTestServerAudioContent(byte[] Bytes, string MediaType); diff --git a/src/ManagedCode.LlmTck.Client/LlmTckChatClient.cs b/src/ManagedCode.LlmTestServer.Client/LlmTestServerChatClient.cs similarity index 76% rename from src/ManagedCode.LlmTck.Client/LlmTckChatClient.cs rename to src/ManagedCode.LlmTestServer.Client/LlmTestServerChatClient.cs index 9b9f010..8fbbd5b 100644 --- a/src/ManagedCode.LlmTck.Client/LlmTckChatClient.cs +++ b/src/ManagedCode.LlmTestServer.Client/LlmTestServerChatClient.cs @@ -3,23 +3,24 @@ using System.Runtime.CompilerServices; using System.Text; using System.Text.Json; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.OpenAI; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.OpenAI; using Microsoft.Extensions.AI; -using ProviderRoutes = ManagedCode.LlmTck.Providers.LlmTckProviderRouteNamespaces; +using ProviderRoutes = ManagedCode.LlmTestServer.Providers.LlmTestServerProviderRouteNamespaces; -namespace ManagedCode.LlmTck.Client; +namespace ManagedCode.LlmTestServer.Client; -public sealed class LlmTckChatClient( +public sealed class LlmTestServerChatClient( HttpClient httpClient, - string defaultModelId = LlmTckKnownModelIds.Gpt41Mini, + string defaultModelId = LlmTestServerKnownModelIds.Gpt41Mini, string? bearerToken = null ) : IChatClient { + private const string _refusalErrorCode = nameof(OpenAiChatMessage.Refusal); private static readonly JsonSerializerOptions _jsonOptions = new(JsonSerializerDefaults.Web); public ChatClientMetadata Metadata { get; } = - new(nameof(LlmTckChatClient), httpClient.BaseAddress, defaultModelId); + new(nameof(LlmTestServerChatClient), httpClient.BaseAddress, defaultModelId); public async Task GetResponseAsync( IEnumerable messages, @@ -40,15 +41,18 @@ public async Task GetResponseAsync( .ConfigureAwait(false); var text = completion?.Choices.FirstOrDefault()?.Message.TextContent ?? string.Empty; - var message = new ChatMessage(ChatRole.Assistant, text); + var message = completion?.Choices.FirstOrDefault()?.Message.Refusal is null + ? new ChatMessage(ChatRole.Assistant, text) : new ChatMessage(ChatRole.Assistant, new List()); AddToolCalls(message.Contents, completion?.Choices.FirstOrDefault()?.Message.ToolCalls); + if (completion?.Choices.FirstOrDefault()?.Message.Refusal is { } refusal) + { message.Contents.Add(new ErrorContent(refusal) { ErrorCode = _refusalErrorCode }); } return new ChatResponse(message) { ModelId = completion?.Model, - CreatedAt = DateTimeOffset.UtcNow, + CreatedAt = completion is null ? null : DateTimeOffset.FromUnixTimeSeconds(completion.Created), ResponseId = completion?.Id, FinishReason = MapFinishReason(completion?.Choices.FirstOrDefault()?.FinishReason), - Usage = completion is null ? null : LlmTckUsageMapper.Map(completion.Usage), + Usage = completion is null ? null : LlmTestServerUsageMapper.Map(completion.Usage), }; } @@ -84,13 +88,14 @@ public async IAsyncEnumerable GetStreamingResponseAsync( .ReadAsStreamAsync(cancellationToken) .ConfigureAwait(false); using var reader = new StreamReader(stream); + StringBuilder? refusal = null; while (true) { var line = await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false); if (line is null) { - yield break; + break; } if (!line.StartsWith("data: ", StringComparison.Ordinal)) @@ -101,29 +106,37 @@ public async IAsyncEnumerable GetStreamingResponseAsync( var payload = line["data: ".Length..]; if (string.Equals(payload, "[DONE]", StringComparison.Ordinal)) { - yield break; + break; } var chunk = JsonSerializer.Deserialize(payload, _jsonOptions); var choice = chunk?.Choices.FirstOrDefault(); var text = choice?.Delta.TextContent; + if (choice?.Delta.Refusal is { } refusalDelta) { (refusal ??= new()).Append(refusalDelta); } if (chunk is not null && (!string.IsNullOrEmpty(text) || choice?.FinishReason is not null || choice?.Delta.ToolCalls is { Count: > 0 } || chunk.Usage is not null)) { - var update = new ChatResponseUpdate(ChatRole.Assistant, text) + var update = new ChatResponseUpdate { + Role = ChatRole.Assistant, ModelId = chunk.Model, ResponseId = chunk.Id, + CreatedAt = DateTimeOffset.FromUnixTimeSeconds(chunk.Created), FinishReason = MapFinishReason(choice?.FinishReason), }; + if (!string.IsNullOrEmpty(text)) { update.Contents.Add(new TextContent(text)); } AddToolCalls(update.Contents, choice?.Delta.ToolCalls); if (chunk.Usage is not null) { - update.Contents.Add(new UsageContent(LlmTckUsageMapper.Map(chunk.Usage))); + update.Contents.Add(new UsageContent(LlmTestServerUsageMapper.Map(chunk.Usage))); } yield return update; } } + if (refusal is not null) + { + yield return new ChatResponseUpdate(ChatRole.Assistant, [new ErrorContent(refusal.ToString()) { ErrorCode = _refusalErrorCode }]); + } } public object? GetService(Type serviceType, object? serviceKey = null) @@ -176,9 +189,15 @@ private static IEnumerable ToWireMessages(ChatMessage message Id = call.CallId, Function = new() { Name = call.Name, Arguments = JsonSerializer.Serialize(call.Arguments ?? new Dictionary()) }, }).ToList(); - if (calls.Count > 0 || message.Contents.OfType().Any()) + var refusals = message.Contents.OfType().Where(error => error.ErrorCode == _refusalErrorCode).ToArray(); + if (calls.Count > 0 || message.Contents.OfType().Any() || refusals.Length > 0) { - yield return OpenAiChatMessage.FromText(message.Role.Value, message.Text) with { ToolCalls = calls.Count > 0 ? calls : null }; + yield return OpenAiChatMessage.FromText(message.Role.Value, message.Text) with + { + Content = refusals.Length > 0 && message.Text.Length == 0 ? JsonSerializer.SerializeToElement((string?)null) : JsonSerializer.SerializeToElement(message.Text), + Refusal = refusals.Length > 0 ? string.Concat(refusals.Select(error => error.Message)) : null, + ToolCalls = calls.Count > 0 ? calls : null, + }; } foreach (var result in message.Contents.OfType()) diff --git a/src/ManagedCode.LlmTestServer.Client/LlmTestServerClient.Conversations.cs b/src/ManagedCode.LlmTestServer.Client/LlmTestServerClient.Conversations.cs new file mode 100644 index 0000000..39930a4 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Client/LlmTestServerClient.Conversations.cs @@ -0,0 +1,30 @@ +using System.Net.Http.Json; +using ManagedCode.LlmTestServer.Control; +using ManagedCode.LlmTestServer.Conversations; + +namespace ManagedCode.LlmTestServer.Client; + +public sealed partial class LlmTestServerClient +{ + public LlmTestServerConversationClient ForConversation(string id) + { + return new(httpClient, id, bearerToken); + } + + public async Task ExplainMatchAsync(LlmTestServerMatchExplanationRequest match, + CancellationToken cancellationToken = default) + { + using var request = CreateJsonRequest(HttpMethod.Post, LlmTestServerControlRoutes.ExplainMatch, match); + using var response = await httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + return (await response.Content.ReadFromJsonAsync(_jsonOptions, cancellationToken).ConfigureAwait(false))!; + } + + public async Task> GetConversationsAsync(CancellationToken cancellationToken = default) + { + using var request = CreateRequest(HttpMethod.Get, LlmTestServerControlRoutes.Conversations); + using var response = await httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + return (await response.Content.ReadFromJsonAsync>(_jsonOptions, cancellationToken).ConfigureAwait(false))!; + } +} diff --git a/src/ManagedCode.LlmTestServer.Client/LlmTestServerClient.Requests.cs b/src/ManagedCode.LlmTestServer.Client/LlmTestServerClient.Requests.cs new file mode 100644 index 0000000..6b47271 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Client/LlmTestServerClient.Requests.cs @@ -0,0 +1,76 @@ +using System.Globalization; +using System.Net.Http.Json; +using ManagedCode.LlmTestServer.Control; +using ManagedCode.LlmTestServer.Diagnostics; + +namespace ManagedCode.LlmTestServer.Client; + +public sealed partial class LlmTestServerClient +{ + public async Task GetRequestsAsync(LlmTestServerRequestQuery? query = null, CancellationToken cancellationToken = default) + { + query ??= new(); + using var request = CreateRequest(HttpMethod.Get, LlmTestServerControlRoutes.Requests + BuildRequestQuery(query)); + using var response = await httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + var entries = await response.Content.ReadFromJsonAsync>(_jsonOptions, cancellationToken).ConfigureAwait(false) ?? []; + long Header(string name) + { + return response.Headers.TryGetValues(name, out var values) ? long.Parse(values.Single(), CultureInfo.InvariantCulture) : 0; + } + + return new() + { + Requests = entries.AsReadOnly(), + TotalMatched = checked((int)Header("X-Total-Count")), + TotalObserved = Header("X-Total-Observed"), + EvictedCount = Header("X-Evicted-Count"), + RetainedByteCount = Header("X-Retained-Byte-Count"), + RetentionTruncated = response.Headers.TryGetValues("X-Retention-Truncated", out var truncated) && truncated.Single() == "true", + FirstSequence = entries.Count == 0 ? null : entries.Min(entry => entry.Sequence), + LastSequence = entries.Count == 0 ? null : entries.Max(entry => entry.Sequence) + }; + } + + public async Task ExportRequestsAsync(LlmTestServerRequestQuery? query = null, CancellationToken cancellationToken = default) + { + using var request = CreateRequest(HttpMethod.Get, LlmTestServerControlRoutes.Requests + "/export" + BuildRequestQuery(query ?? new())); + using var response = await httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + return (await response.Content.ReadFromJsonAsync(_jsonOptions, cancellationToken).ConfigureAwait(false))!; + } + + public async Task ExportRequestsNdjsonAsync(LlmTestServerRequestQuery? query = null, CancellationToken cancellationToken = default) + { + using var request = CreateRequest(HttpMethod.Get, LlmTestServerControlRoutes.Requests + "/export" + BuildRequestQuery(query ?? new()) + "&format=ndjson"); + using var response = await httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + return await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + } + + private static string BuildRequestQuery(LlmTestServerRequestQuery query) + { + LlmTestServerRequestQueries.Apply(new(), query); + var values = new Dictionary + { + ["provider"] = query.Provider, + ["model"] = query.Model, + ["requestId"] = query.RequestId, + ["status"] = query.Status?.ToString(CultureInfo.InvariantCulture), + ["operation"] = query.Operation, + ["scenario"] = query.Scenario, + ["conversation"] = query.Conversation, + ["step"] = query.Step, + ["streaming"] = query.Streaming?.ToString().ToLowerInvariant(), + ["completed"] = query.Completed?.ToString().ToLowerInvariant(), + ["success"] = query.Success?.ToString().ToLowerInvariant(), + ["errorCode"] = query.ErrorCode, + ["from"] = query.From?.ToString("O", CultureInfo.InvariantCulture), + ["to"] = query.To?.ToString("O", CultureInfo.InvariantCulture), + ["offset"] = query.Offset.ToString(CultureInfo.InvariantCulture), + ["limit"] = query.Limit.ToString(CultureInfo.InvariantCulture), + ["order"] = query.Order == LlmTestServerRequestOrder.Ascending ? "asc" : "desc", + }; + return "?" + string.Join("&", values.Where(pair => pair.Value is not null).Select(pair => Uri.EscapeDataString(pair.Key) + "=" + Uri.EscapeDataString(pair.Value!))); + } +} diff --git a/src/ManagedCode.LlmTestServer.Client/LlmTestServerClient.Sessions.cs b/src/ManagedCode.LlmTestServer.Client/LlmTestServerClient.Sessions.cs new file mode 100644 index 0000000..fc18b2f --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Client/LlmTestServerClient.Sessions.cs @@ -0,0 +1,24 @@ +using System.Net.Http.Json; +using ManagedCode.LlmTestServer.Control; +using ManagedCode.LlmTestServer.Sessions; + +namespace ManagedCode.LlmTestServer.Client; + +public sealed partial class LlmTestServerClient +{ + public async Task CreateSessionAsync( + Action? configure = null, + CancellationToken cancellationToken = default) + { + LlmTestServerClientConfigurationBuilder? configuration = null; + if (configure is not null) { configuration = new(); configure(configuration); } + var configured = configuration?.Build(); + using var request = CreateJsonRequest(HttpMethod.Post, LlmTestServerControlRoutes.Sessions, + new LlmTestServerSessionCreateRequest { Configuration = configured }); + using var response = await httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + var session = await response.Content.ReadFromJsonAsync(_jsonOptions, cancellationToken).ConfigureAwait(false) + ?? throw new HttpRequestException("The server returned an empty session response."); + return new LlmTestServerSessionClient(httpClient, session, configured?.RequiredBearerToken ?? bearerToken); + } +} diff --git a/src/ManagedCode.LlmTestServer.Client/LlmTestServerClient.Verification.cs b/src/ManagedCode.LlmTestServer.Client/LlmTestServerClient.Verification.cs new file mode 100644 index 0000000..f49200c --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Client/LlmTestServerClient.Verification.cs @@ -0,0 +1,18 @@ +using System.Net.Http.Json; +using ManagedCode.LlmTestServer.Control; +using ManagedCode.LlmTestServer.Verification; + +namespace ManagedCode.LlmTestServer.Client; + +public sealed partial class LlmTestServerClient +{ + public async Task VerifyAsync(LlmTestServerVerificationRequest verification, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(verification); + using var request = CreateJsonRequest(HttpMethod.Post, LlmTestServerControlRoutes.Verify, verification); + using var response = await httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + return (await response.Content.ReadFromJsonAsync(_jsonOptions, cancellationToken).ConfigureAwait(false))!; + } +} diff --git a/src/ManagedCode.LlmTck.Client/LlmTckClient.cs b/src/ManagedCode.LlmTestServer.Client/LlmTestServerClient.cs similarity index 78% rename from src/ManagedCode.LlmTck.Client/LlmTckClient.cs rename to src/ManagedCode.LlmTestServer.Client/LlmTestServerClient.cs index ac665c7..417c1df 100644 --- a/src/ManagedCode.LlmTck.Client/LlmTckClient.cs +++ b/src/ManagedCode.LlmTestServer.Client/LlmTestServerClient.cs @@ -1,34 +1,34 @@ using System.Net.Http.Headers; using System.Net.Http.Json; using System.Text.Json; -using ManagedCode.LlmTck.Configuration; -using ManagedCode.LlmTck.Control; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.OpenAI; -using ManagedCode.LlmTck.Runtime; +using ManagedCode.LlmTestServer.Configuration; +using ManagedCode.LlmTestServer.Control; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.OpenAI; +using ManagedCode.LlmTestServer.Runtime; using Microsoft.Extensions.AI; -using ProviderRoutes = ManagedCode.LlmTck.Providers.LlmTckProviderRouteNamespaces; +using ProviderRoutes = ManagedCode.LlmTestServer.Providers.LlmTestServerProviderRouteNamespaces; -namespace ManagedCode.LlmTck.Client; +namespace ManagedCode.LlmTestServer.Client; -public sealed class LlmTckClient(HttpClient httpClient, string? bearerToken = null) +public sealed partial class LlmTestServerClient(HttpClient httpClient, string? bearerToken = null) { private static readonly JsonSerializerOptions _jsonOptions = new(JsonSerializerDefaults.Web); - public static LlmTckClient Create(HttpClient httpClient, string? bearerToken = null) + public static LlmTestServerClient Create(HttpClient httpClient, string? bearerToken = null) { ArgumentNullException.ThrowIfNull(httpClient); - return new LlmTckClient(httpClient, bearerToken); + return new LlmTestServerClient(httpClient, bearerToken); } - public LlmTckClient WithBearerToken(string token) + public LlmTestServerClient WithBearerToken(string token) { ArgumentException.ThrowIfNullOrWhiteSpace(token); - return new LlmTckClient(httpClient, token); + return new LlmTestServerClient(httpClient, token); } public Task ConfigureAsync( - Action configure, + Action configure, CancellationToken cancellationToken = default ) { @@ -36,7 +36,7 @@ public Task ConfigureAsync( } public async Task ConfigureAsync( - Action configure, + Action configure, bool resetFirst, CancellationToken cancellationToken = default ) @@ -48,73 +48,73 @@ public async Task ConfigureAsync( await ResetAsync(cancellationToken).ConfigureAwait(false); } - var builder = new LlmTckClientConfigurationBuilder(); + var builder = new LlmTestServerClientConfigurationBuilder(); configure(builder); await ConfigureAsync(builder.Build(), cancellationToken).ConfigureAwait(false); } public async Task ConfigureAsync( - LlmTckConfiguration configuration, + LlmTestServerConfiguration configuration, CancellationToken cancellationToken = default ) { ArgumentNullException.ThrowIfNull(configuration); - using var request = CreateJsonRequest(HttpMethod.Post, LlmTckControlRoutes.Configure, configuration); + using var request = CreateJsonRequest(HttpMethod.Post, LlmTestServerControlRoutes.Configure, configuration); using var response = await httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); response.EnsureSuccessStatusCode(); } public async Task ResetAsync(CancellationToken cancellationToken = default) { - using var request = CreateRequest(HttpMethod.Post, LlmTckControlRoutes.Reset); + using var request = CreateRequest(HttpMethod.Post, LlmTestServerControlRoutes.Reset); using var response = await httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); response.EnsureSuccessStatusCode(); } - public async Task GetAssertionsAsync( + public async Task GetAssertionsAsync( CancellationToken cancellationToken = default ) { - using var request = CreateRequest(HttpMethod.Get, LlmTckControlRoutes.Assertions); + using var request = CreateRequest(HttpMethod.Get, LlmTestServerControlRoutes.Assertions); using var response = await httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); response.EnsureSuccessStatusCode(); return await response .Content - .ReadFromJsonAsync(_jsonOptions, cancellationToken) + .ReadFromJsonAsync(_jsonOptions, cancellationToken) .ConfigureAwait(false) - ?? new LlmTckAssertionSummary(); + ?? new LlmTestServerAssertionSummary(); } - public IChatClient CreateChatClient(string defaultModelId = LlmTckKnownModelIds.Gpt41Mini) + public IChatClient CreateChatClient(string defaultModelId = LlmTestServerKnownModelIds.Gpt41Mini) { - return new LlmTckChatClient(httpClient, defaultModelId, bearerToken); + return new LlmTestServerChatClient(httpClient, defaultModelId, bearerToken); } - public LlmTckCloudflareDecisionClient CreateCloudflareDecisionClient(string accountId) + public LlmTestServerCloudflareDecisionClient CreateCloudflareDecisionClient(string accountId) { return new(httpClient, accountId, bearerToken); } - public LlmTckSystemOneClient CreateSystemOneClient() + public LlmTestServerSystemOneClient CreateSystemOneClient() { return new(httpClient, bearerToken); } public IEmbeddingGenerator> CreateEmbeddingGenerator( - string defaultModelId = LlmTckKnownModelIds.TextEmbedding3Small + string defaultModelId = LlmTestServerKnownModelIds.TextEmbedding3Small ) { - return new LlmTckEmbeddingGenerator(httpClient, defaultModelId, bearerToken); + return new LlmTestServerEmbeddingGenerator(httpClient, defaultModelId, bearerToken); } - public IImageGenerator CreateImageGenerator(string defaultModelId = LlmTckKnownModelIds.GptImage1) + public IImageGenerator CreateImageGenerator(string defaultModelId = LlmTestServerKnownModelIds.GptImage1) { - return new LlmTckImageGenerator(httpClient, defaultModelId, bearerToken); + return new LlmTestServerImageGenerator(httpClient, defaultModelId, bearerToken); } - public Task GenerateAudioAsync( + public Task GenerateAudioAsync( string modelId, string input, CancellationToken cancellationToken @@ -123,7 +123,7 @@ CancellationToken cancellationToken return GenerateAudioAsync(modelId, input, voice: "alloy", cancellationToken); } - public async Task GenerateAudioAsync( + public async Task GenerateAudioAsync( string modelId, string input, string voice = "alloy", @@ -142,7 +142,7 @@ public async Task GenerateAudioAsync( using var response = await httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); response.EnsureSuccessStatusCode(); - return new LlmTckAudioContent( + return new LlmTestServerAudioContent( await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false), response.Content.Headers.ContentType?.MediaType ?? "application/octet-stream" ); diff --git a/src/ManagedCode.LlmTestServer.Client/LlmTestServerClientConfigurationBuilder.cs b/src/ManagedCode.LlmTestServer.Client/LlmTestServerClientConfigurationBuilder.cs new file mode 100644 index 0000000..ae6ca8c --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Client/LlmTestServerClientConfigurationBuilder.cs @@ -0,0 +1,347 @@ +using ManagedCode.LlmTestServer.Configuration; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Scenarios; + +namespace ManagedCode.LlmTestServer.Client; + +public sealed class LlmTestServerClientConfigurationBuilder +{ + private readonly LlmTestServerConfigurationBuilder _builder = new(); + + public LlmTestServerClientConfigurationBuilder UseDeterminism(string seed, DateTimeOffset fixedUtc) + { + _builder.WithDeterminism(seed, fixedUtc); + return this; + } + + public LlmTestServerClientConfigurationBuilder RequireBearerToken(string token) + { + _builder.RequireBearerToken(token); + return this; + } + + public LlmTestServerClientConfigurationBuilder SimulateRateLimitAfter(int allowedRequests) + { + _builder.SimulateRateLimitAfter(allowedRequests); + return this; + } + + public LlmTestServerClientConfigurationBuilder SimulateContentFilter(params string[] blockedTerms) + { + _builder.SimulateContentFilter(blockedTerms); + return this; + } + + public LlmTestServerClientConfigurationBuilder UseModel( + string id, + LlmTestServerModelKind kind, + int reasoningTokens = 0 + ) + { + _builder.AddModel(id, kind, reasoningTokens); + return this; + } + + public LlmTestServerClientConfigurationBuilder UseReasoningChatModel( + string id, + int reasoningTokens + ) + { + _builder.AddReasoningChatModel(id, reasoningTokens); + return this; + } + + public LlmTestServerClientConfigurationBuilder UseGpt55(int reasoningTokens = 0) + { + return UseModel(LlmTestServerKnownModelIds.Gpt55, LlmTestServerModelKind.Chat, reasoningTokens); + } + + public LlmTestServerClientConfigurationBuilder UseGpt55Pro(int reasoningTokens = 0) + { + return UseModel(LlmTestServerKnownModelIds.Gpt55Pro, LlmTestServerModelKind.Chat, reasoningTokens); + } + + public LlmTestServerClientConfigurationBuilder UseGpt54(int reasoningTokens = 0) + { + return UseModel(LlmTestServerKnownModelIds.Gpt54, LlmTestServerModelKind.Chat, reasoningTokens); + } + + public LlmTestServerClientConfigurationBuilder UseGpt54Pro(int reasoningTokens = 0) + { + return UseModel(LlmTestServerKnownModelIds.Gpt54Pro, LlmTestServerModelKind.Chat, reasoningTokens); + } + + public LlmTestServerClientConfigurationBuilder UseGpt54Mini(int reasoningTokens = 0) + { + return UseModel(LlmTestServerKnownModelIds.Gpt54Mini, LlmTestServerModelKind.Chat, reasoningTokens); + } + + public LlmTestServerClientConfigurationBuilder UseGpt54Nano(int reasoningTokens = 0) + { + return UseModel(LlmTestServerKnownModelIds.Gpt54Nano, LlmTestServerModelKind.Chat, reasoningTokens); + } + + public LlmTestServerClientConfigurationBuilder UseGpt5(int reasoningTokens = 0) + { + return UseModel(LlmTestServerKnownModelIds.Gpt5, LlmTestServerModelKind.Chat, reasoningTokens); + } + + public LlmTestServerClientConfigurationBuilder UseGpt5Pro(int reasoningTokens = 0) + { + return UseModel(LlmTestServerKnownModelIds.Gpt5Pro, LlmTestServerModelKind.Chat, reasoningTokens); + } + + public LlmTestServerClientConfigurationBuilder UseGpt5Mini(int reasoningTokens = 0) + { + return UseModel(LlmTestServerKnownModelIds.Gpt5Mini, LlmTestServerModelKind.Chat, reasoningTokens); + } + + public LlmTestServerClientConfigurationBuilder UseGpt5Nano(int reasoningTokens = 0) + { + return UseModel(LlmTestServerKnownModelIds.Gpt5Nano, LlmTestServerModelKind.Chat, reasoningTokens); + } + + public LlmTestServerClientConfigurationBuilder UseGpt41() + { + return UseChatModel(LlmTestServerKnownModelIds.Gpt41); + } + + public LlmTestServerClientConfigurationBuilder UseGpt41Mini() + { + return UseChatModel(LlmTestServerKnownModelIds.Gpt41Mini); + } + + public LlmTestServerClientConfigurationBuilder UseGpt4OMini() + { + return UseChatModel(LlmTestServerKnownModelIds.Gpt4OMini); + } + + public LlmTestServerClientConfigurationBuilder UseTextEmbedding3Small() + { + return UseEmbeddingModel(LlmTestServerKnownModelIds.TextEmbedding3Small); + } + + public LlmTestServerClientConfigurationBuilder UseTextEmbedding3Large() + { + return UseEmbeddingModel(LlmTestServerKnownModelIds.TextEmbedding3Large); + } + + public LlmTestServerClientConfigurationBuilder UseTextEmbeddingAda002() + { + return UseEmbeddingModel(LlmTestServerKnownModelIds.TextEmbeddingAda002); + } + + public LlmTestServerClientConfigurationBuilder UseGptImage2() + { + return UseImageModel(LlmTestServerKnownModelIds.GptImage2); + } + + public LlmTestServerClientConfigurationBuilder UseGptImage15() + { + return UseImageModel(LlmTestServerKnownModelIds.GptImage15); + } + + public LlmTestServerClientConfigurationBuilder UseGptImage1() + { + return UseImageModel(LlmTestServerKnownModelIds.GptImage1); + } + + public LlmTestServerClientConfigurationBuilder UseGptImage1Mini() + { + return UseImageModel(LlmTestServerKnownModelIds.GptImage1Mini); + } + + public LlmTestServerClientConfigurationBuilder UseGpt4OMiniTts() + { + return UseAudioModel(LlmTestServerKnownModelIds.Gpt4OMiniTts); + } + + public LlmTestServerClientConfigurationBuilder UseTts1() + { + return UseAudioModel(LlmTestServerKnownModelIds.Tts1); + } + + public LlmTestServerClientConfigurationBuilder UseTts1Hd() + { + return UseAudioModel(LlmTestServerKnownModelIds.Tts1Hd); + } + + public LlmTestServerClientConfigurationBuilder UseSora2() + { + return UseVideoModel(LlmTestServerKnownModelIds.Sora2); + } + + public LlmTestServerClientConfigurationBuilder UseSora2Pro() + { + return UseVideoModel(LlmTestServerKnownModelIds.Sora2Pro); + } + + public LlmTestServerClientConfigurationBuilder UseDefaultOpenAiModels() + { + _builder.AddDefaultOpenAiModels(); + return this; + } + + public LlmTestServerClientConfigurationBuilder UseCurrentOpenAiChatModels(int reasoningTokens = 0) + { + _builder.AddCurrentOpenAiChatModels(reasoningTokens); + return this; + } + + public LlmTestServerClientConfigurationBuilder UseOpenAiEmbeddingModels() + { + _builder.AddOpenAiEmbeddingModels(); + return this; + } + + public LlmTestServerClientConfigurationBuilder UseOpenAiImageModels() + { + _builder.AddOpenAiImageModels(); + return this; + } + + public LlmTestServerClientConfigurationBuilder UseOpenAiAudioModels() + { + _builder.AddOpenAiAudioModels(); + return this; + } + + public LlmTestServerClientConfigurationBuilder UseOpenAiVideoModels() + { + _builder.AddOpenAiVideoModels(); + return this; + } + + public LlmTestServerClientConfigurationBuilder UseKnownOpenAiModels(int reasoningTokens = 0) + { + _builder.AddKnownOpenAiModels(reasoningTokens); + return this; + } + + public LlmTestServerClientConfigurationBuilder UseChatModel(string id) + { + return UseModel(id, LlmTestServerModelKind.Chat); + } + + public LlmTestServerClientConfigurationBuilder UseEmbeddingModel(string id) + { + return UseModel(id, LlmTestServerModelKind.Embedding); + } + + public LlmTestServerClientConfigurationBuilder UseImageModel(string id) + { + return UseModel(id, LlmTestServerModelKind.Image); + } + + public LlmTestServerClientConfigurationBuilder UseAudioModel(string id) + { + return UseModel(id, LlmTestServerModelKind.Audio); + } + + public LlmTestServerClientConfigurationBuilder UseVideoModel(string id) + { + return UseModel(id, LlmTestServerModelKind.Video); + } + + public LlmTestServerClientConfigurationBuilder UseConversation(string id, Action configure) + { + _builder.AddConversation(id, configure); + return this; + } + + public LlmTestServerClientConfigurationBuilder UseConversationCapacity(int maximumConversations, long maximumBytes) + { + _builder.WithConversationCapacity(maximumConversations, maximumBytes); + return this; + } + + public LlmTestServerClientConfigurationBuilder UseChatScenario( + string id, + Action configure + ) + { + _builder.AddChatScenario(id, configure); + return this; + } + + public LlmTestServerClientConfigurationBuilder UseDecisionModel(string id, ManagedCode.LlmTestServer.Decisions.LlmTestServerDecisionProvider provider) + { + _builder.AddDecisionModel(id, provider); + return this; + } + + public LlmTestServerClientConfigurationBuilder UseDecisionScenario(ManagedCode.LlmTestServer.Decisions.LlmTestServerDecisionScenario scenario) + { + _builder.AddDecisionScenario(scenario); + return this; + } + + public LlmTestServerClientConfigurationBuilder UseDataset( + string id, + Action configure + ) + { + _builder.AddDataset(id, configure); + return this; + } + + public LlmTestServerClientConfigurationBuilder UseEmbeddingVector(params float[] values) + { + _builder.WithDefaultEmbeddingVector(values); + return this; + } + + public LlmTestServerClientConfigurationBuilder UseImageDataUri(string dataUri) + { + _builder.WithDefaultImageDataUri(dataUri); + return this; + } + + public LlmTestServerClientConfigurationBuilder UseAudio(byte[] bytes, string mediaType) + { + _builder.WithDefaultAudio(bytes, mediaType); + return this; + } + + public LlmTestServerClientConfigurationBuilder UseVideo(byte[] bytes, string mediaType) + { + _builder.WithDefaultVideo(bytes, mediaType); + return this; + } + + public LlmTestServerClientConfigurationBuilder UseTranscriptionText(string text) + { + _builder.WithDefaultTranscriptionText(text); + return this; + } + + public LlmTestServerClientConfigurationBuilder UseTranscriptionUsage(LlmTestServerTranscriptionUsage usage) + { + _builder.WithDefaultTranscriptionUsage(usage); + return this; + } + + public LlmTestServerClientConfigurationBuilder WithoutTranscriptionUsage() + { + _builder.WithoutDefaultTranscriptionUsage(); + return this; + } + + public LlmTestServerClientConfigurationBuilder UseTranslationText(string text) + { + _builder.WithDefaultTranslationText(text); + return this; + } + + public LlmTestServerClientConfigurationBuilder UseChatResponseCapacity(int maxResponses, long maxBytes) + { + _builder.WithChatResponseCapacity(maxResponses, maxBytes); + return this; + } + + public LlmTestServerConfiguration Build() + { + return _builder.Build(); + } +} diff --git a/src/ManagedCode.LlmTck.Client/LlmTckCloudflareDecisionClient.cs b/src/ManagedCode.LlmTestServer.Client/LlmTestServerCloudflareDecisionClient.cs similarity index 81% rename from src/ManagedCode.LlmTck.Client/LlmTckCloudflareDecisionClient.cs rename to src/ManagedCode.LlmTestServer.Client/LlmTestServerCloudflareDecisionClient.cs index 197c7f7..7189611 100644 --- a/src/ManagedCode.LlmTck.Client/LlmTckCloudflareDecisionClient.cs +++ b/src/ManagedCode.LlmTestServer.Client/LlmTestServerCloudflareDecisionClient.cs @@ -1,17 +1,17 @@ using System.Net.Http.Headers; using System.Net.Http.Json; -using ManagedCode.LlmTck.Cloudflare; +using ManagedCode.LlmTestServer.Cloudflare; -namespace ManagedCode.LlmTck.Client; +namespace ManagedCode.LlmTestServer.Client; -/// Typed native Clef/Clef-flash client for a TCK host. The caller owns the HttpClient. -public sealed class LlmTckCloudflareDecisionClient +/// Typed native Clef/Clef-flash client for a test-server host. The caller owns the HttpClient. +public sealed class LlmTestServerCloudflareDecisionClient { private readonly HttpClient _httpClient; private readonly string _accountId; private readonly string? _bearerToken; - public LlmTckCloudflareDecisionClient(HttpClient httpClient, string accountId, string? bearerToken = null) + public LlmTestServerCloudflareDecisionClient(HttpClient httpClient, string accountId, string? bearerToken = null) { ArgumentNullException.ThrowIfNull(httpClient); ArgumentException.ThrowIfNullOrWhiteSpace(accountId); diff --git a/src/ManagedCode.LlmTestServer.Client/LlmTestServerConversationClient.cs b/src/ManagedCode.LlmTestServer.Client/LlmTestServerConversationClient.cs new file mode 100644 index 0000000..df59610 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Client/LlmTestServerConversationClient.cs @@ -0,0 +1,21 @@ +using ManagedCode.LlmTestServer.Conversations; + +namespace ManagedCode.LlmTestServer.Client; + +public sealed class LlmTestServerConversationClient : IDisposable +{ + internal LlmTestServerConversationClient(HttpClient transport, string id, string? bearerToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(id); + if (id.Length > 128) { throw new ArgumentException("Conversation IDs are limited to 128 characters.", nameof(id)); } + Id = id; + HttpClient = new HttpClient(new LlmTestServerHeaderTransport(transport, + new Dictionary { [LlmTestServerConversationHeaders.Id] = id })) + { BaseAddress = transport.BaseAddress }; + Control = new LlmTestServerClient(HttpClient, bearerToken); + } + public string Id { get; } + public HttpClient HttpClient { get; } + public LlmTestServerClient Control { get; } + public void Dispose() { HttpClient.Dispose(); } +} diff --git a/src/ManagedCode.LlmTck.Client/LlmTckEmbeddingGenerator.cs b/src/ManagedCode.LlmTestServer.Client/LlmTestServerEmbeddingGenerator.cs similarity index 87% rename from src/ManagedCode.LlmTck.Client/LlmTckEmbeddingGenerator.cs rename to src/ManagedCode.LlmTestServer.Client/LlmTestServerEmbeddingGenerator.cs index 30cb566..0c403b6 100644 --- a/src/ManagedCode.LlmTck.Client/LlmTckEmbeddingGenerator.cs +++ b/src/ManagedCode.LlmTestServer.Client/LlmTestServerEmbeddingGenerator.cs @@ -1,16 +1,16 @@ using System.Net.Http.Headers; using System.Net.Http.Json; using System.Text.Json; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.OpenAI; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.OpenAI; using Microsoft.Extensions.AI; -using ProviderRoutes = ManagedCode.LlmTck.Providers.LlmTckProviderRouteNamespaces; +using ProviderRoutes = ManagedCode.LlmTestServer.Providers.LlmTestServerProviderRouteNamespaces; -namespace ManagedCode.LlmTck.Client; +namespace ManagedCode.LlmTestServer.Client; -public sealed class LlmTckEmbeddingGenerator( +public sealed class LlmTestServerEmbeddingGenerator( HttpClient httpClient, - string defaultModelId = LlmTckKnownModelIds.TextEmbedding3Small, + string defaultModelId = LlmTestServerKnownModelIds.TextEmbedding3Small, string? bearerToken = null ) : IEmbeddingGenerator> { diff --git a/src/ManagedCode.LlmTestServer.Client/LlmTestServerHeaderTransport.cs b/src/ManagedCode.LlmTestServer.Client/LlmTestServerHeaderTransport.cs new file mode 100644 index 0000000..1051bd3 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Client/LlmTestServerHeaderTransport.cs @@ -0,0 +1,16 @@ +namespace ManagedCode.LlmTestServer.Client; + +internal sealed class LlmTestServerHeaderTransport(HttpClient transport, IReadOnlyDictionary headers) : HttpMessageHandler +{ + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + using var forwarded = new HttpRequestMessage(request.Method, request.RequestUri) + { Version = request.Version, VersionPolicy = request.VersionPolicy, Content = request.Content }; + foreach (var header in request.Headers) { forwarded.Headers.TryAddWithoutValidation(header.Key, header.Value); } + foreach (var option in request.Options) { forwarded.Options.Set(new HttpRequestOptionsKey(option.Key), option.Value); } + foreach (var header in headers) + { forwarded.Headers.Remove(header.Key); forwarded.Headers.Add(header.Key, header.Value); } + try { return await transport.SendAsync(forwarded, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); } + finally { forwarded.Content = null; } + } +} diff --git a/src/ManagedCode.LlmTck.Client/LlmTckImageGenerator.cs b/src/ManagedCode.LlmTestServer.Client/LlmTestServerImageGenerator.cs similarity index 87% rename from src/ManagedCode.LlmTck.Client/LlmTckImageGenerator.cs rename to src/ManagedCode.LlmTestServer.Client/LlmTestServerImageGenerator.cs index 8b19185..3e53746 100644 --- a/src/ManagedCode.LlmTck.Client/LlmTckImageGenerator.cs +++ b/src/ManagedCode.LlmTestServer.Client/LlmTestServerImageGenerator.cs @@ -1,16 +1,16 @@ using System.Net.Http.Headers; using System.Net.Http.Json; using System.Text.Json; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.OpenAI; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.OpenAI; using Microsoft.Extensions.AI; -using ProviderRoutes = ManagedCode.LlmTck.Providers.LlmTckProviderRouteNamespaces; +using ProviderRoutes = ManagedCode.LlmTestServer.Providers.LlmTestServerProviderRouteNamespaces; -namespace ManagedCode.LlmTck.Client; +namespace ManagedCode.LlmTestServer.Client; -public sealed class LlmTckImageGenerator( +public sealed class LlmTestServerImageGenerator( HttpClient httpClient, - string defaultModelId = LlmTckKnownModelIds.GptImage1, + string defaultModelId = LlmTestServerKnownModelIds.GptImage1, string? bearerToken = null ) : IImageGenerator { diff --git a/src/ManagedCode.LlmTestServer.Client/LlmTestServerSessionClient.cs b/src/ManagedCode.LlmTestServer.Client/LlmTestServerSessionClient.cs new file mode 100644 index 0000000..d211d4a --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Client/LlmTestServerSessionClient.cs @@ -0,0 +1,32 @@ +using ManagedCode.LlmTestServer.Control; +using ManagedCode.LlmTestServer.Sessions; + +namespace ManagedCode.LlmTestServer.Client; + +public sealed class LlmTestServerSessionClient : IAsyncDisposable +{ + private bool _disposed; + + internal LlmTestServerSessionClient(HttpClient transport, LlmTestServerSession session, string? bearerToken) + { + Session = session; + HttpClient = new HttpClient(new LlmTestServerHeaderTransport(transport, new Dictionary + { [LlmTestServerSessionHeaders.Id] = session.Id, [LlmTestServerSessionHeaders.AccessKey] = session.AccessKey })) + { BaseAddress = transport.BaseAddress }; + Control = new LlmTestServerClient(HttpClient, bearerToken); + } + + public LlmTestServerSession Session { get; } + public HttpClient HttpClient { get; } + public LlmTestServerClient Control { get; } + + public async ValueTask DisposeAsync() + { + if (_disposed) { return; } + using var response = await HttpClient.DeleteAsync(LlmTestServerControlRoutes.Sessions + "/" + Session.Id).ConfigureAwait(false); + if (response.StatusCode != System.Net.HttpStatusCode.NotFound) { response.EnsureSuccessStatusCode(); } + _disposed = true; + HttpClient.Dispose(); + } + +} diff --git a/src/ManagedCode.LlmTck.Client/LlmTckSystemOneClient.cs b/src/ManagedCode.LlmTestServer.Client/LlmTestServerSystemOneClient.cs similarity index 90% rename from src/ManagedCode.LlmTck.Client/LlmTckSystemOneClient.cs rename to src/ManagedCode.LlmTestServer.Client/LlmTestServerSystemOneClient.cs index 491fbde..761e079 100644 --- a/src/ManagedCode.LlmTck.Client/LlmTckSystemOneClient.cs +++ b/src/ManagedCode.LlmTestServer.Client/LlmTestServerSystemOneClient.cs @@ -1,11 +1,11 @@ using System.Net.Http.Headers; using System.Net.Http.Json; -using ManagedCode.LlmTck.SystemOne; +using ManagedCode.LlmTestServer.SystemOne; -namespace ManagedCode.LlmTck.Client; +namespace ManagedCode.LlmTestServer.Client; -/// Typed native Jev/Kev client for a TCK host. The caller owns the shared HttpClient. -public sealed class LlmTckSystemOneClient(HttpClient httpClient, string? bearerToken = null) +/// Typed native Jev/Kev client for a test-server host. The caller owns the shared HttpClient. +public sealed class LlmTestServerSystemOneClient(HttpClient httpClient, string? bearerToken = null) { private const string _path = "systemone/v1/systemone"; private const string _requestPart = "request"; diff --git a/src/ManagedCode.LlmTck.Client/LlmTckUsageMapper.cs b/src/ManagedCode.LlmTestServer.Client/LlmTestServerUsageMapper.cs similarity index 83% rename from src/ManagedCode.LlmTck.Client/LlmTckUsageMapper.cs rename to src/ManagedCode.LlmTestServer.Client/LlmTestServerUsageMapper.cs index 0de0478..8e0efe9 100644 --- a/src/ManagedCode.LlmTck.Client/LlmTckUsageMapper.cs +++ b/src/ManagedCode.LlmTestServer.Client/LlmTestServerUsageMapper.cs @@ -1,9 +1,9 @@ -using ManagedCode.LlmTck.OpenAI; +using ManagedCode.LlmTestServer.OpenAI; using Microsoft.Extensions.AI; -namespace ManagedCode.LlmTck.Client; +namespace ManagedCode.LlmTestServer.Client; -internal static class LlmTckUsageMapper +internal static class LlmTestServerUsageMapper { public static UsageDetails Map(OpenAiUsage usage) { diff --git a/src/ManagedCode.LlmTestServer.Client/ManagedCode.LlmTestServer.Client.csproj b/src/ManagedCode.LlmTestServer.Client/ManagedCode.LlmTestServer.Client.csproj new file mode 100644 index 0000000..e586726 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Client/ManagedCode.LlmTestServer.Client.csproj @@ -0,0 +1,18 @@ + + + ManagedCode.LlmTestServer.Client + Typed control client and assertion helpers for ManagedCode.LlmTestServer test hosts. + + + + + + + + + + + + + + diff --git a/src/ManagedCode.LlmTestServer.Cloudflare/AGENTS.md b/src/ManagedCode.LlmTestServer.Cloudflare/AGENTS.md new file mode 100644 index 0000000..4532165 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Cloudflare/AGENTS.md @@ -0,0 +1,3 @@ +# ManagedCode.LlmTestServer.Cloudflare + +Owns native Cloudflare decision wire contracts and compatibility profiles. Keep deterministic fixtures in `ManagedCode.LlmTestServer` and routes in `ManagedCode.LlmTestServer.Hosting`. DTOs use explicit JSON names; preserve opaque choice IDs and native scores. diff --git a/src/ManagedCode.LlmTestServer.Cloudflare/CloudflareCompatibility.cs b/src/ManagedCode.LlmTestServer.Cloudflare/CloudflareCompatibility.cs new file mode 100644 index 0000000..bd3824e --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Cloudflare/CloudflareCompatibility.cs @@ -0,0 +1,24 @@ +using ManagedCode.LlmTestServer.Providers; +namespace ManagedCode.LlmTestServer.Cloudflare; + +public static class CloudflareCompatibility +{ + public const string ProviderId = LlmTestServerCompatibilityTags.Cloudflare; + public static LlmTestServerProviderProfile Profile { get; } = new() + { + Id = ProviderId, + DisplayName = "Cloudflare Workers AI (Clef)", + Protocol = LlmTestServerProtocolFamily.Cloudflare, + DefaultEndpointPath = "/cloudflare/client/v4/accounts/{accountId}/ai/run/@cf/cloudflare/clef", + Capabilities = [LlmTestServerProviderCapability.Decisions], + CompatibilityTags = [ProviderId], + ApiContract = new() + { + DocumentationUrl = "https://developers.cloudflare.com/workers-ai/models/clef/", + DocumentationRetrievedOn = "2026-10-20", + DocumentationVersion = "native Clef/Clef-flash decisions and embedded PNG/JPEG/WebP images", + Operations = [ new() { Id = LlmTestServerProviderOperationIds.Cloudflare.ClefDecisionsCreate, Method = "POST", Path = "/cloudflare/client/v4/accounts/{accountId}/ai/run/@cf/cloudflare/clef", DocumentationUrl = "https://developers.cloudflare.com/workers-ai/models/clef/", ImplementedByHosting = true, Capabilities = [LlmTestServerProviderCapability.Decisions] }, +new() { Id = LlmTestServerProviderOperationIds.Cloudflare.ClefFlashDecisionsCreate, Method = "POST", Path = "/cloudflare/client/v4/accounts/{accountId}/ai/run/@cf/cloudflare/clef-flash", DocumentationUrl = "https://developers.cloudflare.com/workers-ai/models/clef/", ImplementedByHosting = true, Capabilities = [LlmTestServerProviderCapability.Decisions] } ] + }, + }; +} diff --git a/src/ManagedCode.LlmTck.Cloudflare/CloudflareDecisionErrors.cs b/src/ManagedCode.LlmTestServer.Cloudflare/CloudflareDecisionErrors.cs similarity index 96% rename from src/ManagedCode.LlmTck.Cloudflare/CloudflareDecisionErrors.cs rename to src/ManagedCode.LlmTestServer.Cloudflare/CloudflareDecisionErrors.cs index e6f4685..a34a321 100644 --- a/src/ManagedCode.LlmTck.Cloudflare/CloudflareDecisionErrors.cs +++ b/src/ManagedCode.LlmTestServer.Cloudflare/CloudflareDecisionErrors.cs @@ -1,6 +1,6 @@ using System.Globalization; -namespace ManagedCode.LlmTck.Cloudflare; +namespace ManagedCode.LlmTestServer.Cloudflare; public static class CloudflareDecisionErrors { diff --git a/src/ManagedCode.LlmTck.Cloudflare/CloudflareDecisionImageConverter.cs b/src/ManagedCode.LlmTestServer.Cloudflare/CloudflareDecisionImageConverter.cs similarity index 96% rename from src/ManagedCode.LlmTck.Cloudflare/CloudflareDecisionImageConverter.cs rename to src/ManagedCode.LlmTestServer.Cloudflare/CloudflareDecisionImageConverter.cs index 07f31a0..6ca8f21 100644 --- a/src/ManagedCode.LlmTck.Cloudflare/CloudflareDecisionImageConverter.cs +++ b/src/ManagedCode.LlmTestServer.Cloudflare/CloudflareDecisionImageConverter.cs @@ -1,7 +1,7 @@ using System.Text.Json; using System.Text.Json.Serialization; -namespace ManagedCode.LlmTck.Cloudflare; +namespace ManagedCode.LlmTestServer.Cloudflare; public sealed class CloudflareDecisionImageConverter : JsonConverter { diff --git a/src/ManagedCode.LlmTck.Cloudflare/CloudflareDecisionImageValidation.cs b/src/ManagedCode.LlmTestServer.Cloudflare/CloudflareDecisionImageValidation.cs similarity index 99% rename from src/ManagedCode.LlmTck.Cloudflare/CloudflareDecisionImageValidation.cs rename to src/ManagedCode.LlmTestServer.Cloudflare/CloudflareDecisionImageValidation.cs index 28ea10a..f25c3bf 100644 --- a/src/ManagedCode.LlmTck.Cloudflare/CloudflareDecisionImageValidation.cs +++ b/src/ManagedCode.LlmTestServer.Cloudflare/CloudflareDecisionImageValidation.cs @@ -6,7 +6,7 @@ using MetadataExtractor.Formats.Png; using MetadataExtractor.Formats.WebP; -namespace ManagedCode.LlmTck.Cloudflare; +namespace ManagedCode.LlmTestServer.Cloudflare; public static class CloudflareDecisionImageValidation { diff --git a/src/ManagedCode.LlmTck.Cloudflare/CloudflareDecisionMapper.cs b/src/ManagedCode.LlmTestServer.Cloudflare/CloudflareDecisionMapper.cs similarity index 76% rename from src/ManagedCode.LlmTck.Cloudflare/CloudflareDecisionMapper.cs rename to src/ManagedCode.LlmTestServer.Cloudflare/CloudflareDecisionMapper.cs index 03905d5..10b7d6b 100644 --- a/src/ManagedCode.LlmTck.Cloudflare/CloudflareDecisionMapper.cs +++ b/src/ManagedCode.LlmTestServer.Cloudflare/CloudflareDecisionMapper.cs @@ -1,9 +1,9 @@ using System.Text.Json; using System.Text.Json.Serialization; -using ManagedCode.LlmTck.Decisions; -using ManagedCode.LlmTck.SystemOne; +using ManagedCode.LlmTestServer.Decisions; +using ManagedCode.LlmTestServer.SystemOne; -namespace ManagedCode.LlmTck.Cloudflare; +namespace ManagedCode.LlmTestServer.Cloudflare; public static class CloudflareDecisionMapper { @@ -17,9 +17,9 @@ public static SystemOneDecisionRequest ToSystemOne(CloudflareDecisionRequest req }; } - public static LlmTckDecisionRequest ToRequest(CloudflareDecisionRequest request) + public static LlmTestServerDecisionRequest ToRequest(CloudflareDecisionRequest request) { - var native = SystemOneDecisionMapper.ToRequest(ToSystemOne(request), LlmTckDecisionProvider.Cloudflare); + var native = SystemOneDecisionMapper.ToRequest(ToSystemOne(request), LlmTestServerDecisionProvider.Cloudflare); return request.Images is null ? native : native with { Input = JsonSerializer.Serialize(new CloudflareDecisionInput { State = request.State, Images = request.Images }), diff --git a/src/ManagedCode.LlmTck.Cloudflare/CloudflareDecisionModels.cs b/src/ManagedCode.LlmTestServer.Cloudflare/CloudflareDecisionModels.cs similarity index 88% rename from src/ManagedCode.LlmTck.Cloudflare/CloudflareDecisionModels.cs rename to src/ManagedCode.LlmTestServer.Cloudflare/CloudflareDecisionModels.cs index 014164b..46dc682 100644 --- a/src/ManagedCode.LlmTck.Cloudflare/CloudflareDecisionModels.cs +++ b/src/ManagedCode.LlmTestServer.Cloudflare/CloudflareDecisionModels.cs @@ -1,7 +1,7 @@ using System.Text.Json.Serialization; -using ManagedCode.LlmTck.SystemOne; +using ManagedCode.LlmTestServer.SystemOne; -namespace ManagedCode.LlmTck.Cloudflare; +namespace ManagedCode.LlmTestServer.Cloudflare; public sealed record CloudflareDecisionResponse { diff --git a/src/ManagedCode.LlmTck.Cloudflare/CloudflareDecisionRequest.cs b/src/ManagedCode.LlmTestServer.Cloudflare/CloudflareDecisionRequest.cs similarity index 91% rename from src/ManagedCode.LlmTck.Cloudflare/CloudflareDecisionRequest.cs rename to src/ManagedCode.LlmTestServer.Cloudflare/CloudflareDecisionRequest.cs index f175a77..8104d3d 100644 --- a/src/ManagedCode.LlmTck.Cloudflare/CloudflareDecisionRequest.cs +++ b/src/ManagedCode.LlmTestServer.Cloudflare/CloudflareDecisionRequest.cs @@ -1,8 +1,8 @@ using System.Text.Json; using System.Text.Json.Serialization; -using ManagedCode.LlmTck.SystemOne; +using ManagedCode.LlmTestServer.SystemOne; -namespace ManagedCode.LlmTck.Cloudflare; +namespace ManagedCode.LlmTestServer.Cloudflare; public sealed record CloudflareDecisionRequest { diff --git a/src/ManagedCode.LlmTck.Cloudflare/CloudflareDecisionValidation.cs b/src/ManagedCode.LlmTestServer.Cloudflare/CloudflareDecisionValidation.cs similarity index 89% rename from src/ManagedCode.LlmTck.Cloudflare/CloudflareDecisionValidation.cs rename to src/ManagedCode.LlmTestServer.Cloudflare/CloudflareDecisionValidation.cs index 8b5d581..953ccd3 100644 --- a/src/ManagedCode.LlmTck.Cloudflare/CloudflareDecisionValidation.cs +++ b/src/ManagedCode.LlmTestServer.Cloudflare/CloudflareDecisionValidation.cs @@ -1,16 +1,16 @@ using System.Text.Json; -using ManagedCode.LlmTck.Decisions; -using ManagedCode.LlmTck.Providers; -using ManagedCode.LlmTck.SystemOne; +using ManagedCode.LlmTestServer.Decisions; +using ManagedCode.LlmTestServer.Providers; +using ManagedCode.LlmTestServer.SystemOne; -namespace ManagedCode.LlmTck.Cloudflare; +namespace ManagedCode.LlmTestServer.Cloudflare; public static class CloudflareDecisionValidation { public const int MaxQuestions = 64; public const int MaxQuestionIdLength = 100; - public static LlmTckRequestValidationResult Validate(JsonElement body) + public static LlmTestServerRequestValidationResult Validate(JsonElement body) { var native = ValidateNative(body); return new(native.Error); @@ -18,7 +18,7 @@ public static LlmTckRequestValidationResult Validate(JsonElement body) public static CloudflareDecisionValidationResult ValidateNative(JsonElement body) { - var shape = SystemOneDecisionMapper.Validate(body, LlmTckDecisionProvider.Cloudflare); + var shape = SystemOneDecisionMapper.Validate(body, LlmTestServerDecisionProvider.Cloudflare); if (!shape.IsValid) { return new(shape.Error); } var model = body.GetProperty("model").GetString()!.Trim(); if (model is not ("clef" or "clef-flash")) { return new("Clef requires its short native model selector."); } diff --git a/src/ManagedCode.LlmTck.Cloudflare/CloudflareDecisionValidationResult.cs b/src/ManagedCode.LlmTestServer.Cloudflare/CloudflareDecisionValidationResult.cs similarity index 81% rename from src/ManagedCode.LlmTck.Cloudflare/CloudflareDecisionValidationResult.cs rename to src/ManagedCode.LlmTestServer.Cloudflare/CloudflareDecisionValidationResult.cs index dcd4fef..5e75469 100644 --- a/src/ManagedCode.LlmTck.Cloudflare/CloudflareDecisionValidationResult.cs +++ b/src/ManagedCode.LlmTestServer.Cloudflare/CloudflareDecisionValidationResult.cs @@ -1,4 +1,4 @@ -namespace ManagedCode.LlmTck.Cloudflare; +namespace ManagedCode.LlmTestServer.Cloudflare; public sealed record CloudflareDecisionValidationResult( string? Error = null, int StatusCode = 400, int ErrorCode = CloudflareDecisionErrors.InvalidData) diff --git a/src/ManagedCode.LlmTestServer.Cloudflare/ManagedCode.LlmTestServer.Cloudflare.csproj b/src/ManagedCode.LlmTestServer.Cloudflare/ManagedCode.LlmTestServer.Cloudflare.csproj new file mode 100644 index 0000000..fb763aa --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Cloudflare/ManagedCode.LlmTestServer.Cloudflare.csproj @@ -0,0 +1,5 @@ + + ManagedCode.LlmTestServer.CloudflareNative Cloudflare decision API compatibility for ManagedCode.LlmTestServer. + + + diff --git a/src/ManagedCode.LlmTestServer.Cohere/AGENTS.md b/src/ManagedCode.LlmTestServer.Cohere/AGENTS.md new file mode 100644 index 0000000..517a794 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Cohere/AGENTS.md @@ -0,0 +1,5 @@ +# ManagedCode.LlmTestServer.Cohere + +This package owns Cohere compatibility constants, provider profiles, and Cohere wire contracts. + +Keep deterministic runtime behavior in `ManagedCode.LlmTestServer`; keep HTTP hosting in `ManagedCode.LlmTestServer.Hosting`. diff --git a/src/ManagedCode.LlmTck.Cohere/CohereCompatibility.cs b/src/ManagedCode.LlmTestServer.Cohere/CohereCompatibility.cs similarity index 55% rename from src/ManagedCode.LlmTck.Cohere/CohereCompatibility.cs rename to src/ManagedCode.LlmTestServer.Cohere/CohereCompatibility.cs index f016ff1..09497e9 100644 --- a/src/ManagedCode.LlmTck.Cohere/CohereCompatibility.cs +++ b/src/ManagedCode.LlmTestServer.Cohere/CohereCompatibility.cs @@ -1,25 +1,25 @@ -using ManagedCode.LlmTck.Providers; +using ManagedCode.LlmTestServer.Providers; -namespace ManagedCode.LlmTck.Cohere; +namespace ManagedCode.LlmTestServer.Cohere; public static class CohereCompatibility { - public const string ProviderId = LlmTckCompatibilityTags.Cohere; + public const string ProviderId = LlmTestServerCompatibilityTags.Cohere; - public static LlmTckProviderProfile Profile { get; } = + public static LlmTestServerProviderProfile Profile { get; } = new() { Id = ProviderId, DisplayName = "Cohere", - Protocol = LlmTckProtocolFamily.Cohere, + Protocol = LlmTestServerProtocolFamily.Cohere, DefaultEndpointPath = "/cohere/v2/chat", Capabilities = [ - LlmTckProviderCapability.Chat, - LlmTckProviderCapability.StreamingChat, - LlmTckProviderCapability.Tools, - LlmTckProviderCapability.StructuredOutput, - LlmTckProviderCapability.Embeddings, + LlmTestServerProviderCapability.Chat, + LlmTestServerProviderCapability.StreamingChat, + LlmTestServerProviderCapability.Tools, + LlmTestServerProviderCapability.StructuredOutput, + LlmTestServerProviderCapability.Embeddings, ], CompatibilityTags = [ProviderId], ApiContract = new() @@ -31,7 +31,7 @@ public static class CohereCompatibility [ new() { - Id = LlmTckProviderOperationIds.Cohere.ChatCreate, + Id = LlmTestServerProviderOperationIds.Cohere.ChatCreate, Method = "POST", Path = "/cohere/v2/chat", DocumentationUrl = "https://docs.cohere.com/reference/chat", @@ -39,20 +39,20 @@ public static class CohereCompatibility ImplementedByHosting = true, Capabilities = [ - LlmTckProviderCapability.Chat, - LlmTckProviderCapability.StreamingChat, - LlmTckProviderCapability.Tools, - LlmTckProviderCapability.StructuredOutput, + LlmTestServerProviderCapability.Chat, + LlmTestServerProviderCapability.StreamingChat, + LlmTestServerProviderCapability.Tools, + LlmTestServerProviderCapability.StructuredOutput, ], }, new() { - Id = LlmTckProviderOperationIds.Cohere.EmbedCreate, + Id = LlmTestServerProviderOperationIds.Cohere.EmbedCreate, Method = "POST", Path = "/cohere/v2/embed", DocumentationUrl = "https://docs.cohere.com/reference/embed", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Embeddings], + Capabilities = [LlmTestServerProviderCapability.Embeddings], }, ], }, diff --git a/src/ManagedCode.LlmTck.Cohere/CohereModels.cs b/src/ManagedCode.LlmTestServer.Cohere/CohereModels.cs similarity index 99% rename from src/ManagedCode.LlmTck.Cohere/CohereModels.cs rename to src/ManagedCode.LlmTestServer.Cohere/CohereModels.cs index 05aefcf..1e21341 100644 --- a/src/ManagedCode.LlmTck.Cohere/CohereModels.cs +++ b/src/ManagedCode.LlmTestServer.Cohere/CohereModels.cs @@ -1,7 +1,7 @@ using System.Text.Json; using System.Text.Json.Serialization; -namespace ManagedCode.LlmTck.Cohere; +namespace ManagedCode.LlmTestServer.Cohere; public sealed record CohereChatRequest { diff --git a/src/ManagedCode.LlmTck.Cohere/CohereRequestValidation.cs b/src/ManagedCode.LlmTestServer.Cohere/CohereRequestValidation.cs similarity index 83% rename from src/ManagedCode.LlmTck.Cohere/CohereRequestValidation.cs rename to src/ManagedCode.LlmTestServer.Cohere/CohereRequestValidation.cs index f2ccd54..bebb7fe 100644 --- a/src/ManagedCode.LlmTck.Cohere/CohereRequestValidation.cs +++ b/src/ManagedCode.LlmTestServer.Cohere/CohereRequestValidation.cs @@ -1,12 +1,12 @@ using System.Text.Json; -using ManagedCode.LlmTck.Providers; -using static ManagedCode.LlmTck.ProviderValidation.LlmTckJsonValidation; +using ManagedCode.LlmTestServer.Providers; +using static ManagedCode.LlmTestServer.ProviderValidation.LlmTestServerJsonValidation; -namespace ManagedCode.LlmTck.Cohere; +namespace ManagedCode.LlmTestServer.Cohere; public static class CohereRequestValidation { - public static LlmTckRequestValidationResult Validate(JsonElement body) + public static LlmTestServerRequestValidationResult Validate(JsonElement body) { return Check(body, ValidateBody); } diff --git a/src/ManagedCode.LlmTck.Cohere/CohereWireMapper.cs b/src/ManagedCode.LlmTestServer.Cohere/CohereWireMapper.cs similarity index 71% rename from src/ManagedCode.LlmTck.Cohere/CohereWireMapper.cs rename to src/ManagedCode.LlmTestServer.Cohere/CohereWireMapper.cs index c747db4..3dba82c 100644 --- a/src/ManagedCode.LlmTck.Cohere/CohereWireMapper.cs +++ b/src/ManagedCode.LlmTestServer.Cohere/CohereWireMapper.cs @@ -1,38 +1,39 @@ -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Scenarios; +using ManagedCode.LlmTestServer.Determinism; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Scenarios; -namespace ManagedCode.LlmTck.Cohere; +namespace ManagedCode.LlmTestServer.Cohere; public static class CohereWireMapper { - public static LlmTckChatRequest ToRuntimeRequest(CohereChatRequest request) + public static LlmTestServerChatRequest ToRuntimeRequest(CohereChatRequest request) { return new() { - Tools = request.Tools.Select(tool => new LlmTckToolDefinition { Name = tool.Function.Name, Description = tool.Function.Description, ParametersJson = tool.Function.Parameters.GetRawText() }).ToList(), - ToolChoice = request.ToolChoice switch { "REQUIRED" => LlmTckToolChoice.Required, "NONE" => LlmTckToolChoice.None, _ => LlmTckToolChoice.Auto }, + Tools = request.Tools.Select(tool => new LlmTestServerToolDefinition { Name = tool.Function.Name, Description = tool.Function.Description, ParametersJson = tool.Function.Parameters.GetRawText() }).ToList(), + ToolChoice = request.ToolChoice switch { "REQUIRED" => LlmTestServerToolChoice.Required, "NONE" => LlmTestServerToolChoice.None, _ => LlmTestServerToolChoice.Auto }, RequireJson = request.ResponseFormat?.Type == "json_object", ResponseSchemaJson = request.ResponseFormat?.Schema?.GetRawText(), ModelId = request.Model, Stream = request.Stream, Messages = request .Messages - .Select(message => new LlmTckMessage + .Select(message => new LlmTestServerMessage { Role = message.Role, Content = message.TextContent, ToolCallId = message.ToolCallId, - ToolCalls = message.ToolCalls?.Select(call => new LlmTckToolCall { Id = call.Id, Name = call.Function.Name, ArgumentsJson = call.Function.Arguments }).ToList() ?? [], + ToolCalls = message.ToolCalls?.Select(call => new LlmTestServerToolCall { Id = call.Id, Name = call.Function.Name, ArgumentsJson = call.Function.Arguments }).ToList() ?? [], }) .ToList(), }; } - public static CohereChatResponse ToChatResponse(LlmTckChatResult result) + public static CohereChatResponse ToChatResponse(LlmTestServerChatResult result) { return new() { - Id = CreateResponseId(), + Id = CreateResponseId(result.WireMetadata), FinishReason = result.ToolCalls.Count > 0 ? "TOOL_CALL" : "COMPLETE", Message = new CohereAssistantMessage { @@ -43,12 +44,12 @@ public static CohereChatResponse ToChatResponse(LlmTckChatResult result) }; } - public static object ToMessageStartEvent() + public static object ToMessageStartEvent(LlmTestServerResponseMetadata? metadata = null) { return new { type = "message-start", - id = CreateResponseId(), + id = CreateResponseId(metadata), delta = new { message = new { role = "assistant" }, @@ -97,7 +98,7 @@ public static object ToContentEndEvent() }; } - public static object ToMessageEndEvent(LlmTckChatResult result) + public static object ToMessageEndEvent(LlmTestServerChatResult result) { return new { @@ -110,12 +111,12 @@ public static object ToMessageEndEvent(LlmTckChatResult result) }; } - private static CohereToolCall ToToolCall(LlmTckToolCall call) + private static CohereToolCall ToToolCall(LlmTestServerToolCall call) { return new() { Id = call.Id, Function = new() { Name = call.Name, Arguments = call.ArgumentsJson } }; } - public static IEnumerable ToToolStreamEvents(LlmTckChatResult result) + public static IEnumerable ToToolStreamEvents(LlmTestServerChatResult result) { for (var index = 0; index < result.ToolCalls.Count; index++) { @@ -126,11 +127,11 @@ public static IEnumerable ToToolStreamEvents(LlmTckChatResult result) } } - public static CohereEmbedResponse ToEmbedResponse(IReadOnlyList> vectors) + public static CohereEmbedResponse ToEmbedResponse(IReadOnlyList> vectors, LlmTestServerResponseMetadata? metadata = null) { return new() { - Id = CreateResponseId(), + Id = CreateResponseId(metadata), Embeddings = new CohereEmbeddings { FloatValues = vectors }, }; } @@ -160,7 +161,7 @@ public static List ReadEmbeddingInputs(CohereEmbedRequest request) .ToList(); } - private static CohereUsage CreateUsage(LlmTckTokenUsage usage) + private static CohereUsage CreateUsage(LlmTestServerTokenUsage usage) { var tokens = new CohereTokenUsage { @@ -170,9 +171,9 @@ private static CohereUsage CreateUsage(LlmTckTokenUsage usage) return new() { BilledUnits = tokens, Tokens = tokens }; } - private static string CreateResponseId() + private static string CreateResponseId(LlmTestServerResponseMetadata? metadata) { - return Guid.NewGuid().ToString(); + return new Guid((metadata ?? LlmTestServerResponseMetadata.Live()).IdFor("response")).ToString(); } } diff --git a/src/ManagedCode.LlmTestServer.Cohere/ManagedCode.LlmTestServer.Cohere.csproj b/src/ManagedCode.LlmTestServer.Cohere/ManagedCode.LlmTestServer.Cohere.csproj new file mode 100644 index 0000000..c2be535 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Cohere/ManagedCode.LlmTestServer.Cohere.csproj @@ -0,0 +1,13 @@ + + + ManagedCode.LlmTestServer.Cohere + Cohere compatibility profile and adapters for ManagedCode.LlmTestServer. + + + + + + + + + diff --git a/src/ManagedCode.LlmTestServer.DeepSeek/AGENTS.md b/src/ManagedCode.LlmTestServer.DeepSeek/AGENTS.md new file mode 100644 index 0000000..249b986 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.DeepSeek/AGENTS.md @@ -0,0 +1,5 @@ +# ManagedCode.LlmTestServer.DeepSeek + +This package owns DeepSeek compatibility constants, provider profiles, and DeepSeek wire contracts. + +Keep deterministic runtime behavior in `ManagedCode.LlmTestServer`; keep HTTP hosting in `ManagedCode.LlmTestServer.Hosting`. diff --git a/src/ManagedCode.LlmTck.DeepSeek/DeepSeekCompatibility.cs b/src/ManagedCode.LlmTestServer.DeepSeek/DeepSeekCompatibility.cs similarity index 53% rename from src/ManagedCode.LlmTck.DeepSeek/DeepSeekCompatibility.cs rename to src/ManagedCode.LlmTestServer.DeepSeek/DeepSeekCompatibility.cs index 444fb0b..7318955 100644 --- a/src/ManagedCode.LlmTck.DeepSeek/DeepSeekCompatibility.cs +++ b/src/ManagedCode.LlmTestServer.DeepSeek/DeepSeekCompatibility.cs @@ -1,27 +1,27 @@ -using ManagedCode.LlmTck.Providers; +using ManagedCode.LlmTestServer.Providers; -namespace ManagedCode.LlmTck.DeepSeek; +namespace ManagedCode.LlmTestServer.DeepSeek; public static class DeepSeekCompatibility { - public const string ProviderId = LlmTckCompatibilityTags.DeepSeek; + public const string ProviderId = LlmTestServerCompatibilityTags.DeepSeek; - public static LlmTckProviderProfile Profile { get; } = + public static LlmTestServerProviderProfile Profile { get; } = new() { Id = ProviderId, DisplayName = "DeepSeek", - Protocol = LlmTckProtocolFamily.DeepSeek, + Protocol = LlmTestServerProtocolFamily.DeepSeek, DefaultEndpointPath = "/deepseek/v1/chat/completions", Capabilities = [ - LlmTckProviderCapability.Models, - LlmTckProviderCapability.Chat, - LlmTckProviderCapability.StreamingChat, - LlmTckProviderCapability.Tools, - LlmTckProviderCapability.StructuredOutput, + LlmTestServerProviderCapability.Models, + LlmTestServerProviderCapability.Chat, + LlmTestServerProviderCapability.StreamingChat, + LlmTestServerProviderCapability.Tools, + LlmTestServerProviderCapability.StructuredOutput, ], - CompatibilityTags = [ProviderId, LlmTckCompatibilityTags.OpenAICompatible], + CompatibilityTags = [ProviderId, LlmTestServerCompatibilityTags.OpenAICompatible], ApiContract = new() { DocumentationUrl = "https://api-docs.deepseek.com/", @@ -31,7 +31,7 @@ public static class DeepSeekCompatibility [ new() { - Id = LlmTckProviderOperationIds.DeepSeek.ChatCompletionsCreate, + Id = LlmTestServerProviderOperationIds.DeepSeek.ChatCompletionsCreate, Method = "POST", Path = "/deepseek/v1/chat/completions", DocumentationUrl = @@ -40,20 +40,20 @@ public static class DeepSeekCompatibility ImplementedByHosting = true, Capabilities = [ - LlmTckProviderCapability.Chat, - LlmTckProviderCapability.StreamingChat, - LlmTckProviderCapability.Tools, - LlmTckProviderCapability.StructuredOutput, + LlmTestServerProviderCapability.Chat, + LlmTestServerProviderCapability.StreamingChat, + LlmTestServerProviderCapability.Tools, + LlmTestServerProviderCapability.StructuredOutput, ], }, new() { - Id = LlmTckProviderOperationIds.DeepSeek.ModelsList, + Id = LlmTestServerProviderOperationIds.DeepSeek.ModelsList, Method = "GET", Path = "/deepseek/models", DocumentationUrl = "https://api-docs.deepseek.com/api/list-models", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Models], + Capabilities = [LlmTestServerProviderCapability.Models], }, ], }, diff --git a/src/ManagedCode.LlmTestServer.DeepSeek/ManagedCode.LlmTestServer.DeepSeek.csproj b/src/ManagedCode.LlmTestServer.DeepSeek/ManagedCode.LlmTestServer.DeepSeek.csproj new file mode 100644 index 0000000..716ca5a --- /dev/null +++ b/src/ManagedCode.LlmTestServer.DeepSeek/ManagedCode.LlmTestServer.DeepSeek.csproj @@ -0,0 +1,10 @@ + + + ManagedCode.LlmTestServer.DeepSeek + DeepSeek compatibility profile and adapters for ManagedCode.LlmTestServer. + + + + + + diff --git a/src/ManagedCode.LlmTestServer.Foundry/AGENTS.md b/src/ManagedCode.LlmTestServer.Foundry/AGENTS.md new file mode 100644 index 0000000..0cefcc0 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Foundry/AGENTS.md @@ -0,0 +1,5 @@ +# ManagedCode.LlmTestServer.Foundry + +This package owns Microsoft Foundry compatibility constants, provider profiles, and Foundry wire contracts. + +Keep deterministic runtime behavior in `ManagedCode.LlmTestServer`; keep HTTP hosting in `ManagedCode.LlmTestServer.Hosting`. diff --git a/src/ManagedCode.LlmTestServer.Foundry/ManagedCode.LlmTestServer.Foundry.csproj b/src/ManagedCode.LlmTestServer.Foundry/ManagedCode.LlmTestServer.Foundry.csproj new file mode 100644 index 0000000..88d5f5d --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Foundry/ManagedCode.LlmTestServer.Foundry.csproj @@ -0,0 +1,10 @@ + + + ManagedCode.LlmTestServer.Foundry + Microsoft Foundry compatibility profile and adapters for ManagedCode.LlmTestServer. + + + + + + diff --git a/src/ManagedCode.LlmTck.Foundry/MicrosoftFoundryCompatibility.cs b/src/ManagedCode.LlmTestServer.Foundry/MicrosoftFoundryCompatibility.cs similarity index 60% rename from src/ManagedCode.LlmTck.Foundry/MicrosoftFoundryCompatibility.cs rename to src/ManagedCode.LlmTestServer.Foundry/MicrosoftFoundryCompatibility.cs index c6b1f25..45bb8cc 100644 --- a/src/ManagedCode.LlmTck.Foundry/MicrosoftFoundryCompatibility.cs +++ b/src/ManagedCode.LlmTestServer.Foundry/MicrosoftFoundryCompatibility.cs @@ -1,33 +1,33 @@ -using ManagedCode.LlmTck.Providers; +using ManagedCode.LlmTestServer.Providers; -namespace ManagedCode.LlmTck.Foundry; +namespace ManagedCode.LlmTestServer.Foundry; public static class MicrosoftFoundryCompatibility { - public const string ProviderId = LlmTckCompatibilityTags.MicrosoftFoundry; + public const string ProviderId = LlmTestServerCompatibilityTags.MicrosoftFoundry; - public static LlmTckProviderProfile Profile { get; } = + public static LlmTestServerProviderProfile Profile { get; } = new() { Id = ProviderId, DisplayName = "Microsoft Foundry", - Protocol = LlmTckProtocolFamily.MicrosoftFoundry, + Protocol = LlmTestServerProtocolFamily.MicrosoftFoundry, DefaultEndpointPath = "/microsoft-foundry/models/chat/completions", Capabilities = [ - LlmTckProviderCapability.Chat, - LlmTckProviderCapability.StreamingChat, - LlmTckProviderCapability.Tools, - LlmTckProviderCapability.StructuredOutput, - LlmTckProviderCapability.Embeddings, - LlmTckProviderCapability.Audio, - LlmTckProviderCapability.StreamingAudio, + LlmTestServerProviderCapability.Chat, + LlmTestServerProviderCapability.StreamingChat, + LlmTestServerProviderCapability.Tools, + LlmTestServerProviderCapability.StructuredOutput, + LlmTestServerProviderCapability.Embeddings, + LlmTestServerProviderCapability.Audio, + LlmTestServerProviderCapability.StreamingAudio, ], CompatibilityTags = [ ProviderId, - LlmTckCompatibilityTags.AzureAIFoundry, - LlmTckCompatibilityTags.OpenAICompatible, + LlmTestServerCompatibilityTags.AzureAIFoundry, + LlmTestServerCompatibilityTags.OpenAICompatible, ], ApiContract = new() { @@ -39,51 +39,51 @@ public static class MicrosoftFoundryCompatibility [ new() { - Id = LlmTckProviderOperationIds.MicrosoftFoundry.V1AudioTranscriptionsCreate, + Id = LlmTestServerProviderOperationIds.MicrosoftFoundry.V1AudioTranscriptionsCreate, Method = "POST", Path = "/microsoft-foundry/openai/v1/audio/transcriptions", DocumentationUrl = "https://learn.microsoft.com/en-us/azure/foundry/openai/reference-preview-latest", ApiVersion = "v1 (default) or preview", SupportsStreaming = true, ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Audio, LlmTckProviderCapability.StreamingAudio], + Capabilities = [LlmTestServerProviderCapability.Audio, LlmTestServerProviderCapability.StreamingAudio], }, new() { - Id = LlmTckProviderOperationIds.MicrosoftFoundry.V1ChatCompletionsCreate, + Id = LlmTestServerProviderOperationIds.MicrosoftFoundry.V1ChatCompletionsCreate, Method = "POST", Path = "/microsoft-foundry/openai/v1/chat/completions", DocumentationUrl = "https://learn.microsoft.com/en-us/azure/foundry/openai/api-version-lifecycle", ApiVersion = "v1", SupportsStreaming = true, ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Chat, LlmTckProviderCapability.StreamingChat, LlmTckProviderCapability.Tools, LlmTckProviderCapability.StructuredOutput], + Capabilities = [LlmTestServerProviderCapability.Chat, LlmTestServerProviderCapability.StreamingChat, LlmTestServerProviderCapability.Tools, LlmTestServerProviderCapability.StructuredOutput], }, new() { - Id = LlmTckProviderOperationIds.MicrosoftFoundry.V1ResponsesCreate, + Id = LlmTestServerProviderOperationIds.MicrosoftFoundry.V1ResponsesCreate, Method = "POST", Path = "/microsoft-foundry/openai/v1/responses", DocumentationUrl = "https://learn.microsoft.com/en-us/azure/foundry/openai/api-version-lifecycle", ApiVersion = "v1", SupportsStreaming = true, ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Chat, LlmTckProviderCapability.StreamingChat, LlmTckProviderCapability.Tools, LlmTckProviderCapability.StructuredOutput], + Capabilities = [LlmTestServerProviderCapability.Chat, LlmTestServerProviderCapability.StreamingChat, LlmTestServerProviderCapability.Tools, LlmTestServerProviderCapability.StructuredOutput], }, new() { - Id = LlmTckProviderOperationIds.MicrosoftFoundry.V1EmbeddingsCreate, + Id = LlmTestServerProviderOperationIds.MicrosoftFoundry.V1EmbeddingsCreate, Method = "POST", Path = "/microsoft-foundry/openai/v1/embeddings", DocumentationUrl = "https://learn.microsoft.com/en-us/azure/foundry/openai/api-version-lifecycle", ApiVersion = "v1", SupportsStreaming = false, ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Embeddings], + Capabilities = [LlmTestServerProviderCapability.Embeddings], }, new() { - Id = LlmTckProviderOperationIds.MicrosoftFoundry.ChatCompletionsCreate, + Id = LlmTestServerProviderOperationIds.MicrosoftFoundry.ChatCompletionsCreate, ApiVersion = "2024-05-01-preview", Method = "POST", Path = "/microsoft-foundry/chat/completions", @@ -93,26 +93,26 @@ public static class MicrosoftFoundryCompatibility ImplementedByHosting = true, Capabilities = [ - LlmTckProviderCapability.Chat, - LlmTckProviderCapability.StreamingChat, - LlmTckProviderCapability.Tools, - LlmTckProviderCapability.StructuredOutput, + LlmTestServerProviderCapability.Chat, + LlmTestServerProviderCapability.StreamingChat, + LlmTestServerProviderCapability.Tools, + LlmTestServerProviderCapability.StructuredOutput, ], }, new() { - Id = LlmTckProviderOperationIds.MicrosoftFoundry.EmbeddingsCreate, + Id = LlmTestServerProviderOperationIds.MicrosoftFoundry.EmbeddingsCreate, ApiVersion = "2024-05-01-preview", Method = "POST", Path = "/microsoft-foundry/embeddings", DocumentationUrl = "https://learn.microsoft.com/en-us/rest/api/microsoft-foundry/modelinference/", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Embeddings], + Capabilities = [LlmTestServerProviderCapability.Embeddings], }, new() { - Id = LlmTckProviderOperationIds.MicrosoftFoundry.ModelsChatCompletionsCreate, + Id = LlmTestServerProviderOperationIds.MicrosoftFoundry.ModelsChatCompletionsCreate, ApiVersion = "2024-05-01-preview", Method = "POST", Path = "/microsoft-foundry/models/chat/completions", @@ -122,22 +122,22 @@ public static class MicrosoftFoundryCompatibility ImplementedByHosting = true, Capabilities = [ - LlmTckProviderCapability.Chat, - LlmTckProviderCapability.StreamingChat, - LlmTckProviderCapability.Tools, - LlmTckProviderCapability.StructuredOutput, + LlmTestServerProviderCapability.Chat, + LlmTestServerProviderCapability.StreamingChat, + LlmTestServerProviderCapability.Tools, + LlmTestServerProviderCapability.StructuredOutput, ], }, new() { - Id = LlmTckProviderOperationIds.MicrosoftFoundry.ModelsEmbeddingsCreate, + Id = LlmTestServerProviderOperationIds.MicrosoftFoundry.ModelsEmbeddingsCreate, ApiVersion = "2024-05-01-preview", Method = "POST", Path = "/microsoft-foundry/models/embeddings", DocumentationUrl = "https://learn.microsoft.com/en-us/rest/api/microsoft-foundry/modelinference/", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Embeddings], + Capabilities = [LlmTestServerProviderCapability.Embeddings], }, ], }, diff --git a/src/ManagedCode.LlmTestServer.Gemini/AGENTS.md b/src/ManagedCode.LlmTestServer.Gemini/AGENTS.md new file mode 100644 index 0000000..47341b6 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Gemini/AGENTS.md @@ -0,0 +1,5 @@ +# ManagedCode.LlmTestServer.Gemini + +This package owns Gemini compatibility constants, provider profiles, and Gemini wire contracts. + +Keep deterministic runtime behavior in `ManagedCode.LlmTestServer`; keep HTTP hosting in `ManagedCode.LlmTestServer.Hosting`. diff --git a/src/ManagedCode.LlmTestServer.Gemini/GeminiBatchEmbeddingModels.cs b/src/ManagedCode.LlmTestServer.Gemini/GeminiBatchEmbeddingModels.cs new file mode 100644 index 0000000..4d4e441 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Gemini/GeminiBatchEmbeddingModels.cs @@ -0,0 +1,38 @@ +using System.Text.Json.Serialization; + +namespace ManagedCode.LlmTestServer.Gemini; + +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record GeminiBatchEmbedContentsRequest +{ + [JsonPropertyName("requests")] + public List Requests { get; init; } = []; +} + +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record GeminiBatchEmbedContentRequest +{ + [JsonPropertyName("model")] + public string Model { get; init; } = string.Empty; + + [JsonPropertyName("content")] + public GeminiContent Content { get; init; } = new(); + + [JsonPropertyName("outputDimensionality")] + public int? OutputDimensionality { get; init; } +} + +public sealed record GeminiBatchEmbedContentsResponse +{ + [JsonPropertyName("embeddings")] + public List Embeddings { get; init; } = []; + + [JsonPropertyName("usageMetadata")] + public GeminiEmbeddingUsageMetadata UsageMetadata { get; init; } = new(); +} + +public sealed record GeminiEmbeddingUsageMetadata +{ + [JsonPropertyName("promptTokenCount")] + public int PromptTokenCount { get; init; } +} diff --git a/src/ManagedCode.LlmTestServer.Gemini/GeminiCompatibility.cs b/src/ManagedCode.LlmTestServer.Gemini/GeminiCompatibility.cs new file mode 100644 index 0000000..080a1d2 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Gemini/GeminiCompatibility.cs @@ -0,0 +1,114 @@ +using ManagedCode.LlmTestServer.Providers; + +namespace ManagedCode.LlmTestServer.Gemini; + +public static class GeminiCompatibility +{ + public const string ProviderId = LlmTestServerCompatibilityTags.Gemini; + + public static LlmTestServerProviderProfile Profile { get; } = + new() + { + Id = ProviderId, + DisplayName = "Gemini", + Protocol = LlmTestServerProtocolFamily.Gemini, + DefaultEndpointPath = "/gemini/v1beta/models/{model}:generateContent", + Capabilities = + [ + LlmTestServerProviderCapability.Chat, + LlmTestServerProviderCapability.StreamingChat, + LlmTestServerProviderCapability.Tools, + LlmTestServerProviderCapability.StructuredOutput, + LlmTestServerProviderCapability.Embeddings, + LlmTestServerProviderCapability.Images, + LlmTestServerProviderCapability.Video, + LlmTestServerProviderCapability.Audio, + ], + CompatibilityTags = + [ + ProviderId, + LlmTestServerCompatibilityTags.GoogleAI, + LlmTestServerCompatibilityTags.OpenAICompatible, + ], + ApiContract = new() + { + DocumentationUrl = "https://ai.google.dev/api", + DocumentationRetrievedOn = "2026-09-07", + DocumentationVersion = "v1beta Generative Language API", + Operations = + [ + new() + { + Id = LlmTestServerProviderOperationIds.Gemini.ModelsGenerateContent, + Method = "POST", + Path = "/gemini/v1beta/models/{model}:generateContent", + DocumentationUrl = "https://ai.google.dev/api/generate-content", + ImplementedByHosting = true, + Capabilities = + [ + LlmTestServerProviderCapability.Chat, + LlmTestServerProviderCapability.Tools, + LlmTestServerProviderCapability.StructuredOutput, + LlmTestServerProviderCapability.Images, + LlmTestServerProviderCapability.Audio, + ], + }, + new() + { + Id = LlmTestServerProviderOperationIds.Gemini.ModelsStreamGenerateContent, + Method = "POST", + Path = "/gemini/v1beta/models/{model}:streamGenerateContent", + DocumentationUrl = "https://ai.google.dev/api/generate-content", + SupportsStreaming = true, + ImplementedByHosting = true, + Capabilities = [LlmTestServerProviderCapability.StreamingChat, LlmTestServerProviderCapability.Tools, LlmTestServerProviderCapability.StructuredOutput], + }, + new() + { + Id = LlmTestServerProviderOperationIds.Gemini.ModelsEmbedContent, + Method = "POST", + Path = "/gemini/v1beta/models/{model}:embedContent", + DocumentationUrl = "https://ai.google.dev/api/embeddings", + ImplementedByHosting = true, + Capabilities = [LlmTestServerProviderCapability.Embeddings], + }, + new() + { + Id = LlmTestServerProviderOperationIds.Gemini.ModelsBatchEmbedContents, + Method = "POST", + Path = "/gemini/v1beta/models/{model}:batchEmbedContents", + DocumentationUrl = "https://ai.google.dev/api/embeddings#method:-models.batchembedcontents", + ImplementedByHosting = true, + Capabilities = [LlmTestServerProviderCapability.Embeddings], + }, + new() + { + Id = LlmTestServerProviderOperationIds.Gemini.ModelsPredictLongRunningVideo, + Method = "POST", + Path = "/gemini/v1beta/models/{model}:predictLongRunning", + DocumentationUrl = "https://ai.google.dev/gemini-api/docs/video", + ImplementedByHosting = true, + Capabilities = [LlmTestServerProviderCapability.Video], + }, + new() + { + Id = LlmTestServerProviderOperationIds.Gemini.ModelsOperationsGetVideo, + Method = "GET", + Path = "/gemini/v1beta/models/{model}/operations/{operationId}", + DocumentationUrl = "https://ai.google.dev/gemini-api/docs/video", + ImplementedByHosting = true, + Capabilities = [LlmTestServerProviderCapability.Video], + }, + new() + { + Id = LlmTestServerProviderOperationIds.Gemini.FilesGetGeneratedVideo, + Method = "GET", + Path = "/gemini/v1beta/files/{fileId}", + DocumentationUrl = "https://ai.google.dev/api/files", + ImplementedByHosting = true, + Capabilities = [LlmTestServerProviderCapability.Video], + }, + ], + }, + }; +} diff --git a/src/ManagedCode.LlmTck.Gemini/GeminiModels.cs b/src/ManagedCode.LlmTestServer.Gemini/GeminiModels.cs similarity index 96% rename from src/ManagedCode.LlmTck.Gemini/GeminiModels.cs rename to src/ManagedCode.LlmTestServer.Gemini/GeminiModels.cs index 1b42122..3c53d95 100644 --- a/src/ManagedCode.LlmTck.Gemini/GeminiModels.cs +++ b/src/ManagedCode.LlmTestServer.Gemini/GeminiModels.cs @@ -1,7 +1,7 @@ using System.Text.Json; using System.Text.Json.Serialization; -namespace ManagedCode.LlmTck.Gemini; +namespace ManagedCode.LlmTestServer.Gemini; public sealed record GeminiGenerateContentRequest { @@ -46,6 +46,8 @@ public sealed record GeminiContent public sealed record GeminiPart { + [JsonPropertyName("thoughtSignature")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? ThoughtSignature { get; init; } + [JsonPropertyName("functionCall")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public GeminiFunctionCall? FunctionCall { get; init; } [JsonPropertyName("functionResponse")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public GeminiFunctionResponse? FunctionResponse { get; init; } @@ -88,7 +90,8 @@ public sealed record GeminiGenerateContentResponse public sealed record GeminiCandidate { [JsonPropertyName("content")] - public GeminiContent Content { get; init; } = new(); + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public GeminiContent? Content { get; init; } = new(); [JsonPropertyName("finishReason")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] diff --git a/src/ManagedCode.LlmTck.Gemini/GeminiRequestValidation.cs b/src/ManagedCode.LlmTestServer.Gemini/GeminiRequestValidation.cs similarity index 89% rename from src/ManagedCode.LlmTck.Gemini/GeminiRequestValidation.cs rename to src/ManagedCode.LlmTestServer.Gemini/GeminiRequestValidation.cs index 4916f1c..86ed135 100644 --- a/src/ManagedCode.LlmTck.Gemini/GeminiRequestValidation.cs +++ b/src/ManagedCode.LlmTestServer.Gemini/GeminiRequestValidation.cs @@ -1,12 +1,12 @@ using System.Text.Json; -using ManagedCode.LlmTck.Providers; -using static ManagedCode.LlmTck.ProviderValidation.LlmTckJsonValidation; +using ManagedCode.LlmTestServer.Providers; +using static ManagedCode.LlmTestServer.ProviderValidation.LlmTestServerJsonValidation; -namespace ManagedCode.LlmTck.Gemini; +namespace ManagedCode.LlmTestServer.Gemini; public static class GeminiRequestValidation { - public static LlmTckRequestValidationResult Validate(JsonElement body) + public static LlmTestServerRequestValidationResult Validate(JsonElement body) { return Check(body, ValidateBody); } diff --git a/src/ManagedCode.LlmTck.Gemini/GeminiSchemaMapper.cs b/src/ManagedCode.LlmTestServer.Gemini/GeminiSchemaMapper.cs similarity index 98% rename from src/ManagedCode.LlmTck.Gemini/GeminiSchemaMapper.cs rename to src/ManagedCode.LlmTestServer.Gemini/GeminiSchemaMapper.cs index eb10f11..3c89377 100644 --- a/src/ManagedCode.LlmTck.Gemini/GeminiSchemaMapper.cs +++ b/src/ManagedCode.LlmTestServer.Gemini/GeminiSchemaMapper.cs @@ -2,7 +2,7 @@ using System.Text.Json; using System.Text.Json.Nodes; -namespace ManagedCode.LlmTck.Gemini; +namespace ManagedCode.LlmTestServer.Gemini; /// Adapts Gemini's OpenAPI Schema fields to the runtime JSON Schema evaluator. public static class GeminiSchemaMapper diff --git a/src/ManagedCode.LlmTck.Gemini/GeminiWireMapper.cs b/src/ManagedCode.LlmTestServer.Gemini/GeminiWireMapper.cs similarity index 76% rename from src/ManagedCode.LlmTck.Gemini/GeminiWireMapper.cs rename to src/ManagedCode.LlmTestServer.Gemini/GeminiWireMapper.cs index b74b7ce..98bdc4f 100644 --- a/src/ManagedCode.LlmTck.Gemini/GeminiWireMapper.cs +++ b/src/ManagedCode.LlmTestServer.Gemini/GeminiWireMapper.cs @@ -1,12 +1,13 @@ using System.Text.Json; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Scenarios; +using ManagedCode.LlmTestServer.Determinism; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Scenarios; -namespace ManagedCode.LlmTck.Gemini; +namespace ManagedCode.LlmTestServer.Gemini; public static class GeminiWireMapper { - public static LlmTckChatRequest ToRuntimeRequest( + public static LlmTestServerChatRequest ToRuntimeRequest( string model, GeminiGenerateContentRequest request, bool stream @@ -21,7 +22,7 @@ bool stream { messages.Insert( 0, - new LlmTckMessage + new LlmTestServerMessage { Role = "system", Content = ReadText(request.SystemInstruction), @@ -32,19 +33,20 @@ bool stream return new() { Tools = request.Tools.SelectMany(tool => tool.FunctionDeclarations).Where(tool => request.ToolConfig?.FunctionCallingConfig?.AllowedFunctionNames is not { Count: > 0 } allowed || allowed.Contains(tool.Name)) - .Select(tool => new LlmTckToolDefinition { Name = tool.Name, Description = tool.Description, ParametersJson = tool.ParametersJsonSchema?.GetRawText() ?? GeminiSchemaMapper.ToJsonSchema(tool.Parameters) ?? "{\"type\":\"object\"}" }).ToList(), - ToolChoice = request.ToolConfig?.FunctionCallingConfig?.Mode switch { "ANY" => LlmTckToolChoice.Required, "NONE" => LlmTckToolChoice.None, _ => LlmTckToolChoice.Auto }, + .Select(tool => new LlmTestServerToolDefinition { Name = tool.Name, Description = tool.Description, ParametersJson = tool.ParametersJsonSchema?.GetRawText() ?? GeminiSchemaMapper.ToJsonSchema(tool.Parameters) ?? "{\"type\":\"object\"}" }).ToList(), + ToolChoice = request.ToolConfig?.FunctionCallingConfig?.Mode switch { "ANY" => LlmTestServerToolChoice.Required, "NONE" => LlmTestServerToolChoice.None, _ => LlmTestServerToolChoice.Auto }, RequireJson = request.GenerationConfig?.ResponseMimeType == "application/json", ResponseSchemaJson = request.GenerationConfig?.ResponseJsonSchema?.GetRawText() ?? GeminiSchemaMapper.ToJsonSchema(request.GenerationConfig?.ResponseSchema), + RequireToolContinuationTokens = RequiresToolContinuationTokens(model), ModelId = model, Stream = stream, - PromptCachePolicy = LlmTckPromptCachePolicy.Gemini, + PromptCachePolicy = LlmTestServerPromptCachePolicy.Gemini, Messages = messages, }; } public static GeminiGenerateContentResponse ToGenerateContentResponse( - LlmTckChatResult result, + LlmTestServerChatResult result, string content, string? finishReason = "STOP", string? responseId = null @@ -52,19 +54,19 @@ public static GeminiGenerateContentResponse ToGenerateContentResponse( { var outputTokens = finishReason is not null ? result.Usage.OutputTokens - result.Usage.ReasoningTokens - : LlmTckTokenCounter.CountTextTokens(content); + : LlmTestServerTokenCounter.CountTextTokens(content); return new() { Candidates = [ new GeminiCandidate { - Content = new GeminiContent + Content = finishReason is not null && result.OutputDisposition?.Kind == LlmTestServerOutputDispositionKind.Filtered ? null : new GeminiContent { Role = "model", - Parts = result.ToolCalls.Count > 0 && finishReason is not null ? (content.Length > 0 ? new[] { new GeminiPart { Text = content } } : []).Concat(result.ToolCalls.Select(call => new GeminiPart { FunctionCall = new() { Id = call.Id, Name = call.Name, Args = JsonSerializer.Deserialize(call.ArgumentsJson) } })).ToList() : [new GeminiPart { Text = content }], + Parts = result.ToolCalls.Count > 0 && finishReason is not null ? (content.Length > 0 ? new[] { new GeminiPart { Text = content } } : []).Concat(result.ToolCalls.Select(call => new GeminiPart { ThoughtSignature = call.ContinuationToken, FunctionCall = new() { Id = call.Id, Name = call.Name, Args = JsonSerializer.Deserialize(call.ArgumentsJson) } })).ToList() : [new GeminiPart { Text = content }], }, - FinishReason = finishReason, + FinishReason = finishReason is not null && result.OutputDisposition?.Kind == LlmTestServerOutputDispositionKind.Filtered ? "SAFETY" : finishReason, Index = 0, }, ], @@ -77,13 +79,18 @@ public static GeminiGenerateContentResponse ToGenerateContentResponse( TotalTokenCount = result.Usage.InputTokens + outputTokens + (finishReason is not null ? result.Usage.ReasoningTokens : 0), }, ModelVersion = result.ModelId, - ResponseId = responseId ?? CreateResponseId(), + ResponseId = responseId ?? CreateResponseId(result.WireMetadata), }; } - private static IEnumerable ReadMessages(GeminiContent content) + public static bool RequiresToolContinuationTokens(string model) { - var calls = content.Parts.Where(part => part.FunctionCall is not null).Select(part => new LlmTckToolCall { Id = part.FunctionCall!.Id ?? part.FunctionCall.Name, Name = part.FunctionCall.Name, ArgumentsJson = part.FunctionCall.Args.GetRawText() }).ToList(); + return model == "gemini-3.8-flash"; + } + + private static IEnumerable ReadMessages(GeminiContent content) + { + var calls = content.Parts.Where(part => part.FunctionCall is not null).Select(part => new LlmTestServerToolCall { Id = part.FunctionCall!.Id ?? part.FunctionCall.Name, Name = part.FunctionCall.Name, ArgumentsJson = part.FunctionCall.Args.GetRawText(), ContinuationToken = part.ThoughtSignature }).ToList(); if (calls.Count > 0 || content.Parts.Any(part => part.Text is not null || part.InlineData is not null || part.FileData is not null)) { yield return new() { Role = NormalizeRole(content.Role), Content = ReadText(content), ToolCalls = calls }; @@ -110,7 +117,7 @@ public static GeminiEmbedContentResponse ToEmbedContentResponse(IReadOnlyList + + ManagedCode.LlmTestServer.Gemini + Gemini compatibility profile and adapters for ManagedCode.LlmTestServer. + + + + + + + + + diff --git a/src/ManagedCode.LlmTestServer.Groq/AGENTS.md b/src/ManagedCode.LlmTestServer.Groq/AGENTS.md new file mode 100644 index 0000000..5b2e83f --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Groq/AGENTS.md @@ -0,0 +1,5 @@ +# ManagedCode.LlmTestServer.Groq + +This package owns Groq compatibility constants, provider profiles, and Groq wire contracts. + +Keep deterministic runtime behavior in `ManagedCode.LlmTestServer`; keep HTTP hosting in `ManagedCode.LlmTestServer.Hosting`. diff --git a/src/ManagedCode.LlmTck.Groq/GroqCompatibility.cs b/src/ManagedCode.LlmTestServer.Groq/GroqCompatibility.cs similarity index 57% rename from src/ManagedCode.LlmTck.Groq/GroqCompatibility.cs rename to src/ManagedCode.LlmTestServer.Groq/GroqCompatibility.cs index 7a47a6a..6c3a849 100644 --- a/src/ManagedCode.LlmTck.Groq/GroqCompatibility.cs +++ b/src/ManagedCode.LlmTestServer.Groq/GroqCompatibility.cs @@ -1,28 +1,28 @@ -using ManagedCode.LlmTck.Providers; +using ManagedCode.LlmTestServer.Providers; -namespace ManagedCode.LlmTck.Groq; +namespace ManagedCode.LlmTestServer.Groq; public static class GroqCompatibility { - public const string ProviderId = LlmTckCompatibilityTags.Groq; + public const string ProviderId = LlmTestServerCompatibilityTags.Groq; - public static LlmTckProviderProfile Profile { get; } = + public static LlmTestServerProviderProfile Profile { get; } = new() { Id = ProviderId, DisplayName = "Groq", - Protocol = LlmTckProtocolFamily.Groq, + Protocol = LlmTestServerProtocolFamily.Groq, DefaultEndpointPath = "/groq/openai/v1/chat/completions", Capabilities = [ - LlmTckProviderCapability.Models, - LlmTckProviderCapability.Chat, - LlmTckProviderCapability.StreamingChat, - LlmTckProviderCapability.Tools, - LlmTckProviderCapability.StructuredOutput, - LlmTckProviderCapability.Audio, + LlmTestServerProviderCapability.Models, + LlmTestServerProviderCapability.Chat, + LlmTestServerProviderCapability.StreamingChat, + LlmTestServerProviderCapability.Tools, + LlmTestServerProviderCapability.StructuredOutput, + LlmTestServerProviderCapability.Audio, ], - CompatibilityTags = [ProviderId, LlmTckCompatibilityTags.OpenAICompatible], + CompatibilityTags = [ProviderId, LlmTestServerCompatibilityTags.OpenAICompatible], ApiContract = new() { DocumentationUrl = "https://console.groq.com/docs/api-reference", @@ -32,7 +32,7 @@ public static class GroqCompatibility [ new() { - Id = LlmTckProviderOperationIds.Groq.ChatCompletionsCreate, + Id = LlmTestServerProviderOperationIds.Groq.ChatCompletionsCreate, Method = "POST", Path = "/groq/openai/v1/chat/completions", DocumentationUrl = "https://console.groq.com/docs/api-reference", @@ -40,15 +40,15 @@ public static class GroqCompatibility ImplementedByHosting = true, Capabilities = [ - LlmTckProviderCapability.Chat, - LlmTckProviderCapability.StreamingChat, - LlmTckProviderCapability.Tools, - LlmTckProviderCapability.StructuredOutput, + LlmTestServerProviderCapability.Chat, + LlmTestServerProviderCapability.StreamingChat, + LlmTestServerProviderCapability.Tools, + LlmTestServerProviderCapability.StructuredOutput, ], }, new() { - Id = LlmTckProviderOperationIds.Groq.ResponsesCreate, + Id = LlmTestServerProviderOperationIds.Groq.ResponsesCreate, Method = "POST", Path = "/groq/openai/v1/responses", DocumentationUrl = "https://console.groq.com/docs/api-reference", @@ -56,47 +56,47 @@ public static class GroqCompatibility ImplementedByHosting = true, Capabilities = [ - LlmTckProviderCapability.Chat, - LlmTckProviderCapability.StreamingChat, - LlmTckProviderCapability.Tools, - LlmTckProviderCapability.StructuredOutput, + LlmTestServerProviderCapability.Chat, + LlmTestServerProviderCapability.StreamingChat, + LlmTestServerProviderCapability.Tools, + LlmTestServerProviderCapability.StructuredOutput, ], }, new() { - Id = LlmTckProviderOperationIds.Groq.AudioSpeechCreate, + Id = LlmTestServerProviderOperationIds.Groq.AudioSpeechCreate, Method = "POST", Path = "/groq/openai/v1/audio/speech", DocumentationUrl = "https://console.groq.com/docs/api-reference", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Audio], + Capabilities = [LlmTestServerProviderCapability.Audio], }, new() { - Id = LlmTckProviderOperationIds.Groq.AudioTranscriptionsCreate, + Id = LlmTestServerProviderOperationIds.Groq.AudioTranscriptionsCreate, Method = "POST", Path = "/groq/openai/v1/audio/transcriptions", DocumentationUrl = "https://console.groq.com/docs/api-reference", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Audio], + Capabilities = [LlmTestServerProviderCapability.Audio], }, new() { - Id = LlmTckProviderOperationIds.Groq.AudioTranslationsCreate, + Id = LlmTestServerProviderOperationIds.Groq.AudioTranslationsCreate, Method = "POST", Path = "/groq/openai/v1/audio/translations", DocumentationUrl = "https://console.groq.com/docs/api-reference", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Audio], + Capabilities = [LlmTestServerProviderCapability.Audio], }, new() { - Id = LlmTckProviderOperationIds.Groq.ModelsList, + Id = LlmTestServerProviderOperationIds.Groq.ModelsList, Method = "GET", Path = "/groq/openai/v1/models", DocumentationUrl = "https://console.groq.com/docs/models", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Models], + Capabilities = [LlmTestServerProviderCapability.Models], }, ], }, diff --git a/src/ManagedCode.LlmTestServer.Groq/ManagedCode.LlmTestServer.Groq.csproj b/src/ManagedCode.LlmTestServer.Groq/ManagedCode.LlmTestServer.Groq.csproj new file mode 100644 index 0000000..e6a272b --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Groq/ManagedCode.LlmTestServer.Groq.csproj @@ -0,0 +1,10 @@ + + + ManagedCode.LlmTestServer.Groq + Groq compatibility profile and adapters for ManagedCode.LlmTestServer. + + + + + + diff --git a/src/ManagedCode.LlmTck.Hosting/AGENTS.md b/src/ManagedCode.LlmTestServer.Hosting/AGENTS.md similarity index 64% rename from src/ManagedCode.LlmTck.Hosting/AGENTS.md rename to src/ManagedCode.LlmTestServer.Hosting/AGENTS.md index a5ec95c..ee89474 100644 --- a/src/ManagedCode.LlmTck.Hosting/AGENTS.md +++ b/src/ManagedCode.LlmTestServer.Hosting/AGENTS.md @@ -1,5 +1,5 @@ -# ManagedCode.LlmTck.Hosting +# ManagedCode.LlmTestServer.Hosting -This package maps `ILlmTckRuntime` to HTTP endpoints. +This package maps `ILlmTestServerRuntime` to HTTP endpoints. Keep endpoint behavior thin: parse request, read bearer token, call the runtime, map to provider response shape. Add endpoint tests for every new route or status-code behavior. diff --git a/src/ManagedCode.LlmTck.Hosting/CloudflareDecisionRequestBody.cs b/src/ManagedCode.LlmTestServer.Hosting/CloudflareDecisionRequestBody.cs similarity index 92% rename from src/ManagedCode.LlmTck.Hosting/CloudflareDecisionRequestBody.cs rename to src/ManagedCode.LlmTestServer.Hosting/CloudflareDecisionRequestBody.cs index 0b9547a..bc49f6d 100644 --- a/src/ManagedCode.LlmTck.Hosting/CloudflareDecisionRequestBody.cs +++ b/src/ManagedCode.LlmTestServer.Hosting/CloudflareDecisionRequestBody.cs @@ -1,8 +1,8 @@ using System.Buffers; -using ManagedCode.LlmTck.Cloudflare; +using ManagedCode.LlmTestServer.Cloudflare; using Microsoft.AspNetCore.Http; -namespace ManagedCode.LlmTck.Hosting; +namespace ManagedCode.LlmTestServer.Hosting; internal static class CloudflareDecisionRequestBody { diff --git a/src/ManagedCode.LlmTck.Hosting/LlmTckAdminPage.razor b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerAdminPage.razor similarity index 61% rename from src/ManagedCode.LlmTck.Hosting/LlmTckAdminPage.razor rename to src/ManagedCode.LlmTestServer.Hosting/LlmTestServerAdminPage.razor index db0a665..017068d 100644 --- a/src/ManagedCode.LlmTck.Hosting/LlmTckAdminPage.razor +++ b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerAdminPage.razor @@ -1,4 +1,4 @@ -@namespace ManagedCode.LlmTck.Hosting +@namespace ManagedCode.LlmTestServer.Hosting @page "/" @using Microsoft.AspNetCore.Components.Web @@ -9,8 +9,8 @@ - -LLM TCK · Request dashboard + +LLM Test Server · Request dashboard - + + + diff --git a/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerAdminPanel.razor b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerAdminPanel.razor new file mode 100644 index 0000000..b7d546d --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerAdminPanel.razor @@ -0,0 +1,2094 @@ +@using ManagedCode.LlmTestServer.Diagnostics +@namespace ManagedCode.LlmTestServer.Hosting +@using System.Globalization +@using System.Runtime.CompilerServices +@using System.Text +@using System.Text.Encodings.Web +@using System.Text.Json +@using Microsoft.AspNetCore.Components.Web +@using ManagedCode.LlmTestServer.Control +@using ManagedCode.LlmTestServer.Models +@using ManagedCode.LlmTestServer.Providers +@using ManagedCode.LlmTestServer.Runtime +@using ManagedCode.LlmTestServer.Scenarios +@using Microsoft.Extensions.Options +@using Microsoft.FluentUI.AspNetCore.Components +@implements IAsyncDisposable +@inject ILlmTestServerRuntime Runtime +@inject ILlmTestServerProviderHttpTraceStore TraceStore +@inject IOptions TraceOptions +@inject NavigationManager Navigation + +@{ + var visibleTraces = FilterTraces(); + var providerOptions = GetProviderOptions(); + var modelOptions = GetModelOptions(); +} + +
+
+
+ +
+
Deterministic provider lab
+

LLM Test Server request dashboardLLM Test Server

+
+
+ +
+
+ + @_statusText +
+ + + + +
+
+ + +
+
+
+
Runtime setup & diagnostics
+

Runtime state

+
+
+ @FormatBytes(TraceStore.RetainedByteCount) retained + +
+
+
+ + +
+
+ @InsightCard(AssertionChart()) + @InsightCard(FixtureChart()) +
+

Event timeline

Oldest to newest, colored by result

+
+ @if (_events.Count == 0) + { +

No runtime events yet.

+ } + else + { +
+ @foreach (var runtimeEvent in _events.TakeLast(120)) + { + + } +
+
    +
  • Matched
    • +
  • Unmatched or exhausted
    • +
  • Errors
    • +
+ } +
+
+
+
+
+

Fixtures / models@_models.Count

+
+ @if (_models.Count == 0) + { +
No models loaded.
+ } + else + { +
    + @foreach (var model in _models) + { +
  • @model.Id@model.Kind · @model.OwnedBy
    • + } +
+ } +
+
+ +
+

Provider APIs@_providerRoutes.Count

+
+
    + @foreach (var route in _providerRoutes) + { +
  • @route.Name@route.Route
    • + } +
+
+
+ +
+

Runtime actions4

+
+
    +
  • Refresh stateGET @LlmTestServerControlRoutes.Assertions
    • +
  • List fixturesGET @LlmTestServerControlRoutes.Models
    • +
  • Configure runtimePOST @LlmTestServerControlRoutes.Configure
    • +
  • Reset runtimePOST @LlmTestServerControlRoutes.Reset
    • +
+
+
+ +
+

Assertions@_summary.TotalEvents

+
+
    +
  • Matched@_summary.Matched
    • +
  • Unmatched@_summary.Unmatched
    • +
  • Model not found@_summary.ModelNotFound
    • +
  • Auth failed@_summary.AuthFailed
    • +
  • Scenario exhausted@_summary.ScenarioExhausted
    • +
  • Errors returned@_summary.ErrorsReturned
    • +
+
+
+ +
+

Runtime events@_events.Count

+
+ @if (_events.Count == 0) + { +
No runtime events yet.
+ } + else + { +
    + @foreach (var runtimeEvent in _events.AsEnumerable().Reverse()) + { +
  • + @runtimeEvent.Kind@runtimeEvent.Message + @runtimeEvent.ModelId · @runtimeEvent.Timestamp.ToLocalTime().ToString("HH:mm:ss", CultureInfo.InvariantCulture) +
    • + } +
+ } +
+
+
+
+ + +
@_bannerText
+ +
+ + +
+
+
+

Provider requests

+

@visibleTraces.Count of @_traces.Count · newest first

+
+ +
+ +
+ +
+ + + + +
+
+ + + + @if (visibleTraces.Count == 0) + { +
+
+ @(_traces.Count == 0 ? "No provider requests yet" : "No requests match these filters") + @(_traces.Count == 0 ? "Call a namespaced provider endpoint to start the inbox." : "Clear or change filters to see more calls.") +
+
+ } + else + { +
+ @foreach (var trace in visibleTraces) + { + var usage = UsageFor(trace); + var messages = MessagesFor(trace); + var messageCount = messages.Count; + + } +
+ } +
+ + +
+ +
Served by MapLlmTestServer(); the request journal is bounded and never records authorization headers.
+
+@code { + private const string _overviewTab = "overview"; + private const string _conversationTab = "conversation"; + private const string _wireTab = "wire"; + private const string _streamTab = "stream"; + private const string _runtimeTab = "runtime"; + private const string _analyticsTab = "analytics"; + private const int _collapsedTurnThreshold = 420; + private const int _turnPreviewLimit = 132; + private static readonly TimeSpan _autoRefreshInterval = TimeSpan.FromSeconds(15); + private static readonly IReadOnlyList _providerRoutes = + [ + new("OpenAI", LlmTestServerProviderRouteNamespaces.OpenAI), + new("Azure OpenAI", LlmTestServerProviderRouteNamespaces.AzureOpenAI), + new("Microsoft Foundry", LlmTestServerProviderRouteNamespaces.MicrosoftFoundry), + new("Anthropic", LlmTestServerProviderRouteNamespaces.Anthropic), + new("Gemini", LlmTestServerProviderRouteNamespaces.Gemini), + new("Groq", LlmTestServerProviderRouteNamespaces.Groq), + new("Mistral", LlmTestServerProviderRouteNamespaces.Mistral), + new("Ollama", LlmTestServerProviderRouteNamespaces.Ollama), + new("Cohere", LlmTestServerProviderRouteNamespaces.Cohere), + new("Amazon Bedrock", LlmTestServerProviderRouteNamespaces.Bedrock), + new("OpenRouter", LlmTestServerProviderRouteNamespaces.OpenRouter), + new("DeepSeek", LlmTestServerProviderRouteNamespaces.DeepSeek), + new("Perplexity", LlmTestServerProviderRouteNamespaces.Perplexity), + ]; + + private IReadOnlyList _models = []; + private IReadOnlyList _traces = []; + private IReadOnlyList _events = []; + private IReadOnlyDictionary _eventsByRequestId = + new Dictionary(StringComparer.Ordinal); + private LlmTestServerAssertionSummary _summary = new(); + private LlmTestServerRequestSnapshot _evidenceSnapshot = new(); + private bool _analyticsConversation; + private bool _insightsOpen; + private IReadOnlyDictionary _providerColors = new Dictionary(StringComparer.Ordinal); + private string _bearerToken = string.Empty; + private string _search = string.Empty; + private string _providerFilter = string.Empty; + private string _modelFilter = string.Empty; + private string _outcomeFilter = string.Empty; + private string _activeDetailTab = _overviewTab; + private string? _selectedRequestId; + private bool _mobileDetailsOpen; + private bool _autoRefresh = true; + private bool _resetting; + private bool _runtimeDialogHidden = true; + private bool _interactive; + private bool _restoreRuntimeTriggerFocus; + private ElementReference _runtimeDialogTrigger; + private FluentDialog _runtimeDialog = default!; + private const string ClosedDialogState = "closed"; + private string _statusText = "Connecting…"; + private string _statusClass = string.Empty; + private string? _bannerKind; + private string? _bannerText; + private CancellationTokenSource? _autoRefreshCancellation; + + private string WorkspaceCss => _mobileDetailsOpen ? "shell mobile-reading" : "shell mobile-list"; + private string StatusCss => string.IsNullOrWhiteSpace(_statusClass) + ? "connection" + : $"connection {_statusClass}"; + private string BannerCss => string.IsNullOrWhiteSpace(_bannerText) + ? "notice" + : $"notice show {_bannerKind}"; + private string BannerRole => string.Equals(_bannerKind, "err", StringComparison.Ordinal) + ? "alert" + : "status"; + private string? AccessToken => string.IsNullOrWhiteSpace(_bearerToken) + ? null + : _bearerToken.Trim(); + private string PreviewLimitLabel => $"{FormatBytes(TraceOptions.Value.MaxPreviewBytes)} preview"; + private int SuccessfulTraceCount => _traces.Count(trace => trace.IsSuccess is true); + private int FailedTraceCount => _traces.Count(trace => trace.IsSuccess is false); + private int StreamingTraceCount => _traces.Count(trace => trace.IsStreaming); + private int CacheTraceCount => _traces.Count(HasCacheActivity); + private bool HasActiveFilters => !string.IsNullOrWhiteSpace(_search) + || !string.IsNullOrWhiteSpace(_providerFilter) + || !string.IsNullOrWhiteSpace(_modelFilter) + || !string.IsNullOrWhiteSpace(_outcomeFilter); + private LlmTestServerProviderHttpTrace? SelectedTrace => _selectedRequestId is null + ? null + : _traces.FirstOrDefault(trace => + string.Equals(trace.RequestId, _selectedRequestId, StringComparison.Ordinal) + ); + private IReadOnlyList GetProviderOptions() => _traces + .Select(trace => trace.ProviderId) + .Where(provider => !string.IsNullOrWhiteSpace(provider)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .Order(StringComparer.OrdinalIgnoreCase) + .ToList(); + private IReadOnlyList GetModelOptions() => _traces + .Select(ModelName) + .Where(model => model != "—") + .Distinct(StringComparer.OrdinalIgnoreCase) + .Order(StringComparer.OrdinalIgnoreCase) + .ToList(); + + protected override void OnInitialized() + { + // `/?view=insights` deep-links straight to the traffic charts. + var query = Microsoft.AspNetCore.WebUtilities.QueryHelpers.ParseQuery(new Uri(Navigation.Uri).Query); + _insightsOpen = query.TryGetValue("view", out var view) + && string.Equals(view.ToString(), "insights", StringComparison.OrdinalIgnoreCase); + _mobileDetailsOpen = _insightsOpen; + RefreshState(); + } + + protected override async Task OnAfterRenderAsync(bool firstRender) + { + if (firstRender) + { + _interactive = true; + StateHasChanged(); + StartAutoRefresh(); + } + + if (_restoreRuntimeTriggerFocus) + { + _restoreRuntimeTriggerFocus = false; + await _runtimeDialogTrigger.FocusAsync(); + } + } + + private Task RefreshAsync() + { + RefreshState(); + return Task.CompletedTask; + } + + private async Task ResetAsync() + { + if (!Runtime.IsBearerTokenAccepted(AccessToken)) + { + SetUnauthorizedState(); + return; + } + + _resetting = true; + try + { + TraceStore.Reset(); + await Runtime.ResetAsync(); + RefreshState(); + await CloseRuntimeDialogAsync(); + SetBanner("warn", "Runtime reset · scenarios, events and provider request traces were cleared."); + } + catch (Exception exception) + { + _statusClass = "err"; + _statusText = "Server error"; + SetBanner("err", $"Reset failed: {exception.Message}"); + } + finally + { + _resetting = false; + } + } + + private Task SetAutoRefreshAsync(ChangeEventArgs args) + { + _autoRefresh = args.Value is true + || string.Equals(args.Value?.ToString(), "true", StringComparison.OrdinalIgnoreCase); + if (_autoRefresh) + { + StartAutoRefresh(); + } + else + { + StopAutoRefresh(); + } + + return Task.CompletedTask; + } + + private async Task OpenRuntimeDialogAsync() + { + await _runtimeDialog.ShowAsync(); + _runtimeDialogHidden = false; + } + + private async Task CloseRuntimeDialogAsync() + { + await _runtimeDialog.HideAsync(); + _runtimeDialogHidden = true; + _restoreRuntimeTriggerFocus = true; + } + + private void RuntimeDialogToggled(DialogToggleEventArgs args) + { + if (args.Id == _runtimeDialog.Id && string.Equals(args.NewState, ClosedDialogState, StringComparison.Ordinal)) + { + _runtimeDialogHidden = true; + _restoreRuntimeTriggerFocus = true; + } + } + + private void RefreshState() + { + _statusText = "Loading…"; + if (!Runtime.IsBearerTokenAccepted(AccessToken)) + { + SetUnauthorizedState(); + return; + } + + _models = Runtime.GetModels(); + _summary = Runtime.GetAssertionSummary(); + _events = _summary.Events; + var eventsByRequestId = new Dictionary(StringComparer.Ordinal); + foreach (var runtimeEvent in _events) + { + if (!string.IsNullOrWhiteSpace(runtimeEvent.RequestId)) + { + eventsByRequestId[runtimeEvent.RequestId] = runtimeEvent; + } + } + + _eventsByRequestId = eventsByRequestId; + _evidenceSnapshot = TraceStore.GetEvidenceSnapshot(); + _traces = _evidenceSnapshot.Requests; + _providerColors = LlmTestServerDashboardInsights.CategoricalColors(_traces, trace => trace.ProviderId); + if (_selectedRequestId is null + || !_traces.Any(trace => + string.Equals(trace.RequestId, _selectedRequestId, StringComparison.Ordinal) + )) + { + _selectedRequestId = _traces.FirstOrDefault()?.RequestId; + _activeDetailTab = _overviewTab; + if (_selectedRequestId is null) + { + _mobileDetailsOpen = false; + } + } + + _statusClass = "ok"; + _statusText = _traces.Any(trace => !trace.IsCompleted) ? "Requests in flight" : "Ready"; + ClearBanner(); + } + + private void SetUnauthorizedState() + { + _models = []; + _traces = []; + _evidenceSnapshot = new(); + _analyticsConversation = false; + _events = []; + _eventsByRequestId = new Dictionary(StringComparer.Ordinal); + _summary = new LlmTestServerAssertionSummary(); + _selectedRequestId = null; + _mobileDetailsOpen = false; + _statusClass = "err"; + _statusText = "Unauthorized"; + SetBanner("err", "Enter the configured bearer token to load runtime state and request traces."); + } + + private void StartAutoRefresh() + { + StopAutoRefresh(); + if (!_autoRefresh) + { + return; + } + + var cancellation = new CancellationTokenSource(); + _autoRefreshCancellation = cancellation; + _ = RunAutoRefreshAsync(cancellation.Token); + } + + private void StopAutoRefresh() + { + var cancellation = _autoRefreshCancellation; + _autoRefreshCancellation = null; + if (cancellation is null) + { + return; + } + + cancellation.Cancel(); + cancellation.Dispose(); + } + + private async Task RunAutoRefreshAsync(CancellationToken cancellationToken) + { + try + { + using var timer = new PeriodicTimer(_autoRefreshInterval); + while (await timer.WaitForNextTickAsync(cancellationToken).ConfigureAwait(false)) + { + await InvokeAsync(() => + { + RefreshState(); + StateHasChanged(); + }); + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + } + catch (ObjectDisposedException) when (cancellationToken.IsCancellationRequested) + { + } + } + + private IReadOnlyList FilterTraces() + { + IEnumerable query = _traces; + if (!string.IsNullOrWhiteSpace(_search)) + { + var search = _search.Trim(); + query = query.Where(trace => TraceContains(trace, search)); + } + + if (!string.IsNullOrWhiteSpace(_providerFilter)) + { + query = query.Where(trace => + string.Equals(trace.ProviderId, _providerFilter, StringComparison.OrdinalIgnoreCase) + ); + } + + if (!string.IsNullOrWhiteSpace(_modelFilter)) + { + query = query.Where(trace => + string.Equals(ModelName(trace), _modelFilter, StringComparison.OrdinalIgnoreCase) + ); + } + + query = _outcomeFilter switch + { + "success" => query.Where(trace => trace.IsSuccess is true), + "error" => query.Where(trace => trace.IsSuccess is false), + "stream" => query.Where(trace => trace.IsStreaming), + "cache" => query.Where(HasCacheActivity), + _ => query, + }; + return query.ToList(); + } + + private static bool TraceContains(LlmTestServerProviderHttpTrace trace, string search) + { + return Contains(trace.ProviderId, search) + || Contains(trace.Path, search) + || Contains(trace.OperationId, search) + || Contains(trace.ModelId, search) + || Contains(trace.RequestPreview, search) + || Contains(trace.ResponsePreview, search) + || Contains(trace.RuntimeResponse, search) + || Contains(trace.RuntimeMessage, search); + } + + private static bool Contains(string? value, string search) + { + return value?.Contains(search, StringComparison.OrdinalIgnoreCase) == true; + } + + private void ClearFilters() + { + _search = string.Empty; + _providerFilter = string.Empty; + _modelFilter = string.Empty; + _outcomeFilter = string.Empty; + } + + private void SetOutcomeFilter(string filter) + { + _outcomeFilter = filter; + } + + private bool IsOutcomeFilter(string filter) + { + return string.Equals(_outcomeFilter, filter, StringComparison.Ordinal); + } + + private string RailFilterCss(string filter) + { + return IsOutcomeFilter(filter) ? "rail-item selected" : "rail-item"; + } + + private void SetProviderFilter(string provider) + { + _providerFilter = provider; + } + + private bool IsProviderFilter(string provider) + { + return string.Equals(_providerFilter, provider, StringComparison.OrdinalIgnoreCase); + } + + private string ProviderFilterCss(string provider) + { + return IsProviderFilter(provider) ? "rail-item provider selected" : "rail-item provider"; + } + + private int ProviderTraceCount(string provider) + { + return _traces.Count(trace => + string.Equals(trace.ProviderId, provider, StringComparison.OrdinalIgnoreCase) + ); + } + + private void SelectTrace(string requestId) + { + _insightsOpen = false; + _selectedRequestId = requestId; + _activeDetailTab = _overviewTab; + _mobileDetailsOpen = true; + } + + private void BackToRequests() + { + _mobileDetailsOpen = false; + } + + private bool IsSelected(LlmTestServerProviderHttpTrace trace) + { + return string.Equals(trace.RequestId, _selectedRequestId, StringComparison.Ordinal); + } + + private string InboxItemCss(LlmTestServerProviderHttpTrace trace) + { + return IsSelected(trace) ? "inbox-item selected" : "inbox-item"; + } + + private static bool HasCacheActivity(LlmTestServerProviderHttpTrace trace) + { + return trace.Usage is { } usage + && (usage.CachedInputTokens > 0 || usage.CacheCreationInputTokens > 0); + } + + private bool IsDetailTab(string tab) + { + return string.Equals(_activeDetailTab, tab, StringComparison.Ordinal); + } + + private string DetailTabCss(string tab) + { + return IsDetailTab(tab) ? "detail-tab selected" : "detail-tab"; + } + + private string DetailPanelCss(string tab) + { + return IsDetailTab(tab) ? "detail-panel active" : "detail-panel"; + } + + private void SetDetailTab(string tab) + { + _activeDetailTab = tab; + } + + private static string AnalyticsValue(long value, LlmTestServerDashboardAnalytics analytics) + => analytics.UsageRequestCount > 0 ? value.ToString(CultureInfo.InvariantCulture) : string.Empty; + + private static string AnalyticsNumber(long value, LlmTestServerDashboardAnalytics analytics) + => analytics.UsageRequestCount > 0 ? value.ToString("N0", CultureInfo.InvariantCulture) : "—"; + + private static string AnalyticsRatioValue(decimal? value) => value?.ToString(CultureInfo.InvariantCulture) ?? string.Empty; + + private static string AnalyticsPercent(decimal? value) => value?.ToString("0.0%", CultureInfo.InvariantCulture) ?? "N/A"; + + private static string AnalyticsShare(long value, long total) + => total > 0 ? LlmTestServerDashboardCharts.Percent((double)value / total) : string.Empty; + + private static string TimelineHeight(long tokens, long max) + => $"{LlmTestServerDashboardCharts.Number(Math.Max(3, tokens * 100d / max))}%"; + + private static string ToolMeterWidth(int count, int max) + => $"{LlmTestServerDashboardCharts.Number(count * 100d / max)}%"; + + private static string TimelineTitle(int index, LlmTestServerDashboardRequestPoint point) + { + var header = $"Request #{index + 1} · {point.ModelId ?? "unknown model"} · {point.StartedAt.ToLocalTime():HH:mm:ss}"; + return point.Usage is not { } usage + ? $"{header}\nNo usage reported" + : string.Create(CultureInfo.InvariantCulture, + $"{header}\n{usage.TotalTokens:N0} tokens · {usage.InputTokens:N0} input ({usage.CachedInputTokens:N0} cache read, {usage.CacheCreationInputTokens:N0} cache write) · {usage.OutputTokens:N0} output"); + } + + private static IEnumerable TokenSpark(LlmTestServerTokenUsage usage) + { + var mix = usage.InputComposition is { } composition && composition.TotalTokens == usage.InputTokens + ? new LlmTestServerDashboardAnalytics + { + SystemTokens = composition.SystemTokens, + ToolDefinitionTokens = composition.ToolDefinitionTokens, + UserTokens = composition.UserTokens, + AssistantTokens = composition.AssistantTokens, + ToolCallTokens = composition.ToolCallTokens, + ToolResultTokens = composition.ToolResultTokens, + OtherRoleTokens = composition.OtherTokens, + }.InputMix + : [new("not-itemized", "Input", usage.InputTokens, "var(--mix-not-itemized)")]; + return mix.Append(new("output", "Output", usage.OutputTokens, "var(--output-source)")).Where(part => part.Tokens > 0); + } + + private static string TokenSparkTitle(LlmTestServerTokenUsage usage) + { + var total = Math.Max(1, usage.TotalTokens); + return $"{LlmTestServerDashboardCharts.Tokens(usage.TotalTokens)} tokens\n" + string.Join("\n", TokenSpark(usage).Select(part => + $"{part.Label} · {LlmTestServerDashboardCharts.Tokens(part.Tokens)} · {LlmTestServerDashboardCharts.Percent((double)part.Tokens / total)}")); + } + + private static string NodeTip(LlmTestServerSankeyNode node) + { + return $"{node.Segment.Label}\n{LlmTestServerDashboardCharts.Tokens(node.Segment.Tokens)} tokens · {LlmTestServerDashboardCharts.Percent(node.Share)}"; + } + + private static string ArcTip(LlmTestServerDonutArc arc) + { + return $"{arc.Segment.Label}\n{LlmTestServerDashboardCharts.Tokens(arc.Segment.Tokens)} · {LlmTestServerDashboardCharts.Percent(arc.Share)}"; + } + + private static string SegmentTip(LlmTestServerDashboardSegment segment, long total) + { + return total > 0 + ? $"{segment.Label}\n{LlmTestServerDashboardCharts.Tokens(segment.Tokens)} · {LlmTestServerDashboardCharts.Percent((double)segment.Tokens / total)}" + : $"{segment.Label}\n{LlmTestServerDashboardCharts.Tokens(segment.Tokens)}"; + } + + private string RailTip(string label, int count) + { + return $"{label}\n{count} of {_traces.Count} requests · {LlmTestServerDashboardCharts.Percent(_traces.Count == 0 ? 0 : (double)count / _traces.Count)}"; + } + + private readonly System.Runtime.CompilerServices.ConditionalWeakTable> _messageTokens = new(); + + private int MessageTokens(LlmTestServerMessage message) + { + // Mirrors the runtime's per-message accounting so the bars add up to the reported input. + return _messageTokens.GetValue(message, static item => new StrongBox(LlmTestServerTokenCounter.CountTextTokens( + item.ToolCalls.Count == 0 && item.ToolCallId is null + ? item.Content + : item.Content + " " + item.ToolCallId + " " + string.Join(" ", item.ToolCalls.Select(call => call.Id + " " + call.Name + " " + call.ArgumentsJson))))).Value; + } + + private static RenderFragment MessageBody(LlmTestServerMessage message, string css) => __builder => + { + // Tool results are usually JSON; show them like the wire payloads instead of a single long line. + var content = message.Content.TrimStart(); + if ((message.ToolCallId is not null || string.Equals(message.Role, "tool", StringComparison.OrdinalIgnoreCase)) + && content.Length > 1 && content[0] is '{' or '[') + { + 
@LlmTestServerJsonHighlighter.Highlight(message.Content, false)
+ } + else + { +
@message.Content
+ } + }; + + private static string RoleLabel(LlmTestServerMessage message) + { + return message.ToolCalls.Count > 0 ? $"{message.Role} · tool call" : message.Role; + } + + private static string RoleColor(LlmTestServerMessage message) + { + if (message.ToolCallId is not null || string.Equals(message.Role, "tool", StringComparison.OrdinalIgnoreCase)) + { + return "var(--mix-tool-results)"; + } + + return message.Role.ToLowerInvariant() switch + { + "system" or "developer" => "var(--mix-system)", + "user" => "var(--mix-user)", + "assistant" or "model" or "chatbot" => message.ToolCalls.Count > 0 && string.IsNullOrWhiteSpace(message.Content) + ? "var(--mix-tool-calls)" + : "var(--mix-assistant)", + _ => "var(--mix-other)", + }; + } + + private RenderFragment ConversationStrip(IReadOnlyList messages, LlmTestServerTokenUsage? usage) => __builder => + { + if (messages.Count == 0) + { + return; + } + + var schemas = usage?.InputComposition is { } composition && composition.TotalTokens == usage.InputTokens ? composition.ToolDefinitionTokens : 0; + var total = Math.Max(1, messages.Sum(MessageTokens) + schemas); +
+ +
@messages.Count messages@(schemas > 0 ? " + tool schemas" : string.Empty)@LlmTestServerDashboardCharts.Tokens(total) tokens
+
+ }; + + private RenderFragment MessageMeter(LlmTestServerMessage message, IReadOnlyList messages) => __builder => + { + var tokens = MessageTokens(message); + var max = Math.Max(1, messages.Max(MessageTokens)); + @LlmTestServerDashboardCharts.Tokens(tokens) tok + }; + + private static RenderFragment SizeMeter(long bytes, long max) => __builder => + { + @FormatBytes(bytes) + }; + + private string RailShare(int count) + { + return LlmTestServerDashboardCharts.Number(_traces.Count == 0 ? 0 : (double)count / _traces.Count); + } + + private LlmTestServerInsightChart AssertionChart() + { + return new("assertions", "Assertions", "Runtime outcome of every scripted call", + LlmTestServerDashboardCharts.Tokens(_summary.TotalEvents), "events", + [ + new("matched", "Matched", _summary.Matched, "#3f8d55"), + new("unmatched", "Unmatched", _summary.Unmatched, "#e8b445"), + new("model-not-found", "Model not found", _summary.ModelNotFound, "#ed732e"), + new("auth-failed", "Auth failed", _summary.AuthFailed, "#c94f3d"), + new("scenario-exhausted", "Scenario exhausted", _summary.ScenarioExhausted, "#a8a29e"), + new("errors-returned", "Errors returned", _summary.ErrorsReturned, "#8a2f22"), + ]); + } + + private LlmTestServerInsightChart FixtureChart() + { + string[] colors = ["#2a78d6", "#eb6834", "#1baf7a", "#eda100", "#e87ba4", "#008300", "#6250d6"]; + var kinds = _models.GroupBy(model => model.Kind.ToString()).OrderByDescending(group => group.Count()).ThenBy(group => group.Key, StringComparer.Ordinal).ToArray(); + return new("fixtures", "Model fixtures", "Loaded models by kind", LlmTestServerDashboardCharts.Tokens(_models.Count), + _models.Count == 1 ? "model" : "models", + kinds.Select((group, index) => new LlmTestServerDashboardSegment(group.Key, group.Key, group.Count(), + index < colors.Length ? colors[index] : "#a8a29e")).ToArray()); + } + + private void ToggleInsights() + { + _insightsOpen = !_insightsOpen; + _mobileDetailsOpen = _insightsOpen; + } + + private void CloseInsights() + { + _insightsOpen = false; + _mobileDetailsOpen = false; + } + + private string ProviderColor(string provider) + { + return _providerColors.GetValueOrDefault(provider) ?? "#a8a29e"; + } + + private string? InsightSelection(string? kind) + { + return kind switch + { + LlmTestServerDashboardInsights.ProviderFilter => NullIfEmpty(_providerFilter), + LlmTestServerDashboardInsights.ModelFilter => NullIfEmpty(_modelFilter), + LlmTestServerDashboardInsights.SearchFilter => NullIfEmpty(_search), + LlmTestServerDashboardInsights.OutcomeFilter => _outcomeFilter switch + { + "success" => "success", + "stream" => "stream", + _ => null, + }, + _ => null, + }; + + static string? NullIfEmpty(string value) => string.IsNullOrWhiteSpace(value) ? null : value; + } + + + private RenderFragment InsightCard(LlmTestServerInsightChart chart) => __builder => + { + var v = LlmTestServerInsightCardView.Create(chart, $"insight-{chart.Key}", InsightSelection(chart.FilterKind)); +
+
+
+

@v.Chart.Title

+

@v.Chart.Subtitle

+
+ @if (v.Chart.Form != LlmTestServerInsightForm.Donut && v.Visible.Count > 0) + { + @v.Chart.CenterValue @v.Chart.CenterLabel + } +
+
+ + @if (v.Visible.Count == 0) + { +

Nothing recorded yet.

+ } + else if (v.Chart.Form == LlmTestServerInsightForm.Donut) + { +
+
+ + + @foreach (var arc in v.Arcs) + { + + } + +
@v.Chart.CenterValue@v.Chart.CenterLabel
+
+
    + @foreach (var segment in v.Visible) + { +
  • @InsightLegendRow(v, segment)
    • + } +
+
+ } + else if (v.Chart.Form == LlmTestServerInsightForm.Bars) + { +
    + @foreach (var segment in v.Visible) + { +
  • + +
    • + } +
+ } + else if (v.Chart.Form == LlmTestServerInsightForm.Columns) + { + + } + else + { + +
    + @foreach (var segment in v.Visible) + { +
  • @InsightLegendRow(v, segment)
    • + } +
+ } +
+
+ }; + + private RenderFragment InsightLegendRow(LlmTestServerInsightCardView v, LlmTestServerDashboardSegment segment) => __builder => + { + var share = v.Total > 0 ? LlmTestServerDashboardCharts.Percent((double)segment.Tokens / v.Total) : string.Empty; + if (InsightCanSelect(v, segment)) + { + + } + else + { + @segment.Label@LlmTestServerDashboardCharts.Tokens(segment.Tokens)@share + } + }; + + private static bool InsightCanSelect(LlmTestServerInsightCardView v, LlmTestServerDashboardSegment segment) + { + return v.Chart.FilterKind is not null && segment.Key != LlmTestServerDashboardInsights.OtherKey && segment.Key != "—"; + } + + private void InsightSelect(LlmTestServerInsightCardView v, LlmTestServerDashboardSegment segment) + { + if (InsightCanSelect(v, segment)) + { + ApplyInsightFilter(v.Chart.FilterKind!, segment.Key); + } + } + + private static string InsightPressed(LlmTestServerInsightCardView v, string key) + { + return string.Equals(v.SelectedKey, key, StringComparison.Ordinal) ? "true" : "false"; + } + + private static string InsightArcCss(LlmTestServerInsightCardView v, string key) + { + return v.SelectedKey is null || string.Equals(v.SelectedKey, key, StringComparison.Ordinal) ? "donut-arc" : "donut-arc muted"; + } + + private static string InsightCapsuleCss(LlmTestServerInsightCardView v, string key) + { + return v.SelectedKey is null || string.Equals(v.SelectedKey, key, StringComparison.Ordinal) ? "capsule" : "capsule muted"; + } + + private static string InsightWidth(long value, long max) + { + return $"{LlmTestServerDashboardCharts.Number(Math.Max(value > 0 ? 2 : 0, value * 100d / max))}%"; + } + + private void ApplyInsightFilter(string kind, string key) + { + // Selecting the active segment again clears that filter. + switch (kind) + { + case LlmTestServerDashboardInsights.ProviderFilter: + _providerFilter = IsProviderFilter(key) ? string.Empty : key; + break; + case LlmTestServerDashboardInsights.ModelFilter: + _modelFilter = string.Equals(_modelFilter, key, StringComparison.OrdinalIgnoreCase) ? string.Empty : key; + break; + case LlmTestServerDashboardInsights.SearchFilter: + _search = string.Equals(_search, key, StringComparison.Ordinal) ? string.Empty : key; + break; + case LlmTestServerDashboardInsights.OutcomeFilter: + var outcome = key switch + { + "success" => "success", + "stream" => "stream", + "client-error" or "server-error" or "transport" => "error", + _ => string.Empty, + }; + _outcomeFilter = IsOutcomeFilter(outcome) ? string.Empty : outcome; + break; + } + } + + private static string RecentHeight(long tokens, long max) + { + return $"{LlmTestServerDashboardCharts.Number(Math.Max(6, tokens * 100d / max))}%"; + } + + private void SelectAnalyticsRequest(string requestId) + { + _selectedRequestId = requestId; + _activeDetailTab = _analyticsTab; + } + + private LlmTestServerRuntimeEvent? RuntimeEventFor(LlmTestServerProviderHttpTrace trace) + { + return _eventsByRequestId.GetValueOrDefault(trace.RequestId); + } + + private LlmTestServerTokenUsage? UsageFor(LlmTestServerProviderHttpTrace trace) + { + return trace.Usage ?? RuntimeEventFor(trace)?.Usage; + } + + private IReadOnlyList MessagesFor(LlmTestServerProviderHttpTrace trace) + { + return trace.Messages.Count > 0 + ? trace.Messages + : RuntimeEventFor(trace)?.Messages ?? []; + } + + private IReadOnlyList ChunksFor(LlmTestServerProviderHttpTrace trace) + { + return trace.StreamChunks.Count > 0 + ? trace.StreamChunks + : RuntimeEventFor(trace)?.StreamChunks ?? []; + } + + private string ModelName(LlmTestServerProviderHttpTrace trace) + { + return string.IsNullOrWhiteSpace(trace.ModelId) + ? RuntimeEventFor(trace)?.ModelId ?? "—" + : trace.ModelId; + } + + private string RequestPreview(LlmTestServerProviderHttpTrace trace) + { + var messages = MessagesFor(trace); + var message = messages.LastOrDefault(candidate => + string.Equals(candidate.Role, "user", StringComparison.OrdinalIgnoreCase) + ) + ?? messages.LastOrDefault(); + if (message is not null && !string.IsNullOrWhiteSpace(message.Content)) + { + return TurnPreview(message.Content); + } + + return !string.IsNullOrWhiteSpace(trace.OperationId) + ? trace.OperationId + : trace.Path; + } + + private static string ProviderInitial(string providerId) + { + return ProviderName(providerId) switch + { + "Azure OpenAI" => "Az", + "Microsoft Foundry" => "M", + "Amazon Bedrock" => "B", + "OpenRouter" => "Or", + "DeepSeek" => "D", + "Perplexity" => "P", + { Length: > 0 } name => name[..1], + _ => "?", + }; + } + + private static bool ShouldCollapseTurn(string role, string content) + { + return content.Length > _collapsedTurnThreshold + && (string.Equals(role, "system", StringComparison.OrdinalIgnoreCase) + || string.Equals(role, "tool", StringComparison.OrdinalIgnoreCase)); + } + + private static string TurnPreview(string content) + { + var builder = new StringBuilder(Math.Min(content.Length, _turnPreviewLimit + 1)); + var pendingSpace = false; + var truncated = false; + foreach (var character in content) + { + if (char.IsWhiteSpace(character)) + { + pendingSpace = builder.Length > 0; + continue; + } + + if (pendingSpace) + { + if (builder.Length >= _turnPreviewLimit) + { + truncated = true; + break; + } + + builder.Append(' '); + pendingSpace = false; + } + + if (builder.Length >= _turnPreviewLimit) + { + truncated = true; + break; + } + + builder.Append(character); + } + + return truncated ? builder.Append('…').ToString() : builder.ToString(); + } + + private static string FormatCharacterCount(string content) + { + return $"{content.Length.ToString("N0", CultureInfo.InvariantCulture)} chars"; + } + + private static string ProviderName(string providerId) + { + return providerId switch + { + LlmTestServerCompatibilityTags.OpenAI => "OpenAI", + LlmTestServerCompatibilityTags.AzureOpenAI => "Azure OpenAI", + LlmTestServerCompatibilityTags.MicrosoftFoundry => "Microsoft Foundry", + LlmTestServerCompatibilityTags.Anthropic => "Anthropic", + LlmTestServerCompatibilityTags.Gemini => "Gemini", + LlmTestServerCompatibilityTags.Groq => "Groq", + LlmTestServerCompatibilityTags.Mistral => "Mistral", + LlmTestServerCompatibilityTags.Ollama => "Ollama", + LlmTestServerCompatibilityTags.Cohere => "Cohere", + LlmTestServerCompatibilityTags.Bedrock => "Amazon Bedrock", + LlmTestServerCompatibilityTags.OpenRouter => "OpenRouter", + LlmTestServerCompatibilityTags.DeepSeek => "DeepSeek", + LlmTestServerCompatibilityTags.Perplexity => "Perplexity", + _ => string.IsNullOrWhiteSpace(providerId) ? "Unknown" : providerId, + }; + } + + private static string StatusCssFor(LlmTestServerProviderHttpTrace trace) + { + if (!trace.IsCompleted) + { + return "status-chip warning"; + } + + if (trace.TransportState != LlmTestServerProviderHttpTraceTransportState.Completed) + { + return "status-chip error"; + } + + return trace.StatusCode switch + { + null => "status-chip error", + < 400 => "status-chip success", + >= 400 => "status-chip error", + }; + } + + private static string StatusText(LlmTestServerProviderHttpTrace trace) + { + if (!trace.IsCompleted) + { + return "Running"; + } + + if (trace.TransportState != LlmTestServerProviderHttpTraceTransportState.Completed) + { + return trace.TransportState.ToString(); + } + + return trace.StatusCode switch + { + null => "No status", + 200 => "200 OK", + 201 => "201 Created", + 202 => "202 Accepted", + 204 => "204 Empty", + 400 => "400 Invalid", + 401 => "401 Auth", + 404 => "404 Missing", + 405 => "405 Method", + 409 => "409 Conflict", + 429 => "429 Limited", + >= 500 => $"{trace.StatusCode} Server", + _ => trace.StatusCode.Value.ToString(CultureInfo.InvariantCulture), + }; + } + + private static string InboxStatusText(LlmTestServerProviderHttpTrace trace) + { + if (!trace.IsCompleted) + { + return "Live"; + } + + return trace.StatusCode?.ToString(CultureInfo.InvariantCulture) + ?? trace.TransportState.ToString(); + } + + private static string FormatDuration(double? milliseconds) + { + if (milliseconds is null) + { + return "running"; + } + + return milliseconds < 1 + ? "<1 ms" + : $"{milliseconds.Value:N0} ms"; + } + + private static string FormatBytes(long? bytes) + { + if (bytes is null) + { + return "unknown"; + } + + if (bytes < 1024) + { + return string.Create(CultureInfo.InvariantCulture, $"{bytes.Value:N0} B"); + } + + return string.Create(CultureInfo.InvariantCulture, $"{bytes.Value / 1024d:N1} KiB"); + } + + private static string TurnCss(string role) + { + if (string.Equals(role, "system", StringComparison.OrdinalIgnoreCase)) + { + return "turn system"; + } + + if (string.Equals(role, "user", StringComparison.OrdinalIgnoreCase)) + { + return "turn user"; + } + + if (string.Equals(role, "assistant", StringComparison.OrdinalIgnoreCase)) + { + return "turn assistant"; + } + + return string.Equals(role, "tool", StringComparison.OrdinalIgnoreCase) + ? "turn tool" + : "turn"; + } + + private static string RuntimeResponseTurnCss(LlmTestServerProviderHttpTrace trace) + { + return trace.RuntimeEventKind == LlmTestServerEventKind.ErrorReturned + ? "turn error" + : "turn assistant"; + } + + private static string RuntimeResponseRole(LlmTestServerProviderHttpTrace trace) + { + if (trace.IsSuccess is false || trace.RuntimeEventKind == LlmTestServerEventKind.ErrorReturned) + { + return "provider error"; + } + + if (trace.ResponseToolCalls.Count > 0) + { + return "assistant · tool call"; + } + + return string.IsNullOrWhiteSpace(trace.RuntimeResponse) + ? "provider response" + : "assistant"; + } + + private static string AriaBoolean(bool value) + { + return value ? "true" : "false"; + } + + private static string EventTone(LlmTestServerEventKind kind) + { + return kind switch + { + LlmTestServerEventKind.Matched => "success", + LlmTestServerEventKind.Unmatched + or LlmTestServerEventKind.ScenarioExhausted => "warning", + _ => "error", + }; + } + + + private static string Empty(string? value) + { + return string.IsNullOrWhiteSpace(value) ? "—" : value; + } + + private void SetBanner(string kind, string text) + { + _bannerKind = kind; + _bannerText = text; + } + + private void ClearBanner() + { + _bannerKind = null; + _bannerText = null; + } + + public ValueTask DisposeAsync() + { + StopAutoRefresh(); + return ValueTask.CompletedTask; + } + + private sealed record ProviderRoute(string Name, string Route); +} diff --git a/src/ManagedCode.LlmTck.Hosting/LlmTckBoundedTeeStream.cs b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerBoundedTeeStream.cs similarity index 93% rename from src/ManagedCode.LlmTck.Hosting/LlmTckBoundedTeeStream.cs rename to src/ManagedCode.LlmTestServer.Hosting/LlmTestServerBoundedTeeStream.cs index 3f884eb..3cf7a6a 100644 --- a/src/ManagedCode.LlmTck.Hosting/LlmTckBoundedTeeStream.cs +++ b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerBoundedTeeStream.cs @@ -1,8 +1,8 @@ -namespace ManagedCode.LlmTck.Hosting; +namespace ManagedCode.LlmTestServer.Hosting; -internal sealed class LlmTckBoundedWriteTeeStream(Stream inner, int maxCaptureBytes) : Stream +internal sealed class LlmTestServerBoundedWriteTeeStream(Stream inner, int maxCaptureBytes) : Stream { - private readonly LlmTckBoundedCaptureBuffer _capture = new(maxCaptureBytes); + private readonly LlmTestServerBoundedCaptureBuffer _capture = new(maxCaptureBytes); private int _successfulFlushCount; public long BytesObserved => _capture.BytesObserved; @@ -109,9 +109,9 @@ protected override void Dispose(bool disposing) } } -internal sealed class LlmTckBoundedReadTeeStream(Stream inner, int maxCaptureBytes) : Stream +internal sealed class LlmTestServerBoundedReadTeeStream(Stream inner, int maxCaptureBytes) : Stream { - private readonly LlmTckBoundedCaptureBuffer _capture = new(maxCaptureBytes); + private readonly LlmTestServerBoundedCaptureBuffer _capture = new(maxCaptureBytes); public long BytesObserved => _capture.BytesObserved; @@ -261,7 +261,7 @@ protected override void Dispose(bool disposing) } } -internal sealed class LlmTckBoundedCaptureBuffer(int maxCaptureBytes) : IDisposable +internal sealed class LlmTestServerBoundedCaptureBuffer(int maxCaptureBytes) : IDisposable { private readonly MemoryStream _capture = new(Math.Min(maxCaptureBytes, 4096)); private long _bytesObserved; diff --git a/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerDashboardAnalytics.cs b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerDashboardAnalytics.cs new file mode 100644 index 0000000..c550237 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerDashboardAnalytics.cs @@ -0,0 +1,172 @@ +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Runtime; + +namespace ManagedCode.LlmTestServer.Hosting; + +internal sealed record LlmTestServerDashboardAnalytics +{ + public int RequestCount { get; init; } + public int UsageRequestCount { get; init; } + public int ItemizedRequestCount { get; init; } + public int PendingCount { get; init; } + public long InputTokens { get; init; } + public long OutputTokens { get; init; } + public long TotalTokens { get; init; } + public long ReasoningTokens { get; init; } + public long CacheReadTokens { get; init; } + public long CacheWriteTokens { get; init; } + public long SystemTokens { get; init; } + public long UserTokens { get; init; } + public long AssistantTokens { get; init; } + public long ToolCallTokens { get; init; } + public long ToolResultTokens { get; init; } + public long ToolDefinitionTokens { get; init; } + public long OtherRoleTokens { get; init; } + public long NotItemizedTokens { get; init; } + public int CacheHitCount { get; init; } + public long VisibleOutputTokens => Math.Max(0, OutputTokens - ReasoningTokens); + public long UncachedInputTokens => Math.Max(0, InputTokens - CacheReadTokens - CacheWriteTokens); + public decimal? CacheShare => InputTokens > 0 ? (decimal)CacheReadTokens / InputTokens : null; + public decimal? CacheHitRate => UsageRequestCount > 0 ? (decimal)CacheHitCount / UsageRequestCount : null; + public int CallsReturned => Tools.Sum(tool => tool.CallsReturned); + public int ResultsObserved => Tools.Sum(tool => tool.ResultsObserved); + public int DistinctResultIds { get; init; } + public bool PayloadTruncated { get; init; } + public IReadOnlyList Tools { get; init; } = []; + public IReadOnlyList Requests { get; init; } = []; + + public IReadOnlyList InputMix => + [ + new("system", "System prompt", SystemTokens, "var(--mix-system)"), + new("tool-definitions", "Tool schemas", ToolDefinitionTokens, "var(--mix-tool-definitions)"), + new("user", "User", UserTokens, "var(--mix-user)"), + new("assistant", "Assistant", AssistantTokens, "var(--mix-assistant)"), + new("tool-calls", "Tool calls", ToolCallTokens, "var(--mix-tool-calls)"), + new("tool-results", "Tool results", ToolResultTokens, "var(--mix-tool-results)"), + new("other", "Other roles", OtherRoleTokens, "var(--mix-other)"), + new("not-itemized", "Not itemized", NotItemizedTokens, "var(--mix-not-itemized)"), + ]; + + public IReadOnlyList CacheMix => + [ + new("cache-read", "Cache read", CacheReadTokens, "var(--cache-read)"), + new("cache-write", "Cache write", CacheWriteTokens, "var(--cache-write)"), + new("uncached", "Uncached input", UncachedInputTokens, "var(--cache-none)"), + ]; + + public IReadOnlyList OutputMix => + [ + new("response", "Response", VisibleOutputTokens, "var(--output-visible)"), + new("reasoning", "Reasoning", ReasoningTokens, "var(--output-reasoning)"), + ]; + + public IReadOnlyList ToolMix => + [ + new("tool-definitions", "Schemas", ToolDefinitionTokens, "var(--mix-tool-definitions)"), + new("tool-calls", "Call arguments", ToolCallTokens, "var(--mix-tool-calls)"), + new("tool-results", "Results", ToolResultTokens, "var(--mix-tool-results)"), + ]; + + public IReadOnlyList FlowSources => + [.. InputMix, new("output", "Generated output", OutputTokens, "var(--output-source)")]; + + public IReadOnlyList FlowTargets => [.. CacheMix, .. OutputMix]; + + public static LlmTestServerDashboardAnalytics Create(LlmTestServerRequestSnapshot snapshot, + LlmTestServerProviderHttpTrace selected, bool conversation) + { + var rows = conversation && selected.ConversationIdWasProvided && !string.IsNullOrWhiteSpace(selected.ConversationId) + ? snapshot.Requests.Where(trace => trace.ConversationIdWasProvided + && trace.SessionId == selected.SessionId && trace.ProviderNamespace == selected.ProviderNamespace + && trace.ConversationId == selected.ConversationId).ToArray() + : [selected]; + return FromRows(rows); + } + + public static LlmTestServerDashboardAnalytics FromRows(IReadOnlyList rows) + { + var usage = rows.Where(trace => trace.Usage is not null).Select(trace => trace.Usage!).ToArray(); + // A composition is shown only when it reconciles with the reported input; anything else stays visibly unitemized. + var itemized = usage.Where(item => item.InputComposition?.TotalTokens == item.InputTokens) + .Select(item => item.InputComposition!).ToArray(); + var names = new Dictionary(StringComparer.Ordinal); + var resultIds = new HashSet(StringComparer.Ordinal); + ToolAccumulator Tool(string name) + { + if (!names.TryGetValue(name, out var tool)) { tool = new(); names.Add(name, tool); } + return tool; + } + foreach (var trace in rows) + { + foreach (var call in trace.ResponseToolCalls) { Tool(call.Name).CallsReturned++; } + var priorCallsById = new Dictionary(StringComparer.Ordinal); + foreach (var message in trace.Messages) + { + if (message.Role == "tool" && !string.IsNullOrEmpty(message.ToolCallId)) + { + var tool = Tool(priorCallsById.GetValueOrDefault(message.ToolCallId) + ?? "Unknown tool (context unavailable)"); + tool.ResultsObserved++; + tool.ResultIds.Add(message.ToolCallId); + resultIds.Add(message.ToolCallId); + } + foreach (var call in message.ToolCalls) { priorCallsById[call.Id] = call.Name; } + } + } + return new() + { + RequestCount = rows.Count, + UsageRequestCount = usage.Length, + ItemizedRequestCount = itemized.Length, + PendingCount = rows.Count(trace => !trace.IsCompleted), + InputTokens = usage.Sum(item => (long)item.InputTokens), + OutputTokens = usage.Sum(item => (long)item.OutputTokens), + TotalTokens = usage.Sum(item => (long)item.TotalTokens), + ReasoningTokens = usage.Sum(item => (long)item.ReasoningTokens), + CacheReadTokens = usage.Sum(item => (long)item.CachedInputTokens), + CacheWriteTokens = usage.Sum(item => (long)item.CacheCreationInputTokens), + SystemTokens = itemized.Sum(item => (long)item.SystemTokens), + UserTokens = itemized.Sum(item => (long)item.UserTokens), + AssistantTokens = itemized.Sum(item => (long)item.AssistantTokens), + ToolCallTokens = itemized.Sum(item => (long)item.ToolCallTokens), + ToolResultTokens = itemized.Sum(item => (long)item.ToolResultTokens), + ToolDefinitionTokens = itemized.Sum(item => (long)item.ToolDefinitionTokens), + OtherRoleTokens = itemized.Sum(item => (long)item.OtherTokens), + NotItemizedTokens = usage.Where(item => item.InputComposition?.TotalTokens != item.InputTokens) + .Sum(item => (long)item.InputTokens), + CacheHitCount = usage.Count(item => item.CachedInputTokens > 0), + DistinctResultIds = resultIds.Count, + PayloadTruncated = rows.Any(trace => trace.RequestPreviewTruncated || trace.ResponsePreviewTruncated + || trace.RetentionTruncated || trace.RuntimePayloadTruncated), + Tools = names.OrderBy(pair => pair.Key, StringComparer.Ordinal).Select(pair => new LlmTestServerDashboardToolAnalytics + { + Name = pair.Key, + CallsReturned = pair.Value.CallsReturned, + ResultsObserved = pair.Value.ResultsObserved, + ResultIds = pair.Value.ResultIds.Order(StringComparer.Ordinal).ToArray() + }).ToArray(), + Requests = rows.OrderBy(trace => trace.Sequence).Select(trace => new LlmTestServerDashboardRequestPoint( + trace.RequestId, trace.ModelId, trace.StartedAt, trace.Usage)).ToArray() + }; + } + + private sealed class ToolAccumulator + { + public int CallsReturned { get; set; } + public int ResultsObserved { get; set; } + public HashSet ResultIds { get; } = new(StringComparer.Ordinal); + } +} + +internal sealed record LlmTestServerDashboardToolAnalytics +{ + public string Name { get; init; } = string.Empty; + public int CallsReturned { get; init; } + public int ResultsObserved { get; init; } + public IReadOnlyList ResultIds { get; init; } = []; +} + +internal sealed record LlmTestServerDashboardSegment(string Key, string Label, long Tokens, string Color); + +internal sealed record LlmTestServerDashboardRequestPoint(string RequestId, string? ModelId, DateTimeOffset StartedAt, + LlmTestServerTokenUsage? Usage); diff --git a/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerDashboardCharts.cs b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerDashboardCharts.cs new file mode 100644 index 0000000..f8c5af1 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerDashboardCharts.cs @@ -0,0 +1,161 @@ +using System.Globalization; + +namespace ManagedCode.LlmTestServer.Hosting; + +internal sealed record LlmTestServerSankeyNode(LlmTestServerDashboardSegment Segment, double Y, double Height, double Share) +{ + public double Center => Y + Height / 2; +} + +internal sealed record LlmTestServerSankeyBand(string Key, string Path, string Color, string Title); + +internal sealed record LlmTestServerSankeyLayout( + double Height, + double CenterY, + double CenterHeight, + long Total, + IReadOnlyList Sources, + IReadOnlyList Targets, + IReadOnlyList Bands); + +internal sealed record LlmTestServerDonutArc(LlmTestServerDashboardSegment Segment, string DashArray, string DashOffset, double Share); + +// Server-rendered chart geometry: the dashboard draws these shapes as inline SVG without client-side scripts. +internal static class LlmTestServerDashboardCharts +{ + public const double SankeyWidth = 1000; + public const double SourceX = 236; + public const double CenterX = 494; + public const double TargetX = 752; + public const double NodeWidth = 12; + public const double DonutRadius = 52; + + private const double _flowHeight = 210; + private const double _minNodeHeight = 3; + private const double _nodeGap = 8; + private const double _labelPitch = 40; + private const double _topPadding = 34; + private const double _bottomPadding = 14; + private const double _donutGap = 2.5; + + public static LlmTestServerSankeyLayout? Sankey(IEnumerable sources, + IEnumerable targets) + { + var left = sources.Where(segment => segment.Tokens > 0).ToArray(); + var right = targets.Where(segment => segment.Tokens > 0).ToArray(); + var total = left.Sum(segment => segment.Tokens); + if (total <= 0 || right.Sum(segment => segment.Tokens) != total) + { + return null; + } + + var scale = _flowHeight / total; + var (leftNodes, leftHeight) = Stack(left, scale, total); + var (rightNodes, rightHeight) = Stack(right, scale, total); + var columnHeight = Math.Max(_flowHeight, Math.Max(leftHeight, rightHeight)); + var centerY = _topPadding + (columnHeight - _flowHeight) / 2; + leftNodes = Shift(leftNodes, _topPadding + (columnHeight - leftHeight) / 2); + rightNodes = Shift(rightNodes, _topPadding + (columnHeight - rightHeight) / 2); + + var bands = new List(leftNodes.Count + rightNodes.Count); + var offset = centerY; + foreach (var node in leftNodes) + { + var height = node.Segment.Tokens * scale; + bands.Add(new(node.Segment.Key, Band(SourceX + NodeWidth, node.Y, node.Y + node.Height, CenterX, offset, offset + height), + node.Segment.Color, $"{node.Segment.Label} → request\n{Tokens(node.Segment.Tokens)} tokens · {Percent(node.Share)}")); + offset += height; + } + + offset = centerY; + foreach (var node in rightNodes) + { + var height = node.Segment.Tokens * scale; + bands.Add(new(node.Segment.Key, Band(CenterX + NodeWidth, offset, offset + height, TargetX, node.Y, node.Y + node.Height), + node.Segment.Color, $"Request → {node.Segment.Label}\n{Tokens(node.Segment.Tokens)} tokens · {Percent(node.Share)}")); + offset += height; + } + + return new(_topPadding + columnHeight + _bottomPadding, centerY, _flowHeight, total, leftNodes, rightNodes, bands); + } + + public static IReadOnlyList Donut(IEnumerable segments) + { + var visible = segments.Where(segment => segment.Tokens > 0).ToArray(); + var total = visible.Sum(segment => segment.Tokens); + if (total <= 0) + { + return []; + } + + var circumference = 2 * Math.PI * DonutRadius; + var gap = visible.Length > 1 ? _donutGap : 0; + var arcs = new List(visible.Length); + var start = 0d; + foreach (var segment in visible) + { + var length = segment.Tokens * circumference / total; + var drawn = Math.Max(0.8, length - gap); + arcs.Add(new(segment, $"{Number(drawn)} {Number(circumference - drawn)}", Number(-(start + gap / 2)), + (double)segment.Tokens / total)); + start += length; + } + + return arcs; + } + + public static string Number(double value) + { + return value.ToString("0.##", CultureInfo.InvariantCulture); + } + + public static string Tokens(long value) + { + return value.ToString("N0", CultureInfo.InvariantCulture); + } + + public static string Percent(double share) + { + return share switch + { + <= 0 => "0%", + < 0.001 => "<0.1%", + < 0.1 => share.ToString("0.0%", CultureInfo.InvariantCulture), + _ => share.ToString("0%", CultureInfo.InvariantCulture), + }; + } + + private static (IReadOnlyList Nodes, double Height) Stack( + IReadOnlyList segments, double scale, long total) + { + var nodes = new List(segments.Count); + foreach (var segment in segments) + { + var height = Math.Max(_minNodeHeight, segment.Tokens * scale); + var y = 0d; + if (nodes.Count > 0) + { + // Keep two-line labels readable even when neighbouring nodes are only a few tokens tall. + var previous = nodes[^1]; + y = Math.Max(previous.Y + previous.Height + _nodeGap, previous.Center + _labelPitch - height / 2); + } + + nodes.Add(new(segment, y, height, (double)segment.Tokens / total)); + } + + return (nodes, nodes.Count == 0 ? 0 : nodes[^1].Y + nodes[^1].Height); + } + + private static IReadOnlyList Shift(IReadOnlyList nodes, double offset) + { + return nodes.Select(node => node with { Y = node.Y + offset }).ToArray(); + } + + private static string Band(double x0, double y0, double y1, double x1, double y2, double y3) + { + var middle = (x0 + x1) / 2; + return string.Create(CultureInfo.InvariantCulture, + $"M{Number(x0)} {Number(y0)}C{Number(middle)} {Number(y0)} {Number(middle)} {Number(y2)} {Number(x1)} {Number(y2)}" + + $"L{Number(x1)} {Number(y3)}C{Number(middle)} {Number(y3)} {Number(middle)} {Number(y1)} {Number(x0)} {Number(y1)}Z"); + } +} diff --git a/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerDashboardInsights.cs b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerDashboardInsights.cs new file mode 100644 index 0000000..1ef828a --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerDashboardInsights.cs @@ -0,0 +1,271 @@ +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Runtime; + +namespace ManagedCode.LlmTestServer.Hosting; + +internal enum LlmTestServerInsightForm +{ + Donut, + Bars, + Columns, + Dots, +} + +internal sealed record LlmTestServerInsightChart( + string Key, + string Title, + string Subtitle, + string CenterValue, + string CenterLabel, + IReadOnlyList Segments, + string? FilterKind = null, + LlmTestServerInsightForm Form = LlmTestServerInsightForm.Donut); + +internal sealed record LlmTestServerInsightRequest(string RequestId, long Tokens, string Color, string Outcome, string Title); + +internal sealed record LlmTestServerDashboardInsights( + int RequestCount, + IReadOnlyList Charts, + IReadOnlyList Recent) +{ + public const string ProviderFilter = "provider"; + public const string ModelFilter = "model"; + public const string OutcomeFilter = "outcome"; + public const string SearchFilter = "search"; + public const string OtherKey = "other"; + + // Categorical slots in validated adjacent-pair order; entities past the last slot fold into "Other". + private static readonly string[] _categorical = ["#2a78d6", "#eb6834", "#1baf7a", "#eda100", "#e87ba4", "#008300", "#6250d6"]; + private const string _otherColor = "#a8a29e"; + + public static LlmTestServerDashboardInsights Create( + IReadOnlyList visible, + IReadOnlyList all, + Func providerName, + Func modelName, + Func usageFor, + Func messageCount) + { + var analytics = LlmTestServerDashboardAnalytics.FromRows(visible); + var withUsage = visible.Where(trace => usageFor(trace) is not null).ToArray(); + var completed = visible.Where(trace => trace.DurationMilliseconds is not null).ToArray(); + var outcomes = Outcomes(visible); + var succeeded = outcomes.FirstOrDefault(segment => segment.Key == "success")?.Tokens ?? 0; + var streamed = visible.Count(trace => trace.IsStreaming); + var latencies = completed.Select(trace => trace.DurationMilliseconds!.Value).Order().ToArray(); + var models = Categorical(visible, all, modelName, model => model == "—" ? "No model" : model, _ => 1); + var operations = Categorical(visible, all, Operation, operation => operation, _ => 1); + var totalTokens = withUsage.Sum(trace => (long)usageFor(trace)!.TotalTokens); + + LlmTestServerInsightChart[] built = + [ + new("providers", "Providers", "Requests per provider surface", Count(visible.Count), "requests", + Categorical(visible, all, trace => trace.ProviderId, providerName, _ => 1), ProviderFilter), + new("models", "Models", "Requests per model", Count(models.Count(segment => segment.Key != OtherKey)), + models.Count == 1 ? "model" : "models", models, ModelFilter, LlmTestServerInsightForm.Bars), + new("outcomes", "Outcomes", "Status of each request", + visible.Count == 0 ? "—" : LlmTestServerDashboardCharts.Percent((double)succeeded / visible.Count), "succeeded", + outcomes, OutcomeFilter), + new("operations", "Operations", "Which provider APIs were called", Count(operations.Count), operations.Count == 1 ? "operation" : "operations", + operations, SearchFilter, LlmTestServerInsightForm.Bars), + new("model-tokens", "Tokens by model", "Where the token volume went", + withUsage.Length == 0 ? "—" : LlmTestServerDashboardCharts.Tokens(totalTokens), "tokens", + Categorical(withUsage, all, modelName, model => model == "—" ? "No model" : model, + trace => usageFor(trace)!.TotalTokens), ModelFilter), + new("context-mix", "Context mix", "What all prompts are made of", + analytics.UsageRequestCount == 0 ? "—" : LlmTestServerDashboardCharts.Tokens(analytics.InputTokens), "input", + analytics.InputMix), + new("prompt-cache", "Prompt cache", "How input tokens were served", + analytics.CacheShare is { } share ? LlmTestServerDashboardCharts.Percent((double)share) : "—", "cache read", + analytics.CacheMix), + new("latency", "Latency", "Completed requests by duration", + latencies.Length == 0 ? "—" : FormatLatency(latencies[latencies.Length / 2]), "median", + Latency(visible), Form: LlmTestServerInsightForm.Columns), + new("delivery", "Delivery", "Each dot is 1% of requests", + visible.Count == 0 ? "—" : LlmTestServerDashboardCharts.Percent((double)streamed / visible.Count), "streamed", + [ + new("stream", "Streaming", streamed, "#6fae32"), + new("single", "Single response", visible.Count - streamed, "#e2dbd1"), + ], OutcomeFilter, LlmTestServerInsightForm.Dots), + ]; + var depth = visible.Select(messageCount).Where(count => count > 0).ToArray(); + var deepest = depth.DefaultIfEmpty(0).Max(); + var charts = new List(built.Length + 1); + // Order fills a 3-, 2- or 1-column bento without holes after the double-width recent-requests card. + foreach (var key in new[] { "delivery", "providers", "outcomes", "prompt-cache", "models", "operations", "latency", "model-tokens", "context-mix" }) + { + charts.Add(built.Single(chart => chart.Key == key)); + } + + charts.Add(new("depth", "Conversation depth", "Messages sent per request", Count(deepest), "deepest", + [ + new("single", "1", depth.Count(count => count == 1), "#cfe8a9"), + new("short", "2–4", depth.Count(count => count is >= 2 and <= 4), "#a4d36a"), + new("medium", "5–9", depth.Count(count => count is >= 5 and <= 9), "#6fae32"), + new("long", "10+", depth.Count(count => count >= 10), "#3f7a17"), + ], Form: LlmTestServerInsightForm.Columns)); + var recent = visible.Take(48).Reverse().Select(trace => new LlmTestServerInsightRequest(trace.RequestId, + usageFor(trace)?.TotalTokens ?? 0, OutcomeColor(trace), OutcomeLabel(trace), + $"{modelName(trace)} · {providerName(trace.ProviderId)}\n{(usageFor(trace) is { } usage ? LlmTestServerDashboardCharts.Tokens(usage.TotalTokens) + " tokens" : "No usage reported")} · {OutcomeLabel(trace)}\n{trace.StartedAt.ToLocalTime():HH:mm:ss}")) + .ToArray(); + return new(visible.Count, charts, recent); + } + + public static IReadOnlyDictionary CategoricalColors(IReadOnlyList all, + Func key) + { + return Categorical(all, all, key, value => value, _ => 1) + .Where(segment => segment.Key != OtherKey) + .ToDictionary(segment => segment.Key, segment => segment.Color, StringComparer.Ordinal); + } + + private static IReadOnlyList Categorical( + IReadOnlyList visible, + IReadOnlyList all, + Func key, + Func label, + Func weight) + { + // Slots are ranked over the whole journal, so a filter never repaints the entities that remain visible. + var slots = all.GroupBy(key, StringComparer.Ordinal) + .OrderByDescending(group => group.Count()) + .ThenBy(group => group.Key, StringComparer.Ordinal) + .Take(_categorical.Length) + .Select((group, index) => (group.Key, Index: index)) + .ToDictionary(slot => slot.Key, slot => slot.Index, StringComparer.Ordinal); + var totals = visible.GroupBy(key, StringComparer.Ordinal) + .ToDictionary(group => group.Key, group => group.Sum(weight), StringComparer.Ordinal); + var segments = slots.OrderBy(slot => slot.Value) + .Where(slot => totals.ContainsKey(slot.Key)) + .Select(slot => new LlmTestServerDashboardSegment(slot.Key, label(slot.Key), totals[slot.Key], _categorical[slot.Value])) + .ToList(); + var other = totals.Where(total => !slots.ContainsKey(total.Key)).Sum(total => total.Value); + if (other > 0) + { + segments.Add(new(OtherKey, "Other", other, _otherColor)); + } + + return segments; + } + + private static readonly (string Key, string Label, string Color)[] _outcomes = + [ + ("success", "Succeeded", "#3f8d55"), + ("client-error", "Client errors (4xx)", "#e8b445"), + ("server-error", "Server errors (5xx)", "#c94f3d"), + ("transport", "Transport failures", "#ed732e"), + ("running", "In flight", _otherColor), + ]; + + private static IReadOnlyList Outcomes(IReadOnlyList visible) + { + return _outcomes.Select(outcome => new LlmTestServerDashboardSegment(outcome.Key, outcome.Label, + visible.Count(trace => OutcomeKey(trace) == outcome.Key), outcome.Color)).ToArray(); + } + + private static string OutcomeKey(LlmTestServerProviderHttpTrace trace) + { + if (!trace.IsCompleted) + { + return "running"; + } + + if (trace.TransportState != LlmTestServerProviderHttpTraceTransportState.Completed || trace.StatusCode is null) + { + return "transport"; + } + + return trace.StatusCode switch + { + < 400 => "success", + < 500 => "client-error", + _ => "server-error", + }; + } + + private static string OutcomeColor(LlmTestServerProviderHttpTrace trace) + { + var key = OutcomeKey(trace); + return _outcomes.First(outcome => outcome.Key == key).Color; + } + + private static string OutcomeLabel(LlmTestServerProviderHttpTrace trace) + { + var key = OutcomeKey(trace); + return _outcomes.First(outcome => outcome.Key == key).Label; + } + + private static IReadOnlyList Latency(IReadOnlyList visible) + { + // An ordinal ramp: lighter is faster. + var durations = visible.Select(trace => trace.DurationMilliseconds).ToArray(); + return + [ + new("fast", "<10 ms", durations.Count(value => value < 10), "#86b6ef"), + new("quick", "10–100", durations.Count(value => value is >= 10 and < 100), "#3987e5"), + new("slow", "0.1–1 s", durations.Count(value => value is >= 100 and < 1000), "#256abf"), + new("slowest", "≥1 s", durations.Count(value => value >= 1000), "#104281"), + new("running", "In flight", durations.Count(value => value is null), _otherColor), + ]; + } + + private static string Operation(LlmTestServerProviderHttpTrace trace) + { + return string.IsNullOrWhiteSpace(trace.OperationId) ? trace.Path : trace.OperationId; + } + + private static string Count(int value) + { + return LlmTestServerDashboardCharts.Tokens(value); + } + + private static string FormatLatency(double milliseconds) + { + return milliseconds < 1 ? "<1 ms" : $"{LlmTestServerDashboardCharts.Tokens((long)Math.Round(milliseconds))} ms"; + } +} + +internal sealed record LlmTestServerInsightCardView( + LlmTestServerInsightChart Chart, + string TestId, + string? SelectedKey, + IReadOnlyList Visible, + IReadOnlyList Arcs, + IReadOnlyList Dots, + long Max, + long Total) +{ + private const int _dotCount = 100; + + public string AriaSummary => $"{Chart.Title}: " + string.Join(", ", Visible.Select(segment => + $"{segment.Label} {LlmTestServerDashboardCharts.Tokens(segment.Tokens)}")); + + public static LlmTestServerInsightCardView Create(LlmTestServerInsightChart chart, string testId, string? selectedKey) + { + var visible = chart.Segments.Where(segment => segment.Tokens > 0).ToArray(); + var total = visible.Sum(segment => segment.Tokens); + return new(chart, testId, selectedKey, visible, + chart.Form == LlmTestServerInsightForm.Donut ? LlmTestServerDashboardCharts.Donut(visible) : [], + chart.Form == LlmTestServerInsightForm.Dots ? DotColors(visible, total) : [], + Math.Max(1, chart.Segments.Select(segment => segment.Tokens).DefaultIfEmpty(0).Max()), total); + } + + private static IReadOnlyList DotColors(IReadOnlyList visible, long total) + { + if (total <= 0) + { + return []; + } + + // Largest-remainder rounding keeps the dot count exact. + var exact = visible.Select(segment => (Segment: segment, Value: segment.Tokens * (double)_dotCount / total)).ToArray(); + var counts = exact.Select(item => (int)Math.Floor(item.Value)).ToArray(); + foreach (var index in exact.Select((item, index) => (Remainder: item.Value - Math.Floor(item.Value), Index: index)) + .OrderByDescending(item => item.Remainder).Take(_dotCount - counts.Sum()).Select(item => item.Index)) + { + counts[index]++; + } + + return exact.SelectMany((item, index) => Enumerable.Repeat(item.Segment.Color, counts[index])).ToArray(); + } +} diff --git a/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.Conversations.cs b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.Conversations.cs new file mode 100644 index 0000000..91b469b --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.Conversations.cs @@ -0,0 +1,37 @@ +using ManagedCode.LlmTestServer.Control; +using ManagedCode.LlmTestServer.Conversations; +using ManagedCode.LlmTestServer.Runtime; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Routing; + +namespace ManagedCode.LlmTestServer.Hosting; + +public static partial class LlmTestServerEndpointRouteBuilderExtensions +{ + private static void MapConversationEndpoints(RouteGroupBuilder endpoints) + { + endpoints.MapGet(LlmTestServerControlRoutes.Conversations, (HttpContext context, ILlmTestServerRuntime runtime) => + { + var unauthorized = AuthorizeControlRequest(context, runtime); + if (unauthorized is not null) { return unauthorized; } + var selected = context.Request.Headers[LlmTestServerConversationHeaders.Id].FirstOrDefault(); + var states = ((ILlmTestServerConversationDiagnostics)runtime).GetConversations(); + return Results.Json(selected is null ? states : states.Where(state => state.ConversationId == selected)); + }); + endpoints.MapPost(LlmTestServerControlRoutes.ExplainMatch, async (HttpContext context, ILlmTestServerRuntime runtime, CancellationToken cancellationToken) => + { + var unauthorized = AuthorizeControlRequest(context, runtime); + if (unauthorized is not null) { return unauthorized; } + var read = await ReadJsonAsync(context, cancellationToken).ConfigureAwait(false); + if (read.Error is not null) { return read.Error; } + if (read.Value is null) { return InvalidRequest("Missing match explanation request."); } + try + { + return Results.Json(((ILlmTestServerConversationDiagnostics)runtime).Explain(read.Value.Request, + read.Value.ProviderNamespace, context.Request.Headers[LlmTestServerConversationHeaders.Id].FirstOrDefault(), ReadAccessToken(context))); + } + catch (ArgumentException) { return InvalidRequest("Invalid match explanation request."); } + }); + } +} diff --git a/src/ManagedCode.LlmTck.Hosting/LlmTckEndpointRouteBuilderExtensions.Decisions.cs b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.Decisions.cs similarity index 77% rename from src/ManagedCode.LlmTck.Hosting/LlmTckEndpointRouteBuilderExtensions.Decisions.cs rename to src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.Decisions.cs index 78e784d..5dcca93 100644 --- a/src/ManagedCode.LlmTck.Hosting/LlmTckEndpointRouteBuilderExtensions.Decisions.cs +++ b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.Decisions.cs @@ -1,19 +1,19 @@ using System.Globalization; using System.Text.Json; -using ManagedCode.LlmTck.Cloudflare; -using ManagedCode.LlmTck.Decisions; -using ManagedCode.LlmTck.OpenAI; -using ManagedCode.LlmTck.Providers; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.SystemOne; +using ManagedCode.LlmTestServer.Cloudflare; +using ManagedCode.LlmTestServer.Decisions; +using ManagedCode.LlmTestServer.OpenAI; +using ManagedCode.LlmTestServer.Providers; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.SystemOne; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Routing; -using ProviderRoutes = ManagedCode.LlmTck.Providers.LlmTckProviderRouteNamespaces; +using ProviderRoutes = ManagedCode.LlmTestServer.Providers.LlmTestServerProviderRouteNamespaces; -namespace ManagedCode.LlmTck.Hosting; +namespace ManagedCode.LlmTestServer.Hosting; -public static partial class LlmTckEndpointRouteBuilderExtensions +public static partial class LlmTestServerEndpointRouteBuilderExtensions { private const string _clefModel = "clef"; private const string _clefFlashModel = "clef-flash"; @@ -26,16 +26,16 @@ public static partial class LlmTckEndpointRouteBuilderExtensions private static void MapDecisionEndpoints(RouteGroupBuilder endpoints) { endpoints.MapPost(ProviderRoutes.ForProvider(ProviderRoutes.SystemOne, _systemOnePath), CreateSystemOneDecisionAsync) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.SystemOne.DecisionsCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.SystemOne.DecisionsCreate); endpoints.MapPost(ProviderRoutes.ForProvider(ProviderRoutes.OpenAI, _decisionsPath), CreateOpenAiDecisionAsync) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.OpenAI.DecisionsCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.OpenAI.DecisionsCreate); endpoints.MapPost(ProviderRoutes.ForProvider(ProviderRoutes.Cloudflare, _clefPath), CreateCloudflareDecisionAsync) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.Cloudflare.ClefDecisionsCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.Cloudflare.ClefDecisionsCreate); endpoints.MapPost(ProviderRoutes.ForProvider(ProviderRoutes.Cloudflare, _clefFlashPath), CreateCloudflareDecisionAsync) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.Cloudflare.ClefFlashDecisionsCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.Cloudflare.ClefFlashDecisionsCreate); } - private static async Task CreateOpenAiDecisionAsync(HttpContext context, ILlmTckRuntime runtime, CancellationToken cancellationToken) + private static async Task CreateOpenAiDecisionAsync(HttpContext context, ILlmTestServerRuntime runtime, CancellationToken cancellationToken) { var read = await ReadJsonAsync(context, cancellationToken, OpenAiDecisionMapper.Validate).ConfigureAwait(false); if (read.Error is not null) { return read.Error; } @@ -45,10 +45,10 @@ private static async Task CreateOpenAiDecisionAsync(HttpContext context : Results.Json(OpenAiWireMapper.ToError(result.ErrorCode!, result.ErrorMessage!), statusCode: result.StatusCode); } - private static async Task CreateSystemOneDecisionAsync(HttpContext context, ILlmTckRuntime runtime, CancellationToken cancellationToken) + private static async Task CreateSystemOneDecisionAsync(HttpContext context, ILlmTestServerRuntime runtime, CancellationToken cancellationToken) { SetSystemOneRequestId(context); - var provider = LlmTckDecisionProvider.TypeSafe; + var provider = LlmTestServerDecisionProvider.TypeSafe; var typeSafeValidation = new TypeSafeValidationResult(); JsonReadResult read; if (context.Request.ContentType?.StartsWith("multipart/form-data", StringComparison.OrdinalIgnoreCase) == true) @@ -66,13 +66,13 @@ private static async Task CreateSystemOneDecisionAsync(HttpContext cont read = await ReadJsonAsync(context, SystemOneValidationError, cancellationToken, body => { provider = ResolveSystemOneProvider(body, runtime); - if (provider == LlmTckDecisionProvider.TypeSafe) + if (provider == LlmTestServerDecisionProvider.TypeSafe) { typeSafeValidation = (context.Request.ContentLength ?? context.Request.Body.Length) > TypeSafeRequestValidation.MaxRequestBytes ? new TypeSafeValidationResult("TypeSafe request body exceeds 32 MiB.", StatusCodes.Status413PayloadTooLarge) : TypeSafeRequestValidation.Validate(body, context.Request.Headers[TypeSafeRequestValidation.ImagesHeader] == TypeSafeRequestValidation.Enabled); - return new LlmTckRequestValidationResult(typeSafeValidation.Error); + return new LlmTestServerRequestValidationResult(typeSafeValidation.Error); } return SystemOneDecisionMapper.Validate(body, provider); }).ConfigureAwait(false); @@ -88,29 +88,29 @@ private static async Task CreateSystemOneDecisionAsync(HttpContext cont var result = await runtime.DecideAsync(SystemOneDecisionMapper.ToRequest(read.Value, provider), ReadAccessToken(context), cancellationToken).ConfigureAwait(false); if (!result.IsSuccess) { - if (provider == LlmTckDecisionProvider.Kev && result.StatusCode == StatusCodes.Status401Unauthorized) + if (provider == LlmTestServerDecisionProvider.Kev && result.StatusCode == StatusCodes.Status401Unauthorized) { context.Response.Headers.WWWAuthenticate = "Bearer"; } return Results.Json(new { detail = result.ErrorMessage }, statusCode: result.StatusCode); } if (result.Answers.Values.Any(answer => answer.Refused)) { return Results.Json(new { detail = "SystemOne does not support refusal answer fixtures." }, statusCode: StatusCodes.Status409Conflict); } - if (provider == LlmTckDecisionProvider.Kev) + if (provider == LlmTestServerDecisionProvider.Kev) { context.Response.Headers["server-timing"] = "app;dur=" + Math.Round(result.Metadata.LatencyMilliseconds, 1).ToString("F1", CultureInfo.InvariantCulture); } - return Results.Json(SystemOneDecisionMapper.ToResponse(read.Value, result, provider == LlmTckDecisionProvider.Kev, provider), SystemOneNativeJson.Options); + return Results.Json(SystemOneDecisionMapper.ToResponse(read.Value, result, provider == LlmTestServerDecisionProvider.Kev, provider), SystemOneNativeJson.Options); } - private static LlmTckDecisionProvider ResolveSystemOneProvider(JsonElement body, ILlmTckRuntime runtime) + private static LlmTestServerDecisionProvider ResolveSystemOneProvider(JsonElement body, ILlmTestServerRuntime runtime) { var modelId = body.ValueKind == JsonValueKind.Object && body.TryGetProperty("model", out var model) && model.ValueKind == JsonValueKind.String ? model.GetString() : SystemOneDecisionMapper.KevDefaultModel; var configured = runtime.GetModels().FirstOrDefault(item => item.Id == modelId); - return configured?.DecisionProvider == LlmTckDecisionProvider.Kev ? LlmTckDecisionProvider.Kev : LlmTckDecisionProvider.TypeSafe; + return configured?.DecisionProvider == LlmTestServerDecisionProvider.Kev ? LlmTestServerDecisionProvider.Kev : LlmTestServerDecisionProvider.TypeSafe; } private static void SetSystemOneRequestId(HttpContext context) { var supplied = context.Request.Headers[_typeSafeRequestIdHeader].ToString(); - context.Response.Headers[_typeSafeRequestIdHeader] = string.IsNullOrEmpty(supplied) ? Guid.NewGuid().ToString("N") : supplied; + context.Response.Headers[_typeSafeRequestIdHeader] = string.IsNullOrEmpty(supplied) ? WireMetadata(context).IdFor("request") : supplied; } private static IResult SystemOneValidationError(string message) @@ -118,7 +118,7 @@ private static IResult SystemOneValidationError(string message) return Results.Json(new { detail = new[] { new { loc = new object[] { "body" }, msg = message, type = "value_error" } } }, statusCode: StatusCodes.Status422UnprocessableEntity); } - private static async Task CreateCloudflareDecisionAsync(HttpContext context, ILlmTckRuntime runtime, CancellationToken cancellationToken) + private static async Task CreateCloudflareDecisionAsync(HttpContext context, ILlmTestServerRuntime runtime, CancellationToken cancellationToken) { if (!await CloudflareDecisionRequestBody.FitsLimitAsync(context.Request, cancellationToken).ConfigureAwait(false)) { return CloudflareDecisionFailure(StatusCodes.Status413PayloadTooLarge, "Workers AI request exceeds the native body limit.", "3006"); } @@ -128,20 +128,20 @@ private static async Task CreateCloudflareDecisionAsync(HttpContext con cancellationToken, body => { validation = CloudflareDecisionValidation.ValidateNative(body); - return new LlmTckRequestValidationResult(validation.Error); + return new LlmTestServerRequestValidationResult(validation.Error); }).ConfigureAwait(false); if (read.Error is not null) { return read.Error; } if (read.Value is null) { return CloudflareDecisionFailure(StatusCodes.Status400BadRequest, "Missing decision body."); } var native = CloudflareDecisionMapper.ToSystemOne(read.Value); var expected = context.Request.Path.Value!.EndsWith(_clefFlashModel, StringComparison.Ordinal) ? _clefFlashModel : _clefModel; if (native.Model != expected) { return CloudflareDecisionFailure(StatusCodes.Status400BadRequest, "Body model must match the Workers AI route."); } - var result = await runtime.DecideAsync(CloudflareDecisionMapper.ToRequest(read.Value) with { Provider = LlmTckDecisionProvider.Cloudflare }, ReadAccessToken(context), cancellationToken).ConfigureAwait(false); + var result = await runtime.DecideAsync(CloudflareDecisionMapper.ToRequest(read.Value) with { Provider = LlmTestServerDecisionProvider.Cloudflare }, ReadAccessToken(context), cancellationToken).ConfigureAwait(false); if (!result.IsSuccess) { return CloudflareDecisionFailure(result.StatusCode, result.ErrorMessage!, result.ErrorCode); } if (result.Answers.Values.Any(answer => answer.Refused)) { return CloudflareDecisionFailure(StatusCodes.Status409Conflict, "SystemOne does not support refusal answer fixtures."); } return Results.Json(new CloudflareDecisionResponse { Success = true, - Result = SystemOneDecisionMapper.ToResponse(native, result, true, LlmTckDecisionProvider.Cloudflare) + Result = SystemOneDecisionMapper.ToResponse(native, result, true, LlmTestServerDecisionProvider.Cloudflare) }); } diff --git a/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.GeminiBatchEmbeddings.cs b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.GeminiBatchEmbeddings.cs new file mode 100644 index 0000000..479e75d --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.GeminiBatchEmbeddings.cs @@ -0,0 +1,40 @@ +using ManagedCode.LlmTestServer.Gemini; +using ManagedCode.LlmTestServer.Runtime; +using Microsoft.AspNetCore.Http; + +namespace ManagedCode.LlmTestServer.Hosting; + +public static partial class LlmTestServerEndpointRouteBuilderExtensions +{ + private static async Task CreateGeminiBatchEmbeddingsAsync(string model, HttpContext context, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken) + { + var read = await ReadJsonAsync(context, GeminiInvalidRequest, cancellationToken).ConfigureAwait(false); + if (read.Error is not null) { return read.Error; } + if (read.Value?.Requests is not { Count: > 0 and <= 128 } requests) + { return GeminiInvalidRequest("A batch requires between 1 and 128 embedding requests."); } + var inputs = new List(); + foreach (var request in requests) + { + if (request is null || request.Model != "models/" + model) + { return GeminiInvalidRequest("Every embedding request model must match the batch path model."); } + var validation = ValidateGeminiEmbeddingRequest(model, new() { Model = request.Model, Content = request.Content }); + if (validation is not null) { return validation; } + if (request.Content.Parts.Count > 128 || request.Content.Parts.Any(part => part.Text is null + || part.FunctionCall is not null || part.FunctionResponse is not null || part.InlineData is not null + || part.InlineDataSnakeCase is not null || part.FileData is not null || part.FileDataSnakeCase is not null)) + { return GeminiInvalidRequest("This embedding fixture supports only text parts."); } + if (request.OutputDimensionality is < 128 or > 3072) + { return GeminiInvalidRequest("The supported reduced embedding dimensionality is between 128 and 3072."); } + inputs.Add(GeminiWireMapper.ReadText(request.Content)); + } + var result = await runtime.CreateEmbeddingAsync(model, inputs, ReadAccessToken(context), requests.Select(request => request.OutputDimensionality).ToArray(), cancellationToken).ConfigureAwait(false); + if (!result.IsSuccess) + { return Results.Json(GeminiWireMapper.ToError(result.StatusCode, result.ErrorMessage!), statusCode: result.StatusCode); } + return Results.Json(new GeminiBatchEmbedContentsResponse + { + Embeddings = result.Vectors.Select(vector => new GeminiContentEmbedding { Values = vector }).ToList(), + UsageMetadata = new() { PromptTokenCount = inputs.Sum(LlmTestServerTokenCounter.CountTextTokens) }, + }); + } +} diff --git a/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.RequestHeaders.cs b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.RequestHeaders.cs new file mode 100644 index 0000000..32ec742 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.RequestHeaders.cs @@ -0,0 +1,20 @@ +using ManagedCode.LlmTestServer.Conversations; +using ManagedCode.LlmTestServer.Diagnostics; +using Microsoft.AspNetCore.Http; + +namespace ManagedCode.LlmTestServer.Hosting; + +public static partial class LlmTestServerEndpointRouteBuilderExtensions +{ + private static IResult? ValidateCorrelationHeaders(HttpContext context) + { + foreach (var name in new[] { LlmTestServerConversationHeaders.Id, LlmTestServerRequestHeaders.RetryGroup }) + { + var values = context.Request.Headers[name]; + if (context.Request.Headers.ContainsKey(name) + && (values.Count != 1 || string.IsNullOrWhiteSpace(values[0]) || values[0]!.Length > 128)) + { return InvalidRequest("Supply one nonempty correlation ID of at most 128 characters per header."); } + } + return null; + } +} diff --git a/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.Requests.cs b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.Requests.cs new file mode 100644 index 0000000..e64a167 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.Requests.cs @@ -0,0 +1,115 @@ +using System.Globalization; +using System.Text.Json; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Runtime; +using Microsoft.AspNetCore.Http; + +namespace ManagedCode.LlmTestServer.Hosting; + +public static partial class LlmTestServerEndpointRouteBuilderExtensions +{ + private static IResult GetRequests(HttpContext context, ILlmTestServerRuntime runtime, ILlmTestServerProviderHttpTraceStore traceStore) + { + return QueryRequests(context, runtime, traceStore, export: false); + } + + private static IResult ExportRequests(HttpContext context, ILlmTestServerRuntime runtime, ILlmTestServerProviderHttpTraceStore traceStore) + { + return QueryRequests(context, runtime, traceStore, export: true); + } + + private static IResult QueryRequests(HttpContext context, ILlmTestServerRuntime runtime, ILlmTestServerProviderHttpTraceStore traceStore, bool export) + { + var unauthorized = AuthorizeControlRequest(context, runtime); + if (unauthorized is not null) { return unauthorized; } + try + { + var query = ReadRequestQuery(context.Request); + var evidence = LlmTestServerRequestQueries.Apply(traceStore.GetEvidenceSnapshot(), query); + context.Response.Headers["X-Total-Count"] = evidence.TotalMatched.ToString(CultureInfo.InvariantCulture); + context.Response.Headers["X-Total-Observed"] = evidence.TotalObserved.ToString(CultureInfo.InvariantCulture); + context.Response.Headers["X-Evicted-Count"] = evidence.EvictedCount.ToString(CultureInfo.InvariantCulture); + context.Response.Headers["X-Retained-Byte-Count"] = evidence.RetainedByteCount.ToString(CultureInfo.InvariantCulture); + context.Response.Headers["X-Retention-Truncated"] = evidence.RetentionTruncated ? "true" : "false"; + if (!export) { return Results.Json(evidence.Requests); } + var format = ReadSingle(context.Request, "format") ?? "json"; + var envelope = new LlmTestServerRequestExport { Query = query, Evidence = evidence }; + if (format == "json") { return Results.Json(envelope); } + if (format != "ndjson") { return InvalidRequest("format must be json or ndjson."); } + return new RequestNdjsonResult(envelope); + } + catch (ArgumentException) { return InvalidRequest("Invalid request query parameters."); } + } + + private static LlmTestServerRequestQuery ReadRequestQuery(HttpRequest request) + { + int? Number(string key) + { + return ReadSingle(request, key) is { } value ? int.Parse(value, NumberStyles.Integer, CultureInfo.InvariantCulture) : null; + } + + bool? Boolean(string key) + { + return ReadSingle(request, key) is { } value ? bool.Parse(value) : null; + } + + DateTimeOffset? Date(string key) + { + return ReadSingle(request, key) is { } value ? DateTimeOffset.Parse(value, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind) : null; + } + + try + { + var order = ReadSingle(request, "order") ?? "desc"; + return new() + { + Provider = ReadSingle(request, "provider"), + Model = ReadSingle(request, "model"), + RequestId = ReadSingle(request, "requestId"), + Status = Number("status"), + Offset = Number("offset") ?? 0, + Limit = Number("limit") ?? 500, + Operation = ReadSingle(request, "operation"), + Scenario = ReadSingle(request, "scenario"), + Conversation = ReadSingle(request, "conversation"), + Step = ReadSingle(request, "step"), + Streaming = Boolean("streaming"), + Completed = Boolean("completed"), + Success = Boolean("success"), + ErrorCode = ReadSingle(request, "errorCode"), + From = Date("from"), + To = Date("to"), + Order = order switch { "asc" => LlmTestServerRequestOrder.Ascending, "desc" => LlmTestServerRequestOrder.Descending, _ => throw new ArgumentException("Invalid order.") } + }; + } + catch (FormatException exception) { throw new ArgumentException("Invalid request query format.", exception); } + catch (OverflowException exception) { throw new ArgumentException("Invalid request query number.", exception); } + } + + private static string? ReadSingle(HttpRequest request, string name) + { + var values = request.Query[name]; + if (values.Count > 1 || values.Count == 1 && string.IsNullOrWhiteSpace(values[0])) { throw new ArgumentException("Query values must occur once and be nonblank.", nameof(name)); } + return values.Count == 0 ? null : values[0]; + } + + private sealed class RequestNdjsonResult(LlmTestServerRequestExport export) : IResult + { + public async Task ExecuteAsync(HttpContext context) + { + context.Response.ContentType = "application/x-ndjson; charset=utf-8"; + var options = new JsonSerializerOptions(JsonSerializerDefaults.Web); + await context.Response.WriteAsync(JsonSerializer.Serialize(new + { + type = "metadata", + export.SchemaVersion, + export.Query, + evidence = export.Evidence with { Requests = [] } + }, options) + "\n", context.RequestAborted).ConfigureAwait(false); + foreach (var entry in export.Evidence.Requests) + { + await context.Response.WriteAsync(JsonSerializer.Serialize(new { type = "request", request = entry }, options) + "\n", context.RequestAborted).ConfigureAwait(false); + } + } + } +} diff --git a/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.Sessions.cs b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.Sessions.cs new file mode 100644 index 0000000..c944bde --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.Sessions.cs @@ -0,0 +1,56 @@ +using System.Text.Json; +using ManagedCode.LlmTestServer.Control; +using ManagedCode.LlmTestServer.Sessions; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Routing; + +namespace ManagedCode.LlmTestServer.Hosting; + +public static partial class LlmTestServerEndpointRouteBuilderExtensions +{ + private static void MapSessionEndpoints(IEndpointRouteBuilder endpoints) + { + endpoints.MapPost(LlmTestServerControlRoutes.Sessions, async (HttpContext context, LlmTestServerSessionRegistry registry, CancellationToken cancellationToken) => + { + var unauthorized = AuthorizeControlRequest(context, registry.Default.Runtime); + if (unauthorized is not null) { return unauthorized; } + if (context.Request.Headers.ContainsKey(LlmTestServerSessionHeaders.Id) + || context.Request.Headers.ContainsKey(LlmTestServerSessionHeaders.AccessKey)) + { return InvalidRequest("Create sessions through the default management client."); } + var read = await ReadJsonAsync(context, cancellationToken).ConfigureAwait(false); + if (read.Error is not null) { return read.Error; } + if (read.Value is null) { return InvalidRequest("Missing session body."); } + try + { + var session = registry.Create(read.Value.Configuration); + context.Response.Headers.CacheControl = "no-store"; + return session is null ? SessionCapacityExceeded() + : Results.Created(LlmTestServerControlRoutes.Sessions + "/" + session.Id, session); + } + catch (ArgumentException) { return InvalidRequest("Invalid session configuration."); } + catch (JsonException) { return InvalidRequest("Invalid JSON fixture in session configuration."); } + }); + endpoints.MapDelete(LlmTestServerControlRoutes.Sessions + "/{id}", async (string id, HttpContext context, LlmTestServerSessionRegistry registry) => + { + var state = registry.Resolve(context); + if (state is null || ReferenceEquals(state, registry.Default)) { return SessionNotFound(); } + var removed = registry.DeleteAsync(id, state); + if (removed is null) { return SessionNotFound(); } + await removed.ConfigureAwait(false); + return Results.NoContent(); + }); + } + + private static IResult SessionNotFound() + { + return Results.Json( + new { error = new { code = "session_not_found", message = "The test session is unavailable with the supplied session credentials." } }, statusCode: 404); + } + + private static IResult SessionCapacityExceeded() + { + return Results.Json( + new { error = new { code = "session_capacity_exceeded", message = "The configured test session capacity is exhausted." } }, statusCode: 409); + } +} diff --git a/src/ManagedCode.LlmTck.Hosting/LlmTckEndpointRouteBuilderExtensions.Streaming.cs b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.Streaming.cs similarity index 62% rename from src/ManagedCode.LlmTck.Hosting/LlmTckEndpointRouteBuilderExtensions.Streaming.cs rename to src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.Streaming.cs index 6d971f6..d407f04 100644 --- a/src/ManagedCode.LlmTck.Hosting/LlmTckEndpointRouteBuilderExtensions.Streaming.cs +++ b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.Streaming.cs @@ -1,21 +1,21 @@ using System.Text.Json; -using ManagedCode.LlmTck.Anthropic; -using ManagedCode.LlmTck.Bedrock; -using ManagedCode.LlmTck.Cohere; -using ManagedCode.LlmTck.Gemini; -using ManagedCode.LlmTck.Ollama; -using ManagedCode.LlmTck.OpenAI; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Scenarios; +using ManagedCode.LlmTestServer.Anthropic; +using ManagedCode.LlmTestServer.Bedrock; +using ManagedCode.LlmTestServer.Cohere; +using ManagedCode.LlmTestServer.Gemini; +using ManagedCode.LlmTestServer.Ollama; +using ManagedCode.LlmTestServer.OpenAI; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Scenarios; using Microsoft.AspNetCore.Http; -namespace ManagedCode.LlmTck.Hosting; +namespace ManagedCode.LlmTestServer.Hosting; -public static partial class LlmTckEndpointRouteBuilderExtensions +public static partial class LlmTestServerEndpointRouteBuilderExtensions { private static async Task WriteStreamingChatAsync( HttpContext context, - LlmTckChatResult result, + LlmTestServerChatResult result, bool includeUsage, OpenAiCacheUsageShape cacheUsageShape, CancellationToken cancellationToken @@ -23,27 +23,39 @@ CancellationToken cancellationToken { context.Response.StatusCode = StatusCodes.Status200OK; context.Response.ContentType = "text/event-stream"; - var responseId = OpenAiWireMapper.CreateResponseId(); - var created = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); + var responseId = OpenAiWireMapper.CreateResponseId(result.WireMetadata); + var created = (result.WireMetadata ?? WireMetadata(context)).CreatedAt.ToUnixTimeSeconds(); + var isDeepSeek = context.Request.Path.Equals("/deepseek/v1/chat/completions", StringComparison.Ordinal); + var isModelInference = context.Request.Path.Equals("/microsoft-foundry/chat/completions", StringComparison.Ordinal) + || context.Request.Path.Equals("/microsoft-foundry/models/chat/completions", StringComparison.Ordinal); + LlmTestServerStreamDeliveryEvidence.Begin(context, result.StreamDelivery); foreach (var chunk in result.ToolCalls.Count > 0 ? new List { result.Content } : result.StreamChunks) { - var payload = JsonSerializer.Serialize( - OpenAiWireMapper.ToChatChunk(result, chunk, responseId, created), - _jsonOptions - ); + var contentChunk = OpenAiWireMapper.ToChatChunk(result, chunk, responseId, created); + var payload = JsonSerializer.Serialize(contentChunk, _jsonOptions); + if (isDeepSeek && !includeUsage) + { + var frame = JsonSerializer.SerializeToNode(contentChunk, _jsonOptions)!.AsObject(); + frame.Remove("usage"); + payload = frame.ToJsonString(_jsonOptions); + } await context.Response.WriteAsync($"data: {payload}\n\n", cancellationToken) .ConfigureAwait(false); await context.Response.Body.FlushAsync(cancellationToken).ConfigureAwait(false); + if (!await LlmTestServerStreamDeliveryEvidence.AfterContentFrameAsync(context, chunk, cancellationToken).ConfigureAwait(false)) { return; } } - var finalPayload = JsonSerializer.Serialize( - OpenAiWireMapper.ToChatChunk(result, string.Empty, responseId, created, result.ToolCalls.Count > 0 ? "tool_calls" : "stop"), - _jsonOptions - ); + var finalChunk = OpenAiWireMapper.ToChatChunk(result, string.Empty, responseId, created, + result.OutputDisposition?.Kind == LlmTestServerOutputDispositionKind.Filtered ? "content_filter" : result.ToolCalls.Count > 0 ? "tool_calls" : "stop"); + if (isDeepSeek || isModelInference) + { + finalChunk = finalChunk with { Usage = OpenAiWireMapper.ToChatResponse(result, cacheUsageShape).Usage }; + } + var finalPayload = JsonSerializer.Serialize(finalChunk, _jsonOptions); await context.Response.WriteAsync($"data: {finalPayload}\n\n", cancellationToken) .ConfigureAwait(false); - if (includeUsage) + if (includeUsage && !isDeepSeek && !isModelInference) { var usageChunk = new OpenAiChatCompletionChunk { @@ -57,24 +69,36 @@ await context.Response.WriteAsync($"data: {JsonSerializer.Serialize(usageChunk, } await context.Response.WriteAsync("data: [DONE]\n\n", cancellationToken).ConfigureAwait(false); + await context.Response.Body.FlushAsync(cancellationToken).ConfigureAwait(false); + context.MarkLlmTestServerTerminalMarker(); } private static async Task WriteStreamingResponseAsync( HttpContext context, - LlmTckChatResult result, + LlmTestServerChatResult result, string responseId, long created, OpenAiCacheUsageShape cacheUsageShape, - CancellationToken cancellationToken + CancellationToken cancellationToken, + IReadOnlyCollection? includedProperties = null ) { context.Response.StatusCode = StatusCodes.Status200OK; context.Response.ContentType = "text/event-stream"; - foreach (var streamEvent in OpenAiWireMapper.ToResponseStreamEvents(result, responseId, created, cacheUsageShape)) + LlmTestServerStreamDeliveryEvidence.Begin(context, result.StreamDelivery); + foreach (var streamEvent in OpenAiWireMapper.ToResponseStreamEvents(result, responseId, created, cacheUsageShape, includedProperties)) { await WriteResponseSseDataAsync(context, streamEvent, cancellationToken).ConfigureAwait(false); + if (result.StreamDelivery is not null) + { + var frame = JsonSerializer.SerializeToElement(streamEvent, _jsonOptions); + if (frame.GetProperty("type").GetString() == "response.output_text.delta" + && !await LlmTestServerStreamDeliveryEvidence.AfterContentFrameAsync(context, frame.GetProperty("delta").GetString()!, cancellationToken).ConfigureAwait(false)) { return; } + } } + await context.Response.Body.FlushAsync(cancellationToken).ConfigureAwait(false); + context.MarkLlmTestServerTerminalMarker(); } private static async Task WriteResponseSseDataAsync( @@ -91,7 +115,7 @@ await context.Response.WriteAsync($"data: {payload}\n\n", cancellationToken) private static async Task WriteStreamingTranscriptionAsync( HttpContext context, - LlmTckTranscriptionResult result, + LlmTestServerTranscriptionResult result, CancellationToken cancellationToken ) { @@ -110,6 +134,8 @@ await WriteResponseSseDataAsync( cancellationToken ) .ConfigureAwait(false); + await context.Response.Body.FlushAsync(cancellationToken).ConfigureAwait(false); + context.MarkLlmTestServerTerminalMarker(); } private static async Task WriteImageStreamingAsync( @@ -135,7 +161,8 @@ await WriteResponseSseDataAsync( background, outputFormat, quality, - size + size, + WireMetadata(context) ), cancellationToken ) @@ -149,16 +176,19 @@ await WriteResponseSseDataAsync( background, outputFormat, quality, - size + size, + WireMetadata(context) ), cancellationToken ) .ConfigureAwait(false); + await context.Response.Body.FlushAsync(cancellationToken).ConfigureAwait(false); + context.MarkLlmTestServerTerminalMarker(); } private static async Task WriteOllamaStreamingChatAsync( HttpContext context, - LlmTckChatResult result, + LlmTestServerChatResult result, CancellationToken cancellationToken ) { @@ -182,11 +212,13 @@ await context.Response.WriteAsync($"{payload}\n", cancellationToken) ); await context.Response.WriteAsync($"{finalPayload}\n", cancellationToken) .ConfigureAwait(false); + await context.Response.Body.FlushAsync(cancellationToken).ConfigureAwait(false); + context.MarkLlmTestServerTerminalMarker(); } private static async Task WriteAnthropicStreamingMessageAsync( HttpContext context, - LlmTckChatResult result, + LlmTestServerChatResult result, CancellationToken cancellationToken, bool cacheOnly = false ) @@ -194,6 +226,7 @@ private static async Task WriteAnthropicStreamingMessageAsync( context.Response.StatusCode = StatusCodes.Status200OK; context.Response.ContentType = "text/event-stream"; + LlmTestServerStreamDeliveryEvidence.Begin(context, result.StreamDelivery); await WriteAnthropicSseEventAsync( context, "message_start", @@ -201,42 +234,54 @@ await WriteAnthropicSseEventAsync( cancellationToken ) .ConfigureAwait(false); - if (!cacheOnly && (result.ToolCalls.Count == 0 || result.Content.Length > 0)) + if (!cacheOnly && result.OutputItems.Count > 0) { - await WriteAnthropicSseEventAsync( - context, - "content_block_start", - AnthropicWireMapper.ToContentBlockStartEvent(), - cancellationToken - ) - .ConfigureAwait(false); - - foreach (var chunk in result.StreamChunks) + foreach (var streamEvent in AnthropicWireMapper.ToOrderedContentStreamEvents(result)) + { + var element = JsonSerializer.SerializeToElement(streamEvent, _jsonOptions); + await WriteAnthropicSseEventAsync(context, element.GetProperty("type").GetString()!, streamEvent, cancellationToken).ConfigureAwait(false); + } + } + else + { + if (!cacheOnly && (result.ToolCalls.Count == 0 || result.Content.Length > 0)) { await WriteAnthropicSseEventAsync( context, - "content_block_delta", - AnthropicWireMapper.ToContentBlockDeltaEvent(chunk), + "content_block_start", + AnthropicWireMapper.ToContentBlockStartEvent(), cancellationToken ) .ConfigureAwait(false); - } - await WriteAnthropicSseEventAsync( - context, - "content_block_stop", - AnthropicWireMapper.ToContentBlockStopEvent(), - cancellationToken - ) - .ConfigureAwait(false); - } + foreach (var chunk in result.StreamChunks) + { + await WriteAnthropicSseEventAsync( + context, + "content_block_delta", + AnthropicWireMapper.ToContentBlockDeltaEvent(chunk), + cancellationToken + ) + .ConfigureAwait(false); + if (!await LlmTestServerStreamDeliveryEvidence.AfterContentFrameAsync(context, chunk, cancellationToken).ConfigureAwait(false)) { return; } + } - if (!cacheOnly && result.ToolCalls.Count > 0) - { - foreach (var streamEvent in AnthropicWireMapper.ToToolStreamEvents(result, result.Content.Length > 0 ? 1 : 0)) + await WriteAnthropicSseEventAsync( + context, + "content_block_stop", + AnthropicWireMapper.ToContentBlockStopEvent(), + cancellationToken + ) + .ConfigureAwait(false); + } + + if (!cacheOnly && result.ToolCalls.Count > 0) { - var element = JsonSerializer.SerializeToElement(streamEvent, _jsonOptions); - await WriteAnthropicSseEventAsync(context, element.GetProperty("type").GetString()!, streamEvent, cancellationToken).ConfigureAwait(false); + foreach (var streamEvent in AnthropicWireMapper.ToToolStreamEvents(result, result.Content.Length > 0 ? 1 : 0)) + { + var element = JsonSerializer.SerializeToElement(streamEvent, _jsonOptions); + await WriteAnthropicSseEventAsync(context, element.GetProperty("type").GetString()!, streamEvent, cancellationToken).ConfigureAwait(false); + } } } await WriteAnthropicSseEventAsync( @@ -253,6 +298,8 @@ await WriteAnthropicSseEventAsync( cancellationToken ) .ConfigureAwait(false); + await context.Response.Body.FlushAsync(cancellationToken).ConfigureAwait(false); + context.MarkLlmTestServerTerminalMarker(); } private static async Task WriteAnthropicSseEventAsync( @@ -272,7 +319,7 @@ await context.Response.WriteAsync($"data: {payload}\n\n", cancellationToken) private static async Task WriteCohereStreamingChatAsync( HttpContext context, - LlmTckChatResult result, + LlmTestServerChatResult result, CancellationToken cancellationToken ) { @@ -282,7 +329,7 @@ CancellationToken cancellationToken await WriteCohereSseEventAsync( context, "message-start", - CohereWireMapper.ToMessageStartEvent(), + CohereWireMapper.ToMessageStartEvent(result.WireMetadata), cancellationToken ) .ConfigureAwait(false); @@ -330,6 +377,8 @@ await WriteCohereSseEventAsync( cancellationToken ) .ConfigureAwait(false); + await context.Response.Body.FlushAsync(cancellationToken).ConfigureAwait(false); + context.MarkLlmTestServerTerminalMarker(); } private static async Task WriteCohereSseEventAsync( @@ -349,19 +398,20 @@ await context.Response.WriteAsync($"data: {payload}\n\n", cancellationToken) private static async Task WriteGeminiStreamingContentAsync( HttpContext context, - LlmTckChatResult result, + LlmTestServerChatResult result, CancellationToken cancellationToken ) { context.Response.StatusCode = StatusCodes.Status200OK; var sse = string.Equals(context.Request.Query["alt"], "sse", StringComparison.Ordinal); context.Response.ContentType = sse ? "text/event-stream" : "application/json"; - var responseId = $"gemini_{Guid.NewGuid():N}"; + var responseId = "gemini-" + (result.WireMetadata ?? WireMetadata(context)).IdFor("response"); if (!sse) { await context.Response.WriteAsync("[", cancellationToken).ConfigureAwait(false); } + LlmTestServerStreamDeliveryEvidence.Begin(context, result.StreamDelivery); var first = true; foreach (var chunk in ReadStreamChunks(result)) { @@ -372,6 +422,7 @@ CancellationToken cancellationToken await context.Response.WriteAsync(sse ? $"data: {payload}\n\n" : (first ? payload : $",{payload}"), cancellationToken).ConfigureAwait(false); await context.Response.Body.FlushAsync(cancellationToken).ConfigureAwait(false); first = false; + if (!await LlmTestServerStreamDeliveryEvidence.AfterContentFrameAsync(context, chunk, cancellationToken).ConfigureAwait(false)) { return; } } var finalPayload = JsonSerializer.Serialize( @@ -379,11 +430,13 @@ CancellationToken cancellationToken _jsonOptions ); await context.Response.WriteAsync(sse ? $"data: {finalPayload}\n\n" : $",{finalPayload}]", cancellationToken).ConfigureAwait(false); + await context.Response.Body.FlushAsync(cancellationToken).ConfigureAwait(false); + context.MarkLlmTestServerTerminalMarker(); } private static async Task WriteBedrockConverseStreamAsync( HttpContext context, - LlmTckChatResult result, + LlmTestServerChatResult result, CancellationToken cancellationToken ) { @@ -441,11 +494,13 @@ await WriteBedrockEventAsync( cancellationToken ) .ConfigureAwait(false); + await context.Response.Body.FlushAsync(cancellationToken).ConfigureAwait(false); + context.MarkLlmTestServerTerminalMarker(); } private static async Task WriteBedrockInvokeModelStreamAsync( HttpContext context, - LlmTckChatResult result, + LlmTestServerChatResult result, CancellationToken cancellationToken ) { @@ -476,7 +531,7 @@ await context.Response.Body.WriteAsync(payload, cancellationToken) await context.Response.Body.FlushAsync(cancellationToken).ConfigureAwait(false); } - private static IReadOnlyList ReadStreamChunks(LlmTckChatResult result) + private static IReadOnlyList ReadStreamChunks(LlmTestServerChatResult result) { return result.StreamChunks.Count == 0 ? [result.Content] : result.StreamChunks; } @@ -490,14 +545,56 @@ private static IReadOnlyList ReadStreamChunks(LlmTckChatResult result) : null; } - private static LlmTckChatRequest CorrelateChatRequest( + private static LlmTestServerChatRequest CorrelateChatRequest( HttpContext context, - LlmTckChatRequest request + LlmTestServerChatRequest request ) { return request with { - RequestId = context.GetLlmTckProviderHttpTraceId() ?? context.TraceIdentifier, + RequestId = context.GetLlmTestServerProviderHttpTraceId() ?? context.TraceIdentifier, + StreamDeliverySurface = ResolveStreamDeliverySurface(context), + OutputDispositionSurface = ResolveOutputDispositionSurface(context), + }; + } + + private static LlmTestServerOutputDispositionSurface? ResolveOutputDispositionSurface(HttpContext context) + { + return context.Request.Path.Value switch + { + "/openai/v1/chat/completions" => LlmTestServerOutputDispositionSurface.OpenAiChatCompletions, + "/openai/v1/responses" => LlmTestServerOutputDispositionSurface.OpenAiResponses, + "/deepseek/v1/chat/completions" => LlmTestServerOutputDispositionSurface.DeepSeekChatCompletions, + "/azure-openai/openai/v1/chat/completions" => LlmTestServerOutputDispositionSurface.AzureOpenAiV1ChatCompletions, + "/azure-openai/openai/v1/responses" => LlmTestServerOutputDispositionSurface.AzureOpenAiV1Responses, + "/microsoft-foundry/openai/v1/chat/completions" => LlmTestServerOutputDispositionSurface.MicrosoftFoundryV1ChatCompletions, + "/microsoft-foundry/openai/v1/responses" => LlmTestServerOutputDispositionSurface.MicrosoftFoundryV1Responses, + "/microsoft-foundry/chat/completions" => LlmTestServerOutputDispositionSurface.MicrosoftFoundryChatCompletions, + "/microsoft-foundry/models/chat/completions" => LlmTestServerOutputDispositionSurface.MicrosoftFoundryModelsChatCompletions, + var path when context.Request.RouteValues["deployment"] is string deployment + && path == "/azure-openai/openai/deployments/" + deployment + "/chat/completions" + => LlmTestServerOutputDispositionSurface.AzureOpenAiDeploymentChatCompletions, + "/anthropic/v1/messages" => LlmTestServerOutputDispositionSurface.AnthropicMessages, + var path when context.Request.RouteValues["model"] is string model + && (path == "/gemini/v1beta/models/" + model + ":generateContent" + || path == "/gemini/v1beta/models/" + model + ":streamGenerateContent") + => LlmTestServerOutputDispositionSurface.GeminiGenerateContent, + _ => null, + }; + } + + private static LlmTestServerStreamDeliverySurface? ResolveStreamDeliverySurface(HttpContext context) + { + return context.Request.Path.Value switch + { + "/openai/v1/chat/completions" => LlmTestServerStreamDeliverySurface.OpenAiChatCompletions, + "/openai/v1/responses" => LlmTestServerStreamDeliverySurface.OpenAiResponses, + "/anthropic/v1/messages" => LlmTestServerStreamDeliverySurface.AnthropicMessages, + var path when context.Request.RouteValues["model"] is string model + && path == "/gemini/v1beta/models/" + model + ":streamGenerateContent" + && context.Request.Query["alt"].Count == 1 && context.Request.Query["alt"][0] == "sse" + => LlmTestServerStreamDeliverySurface.GeminiGenerateContent, + _ => null, }; } diff --git a/src/ManagedCode.LlmTck.Hosting/LlmTckEndpointRouteBuilderExtensions.Validation.cs b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.Validation.cs similarity index 87% rename from src/ManagedCode.LlmTck.Hosting/LlmTckEndpointRouteBuilderExtensions.Validation.cs rename to src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.Validation.cs index 1fe26c6..2ca866d 100644 --- a/src/ManagedCode.LlmTck.Hosting/LlmTckEndpointRouteBuilderExtensions.Validation.cs +++ b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.Validation.cs @@ -1,43 +1,43 @@ using System.Text.Json; -using ManagedCode.LlmTck.Anthropic; -using ManagedCode.LlmTck.Bedrock; -using ManagedCode.LlmTck.Cohere; -using ManagedCode.LlmTck.Gemini; -using ManagedCode.LlmTck.Ollama; -using ManagedCode.LlmTck.OpenAI; -using ManagedCode.LlmTck.Runtime; +using ManagedCode.LlmTestServer.Anthropic; +using ManagedCode.LlmTestServer.Bedrock; +using ManagedCode.LlmTestServer.Cohere; +using ManagedCode.LlmTestServer.Gemini; +using ManagedCode.LlmTestServer.Ollama; +using ManagedCode.LlmTestServer.OpenAI; +using ManagedCode.LlmTestServer.Runtime; using Microsoft.AspNetCore.Http; -using ProviderRoutes = ManagedCode.LlmTck.Providers.LlmTckProviderRouteNamespaces; +using ProviderRoutes = ManagedCode.LlmTestServer.Providers.LlmTestServerProviderRouteNamespaces; -namespace ManagedCode.LlmTck.Hosting; +namespace ManagedCode.LlmTestServer.Hosting; -public static partial class LlmTckEndpointRouteBuilderExtensions +public static partial class LlmTestServerEndpointRouteBuilderExtensions { private static IResult? ValidateAnthropicVersion(HttpContext context) { var version = context.Request.Headers[AnthropicWireMapper.VersionHeaderName].ToString(); return string.Equals(version, AnthropicWireMapper.SupportedVersion, StringComparison.Ordinal) ? null - : AnthropicInvalidRequest( + : AnthropicInvalidRequest(context, $"The {AnthropicWireMapper.VersionHeaderName} header must be {AnthropicWireMapper.SupportedVersion}." ); } - private static IResult? ValidateAnthropicMessageRequest(AnthropicMessagesRequest request) + private static IResult? ValidateAnthropicMessageRequest(AnthropicMessagesRequest request, HttpContext context) { if (string.IsNullOrWhiteSpace(request.Model)) { - return AnthropicInvalidRequest("Missing message model."); + return AnthropicInvalidRequest(context, "Missing message model."); } if (request.MaxTokens is null or < 0) { - return AnthropicInvalidRequest("Missing or invalid max_tokens."); + return AnthropicInvalidRequest(context, "Missing or invalid max_tokens."); } if (request.Messages is null || request.Messages.Count == 0) { - return AnthropicInvalidRequest("At least one message is required."); + return AnthropicInvalidRequest(context, "At least one message is required."); } return request.Messages.Any(message => @@ -45,7 +45,7 @@ message is null || message.Role is not ("user" or "assistant") || message.Content.ValueKind is JsonValueKind.Undefined or JsonValueKind.Null ) - ? AnthropicInvalidRequest("Every message requires a user or assistant role and content.") + ? AnthropicInvalidRequest(context, "Every message requires a user or assistant role and content.") : null; } @@ -145,9 +145,26 @@ GeminiGenerateContentRequest request return GeminiInvalidRequest("System instruction requires valid parts."); } - return request.Contents.Any(content => content is null || content.Parts is null || content.Parts.Count == 0 || content.Parts.Any(part => part is null)) - ? GeminiInvalidRequest("Every content item requires at least one part.") - : null; + if (request.Contents.Any(content => content is null || content.Parts is null || content.Parts.Count == 0 || content.Parts.Any(part => part is null))) + { return GeminiInvalidRequest("Every content item requires at least one part."); } + foreach (var part in request.Contents.SelectMany(content => content.Parts)) + { + if (part.ThoughtSignature is { } signature && (signature.Length == 0 || signature.Length > 16384 + || signature.Any(char.IsWhiteSpace) || !Convert.TryFromBase64String(signature, new byte[signature.Length], out var written) || written == 0)) + { return GeminiInvalidRequest("Thought signatures require bounded, nonempty base64."); } + } + if (GeminiWireMapper.RequiresToolContinuationTokens(model)) + { + var start = request.Contents.FindLastIndex(content => content.Role == "user" && content.Parts.Any(part => + part.Text is not null || part.InlineData is not null || part.InlineDataSnakeCase is not null || part.FileData is not null || part.FileDataSnakeCase is not null)); + foreach (var content in request.Contents.Skip(Math.Max(0, start + 1)).Where(content => content.Role == "model")) + { + var firstCall = content.Parts.FirstOrDefault(part => part.FunctionCall is not null); + if (firstCall is not null && firstCall.ThoughtSignature is null) + { return GeminiInvalidRequest("The first function call in each current-turn step requires its original thought signature."); } + } + } + return null; } private static IResult? ValidateGeminiEmbeddingRequest( @@ -246,7 +263,7 @@ message is null return request.Messages.Any(message => message is null || string.IsNullOrWhiteSpace(message.Role) - || (message.Content.ValueKind is JsonValueKind.Undefined or JsonValueKind.Null && message.ToolCalls is not { Count: > 0 }) + || (message.Content.ValueKind is JsonValueKind.Undefined or JsonValueKind.Null && message.ToolCalls is not { Count: > 0 } && message.RefusalTextContent is null) ) ? InvalidRequest("Every chat message requires a role and content.") : null; @@ -409,7 +426,7 @@ string[] allowedResponseFormats : InvalidRequest("Unsupported audio response_format."); } - private static IResult ToOpenAiVideoError(LlmTckVideoResult result) + private static IResult ToOpenAiVideoError(LlmTestServerVideoResult result) { return Results.Json( OpenAiWireMapper.ToError(result.ErrorCode!, result.ErrorMessage!), @@ -417,7 +434,7 @@ private static IResult ToOpenAiVideoError(LlmTckVideoResult result) ); } - private static IResult ToGeminiVideoError(LlmTckVideoResult result) + private static IResult ToGeminiVideoError(LlmTestServerVideoResult result) { return Results.Json( GeminiWireMapper.ToError(result.StatusCode, result.ErrorMessage!), @@ -555,10 +572,10 @@ private static IResult InvalidRequest(string message) ); } - private static IResult AnthropicInvalidRequest(string message) + private static IResult AnthropicInvalidRequest(HttpContext context, string message) { return Results.Json( - AnthropicWireMapper.ToError("invalid_request_error", message), + AnthropicWireMapper.ToError("invalid_request_error", message, WireMetadata(context)), statusCode: StatusCodes.Status400BadRequest ); } @@ -616,7 +633,7 @@ _ when string.Equals(code, "invalid_request", StringComparison.Ordinal) => "inva private readonly record struct FormReadResult(AudioFormRequest Value, IResult? Error); - private readonly record struct VideoFixtureReadResult(LlmTckVideoResult? Value, IResult? Error); + private readonly record struct VideoFixtureReadResult(LlmTestServerVideoResult? Value, IResult? Error); private readonly record struct AudioFormRequest( string Model, diff --git a/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.Verification.cs b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.Verification.cs new file mode 100644 index 0000000..f684591 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.Verification.cs @@ -0,0 +1,40 @@ +using ManagedCode.LlmTestServer.Control; +using ManagedCode.LlmTestServer.Conversations; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Verification; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Routing; + +namespace ManagedCode.LlmTestServer.Hosting; + +public static partial class LlmTestServerEndpointRouteBuilderExtensions +{ + private static void MapVerificationEndpoints(RouteGroupBuilder endpoints) + { + endpoints.MapPost(LlmTestServerControlRoutes.Verify, async (HttpContext context, + ILlmTestServerRuntime runtime, ILlmTestServerProviderHttpTraceStore traces, CancellationToken cancellationToken) => + { + var unauthorized = AuthorizeControlRequest(context, runtime); + if (unauthorized is not null) { return unauthorized; } + var read = await ReadJsonAsync(context, cancellationToken).ConfigureAwait(false); + if (read.Error is not null) { return read.Error; } + if (read.Value is null) { return InvalidRequest("Missing verification expectations."); } + try + { + var snapshot = traces.GetEvidenceSnapshot(); + var selectedConversation = context.Request.Headers[LlmTestServerConversationHeaders.Id].FirstOrDefault(); + if (selectedConversation is not null) + { + var requests = snapshot.Requests.Where(trace => trace.ConversationId == selectedConversation).ToArray(); + snapshot = snapshot with { Requests = requests, TotalMatched = requests.Length }; + } + var conversations = ((ILlmTestServerConversationDiagnostics)runtime).GetConversations(); + if (selectedConversation is not null) + { conversations = conversations.Where(conversation => conversation.ConversationId == selectedConversation).ToArray(); } + return Results.Json(LlmTestServerVerificationEvaluator.Verify(read.Value, snapshot, conversations)); + } + catch (ArgumentException) { return InvalidRequest("Invalid verification expectations."); } + }); + } +} diff --git a/src/ManagedCode.LlmTck.Hosting/LlmTckEndpointRouteBuilderExtensions.Videos.cs b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.Videos.cs similarity index 88% rename from src/ManagedCode.LlmTck.Hosting/LlmTckEndpointRouteBuilderExtensions.Videos.cs rename to src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.Videos.cs index 5ce3ea4..fedee87 100644 --- a/src/ManagedCode.LlmTck.Hosting/LlmTckEndpointRouteBuilderExtensions.Videos.cs +++ b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.Videos.cs @@ -1,16 +1,16 @@ -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.OpenAI; -using ManagedCode.LlmTck.Runtime; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.OpenAI; +using ManagedCode.LlmTestServer.Runtime; using Microsoft.AspNetCore.Http; -using ProviderRoutes = ManagedCode.LlmTck.Providers.LlmTckProviderRouteNamespaces; +using ProviderRoutes = ManagedCode.LlmTestServer.Providers.LlmTestServerProviderRouteNamespaces; -namespace ManagedCode.LlmTck.Hosting; +namespace ManagedCode.LlmTestServer.Hosting; -public static partial class LlmTckEndpointRouteBuilderExtensions +public static partial class LlmTestServerEndpointRouteBuilderExtensions { private static async Task CreateOpenAiVideoAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -46,7 +46,7 @@ CancellationToken cancellationToken return ToOpenAiVideoError(result); } - result = ((ILlmTckVideoStore)runtime).StoreVideo(ProviderRoutes.OpenAI, result with + result = ((ILlmTestServerVideoStore)runtime).StoreVideo(ProviderRoutes.OpenAI, result with { Seconds = read.Value.Seconds ?? result.Seconds, Size = read.Value.Size ?? result.Size, @@ -56,7 +56,7 @@ CancellationToken cancellationToken private static IResult ListOpenAiVideosAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -73,7 +73,7 @@ CancellationToken cancellationToken return unauthorized; } - var videos = ((ILlmTckVideoStore)runtime).ListVideos(ProviderRoutes.OpenAI); + var videos = ((ILlmTestServerVideoStore)runtime).ListVideos(ProviderRoutes.OpenAI); var ordered = context.Request.Query["order"] == "asc" ? videos : videos.Reverse(); var after = context.Request.Query["after"].ToString(); if (!string.IsNullOrEmpty(after)) @@ -96,7 +96,7 @@ CancellationToken cancellationToken private static async Task GetOpenAiVideoAsync( string videoId, HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -113,7 +113,7 @@ CancellationToken cancellationToken private static IResult DeleteOpenAiVideoAsync( string videoId, HttpContext context, - ILlmTckRuntime runtime + ILlmTestServerRuntime runtime ) { if (string.IsNullOrWhiteSpace(videoId)) @@ -122,14 +122,14 @@ ILlmTckRuntime runtime } var unauthorized = AuthorizeControlRequest(context, runtime); - return unauthorized ?? (((ILlmTckVideoStore)runtime).DeleteVideo(ProviderRoutes.OpenAI, videoId) + return unauthorized ?? (((ILlmTestServerVideoStore)runtime).DeleteVideo(ProviderRoutes.OpenAI, videoId) ? Results.Json(OpenAiWireMapper.ToVideoDeleteResponse(videoId)) : VideoNotFound()); } private static async Task GetOpenAiVideoContentAsync( string videoId, HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -160,7 +160,7 @@ CancellationToken cancellationToken private static async Task EditOpenAiVideoAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -197,7 +197,7 @@ CancellationToken cancellationToken private static async Task ExtendOpenAiVideoAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -240,7 +240,7 @@ CancellationToken cancellationToken private static async Task RemixOpenAiVideoAsync( string videoId, HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -274,7 +274,7 @@ CancellationToken cancellationToken private static async Task CreateOpenAiVideoCharacterAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -311,9 +311,9 @@ CancellationToken cancellationToken return Results.Json( OpenAiWireMapper.ToVideoCharacterResponse( - "char_llm_tck", + "char_llm_test_server", name, - DateTimeOffset.UtcNow.ToUnixTimeSeconds() + WireMetadata(context).CreatedAt.ToUnixTimeSeconds() ) ); } @@ -321,7 +321,7 @@ CancellationToken cancellationToken private static IResult GetOpenAiVideoCharacterAsync( string characterId, HttpContext context, - ILlmTckRuntime runtime + ILlmTestServerRuntime runtime ) { if (string.IsNullOrWhiteSpace(characterId)) @@ -334,15 +334,15 @@ ILlmTckRuntime runtime ?? Results.Json( OpenAiWireMapper.ToVideoCharacterResponse( characterId, - "LLM TCK Character", - DateTimeOffset.UtcNow.ToUnixTimeSeconds() + "LLM Test Server Character", + WireMetadata(context).CreatedAt.ToUnixTimeSeconds() ) ); } private static async Task GenerateOpenAiVideoFromDefaultModelAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, string prompt, string? seconds, string? remixedFromVideoId, @@ -361,7 +361,7 @@ CancellationToken cancellationToken return ToOpenAiVideoError(generated); } - var result = ((ILlmTckVideoStore)runtime).StoreVideo(ProviderRoutes.OpenAI, generated with + var result = ((ILlmTestServerVideoStore)runtime).StoreVideo(ProviderRoutes.OpenAI, generated with { Seconds = seconds ?? source.Value.Seconds, Size = source.Value.Size, @@ -372,7 +372,7 @@ CancellationToken cancellationToken private static async Task CreateAzureOpenAiVideoJobAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -408,7 +408,7 @@ CancellationToken cancellationToken return ToOpenAiVideoError(result); } - result = ((ILlmTckVideoStore)runtime).StoreVideo(ProviderRoutes.AzureOpenAI, result with + result = ((ILlmTestServerVideoStore)runtime).StoreVideo(ProviderRoutes.AzureOpenAI, result with { Size = $"{read.Value.Width}x{read.Value.Height}", Seconds = $"{read.Value.NSeconds}", @@ -418,7 +418,7 @@ CancellationToken cancellationToken private static IResult ListAzureOpenAiVideoJobsAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -429,7 +429,7 @@ CancellationToken cancellationToken return unauthorized; } - var data = ((ILlmTckVideoStore)runtime).ListVideos(ProviderRoutes.AzureOpenAI) + var data = ((ILlmTestServerVideoStore)runtime).ListVideos(ProviderRoutes.AzureOpenAI) .Select(OpenAiWireMapper.ToAzureVideoGenerationJobResponse).ToList(); return Results.Json(new AzureVideoGenerationJobListResponse { @@ -442,7 +442,7 @@ CancellationToken cancellationToken private static async Task GetAzureOpenAiVideoJobAsync( string jobId, HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -463,7 +463,7 @@ CancellationToken cancellationToken private static IResult DeleteAzureOpenAiVideoJobAsync( string jobId, HttpContext context, - ILlmTckRuntime runtime + ILlmTestServerRuntime runtime ) { if (string.IsNullOrWhiteSpace(jobId)) @@ -472,14 +472,14 @@ ILlmTckRuntime runtime } var unauthorized = AuthorizeControlRequest(context, runtime); - return unauthorized ?? (((ILlmTckVideoStore)runtime).DeleteVideo(ProviderRoutes.AzureOpenAI, jobId) + return unauthorized ?? (((ILlmTestServerVideoStore)runtime).DeleteVideo(ProviderRoutes.AzureOpenAI, jobId) ? Results.NoContent() : VideoNotFound()); } private static async Task GetAzureOpenAiVideoGenerationAsync( string generationId, HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -500,7 +500,7 @@ CancellationToken cancellationToken private static async Task GetAzureOpenAiVideoThumbnailAsync( string generationId, HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -516,7 +516,7 @@ CancellationToken cancellationToken private static async Task GetAzureOpenAiVideoContentAsync( string generationId, HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -532,7 +532,7 @@ CancellationToken cancellationToken private static async Task HeadAzureOpenAiVideoContentAsync( string generationId, HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -554,7 +554,7 @@ CancellationToken cancellationToken } private static Task ReadStoredVideoAsync( - HttpContext context, ILlmTckRuntime runtime, string provider, string id, bool generation, + HttpContext context, ILlmTestServerRuntime runtime, string provider, string id, bool generation, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); @@ -564,7 +564,7 @@ private static Task ReadStoredVideoAsync( return Task.FromResult(new VideoFixtureReadResult(null, unauthorized)); } - var video = ((ILlmTckVideoStore)runtime).FindVideo(provider, id, generation); + var video = ((ILlmTestServerVideoStore)runtime).FindVideo(provider, id, generation); return Task.FromResult(new VideoFixtureReadResult(video, video is null ? VideoNotFound() : null)); } @@ -576,15 +576,15 @@ private static IResult VideoNotFound() private static async Task GenerateDefaultVideoFixtureAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, string prompt, CancellationToken cancellationToken ) { var modelId = runtime .GetModels() - .FirstOrDefault(model => model.Kind == LlmTckModelKind.Video) - ?.Id ?? LlmTckKnownModelIds.Sora2; + .FirstOrDefault(model => model.Kind == LlmTestServerModelKind.Video) + ?.Id ?? LlmTestServerKnownModelIds.Sora2; var result = await runtime .GenerateVideoAsync(modelId, prompt, ReadAccessToken(context), cancellationToken) .ConfigureAwait(false); @@ -596,7 +596,7 @@ CancellationToken cancellationToken private static async Task GenerateAudioAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -612,7 +612,7 @@ CancellationToken cancellationToken private static async Task GenerateGroqAudioAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { diff --git a/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.WireMetadata.cs b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.WireMetadata.cs new file mode 100644 index 0000000..9de0984 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.WireMetadata.cs @@ -0,0 +1,15 @@ +using ManagedCode.LlmTestServer.Determinism; +using ManagedCode.LlmTestServer.Runtime; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DependencyInjection; + +namespace ManagedCode.LlmTestServer.Hosting; + +public static partial class LlmTestServerEndpointRouteBuilderExtensions +{ + private static LlmTestServerResponseMetadata WireMetadata(HttpContext context) + { + return context.RequestServices.GetRequiredService() is ILlmTestServerWireMetadataSource source + ? source.GetResponseMetadata() : LlmTestServerResponseMetadata.Live(); + } +} diff --git a/src/ManagedCode.LlmTck.Hosting/LlmTckEndpointRouteBuilderExtensions.cs b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.cs similarity index 80% rename from src/ManagedCode.LlmTck.Hosting/LlmTckEndpointRouteBuilderExtensions.cs rename to src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.cs index 7944c0b..e3a70cd 100644 --- a/src/ManagedCode.LlmTck.Hosting/LlmTckEndpointRouteBuilderExtensions.cs +++ b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerEndpointRouteBuilderExtensions.cs @@ -1,18 +1,18 @@ using System.Buffers; using System.Text.Json; using System.Text.Json.Serialization; -using ManagedCode.LlmTck.Anthropic; -using ManagedCode.LlmTck.Bedrock; -using ManagedCode.LlmTck.Cohere; -using ManagedCode.LlmTck.Configuration; -using ManagedCode.LlmTck.Control; -using ManagedCode.LlmTck.Gemini; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.Ollama; -using ManagedCode.LlmTck.OpenAI; -using ManagedCode.LlmTck.Providers; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Scenarios; +using ManagedCode.LlmTestServer.Anthropic; +using ManagedCode.LlmTestServer.Bedrock; +using ManagedCode.LlmTestServer.Cohere; +using ManagedCode.LlmTestServer.Configuration; +using ManagedCode.LlmTestServer.Control; +using ManagedCode.LlmTestServer.Gemini; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Ollama; +using ManagedCode.LlmTestServer.OpenAI; +using ManagedCode.LlmTestServer.Providers; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Scenarios; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Hosting.StaticWebAssets; @@ -24,11 +24,11 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.FluentUI.AspNetCore.Components; -using ProviderRoutes = ManagedCode.LlmTck.Providers.LlmTckProviderRouteNamespaces; +using ProviderRoutes = ManagedCode.LlmTestServer.Providers.LlmTestServerProviderRouteNamespaces; -namespace ManagedCode.LlmTck.Hosting; +namespace ManagedCode.LlmTestServer.Hosting; -public static partial class LlmTckEndpointRouteBuilderExtensions +public static partial class LlmTestServerEndpointRouteBuilderExtensions { private static readonly JsonSerializerOptions _jsonOptions = CreateJsonOptions(); private static readonly byte[] _defaultJpegBytes = [0xFF, 0xD8, 0xFF, 0xD9]; @@ -124,19 +124,30 @@ public static partial class LlmTckEndpointRouteBuilderExtensions ProviderRoutes.Perplexity, ]; - public static IServiceCollection AddLlmTck( + public static IServiceCollection AddLlmTestServer( this IServiceCollection services, - Action? configure = null + Action? configure = null ) { ArgumentNullException.ThrowIfNull(services); - var builder = new LlmTckConfigurationBuilder(); + var builder = new LlmTestServerConfigurationBuilder(); configure?.Invoke(builder); var configuration = builder.Build(); - services.AddSingleton(_ => new LlmTckRuntime(configuration)); - services.AddLlmTckProviderHttpTracing(); + services.AddLlmTestServerProviderHttpTracing(); + services.AddHttpContextAccessor(); + services.AddOptions() + .Validate(value => value.MaxSessions > 0 && value.MaxReservedStateBytes > 0 && value.MaxConfigurationBytes > 0, + "Session count, state budget, and configuration limit must be positive."); + services.AddSingleton(); + services.AddSingleton(provider => new LlmTestServerSessionRegistry(configuration, + provider.GetRequiredService(), + provider.GetRequiredService>(), + provider.GetRequiredService>())); + services.AddScoped(); + services.AddScoped(provider => provider.GetRequiredService().State.Runtime); + services.AddScoped(provider => provider.GetRequiredService().State.TraceStore); services.AddHttpClient(); services.AddFluentUIComponents(); services @@ -146,31 +157,48 @@ public static IServiceCollection AddLlmTck( return services; } - public static IEndpointRouteBuilder MapLlmTck(this IEndpointRouteBuilder endpoints) + public static IEndpointRouteBuilder MapLlmTestServer(this IEndpointRouteBuilder endpoints) { ArgumentNullException.ThrowIfNull(endpoints); MapStaticAssetsIfAvailable(endpoints); endpoints - .MapRazorComponents() + .MapRazorComponents() .AddInteractiveServerRenderMode() .DisableAntiforgery(); - endpoints.MapGet( - LlmTckControlRoutes.Models, - (HttpContext context, ILlmTckRuntime runtime) => + MapSessionEndpoints(endpoints); + var scopedEndpoints = endpoints.MapGroup(string.Empty); + scopedEndpoints.AddEndpointFilter(async (invocation, next) => + { + var registry = invocation.HttpContext.RequestServices.GetRequiredService(); + if (registry.Resolve(invocation.HttpContext) is null) { return SessionNotFound(); } + if (!_providerRouteNamespaces.Any(prefix => invocation.HttpContext.Request.Path.StartsWithSegments(prefix))) + { + var error = ValidateCorrelationHeaders(invocation.HttpContext); + if (error is not null) { return error; } + } + return await next(invocation).ConfigureAwait(false); + }); + scopedEndpoints.MapGet( + LlmTestServerControlRoutes.Models, + (HttpContext context, ILlmTestServerRuntime runtime) => AuthorizeControlRequest(context, runtime) ?? Results.Json(runtime.GetModels()) ); - endpoints.MapGet( - LlmTckControlRoutes.Assertions, - (HttpContext context, ILlmTckRuntime runtime) => + scopedEndpoints.MapGet( + LlmTestServerControlRoutes.Assertions, + (HttpContext context, ILlmTestServerRuntime runtime) => AuthorizeControlRequest(context, runtime) ?? Results.Json(runtime.GetAssertionSummary()) ); - endpoints.MapPost( - LlmTckControlRoutes.Reset, + scopedEndpoints.MapGet(LlmTestServerControlRoutes.Requests, GetRequests); + MapConversationEndpoints(scopedEndpoints); + MapVerificationEndpoints(scopedEndpoints); + scopedEndpoints.MapGet(LlmTestServerControlRoutes.Requests + "/export", ExportRequests); + scopedEndpoints.MapPost( + LlmTestServerControlRoutes.Reset, async ( HttpContext context, - ILlmTckRuntime runtime, - ILlmTckProviderHttpTraceStore traceStore, + ILlmTestServerRuntime runtime, + ILlmTestServerProviderHttpTraceStore traceStore, CancellationToken cancellationToken ) => { @@ -186,13 +214,13 @@ CancellationToken cancellationToken return Results.Ok(new { status = "reset" }); } ); - endpoints.MapPost(LlmTckControlRoutes.Configure, ConfigureAsync); + scopedEndpoints.MapPost(LlmTestServerControlRoutes.Configure, ConfigureAsync); - var providerEndpoints = endpoints.MapGroup(string.Empty); - providerEndpoints.AddLlmTckProviderHttpTracing(); + var providerEndpoints = scopedEndpoints.MapGroup(string.Empty); + providerEndpoints.AddLlmTestServerProviderHttpTracing(); providerEndpoints.AddEndpointFilter(async (invocation, next) => { - var error = LlmTckRequestPolicy.ValidateApiVersion(invocation.HttpContext); + var error = ValidateCorrelationHeaders(invocation.HttpContext) ?? LlmTestServerRequestPolicy.ValidateApiVersion(invocation.HttpContext); return error ?? await next(invocation).ConfigureAwait(false); }); @@ -201,228 +229,228 @@ CancellationToken cancellationToken ProviderRoutes.ForProvider(ProviderRoutes.AzureOpenAI, "/openai/v1/chat/completions"), CompleteOpenAiChatAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.AzureOpenAI.V1ChatCompletionsCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.AzureOpenAI.V1ChatCompletionsCreate); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.AzureOpenAI, "/openai/v1/responses"), CreateOpenAiResponseAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.AzureOpenAI.V1ResponsesCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.AzureOpenAI.V1ResponsesCreate); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.AzureOpenAI, "/openai/v1/audio/transcriptions"), TranscribeOpenAiAudioAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.AzureOpenAI.V1AudioTranscriptionsCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.AzureOpenAI.V1AudioTranscriptionsCreate); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.AzureOpenAI, "/openai/v1/embeddings"), CreateEmbeddingAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.AzureOpenAI.V1EmbeddingsCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.AzureOpenAI.V1EmbeddingsCreate); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.AzureOpenAI, "/openai/v1/images/generations"), GenerateImageAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.AzureOpenAI.V1ImagesCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.AzureOpenAI.V1ImagesCreate); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.MicrosoftFoundry, "/openai/v1/chat/completions"), CompleteOpenAiChatAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.MicrosoftFoundry.V1ChatCompletionsCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.MicrosoftFoundry.V1ChatCompletionsCreate); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.MicrosoftFoundry, "/openai/v1/responses"), CreateOpenAiResponseAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.MicrosoftFoundry.V1ResponsesCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.MicrosoftFoundry.V1ResponsesCreate); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.MicrosoftFoundry, "/openai/v1/audio/transcriptions"), TranscribeOpenAiAudioAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.MicrosoftFoundry.V1AudioTranscriptionsCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.MicrosoftFoundry.V1AudioTranscriptionsCreate); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.MicrosoftFoundry, "/openai/v1/embeddings"), CreateEmbeddingAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.MicrosoftFoundry.V1EmbeddingsCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.MicrosoftFoundry.V1EmbeddingsCreate); providerEndpoints .MapGet( ProviderRoutes.ForProvider(ProviderRoutes.OpenAI, "/v1/models"), - (ILlmTckRuntime runtime) => - Results.Json(OpenAiWireMapper.ToModelsResponse(runtime.GetModels())) + (HttpContext context, ILlmTestServerRuntime runtime) => + AuthorizeControlRequest(context, runtime) ?? Results.Json(OpenAiWireMapper.ToModelsResponse(runtime.GetModels())) ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.OpenAI.ModelsList); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.OpenAI.ModelsList); providerEndpoints .MapGet( ProviderRoutes.ForProvider(ProviderRoutes.Groq, "/openai/v1/models"), - (ILlmTckRuntime runtime) => - Results.Json(OpenAiWireMapper.ToModelsResponse(runtime.GetModels())) + (HttpContext context, ILlmTestServerRuntime runtime) => + AuthorizeControlRequest(context, runtime) ?? Results.Json(OpenAiWireMapper.ToModelsResponse(runtime.GetModels())) ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.Groq.ModelsList); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.Groq.ModelsList); providerEndpoints .MapGet( ProviderRoutes.ForProvider(ProviderRoutes.OpenRouter, "/api/v1/models"), - (ILlmTckRuntime runtime) => - Results.Json(OpenAiWireMapper.ToModelsResponse(runtime.GetModels())) + (HttpContext context, ILlmTestServerRuntime runtime) => + AuthorizeControlRequest(context, runtime) ?? Results.Json(OpenAiWireMapper.ToModelsResponse(runtime.GetModels())) ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.OpenRouter.ModelsList); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.OpenRouter.ModelsList); providerEndpoints .MapGet( ProviderRoutes.ForProvider(ProviderRoutes.DeepSeek, "/models"), - (ILlmTckRuntime runtime) => - Results.Json(OpenAiWireMapper.ToModelsResponse(runtime.GetModels())) + (HttpContext context, ILlmTestServerRuntime runtime) => + AuthorizeControlRequest(context, runtime) ?? Results.Json(OpenAiWireMapper.ToModelsResponse(runtime.GetModels())) ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.DeepSeek.ModelsList); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.DeepSeek.ModelsList); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.OpenAI, "/v1/chat/completions"), CompleteOpenAiChatAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.OpenAI.ChatCompletionsCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.OpenAI.ChatCompletionsCreate); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.Anthropic, "/v1/messages"), CompleteAnthropicMessageAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.Anthropic.MessagesCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.Anthropic.MessagesCreate); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.Perplexity, "/v1/sonar"), CompleteChatAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.Perplexity.SonarCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.Perplexity.SonarCreate); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.OpenAI, "/v1/responses"), CreateOpenAiResponseAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.OpenAI.ResponsesCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.OpenAI.ResponsesCreate); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.Groq, "/openai/v1/chat/completions"), CompleteGroqChatAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.Groq.ChatCompletionsCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.Groq.ChatCompletionsCreate); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.Groq, "/openai/v1/responses"), CreateGroqResponseAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.Groq.ResponsesCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.Groq.ResponsesCreate); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.Groq, "/openai/v1/audio/speech"), GenerateGroqAudioAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.Groq.AudioSpeechCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.Groq.AudioSpeechCreate); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.Groq, "/openai/v1/audio/transcriptions"), TranscribeGroqAudioAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.Groq.AudioTranscriptionsCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.Groq.AudioTranscriptionsCreate); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.Groq, "/openai/v1/audio/translations"), TranslateGroqAudioAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.Groq.AudioTranslationsCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.Groq.AudioTranslationsCreate); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.OpenRouter, "/api/v1/chat/completions"), CompleteOpenRouterChatAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.OpenRouter.ChatCompletionsCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.OpenRouter.ChatCompletionsCreate); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.OpenRouter, "/api/v1/responses"), CreateOpenRouterResponseAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.OpenRouter.ResponsesCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.OpenRouter.ResponsesCreate); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.Mistral, "/v1/chat/completions"), CompleteMistralChatAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.Mistral.ChatComplete); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.Mistral.ChatComplete); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.Mistral, "/v1/embeddings"), CreateEmbeddingAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.Mistral.EmbeddingsCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.Mistral.EmbeddingsCreate); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.DeepSeek, "/v1/chat/completions"), CompleteDeepSeekChatAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.DeepSeek.ChatCompletionsCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.DeepSeek.ChatCompletionsCreate); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.OpenAI, "/v1/embeddings"), CreateEmbeddingAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.OpenAI.EmbeddingsCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.OpenAI.EmbeddingsCreate); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.OpenAI, "/v1/images/generations"), GenerateImageAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.OpenAI.ImagesCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.OpenAI.ImagesCreate); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.OpenAI, "/v1/images/edits"), EditImageAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.OpenAI.ImagesEditsCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.OpenAI.ImagesEditsCreate); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.OpenAI, "/v1/images/variations"), CreateImageVariationAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.OpenAI.ImagesVariationsCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.OpenAI.ImagesVariationsCreate); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.OpenAI, "/v1/audio/speech"), GenerateAudioAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.OpenAI.AudioSpeechCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.OpenAI.AudioSpeechCreate); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.OpenAI, "/v1/audio/transcriptions"), TranscribeOpenAiAudioAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.OpenAI.AudioTranscriptionsCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.OpenAI.AudioTranscriptionsCreate); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.OpenAI, "/v1/audio/translations"), TranslateOpenAiAudioAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.OpenAI.AudioTranslationsCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.OpenAI.AudioTranslationsCreate); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.OpenAI, "/v1/videos"), CreateOpenAiVideoAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.OpenAI.VideosCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.OpenAI.VideosCreate); providerEndpoints .MapGet( ProviderRoutes.ForProvider(ProviderRoutes.OpenAI, "/v1/videos"), ListOpenAiVideosAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.OpenAI.VideosList); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.OpenAI.VideosList); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.OpenAI, "/v1/videos/characters"), CreateOpenAiVideoCharacterAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.OpenAI.VideosCharactersCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.OpenAI.VideosCharactersCreate); providerEndpoints .MapGet( ProviderRoutes.ForProvider( @@ -431,31 +459,31 @@ CancellationToken cancellationToken ), GetOpenAiVideoCharacterAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.OpenAI.VideosCharactersRetrieve); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.OpenAI.VideosCharactersRetrieve); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.OpenAI, "/v1/videos/edits"), EditOpenAiVideoAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.OpenAI.VideosEditsCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.OpenAI.VideosEditsCreate); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.OpenAI, "/v1/videos/extensions"), ExtendOpenAiVideoAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.OpenAI.VideosExtensionsCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.OpenAI.VideosExtensionsCreate); providerEndpoints .MapGet( ProviderRoutes.ForProvider(ProviderRoutes.OpenAI, "/v1/videos/{videoId}"), GetOpenAiVideoAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.OpenAI.VideosRetrieve); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.OpenAI.VideosRetrieve); providerEndpoints .MapDelete( ProviderRoutes.ForProvider(ProviderRoutes.OpenAI, "/v1/videos/{videoId}"), DeleteOpenAiVideoAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.OpenAI.VideosDelete); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.OpenAI.VideosDelete); providerEndpoints .MapGet( ProviderRoutes.ForProvider( @@ -464,7 +492,7 @@ CancellationToken cancellationToken ), GetOpenAiVideoContentAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.OpenAI.VideosContentRetrieve); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.OpenAI.VideosContentRetrieve); providerEndpoints .MapPost( ProviderRoutes.ForProvider( @@ -473,31 +501,31 @@ CancellationToken cancellationToken ), RemixOpenAiVideoAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.OpenAI.VideosRemix); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.OpenAI.VideosRemix); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.Ollama, "/api/chat"), CompleteOllamaChatAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.Ollama.ChatCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.Ollama.ChatCreate); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.Ollama, "/api/embed"), CreateOllamaEmbeddingAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.Ollama.EmbeddingsCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.Ollama.EmbeddingsCreate); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.Cohere, "/v2/chat"), CompleteCohereChatAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.Cohere.ChatCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.Cohere.ChatCreate); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.Cohere, "/v2/embed"), CreateCohereEmbeddingAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.Cohere.EmbedCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.Cohere.EmbedCreate); providerEndpoints .MapPost( ProviderRoutes.ForProvider( @@ -506,7 +534,7 @@ CancellationToken cancellationToken ), CompleteGeminiContentAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.Gemini.ModelsGenerateContent); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.Gemini.ModelsGenerateContent); providerEndpoints .MapPost( ProviderRoutes.ForProvider( @@ -515,8 +543,8 @@ CancellationToken cancellationToken ), StreamGeminiContentAsync ) - .WithLlmTckProviderOperation( - LlmTckProviderOperationIds.Gemini.ModelsStreamGenerateContent + .WithLlmTestServerProviderOperation( + LlmTestServerProviderOperationIds.Gemini.ModelsStreamGenerateContent ); providerEndpoints .MapPost( @@ -526,7 +554,13 @@ CancellationToken cancellationToken ), CreateGeminiEmbeddingAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.Gemini.ModelsEmbedContent); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.Gemini.ModelsEmbedContent); + providerEndpoints + .MapPost( + ProviderRoutes.ForProvider(ProviderRoutes.Gemini, "/v1beta/models/{model}:batchEmbedContents"), + CreateGeminiBatchEmbeddingsAsync + ) + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.Gemini.ModelsBatchEmbedContents); providerEndpoints .MapPost( ProviderRoutes.ForProvider( @@ -535,8 +569,8 @@ CancellationToken cancellationToken ), StartGeminiVideoOperationAsync ) - .WithLlmTckProviderOperation( - LlmTckProviderOperationIds.Gemini.ModelsPredictLongRunningVideo + .WithLlmTestServerProviderOperation( + LlmTestServerProviderOperationIds.Gemini.ModelsPredictLongRunningVideo ); providerEndpoints .MapGet( @@ -546,21 +580,21 @@ CancellationToken cancellationToken ), GetGeminiVideoOperationAsync ) - .WithLlmTckProviderOperation( - LlmTckProviderOperationIds.Gemini.ModelsOperationsGetVideo + .WithLlmTestServerProviderOperation( + LlmTestServerProviderOperationIds.Gemini.ModelsOperationsGetVideo ); providerEndpoints .MapGet( ProviderRoutes.ForProvider(ProviderRoutes.Gemini, "/v1beta/files/{fileId}"), GetGeminiFileAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.Gemini.FilesGetGeneratedVideo); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.Gemini.FilesGetGeneratedVideo); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.Bedrock, "/model/{modelId}/converse"), CompleteBedrockConverseAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.Bedrock.Converse); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.Bedrock.Converse); providerEndpoints .MapPost( ProviderRoutes.ForProvider( @@ -569,13 +603,13 @@ CancellationToken cancellationToken ), CompleteBedrockConverseStreamAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.Bedrock.ConverseStream); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.Bedrock.ConverseStream); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.Bedrock, "/model/{modelId}/invoke"), InvokeBedrockModelAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.Bedrock.InvokeModel); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.Bedrock.InvokeModel); providerEndpoints .MapPost( ProviderRoutes.ForProvider( @@ -584,24 +618,24 @@ CancellationToken cancellationToken ), InvokeBedrockModelWithResponseStreamAsync ) - .WithLlmTckProviderOperation( - LlmTckProviderOperationIds.Bedrock.InvokeModelWithResponseStream + .WithLlmTestServerProviderOperation( + LlmTestServerProviderOperationIds.Bedrock.InvokeModelWithResponseStream ); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.MicrosoftFoundry, "/chat/completions"), CompleteFoundryChatAsync ) - .WithLlmTckProviderOperation( - LlmTckProviderOperationIds.MicrosoftFoundry.ChatCompletionsCreate + .WithLlmTestServerProviderOperation( + LlmTestServerProviderOperationIds.MicrosoftFoundry.ChatCompletionsCreate ); providerEndpoints .MapPost( ProviderRoutes.ForProvider(ProviderRoutes.MicrosoftFoundry, "/embeddings"), CreateEmbeddingAsync ) - .WithLlmTckProviderOperation( - LlmTckProviderOperationIds.MicrosoftFoundry.EmbeddingsCreate + .WithLlmTestServerProviderOperation( + LlmTestServerProviderOperationIds.MicrosoftFoundry.EmbeddingsCreate ); providerEndpoints .MapPost( @@ -611,8 +645,8 @@ CancellationToken cancellationToken ), CompleteFoundryChatAsync ) - .WithLlmTckProviderOperation( - LlmTckProviderOperationIds.MicrosoftFoundry.ModelsChatCompletionsCreate + .WithLlmTestServerProviderOperation( + LlmTestServerProviderOperationIds.MicrosoftFoundry.ModelsChatCompletionsCreate ); providerEndpoints .MapPost( @@ -622,8 +656,8 @@ CancellationToken cancellationToken ), CreateEmbeddingAsync ) - .WithLlmTckProviderOperation( - LlmTckProviderOperationIds.MicrosoftFoundry.ModelsEmbeddingsCreate + .WithLlmTestServerProviderOperation( + LlmTestServerProviderOperationIds.MicrosoftFoundry.ModelsEmbeddingsCreate ); providerEndpoints .MapPost( @@ -633,8 +667,8 @@ CancellationToken cancellationToken ), CompleteAzureOpenAiChatAsync ) - .WithLlmTckProviderOperation( - LlmTckProviderOperationIds.AzureOpenAI.ChatCompletionsCreate + .WithLlmTestServerProviderOperation( + LlmTestServerProviderOperationIds.AzureOpenAI.ChatCompletionsCreate ); providerEndpoints .MapPost( @@ -644,8 +678,8 @@ CancellationToken cancellationToken ), CreateAzureOpenAiEmbeddingAsync ) - .WithLlmTckProviderOperation( - LlmTckProviderOperationIds.AzureOpenAI.EmbeddingsCreate + .WithLlmTestServerProviderOperation( + LlmTestServerProviderOperationIds.AzureOpenAI.EmbeddingsCreate ); providerEndpoints .MapPost( @@ -655,7 +689,7 @@ CancellationToken cancellationToken ), GenerateAzureOpenAiImageAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.AzureOpenAI.ImagesCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.AzureOpenAI.ImagesCreate); providerEndpoints .MapPost( ProviderRoutes.ForProvider( @@ -664,7 +698,7 @@ CancellationToken cancellationToken ), GenerateAzureOpenAiAudioAsync ) - .WithLlmTckProviderOperation(LlmTckProviderOperationIds.AzureOpenAI.AudioSpeechCreate); + .WithLlmTestServerProviderOperation(LlmTestServerProviderOperationIds.AzureOpenAI.AudioSpeechCreate); providerEndpoints .MapPost( ProviderRoutes.ForProvider( @@ -673,8 +707,8 @@ CancellationToken cancellationToken ), TranscribeAzureOpenAiAudioAsync ) - .WithLlmTckProviderOperation( - LlmTckProviderOperationIds.AzureOpenAI.AudioTranscriptionsCreate + .WithLlmTestServerProviderOperation( + LlmTestServerProviderOperationIds.AzureOpenAI.AudioTranscriptionsCreate ); providerEndpoints .MapPost( @@ -684,8 +718,8 @@ CancellationToken cancellationToken ), TranslateAzureOpenAiAudioAsync ) - .WithLlmTckProviderOperation( - LlmTckProviderOperationIds.AzureOpenAI.AudioTranslationsCreate + .WithLlmTestServerProviderOperation( + LlmTestServerProviderOperationIds.AzureOpenAI.AudioTranslationsCreate ); providerEndpoints .MapPost( @@ -695,8 +729,8 @@ CancellationToken cancellationToken ), CreateAzureOpenAiVideoJobAsync ) - .WithLlmTckProviderOperation( - LlmTckProviderOperationIds.AzureOpenAI.VideoGenerationJobsCreate + .WithLlmTestServerProviderOperation( + LlmTestServerProviderOperationIds.AzureOpenAI.VideoGenerationJobsCreate ); providerEndpoints .MapGet( @@ -706,8 +740,8 @@ CancellationToken cancellationToken ), ListAzureOpenAiVideoJobsAsync ) - .WithLlmTckProviderOperation( - LlmTckProviderOperationIds.AzureOpenAI.VideoGenerationJobsList + .WithLlmTestServerProviderOperation( + LlmTestServerProviderOperationIds.AzureOpenAI.VideoGenerationJobsList ); providerEndpoints .MapGet( @@ -717,8 +751,8 @@ CancellationToken cancellationToken ), GetAzureOpenAiVideoJobAsync ) - .WithLlmTckProviderOperation( - LlmTckProviderOperationIds.AzureOpenAI.VideoGenerationJobsRetrieve + .WithLlmTestServerProviderOperation( + LlmTestServerProviderOperationIds.AzureOpenAI.VideoGenerationJobsRetrieve ); providerEndpoints .MapDelete( @@ -728,8 +762,8 @@ CancellationToken cancellationToken ), DeleteAzureOpenAiVideoJobAsync ) - .WithLlmTckProviderOperation( - LlmTckProviderOperationIds.AzureOpenAI.VideoGenerationJobsDelete + .WithLlmTestServerProviderOperation( + LlmTestServerProviderOperationIds.AzureOpenAI.VideoGenerationJobsDelete ); providerEndpoints .MapGet( @@ -739,8 +773,8 @@ CancellationToken cancellationToken ), GetAzureOpenAiVideoGenerationAsync ) - .WithLlmTckProviderOperation( - LlmTckProviderOperationIds.AzureOpenAI.VideoGenerationsRetrieve + .WithLlmTestServerProviderOperation( + LlmTestServerProviderOperationIds.AzureOpenAI.VideoGenerationsRetrieve ); providerEndpoints .MapGet( @@ -750,8 +784,8 @@ CancellationToken cancellationToken ), GetAzureOpenAiVideoThumbnailAsync ) - .WithLlmTckProviderOperation( - LlmTckProviderOperationIds.AzureOpenAI.VideoGenerationsThumbnailRetrieve + .WithLlmTestServerProviderOperation( + LlmTestServerProviderOperationIds.AzureOpenAI.VideoGenerationsThumbnailRetrieve ); providerEndpoints .MapGet( @@ -761,8 +795,8 @@ CancellationToken cancellationToken ), GetAzureOpenAiVideoContentAsync ) - .WithLlmTckProviderOperation( - LlmTckProviderOperationIds.AzureOpenAI.VideoGenerationsContentRetrieve + .WithLlmTestServerProviderOperation( + LlmTestServerProviderOperationIds.AzureOpenAI.VideoGenerationsContentRetrieve ); providerEndpoints .MapMethods( @@ -773,8 +807,8 @@ CancellationToken cancellationToken ["HEAD"], HeadAzureOpenAiVideoContentAsync ) - .WithLlmTckProviderOperation( - LlmTckProviderOperationIds.AzureOpenAI.VideoGenerationsContentHead + .WithLlmTestServerProviderOperation( + LlmTestServerProviderOperationIds.AzureOpenAI.VideoGenerationsContentHead ); MapDecisionEndpoints(providerEndpoints); @@ -792,14 +826,14 @@ private static void MapProviderFallbacks(RouteGroupBuilder providerEndpoints) $"{providerNamespace}/{{**unmatchedProviderPath}}", HandleUnmatchedProviderRequestAsync ) - .WithMetadata(new LlmTckProviderFallbackMetadata()); + .WithMetadata(new LlmTestServerProviderFallbackMetadata()); } } private static async Task HandleUnmatchedProviderRequestAsync( HttpContext context, EndpointDataSource endpointDataSource, - IOptions traceOptions, + IOptions traceOptions, CancellationToken cancellationToken ) { @@ -813,7 +847,7 @@ await ReadProviderFallbackPreviewAsync( var allowedMethods = endpointDataSource .Endpoints.OfType() .Where(endpoint => - endpoint.Metadata.GetMetadata() is not null + endpoint.Metadata.GetMetadata() is not null && RouteMatchesPath(endpoint, context.Request.Path) ) .SelectMany(endpoint => @@ -878,8 +912,8 @@ CancellationToken cancellationToken private static async Task ConfigureAsync( HttpContext context, - ILlmTckRuntime runtime, - ILlmTckProviderHttpTraceStore traceStore, + ILlmTestServerRuntime runtime, + ILlmTestServerProviderHttpTraceStore traceStore, CancellationToken cancellationToken ) { @@ -889,7 +923,7 @@ CancellationToken cancellationToken return unauthorized; } - var read = await ReadJsonAsync(context, cancellationToken) + var read = await ReadJsonAsync(context, cancellationToken) .ConfigureAwait(false); if (read.Error is not null) { @@ -904,12 +938,26 @@ CancellationToken cancellationToken cancellationToken.ThrowIfCancellationRequested(); try { - await runtime.ConfigureAsync(read.Value, CancellationToken.None).ConfigureAwait(false); + var registry = context.RequestServices.GetRequiredService(); + var state = context.RequestServices.GetRequiredService().State; + var configured = ReferenceEquals(state, registry.Default) + ? runtime.ConfigureAsync(read.Value, CancellationToken.None) : registry.ConfigureAsync(state, read.Value); + if (configured is null) { return SessionNotFound(); } + await configured.ConfigureAwait(false); } catch (ArgumentException) { return InvalidRequest("Invalid runtime configuration."); } + catch (JsonException) + { + return InvalidRequest("Invalid JSON fixture in runtime configuration."); + } + + catch (InvalidOperationException) + { + return SessionCapacityExceeded(); + } traceStore.Reset(); return Results.Ok(new { status = "configured" }); @@ -917,7 +965,7 @@ CancellationToken cancellationToken private static async Task CompleteChatAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -925,7 +973,7 @@ CancellationToken cancellationToken context, runtime, null, - LlmTckPromptCachePolicy.None, + LlmTestServerPromptCachePolicy.None, OpenAiCacheUsageShape.None, cancellationToken ) @@ -934,7 +982,7 @@ CancellationToken cancellationToken private static async Task CompleteOpenAiChatAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -942,7 +990,7 @@ CancellationToken cancellationToken context, runtime, null, - LlmTckPromptCachePolicy.OpenAiCompatible, + LlmTestServerPromptCachePolicy.OpenAiCompatible, OpenAiCacheUsageShape.PromptTokensDetails, cancellationToken ) @@ -951,7 +999,7 @@ CancellationToken cancellationToken private static async Task CompleteGroqChatAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -959,7 +1007,7 @@ CancellationToken cancellationToken context, runtime, null, - LlmTckPromptCachePolicy.OpenAiCompatible, + LlmTestServerPromptCachePolicy.OpenAiCompatible, OpenAiCacheUsageShape.PromptTokensDetails, cancellationToken ) @@ -968,7 +1016,7 @@ CancellationToken cancellationToken private static async Task CompleteMistralChatAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -976,7 +1024,7 @@ CancellationToken cancellationToken context, runtime, null, - LlmTckPromptCachePolicy.Mistral, + LlmTestServerPromptCachePolicy.Mistral, OpenAiCacheUsageShape.PromptTokensDetails, cancellationToken ) @@ -985,7 +1033,7 @@ CancellationToken cancellationToken private static async Task CompleteOpenRouterChatAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -993,7 +1041,7 @@ CancellationToken cancellationToken context, runtime, null, - LlmTckPromptCachePolicy.OpenRouter, + LlmTestServerPromptCachePolicy.OpenRouter, OpenAiCacheUsageShape.PromptTokensDetailsWithCacheWriteTokens, cancellationToken ) @@ -1002,7 +1050,7 @@ CancellationToken cancellationToken private static async Task CompleteDeepSeekChatAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -1010,7 +1058,7 @@ CancellationToken cancellationToken context, runtime, null, - LlmTckPromptCachePolicy.DeepSeek, + LlmTestServerPromptCachePolicy.DeepSeek, OpenAiCacheUsageShape.DeepSeekPromptCache, cancellationToken ) @@ -1019,7 +1067,7 @@ CancellationToken cancellationToken private static async Task CompleteFoundryChatAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -1027,7 +1075,7 @@ CancellationToken cancellationToken context, runtime, null, - LlmTckPromptCachePolicy.OpenAiCompatible, + LlmTestServerPromptCachePolicy.OpenAiCompatible, OpenAiCacheUsageShape.PromptTokensDetails, cancellationToken ) @@ -1037,7 +1085,7 @@ CancellationToken cancellationToken private static async Task CompleteAzureOpenAiChatAsync( string deployment, HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -1045,7 +1093,7 @@ CancellationToken cancellationToken context, runtime, deployment, - LlmTckPromptCachePolicy.OpenAiCompatible, + LlmTestServerPromptCachePolicy.OpenAiCompatible, OpenAiCacheUsageShape.PromptTokensDetails, cancellationToken ) @@ -1054,14 +1102,14 @@ CancellationToken cancellationToken private static async Task CreateOpenAiResponseAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { return await CreateResponseCoreAsync( context, runtime, - LlmTckPromptCachePolicy.OpenAiCompatible, + LlmTestServerPromptCachePolicy.OpenAiCompatible, OpenAiCacheUsageShape.PromptTokensDetails, cancellationToken ) @@ -1070,14 +1118,14 @@ CancellationToken cancellationToken private static async Task CreateGroqResponseAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { return await CreateResponseCoreAsync( context, runtime, - LlmTckPromptCachePolicy.OpenAiCompatible, + LlmTestServerPromptCachePolicy.OpenAiCompatible, OpenAiCacheUsageShape.PromptTokensDetails, cancellationToken ) @@ -1086,14 +1134,14 @@ CancellationToken cancellationToken private static async Task CreateOpenRouterResponseAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { return await CreateResponseCoreAsync( context, runtime, - LlmTckPromptCachePolicy.OpenRouter, + LlmTestServerPromptCachePolicy.OpenRouter, OpenAiCacheUsageShape.PromptTokensDetailsWithCacheWriteTokens, cancellationToken ) @@ -1102,13 +1150,15 @@ CancellationToken cancellationToken private static async Task CreateResponseCoreAsync( HttpContext context, - ILlmTckRuntime runtime, - LlmTckPromptCachePolicy promptCachePolicy, + ILlmTestServerRuntime runtime, + LlmTestServerPromptCachePolicy promptCachePolicy, OpenAiCacheUsageShape cacheUsageShape, CancellationToken cancellationToken ) { - var read = await ReadJsonAsync(context, cancellationToken, validation: OpenAiRequestValidation.ValidateResponse) + var allowNativeTools = context.Request.Path.Equals("/openai/v1/responses", StringComparison.Ordinal); + var read = await ReadJsonAsync(context, cancellationToken, + validation: allowNativeTools ? OpenAiRequestValidation.ValidateNativeResponse : OpenAiRequestValidation.ValidateResponse) .ConfigureAwait(false); if (read.Error is not null) { @@ -1121,13 +1171,13 @@ CancellationToken cancellationToken } var request = read.Value; - if (promptCachePolicy == LlmTckPromptCachePolicy.OpenRouter + if (promptCachePolicy == LlmTestServerPromptCachePolicy.OpenRouter && (request.Store == true || request.PreviousResponseId is not null)) { return InvalidRequest("OpenRouter Responses is stateless: store=true and previous_response_id are not supported."); } - if (promptCachePolicy == LlmTckPromptCachePolicy.OpenRouter + if (promptCachePolicy == LlmTestServerPromptCachePolicy.OpenRouter && string.IsNullOrWhiteSpace(request.SessionId) && context.Request.Headers.TryGetValue("x-session-id", out var sessionId) && !string.IsNullOrWhiteSpace(sessionId.ToString())) @@ -1145,11 +1195,11 @@ CancellationToken cancellationToken .CompleteChatAsync( CorrelateChatRequest( context, - OpenAiWireMapper.ToRuntimeRequest(request, promptCachePolicy) with + OpenAiWireMapper.ToRuntimeRequest(request, promptCachePolicy, allowNativeTools) with { HistoryNamespace = context.Request.Path.Value!.Split('/', StringSplitOptions.RemoveEmptyEntries)[0], PreviousResponseId = request.PreviousResponseId, - StoreResponse = promptCachePolicy != LlmTckPromptCachePolicy.OpenRouter && request.Store != false + StoreResponse = promptCachePolicy != LlmTestServerPromptCachePolicy.OpenRouter && request.Store != false } ), ReadAccessToken(context), @@ -1165,8 +1215,8 @@ CancellationToken cancellationToken ); } - var responseId = result.ResponseId ?? $"resp_{Guid.NewGuid():N}"; - var created = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); + var responseId = result.ResponseId ?? "resp_" + (result.WireMetadata ?? WireMetadata(context)).IdFor("response"); + var created = (result.WireMetadata ?? WireMetadata(context)).CreatedAt.ToUnixTimeSeconds(); if (request.Stream) { await WriteStreamingResponseAsync( @@ -1175,18 +1225,19 @@ await WriteStreamingResponseAsync( responseId, created, cacheUsageShape, - cancellationToken + cancellationToken, + request.Include ) .ConfigureAwait(false); return Results.Empty; } - return Results.Json(OpenAiWireMapper.ToResponse(result, responseId, created, cacheUsageShape)); + return Results.Json(OpenAiWireMapper.ToResponse(result, responseId, created, cacheUsageShape, request.Include)); } private static async Task CompleteAnthropicMessageAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -1198,7 +1249,7 @@ CancellationToken cancellationToken var read = await ReadJsonAsync( context, - AnthropicInvalidRequest, + message => AnthropicInvalidRequest(context, message), cancellationToken, validation: AnthropicRequestValidation.Validate ) .ConfigureAwait(false); @@ -1209,10 +1260,10 @@ CancellationToken cancellationToken if (read.Value is null) { - return AnthropicInvalidRequest("Missing message body."); + return AnthropicInvalidRequest(context, "Missing message body."); } - var validationError = ValidateAnthropicMessageRequest(read.Value); + var validationError = ValidateAnthropicMessageRequest(read.Value, context); if (validationError is not null) { return validationError; @@ -1231,7 +1282,8 @@ CancellationToken cancellationToken return Results.Json( AnthropicWireMapper.ToError( ToAnthropicErrorType(result.StatusCode, result.ErrorCode!), - result.ErrorMessage! + result.ErrorMessage!, + result.WireMetadata ), statusCode: result.StatusCode ); @@ -1249,7 +1301,7 @@ await WriteAnthropicStreamingMessageAsync(context, result, cancellationToken, re private static async Task CompleteOllamaChatAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -1303,7 +1355,7 @@ await WriteOllamaStreamingChatAsync(context, result, cancellationToken) private static async Task CompleteCohereChatAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -1358,7 +1410,7 @@ await WriteCohereStreamingChatAsync(context, result, cancellationToken) private static async Task CompleteGeminiContentAsync( string model, HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -1375,7 +1427,7 @@ CancellationToken cancellationToken private static async Task StreamGeminiContentAsync( string model, HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -1392,7 +1444,7 @@ CancellationToken cancellationToken private static async Task CompleteGeminiContentCoreAsync( string model, HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, bool stream, CancellationToken cancellationToken ) @@ -1451,7 +1503,7 @@ await WriteGeminiStreamingContentAsync(context, result, cancellationToken) private static async Task CompleteBedrockConverseAsync( string modelId, HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -1499,7 +1551,7 @@ CancellationToken cancellationToken private static async Task CompleteBedrockConverseStreamAsync( string modelId, HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -1552,7 +1604,7 @@ await WriteBedrockConverseStreamAsync(context, result, cancellationToken) private static async Task InvokeBedrockModelAsync( string modelId, HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -1584,10 +1636,10 @@ CancellationToken cancellationToken .FirstOrDefault(model => string.Equals(model.Id, modelId, StringComparison.OrdinalIgnoreCase)) ?.Kind; - if (modelKind == LlmTckModelKind.Embedding) + if (modelKind == LlmTestServerModelKind.Embedding) { var embedding = await runtime - .CreateEmbeddingAsync(modelId, [input], ReadAccessToken(context), cancellationToken) + .CreateEmbeddingAsync(modelId, [input], ReadAccessToken(context), cancellationToken: cancellationToken) .ConfigureAwait(false); return embedding.IsSuccess @@ -1600,7 +1652,7 @@ CancellationToken cancellationToken ); } - if (modelKind == LlmTckModelKind.Image) + if (modelKind == LlmTestServerModelKind.Image) { var image = await runtime .GenerateImageAsync(modelId, input, ReadAccessToken(context), cancellationToken) @@ -1636,7 +1688,7 @@ CancellationToken cancellationToken private static async Task InvokeBedrockModelWithResponseStreamAsync( string modelId, HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -1688,9 +1740,9 @@ await WriteBedrockInvokeModelStreamAsync(context, result, cancellationToken) private static async Task CompleteChatCoreAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, string? modelOverride, - LlmTckPromptCachePolicy promptCachePolicy, + LlmTestServerPromptCachePolicy promptCachePolicy, OpenAiCacheUsageShape cacheUsageShape, CancellationToken cancellationToken ) @@ -1710,7 +1762,7 @@ CancellationToken cancellationToken var request = string.IsNullOrWhiteSpace(modelOverride) ? read.Value : read.Value with { Model = modelOverride }; - if (promptCachePolicy == LlmTckPromptCachePolicy.OpenRouter + if (promptCachePolicy == LlmTestServerPromptCachePolicy.OpenRouter && string.IsNullOrWhiteSpace(request.SessionId) && context.Request.Headers.TryGetValue("x-session-id", out var sessionId) && !string.IsNullOrWhiteSpace(sessionId.ToString())) @@ -1718,6 +1770,11 @@ CancellationToken cancellationToken request = request with { SessionId = sessionId.ToString() }; } + if (promptCachePolicy == LlmTestServerPromptCachePolicy.DeepSeek && !request.Stream && request.StreamOptions is not null) + { + return InvalidRequest("DeepSeek stream_options requires stream: true."); + } + var validationError = ValidateChatRequest(request); if (validationError is not null) { @@ -1754,7 +1811,7 @@ CancellationToken cancellationToken private static async Task CreateEmbeddingAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -1765,7 +1822,7 @@ CancellationToken cancellationToken private static async Task CreateAzureOpenAiEmbeddingAsync( string deployment, HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -1775,7 +1832,7 @@ CancellationToken cancellationToken private static async Task CreateEmbeddingCoreAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, string? modelOverride, CancellationToken cancellationToken ) @@ -1803,7 +1860,7 @@ CancellationToken cancellationToken var inputs = ReadEmbeddingInputs(request.Input); var result = await runtime - .CreateEmbeddingAsync(request.Model, inputs, ReadAccessToken(context), cancellationToken) + .CreateEmbeddingAsync(request.Model, inputs, ReadAccessToken(context), cancellationToken: cancellationToken) .ConfigureAwait(false); return result.IsSuccess @@ -1811,7 +1868,8 @@ CancellationToken cancellationToken OpenAiWireMapper.ToEmbeddingResponse( request.Model, result.Vectors, - request.UsesBase64Encoding + request.UsesBase64Encoding, + result.WireMetadata ) ) : Results.Json( @@ -1822,7 +1880,7 @@ CancellationToken cancellationToken private static async Task CreateOllamaEmbeddingAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -1854,7 +1912,7 @@ CancellationToken cancellationToken read.Value.Model, inputs, ReadAccessToken(context), - cancellationToken + cancellationToken: cancellationToken ) .ConfigureAwait(false); @@ -1868,7 +1926,7 @@ CancellationToken cancellationToken private static async Task CreateCohereEmbeddingAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -1900,12 +1958,12 @@ CancellationToken cancellationToken read.Value.Model, inputs, ReadAccessToken(context), - cancellationToken + cancellationToken: cancellationToken ) .ConfigureAwait(false); return result.IsSuccess - ? Results.Json(CohereWireMapper.ToEmbedResponse(result.Vectors)) + ? Results.Json(CohereWireMapper.ToEmbedResponse(result.Vectors, result.WireMetadata)) : Results.Json( CohereWireMapper.ToError(result.ErrorMessage!), statusCode: result.StatusCode @@ -1915,7 +1973,7 @@ CancellationToken cancellationToken private static async Task CreateGeminiEmbeddingAsync( string model, HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -1946,7 +2004,7 @@ CancellationToken cancellationToken model, [GeminiWireMapper.ReadText(read.Value.Content)], ReadAccessToken(context), - cancellationToken + cancellationToken: cancellationToken ) .ConfigureAwait(false); @@ -1961,7 +2019,7 @@ CancellationToken cancellationToken private static async Task StartGeminiVideoOperationAsync( string model, HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -2011,7 +2069,7 @@ private static async Task GetGeminiVideoOperationAsync( string model, string operationId, HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -2052,7 +2110,7 @@ CancellationToken cancellationToken private static async Task GetGeminiFileAsync( string fileId, HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -2063,7 +2121,7 @@ CancellationToken cancellationToken var modelId = runtime .GetModels() - .FirstOrDefault(model => model.Kind == LlmTckModelKind.Video) + .FirstOrDefault(model => model.Kind == LlmTestServerModelKind.Video) ?.Id; if (string.IsNullOrWhiteSpace(modelId)) { @@ -2107,7 +2165,7 @@ CancellationToken cancellationToken private static async Task GenerateImageAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -2124,7 +2182,7 @@ CancellationToken cancellationToken private static async Task GenerateAzureOpenAiImageAsync( string deployment, HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -2140,7 +2198,7 @@ CancellationToken cancellationToken private static async Task GenerateImageCoreAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, string? modelOverride, bool allowStreaming, CancellationToken cancellationToken @@ -2206,14 +2264,15 @@ await WriteImageStreamingAsync( request.Background, request.OutputFormat, request.Quality, - request.Size + request.Size, + result.WireMetadata ) ); } private static async Task EditImageAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -2263,14 +2322,15 @@ await WriteImageStreamingAsync( request.Background, request.OutputFormat, request.Quality, - request.Size + request.Size, + result.WireMetadata ) ); } private static async Task CreateImageVariationAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -2298,7 +2358,7 @@ CancellationToken cancellationToken .ConfigureAwait(false); return result.IsSuccess - ? Results.Json(OpenAiWireMapper.ToImageResponse(result.DataUri, size: request.Size)) + ? Results.Json(OpenAiWireMapper.ToImageResponse(result.DataUri, size: request.Size, metadata: result.WireMetadata)) : Results.Json( OpenAiWireMapper.ToError(result.ErrorCode!, result.ErrorMessage!), statusCode: result.StatusCode @@ -2308,7 +2368,7 @@ CancellationToken cancellationToken private static async Task GenerateAzureOpenAiAudioAsync( string deployment, HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -2324,7 +2384,7 @@ CancellationToken cancellationToken private static async Task TranscribeOpenAiAudioAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -2343,7 +2403,7 @@ CancellationToken cancellationToken private static async Task TranscribeGroqAudioAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -2363,7 +2423,7 @@ CancellationToken cancellationToken private static async Task TranscribeAzureOpenAiAudioAsync( string deployment, HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -2382,7 +2442,7 @@ CancellationToken cancellationToken private static async Task TranslateOpenAiAudioAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -2400,7 +2460,7 @@ CancellationToken cancellationToken private static async Task TranslateGroqAudioAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -2419,7 +2479,7 @@ CancellationToken cancellationToken private static async Task TranslateAzureOpenAiAudioAsync( string deployment, HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, CancellationToken cancellationToken ) { @@ -2437,7 +2497,7 @@ CancellationToken cancellationToken private static async Task GenerateAudioCoreAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, string? modelOverride, string[] allowedResponseFormats, CancellationToken cancellationToken @@ -2478,7 +2538,7 @@ CancellationToken cancellationToken private static async Task TranscribeAudioCoreAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, string? modelOverride, bool allowUrlInput, bool allowStreaming, @@ -2543,7 +2603,7 @@ await WriteStreamingTranscriptionAsync(context, result, cancellationToken) private static async Task TranslateAudioCoreAsync( HttpContext context, - ILlmTckRuntime runtime, + ILlmTestServerRuntime runtime, string? modelOverride, bool allowUrlInput, string[] allowedResponseFormats, @@ -2700,7 +2760,7 @@ CancellationToken cancellationToken return new( new OpenAiVideoCreateRequest { - Model = string.IsNullOrWhiteSpace(model) ? LlmTckKnownModelIds.Sora2 : model, + Model = string.IsNullOrWhiteSpace(model) ? LlmTestServerKnownModelIds.Sora2 : model, Prompt = form["prompt"].ToString(), Seconds = EmptyToNull(form["seconds"].ToString()), Size = EmptyToNull(form["size"].ToString()), @@ -2738,7 +2798,7 @@ CancellationToken cancellationToken return new( new OpenAiImageEditRequest { - Model = string.IsNullOrWhiteSpace(model) ? LlmTckKnownModelIds.GptImage1 : model, + Model = string.IsNullOrWhiteSpace(model) ? LlmTestServerKnownModelIds.GptImage1 : model, Prompt = form["prompt"].ToString(), Images = ReadImageReferences(form), Stream = bool.TryParse(form["stream"].ToString(), out var stream) && stream, @@ -2822,14 +2882,14 @@ string[] allowedResponseFormats ); } - private static IResult? AuthorizeControlRequest(HttpContext context, ILlmTckRuntime runtime) + private static IResult? AuthorizeControlRequest(HttpContext context, ILlmTestServerRuntime runtime) { return runtime.IsBearerTokenAccepted(ReadAccessToken(context)) ? null : Results.Json( OpenAiWireMapper.ToError( "invalid_api_key", - "The supplied bearer token did not match the configured LLM TCK token." + "The supplied bearer token did not match the configured LLM Test Server token." ), statusCode: StatusCodes.Status401Unauthorized ); @@ -2884,10 +2944,10 @@ string runtimeManifestPath { endpoints.ServiceProvider .GetService() - ?.CreateLogger(typeof(LlmTckEndpointRouteBuilderExtensions)) + ?.CreateLogger(typeof(LlmTestServerEndpointRouteBuilderExtensions)) .LogWarning( exception, - "Skipping LLM TCK static web assets runtime manifest because it references a missing content root." + "Skipping LLM Test Server static web assets runtime manifest because it references a missing content root." ); } } @@ -2895,7 +2955,7 @@ string runtimeManifestPath private static async Task> ReadJsonAsync( HttpContext context, CancellationToken cancellationToken, - Func? validation = null + Func? validation = null ) { return await ReadJsonAsync(context, InvalidRequest, cancellationToken, validation) @@ -2906,18 +2966,18 @@ private static async Task> ReadJsonAsync( HttpContext context, Func invalidRequest, CancellationToken cancellationToken, - Func? validation = null + Func? validation = null ) { try { var body = await context.Request.ReadFromJsonAsync(_jsonOptions, cancellationToken) .ConfigureAwait(false); - if (LlmTckRequestPolicy.IsChatRequest()) + if (LlmTestServerRequestPolicy.IsChatRequest()) { - if (typeof(T) == typeof(JsonElement) && LlmTckRequestPolicy.HasUnsupportedChatFeatures(body)) + if (typeof(T) == typeof(JsonElement) && LlmTestServerRequestPolicy.HasUnsupportedChatFeatures(body)) { - return new JsonReadResult(default, invalidRequest("Tool calls and structured output fixtures are not supported by LlmTck on this operation.")); + return new JsonReadResult(default, invalidRequest("Tool calls and structured output fixtures are not supported by LlmTestServer on this operation.")); } if (context.Request.Path.StartsWithSegments("/perplexity") && body.TryGetProperty("tools", out var tools) && tools.ValueKind == JsonValueKind.Array && tools.GetArrayLength() > 0) @@ -2946,10 +3006,10 @@ private static async Task> ReadJsonAsync( private static JsonSerializerOptions CreateJsonOptions() { var options = new JsonSerializerOptions(JsonSerializerDefaults.Web); - options.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase)); - options.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase)); - options.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase)); - options.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase)); + options.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase)); + options.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase)); + options.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase)); + options.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase)); return options; } } diff --git a/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerHostingConfiguration.cs b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerHostingConfiguration.cs new file mode 100644 index 0000000..98e4c65 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerHostingConfiguration.cs @@ -0,0 +1,16 @@ +using Microsoft.Extensions.Configuration; + +namespace ManagedCode.LlmTestServer.Hosting; + +/// Configuration names shared by packaged hosts and consumer applications. +public static class LlmTestServerHostingConfiguration +{ + public const string RequiredBearerTokenKey = "LlmTestServer:RequiredBearerToken"; + + /// Reads the canonical host authentication setting. + public static string? GetRequiredBearerToken(IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(configuration); + return configuration[RequiredBearerTokenKey]; + } +} diff --git a/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerJsonHighlighter.cs b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerJsonHighlighter.cs new file mode 100644 index 0000000..c9367c4 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerJsonHighlighter.cs @@ -0,0 +1,78 @@ +using System.Text; +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Text.RegularExpressions; +using Microsoft.AspNetCore.Components; + +namespace ManagedCode.LlmTestServer.Hosting; + +// Renders payload previews as indented JSON with token classes; anything that is not JSON stays plain text. +internal static partial class LlmTestServerJsonHighlighter +{ + private static readonly JsonSerializerOptions _prettyJsonOptions = new() + { + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + WriteIndented = true, + }; + + public static MarkupString Highlight(string? payload, bool truncated) + { + if (string.IsNullOrWhiteSpace(payload)) + { + return new MarkupString("No body captured."); + } + + var suffix = truncated ? "\n\n[preview truncated]" : string.Empty; + string pretty; + try + { + using var document = JsonDocument.Parse(payload); + pretty = JsonSerializer.Serialize(document.RootElement, _prettyJsonOptions); + } + catch (JsonException) + { + return new MarkupString(Encode(payload + suffix)); + } + + var builder = new StringBuilder(pretty.Length * 2); + var last = 0; + foreach (Match match in JsonToken().Matches(pretty)) + { + builder.Append(Encode(pretty[last..match.Index])); + if (match.Groups["string"].Success) + { + var key = match.Groups["colon"].Success; + Wrap(builder, key ? "j-key" : "j-str", match.Groups["string"].Value); + if (key) + { + builder.Append(Encode(match.Groups["colon"].Value)); + } + } + else + { + Wrap(builder, match.Groups["literal"].Success ? "j-lit" : "j-num", match.Value); + } + + last = match.Index + match.Length; + } + + return new MarkupString(builder.Append(Encode(pretty[last..])).Append(Encode(suffix)).ToString()); + } + + private static void Wrap(StringBuilder builder, string css, string value) + { + builder.Append("").Append(Encode(value)).Append(""); + } + + // Only markup-significant characters are escaped so Unicode text stays readable in the page source. + private static string Encode(string value) + { + return value.Replace("&", "&", StringComparison.Ordinal) + .Replace("<", "<", StringComparison.Ordinal) + .Replace(">", ">", StringComparison.Ordinal) + .Replace("\"", """, StringComparison.Ordinal); + } + + [GeneratedRegex("""(?"(?:\\.|[^"\\])*")(?\s*:)?|(?\b(?:true|false|null)\b)|(?-?\b\d+(?:\.\d+)?(?:[eE][+-]?\d+)?\b)""")] + private static partial Regex JsonToken(); +} diff --git a/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerProviderHttpTrace.cs b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerProviderHttpTrace.cs new file mode 100644 index 0000000..04d6e67 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerProviderHttpTrace.cs @@ -0,0 +1,129 @@ +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Scenarios; + +namespace ManagedCode.LlmTestServer.Hosting; + +public sealed record LlmTestServerProviderHttpTraceCompletion +{ + public string? RetryAfter { get; init; } + + public LlmTestServerStreamDelivery? StreamDelivery { get; init; } + public bool StreamDeliveryApplied { get; init; } + public int EmittedContentFrames { get; init; } + + public bool? TerminalMarkerObserved { get; init; } + + public double? CompletedMonotonicSeconds { get; init; } + + public DateTimeOffset CompletedAt { get; init; } + + public double DurationMilliseconds { get; init; } + + public int? StatusCode { get; init; } + + public LlmTestServerProviderHttpTraceTransportState TransportState { get; init; } + + public string? Route { get; init; } + + public string? OperationId { get; init; } + + public bool IsStreaming { get; init; } + + public string? ResponseContentType { get; init; } + + public long ResponseByteCount { get; init; } + + public string? ResponsePreview { get; init; } + + public bool ResponsePreviewTruncated { get; init; } + + public string? TransportError { get; init; } +} + +public sealed record LlmTestServerProviderHttpTraceRequestCapture +{ + public long? ByteCount { get; init; } + + public string? Preview { get; init; } + + public bool PreviewTruncated { get; init; } + + public string? ModelId { get; init; } +} + +public sealed record LlmTestServerProviderHttpTraceEnrichment +{ + public LlmTestServerOutputDisposition? OutputDisposition { get; init; } + public string? WireClockMode { get; init; } + public DateTimeOffset? WireCreatedAt { get; init; } + public long? WireSequence { get; init; } + public IReadOnlyList? NativeTools { get; init; } + public IReadOnlyList? ResponseOutputItems { get; init; } + public IReadOnlyList? ResponseToolCalls { get; init; } + public string? ConversationId { get; init; } + public string? StepId { get; init; } + public int? TurnIndex { get; init; } + public bool? IsReplay { get; init; } + + public string? OperationId { get; init; } + + public string? ModelId { get; init; } + + public string? ScenarioId { get; init; } + + public LlmTestServerEventKind? RuntimeEventKind { get; init; } + + public string? RuntimeMessage { get; init; } + + public string? RuntimeResponse { get; init; } + + public string? ErrorCode { get; init; } + + public string? ErrorMessage { get; init; } + + public LlmTestServerTokenUsage? Usage { get; init; } + + public IReadOnlyList? Messages { get; init; } + + public IReadOnlyList? StreamChunks { get; init; } + + public bool? IsStreaming { get; init; } + + public bool? RuntimePayloadTruncated { get; init; } +} + +public sealed class LlmTestServerProviderHttpTraceOptions +{ + public const int DefaultMaxEntries = 500; + public const int DefaultMaxPreviewBytes = 64 * 1024; + public const int DefaultMaxRetainedBytes = 16 * 1024 * 1024; + public const string DefaultRequestIdHeaderName = "x-llm-test-server-request-id"; + + public int MaxEntries { get; set; } = DefaultMaxEntries; + + public int MaxPreviewBytes { get; set; } = DefaultMaxPreviewBytes; + + public int MaxRetainedBytes { get; set; } = DefaultMaxRetainedBytes; + + public string RequestIdHeaderName { get; set; } = DefaultRequestIdHeaderName; +} + +public interface ILlmTestServerProviderHttpTraceStore +{ + long RetainedByteCount { get; } + + IReadOnlyList GetSnapshot(); + + LlmTestServerRequestSnapshot GetEvidenceSnapshot(); + + void Start(LlmTestServerProviderHttpTrace trace); + + bool TryCaptureRequest(string requestId, LlmTestServerProviderHttpTraceRequestCapture capture); + + bool TryComplete(string requestId, LlmTestServerProviderHttpTraceCompletion completion); + + bool TryEnrich(string requestId, LlmTestServerProviderHttpTraceEnrichment enrichment); + + void Reset(); +} diff --git a/src/ManagedCode.LlmTck.Hosting/LlmTckProviderHttpTraceExtensions.cs b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerProviderHttpTraceExtensions.cs similarity index 62% rename from src/ManagedCode.LlmTck.Hosting/LlmTckProviderHttpTraceExtensions.cs rename to src/ManagedCode.LlmTestServer.Hosting/LlmTestServerProviderHttpTraceExtensions.cs index 555745b..87048e4 100644 --- a/src/ManagedCode.LlmTck.Hosting/LlmTckProviderHttpTraceExtensions.cs +++ b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerProviderHttpTraceExtensions.cs @@ -5,24 +5,24 @@ using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; -namespace ManagedCode.LlmTck.Hosting; +namespace ManagedCode.LlmTestServer.Hosting; -public static class LlmTckProviderHttpTraceExtensions +public static class LlmTestServerProviderHttpTraceExtensions { - private const string _requestIdItemKey = "ManagedCode.LlmTck.Hosting.ProviderHttpTrace.RequestId"; + private const string _requestIdItemKey = "ManagedCode.LlmTestServer.Hosting.ProviderHttpTrace.RequestId"; private const int _absoluteMaxPreviewBytes = 4 * 1024 * 1024; private const int _minimumMaxRetainedBytes = 512; private const int _absoluteMaxRetainedBytes = 256 * 1024 * 1024; - public static IServiceCollection AddLlmTckProviderHttpTracing( + public static IServiceCollection AddLlmTestServerProviderHttpTracing( this IServiceCollection services, - Action? configure = null + Action? configure = null ) { ArgumentNullException.ThrowIfNull(services); var options = services - .AddOptions() + .AddOptions() .Validate(value => value.MaxEntries > 0, "MaxEntries must be greater than zero.") .Validate( value => value.MaxPreviewBytes is > 0 and <= _absoluteMaxPreviewBytes, @@ -48,24 +48,24 @@ public static IServiceCollection AddLlmTckProviderHttpTracing( options.Configure(configure); } - services.TryAddSingleton(); - services.TryAddTransient(); + services.TryAddSingleton(); + services.TryAddTransient(); return services; } - public static RouteGroupBuilder AddLlmTckProviderHttpTracing(this RouteGroupBuilder group) + public static RouteGroupBuilder AddLlmTestServerProviderHttpTracing(this RouteGroupBuilder group) { ArgumentNullException.ThrowIfNull(group); group.AddEndpointFilterFactory( static (_, next) => async invocationContext => { - LlmTckProviderHttpTraceFilter filter; + LlmTestServerProviderHttpTraceFilter filter; try { filter = invocationContext .HttpContext .RequestServices - .GetRequiredService(); + .GetRequiredService(); } catch (Exception exception) { @@ -81,23 +81,34 @@ public static RouteGroupBuilder AddLlmTckProviderHttpTracing(this RouteGroupBuil return group; } - public static string? GetLlmTckProviderHttpTraceId(this HttpContext context) + public static void MarkLlmTestServerTerminalMarker(this HttpContext context) + { + ArgumentNullException.ThrowIfNull(context); + context.Items["ManagedCode.LlmTestServer.TerminalMarker"] = true; + } + + internal static bool? HasLlmTestServerTerminalMarker(this HttpContext context) + { + return context.Items.ContainsKey("ManagedCode.LlmTestServer.TerminalMarker") ? true : null; + } + + public static string? GetLlmTestServerProviderHttpTraceId(this HttpContext context) { ArgumentNullException.ThrowIfNull(context); return context.Items.TryGetValue(_requestIdItemKey, out var value) ? value as string : null; } - public static RouteHandlerBuilder WithLlmTckProviderOperation( + public static RouteHandlerBuilder WithLlmTestServerProviderOperation( this RouteHandlerBuilder builder, string operationId ) { ArgumentNullException.ThrowIfNull(builder); ArgumentException.ThrowIfNullOrWhiteSpace(operationId); - return builder.WithMetadata(new LlmTckProviderOperationMetadata(operationId)); + return builder.WithMetadata(new LlmTestServerProviderOperationMetadata(operationId)); } - internal static void SetLlmTckProviderHttpTraceId(HttpContext context, string requestId) + internal static void SetLlmTestServerProviderHttpTraceId(HttpContext context, string requestId) { context.Items[_requestIdItemKey] = requestId; } @@ -112,10 +123,10 @@ Exception exception context .RequestServices .GetService() - ?.CreateLogger(typeof(LlmTckProviderHttpTraceFilter).FullName!) + ?.CreateLogger(typeof(LlmTestServerProviderHttpTraceFilter).FullName!) .LogWarning( exception, - "LLM TCK provider HTTP tracing could not be resolved; provider behavior continues unchanged." + "LLM Test Server provider HTTP tracing could not be resolved; provider behavior continues unchanged." ); } catch (Exception) @@ -124,6 +135,6 @@ Exception exception } } -public sealed record LlmTckProviderOperationMetadata(string OperationId); +public sealed record LlmTestServerProviderOperationMetadata(string OperationId); -public sealed record LlmTckProviderFallbackMetadata; +public sealed record LlmTestServerProviderFallbackMetadata; diff --git a/src/ManagedCode.LlmTck.Hosting/LlmTckProviderHttpTraceFilter.cs b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerProviderHttpTraceFilter.cs similarity index 60% rename from src/ManagedCode.LlmTck.Hosting/LlmTckProviderHttpTraceFilter.cs rename to src/ManagedCode.LlmTestServer.Hosting/LlmTestServerProviderHttpTraceFilter.cs index ab20c08..93d1003 100644 --- a/src/ManagedCode.LlmTck.Hosting/LlmTckProviderHttpTraceFilter.cs +++ b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerProviderHttpTraceFilter.cs @@ -1,6 +1,8 @@ using System.Diagnostics; -using ManagedCode.LlmTck.Providers; -using ManagedCode.LlmTck.Runtime; +using System.Text.Json; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Providers; +using ManagedCode.LlmTestServer.Runtime; using Microsoft.AspNetCore.Connections; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Http.Features; @@ -8,38 +10,38 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -namespace ManagedCode.LlmTck.Hosting; +namespace ManagedCode.LlmTestServer.Hosting; -public sealed class LlmTckProviderHttpTraceFilter( - ILlmTckProviderHttpTraceStore traceStore, - ILlmTckRuntime runtime, - IOptions options, - ILogger logger +public sealed class LlmTestServerProviderHttpTraceFilter( + ILlmTestServerProviderHttpTraceStore traceStore, + ILlmTestServerRuntime runtime, + IOptions options, + ILogger logger ) : IEndpointFilter { private static readonly ProviderRoute[] _providerRoutes = [ - new(LlmTckProviderRouteNamespaces.SystemOne, LlmTckCompatibilityTags.SystemOne), - new(LlmTckProviderRouteNamespaces.Cloudflare, LlmTckCompatibilityTags.Cloudflare), - new(LlmTckProviderRouteNamespaces.Anthropic, LlmTckCompatibilityTags.Anthropic), - new(LlmTckProviderRouteNamespaces.AzureOpenAI, LlmTckCompatibilityTags.AzureOpenAI), - new(LlmTckProviderRouteNamespaces.Bedrock, LlmTckCompatibilityTags.Bedrock), - new(LlmTckProviderRouteNamespaces.Cohere, LlmTckCompatibilityTags.Cohere), - new(LlmTckProviderRouteNamespaces.DeepSeek, LlmTckCompatibilityTags.DeepSeek), - new(LlmTckProviderRouteNamespaces.Gemini, LlmTckCompatibilityTags.Gemini), - new(LlmTckProviderRouteNamespaces.Groq, LlmTckCompatibilityTags.Groq), + new(LlmTestServerProviderRouteNamespaces.SystemOne, LlmTestServerCompatibilityTags.SystemOne), + new(LlmTestServerProviderRouteNamespaces.Cloudflare, LlmTestServerCompatibilityTags.Cloudflare), + new(LlmTestServerProviderRouteNamespaces.Anthropic, LlmTestServerCompatibilityTags.Anthropic), + new(LlmTestServerProviderRouteNamespaces.AzureOpenAI, LlmTestServerCompatibilityTags.AzureOpenAI), + new(LlmTestServerProviderRouteNamespaces.Bedrock, LlmTestServerCompatibilityTags.Bedrock), + new(LlmTestServerProviderRouteNamespaces.Cohere, LlmTestServerCompatibilityTags.Cohere), + new(LlmTestServerProviderRouteNamespaces.DeepSeek, LlmTestServerCompatibilityTags.DeepSeek), + new(LlmTestServerProviderRouteNamespaces.Gemini, LlmTestServerCompatibilityTags.Gemini), + new(LlmTestServerProviderRouteNamespaces.Groq, LlmTestServerCompatibilityTags.Groq), new( - LlmTckProviderRouteNamespaces.MicrosoftFoundry, - LlmTckCompatibilityTags.MicrosoftFoundry + LlmTestServerProviderRouteNamespaces.MicrosoftFoundry, + LlmTestServerCompatibilityTags.MicrosoftFoundry ), - new(LlmTckProviderRouteNamespaces.Mistral, LlmTckCompatibilityTags.Mistral), - new(LlmTckProviderRouteNamespaces.Ollama, LlmTckCompatibilityTags.Ollama), - new(LlmTckProviderRouteNamespaces.OpenAI, LlmTckCompatibilityTags.OpenAI), - new(LlmTckProviderRouteNamespaces.OpenRouter, LlmTckCompatibilityTags.OpenRouter), - new(LlmTckProviderRouteNamespaces.Perplexity, LlmTckCompatibilityTags.Perplexity), + new(LlmTestServerProviderRouteNamespaces.Mistral, LlmTestServerCompatibilityTags.Mistral), + new(LlmTestServerProviderRouteNamespaces.Ollama, LlmTestServerCompatibilityTags.Ollama), + new(LlmTestServerProviderRouteNamespaces.OpenAI, LlmTestServerCompatibilityTags.OpenAI), + new(LlmTestServerProviderRouteNamespaces.OpenRouter, LlmTestServerCompatibilityTags.OpenRouter), + new(LlmTestServerProviderRouteNamespaces.Perplexity, LlmTestServerCompatibilityTags.Perplexity), ]; - private readonly LlmTckProviderHttpTraceOptions _options = options.Value; + private readonly LlmTestServerProviderHttpTraceOptions _options = options.Value; public async ValueTask InvokeAsync( EndpointFilterInvocationContext context, @@ -62,7 +64,17 @@ EndpointFilterDelegate next return await next(context).ConfigureAwait(false); } - var runtimeScope = TryBeginRuntimeScope(session.RequestId); + var runtimeScope = TryBeginRuntimeScope(session.RequestId, httpContext.Request.Headers[ManagedCode.LlmTestServer.Conversations.LlmTestServerConversationHeaders.Id].FirstOrDefault(), provider.Value.Namespace); + if (runtime is ILlmTestServerWireMetadataSource wireMetadataSource) + { + var metadata = wireMetadataSource.GetResponseMetadata(); + traceStore.TryEnrich(session.RequestId, new LlmTestServerProviderHttpTraceEnrichment + { WireClockMode = metadata.ClockMode, WireCreatedAt = metadata.CreatedAt, WireSequence = metadata.Sequence }); + if (metadata.ClockMode == "fixed") + { + httpContext.Response.Headers.Date = metadata.CreatedAt.ToString("R", System.Globalization.CultureInfo.InvariantCulture); + } + } try { object? result; @@ -77,10 +89,16 @@ EndpointFilterDelegate next throw; } + if (runtime is ILlmTestServerRuntimeEventLookup eventLookup + && eventLookup.TryGetEvent(session.RequestId, out var runtimeEvent) + && runtimeEvent?.RetryAfterSeconds is { } retryAfter) + { + httpContext.Response.Headers.RetryAfter = retryAfter.ToString(System.Globalization.CultureInfo.InvariantCulture); + } session.CaptureRequestBestEffort(); if (result is IResult httpResult) { - return new LlmTckProviderHttpTraceResult(httpResult, session); + return new LlmTestServerProviderHttpTraceResult(httpResult, session); } session.RegisterCompletionFallback(); @@ -95,21 +113,21 @@ EndpointFilterDelegate next private TraceSession? TryStartSession(HttpContext context, ProviderRoute provider) { var requestId = context.TraceIdentifier; - var requestContentType = LlmTckProviderHttpTracePayloads.NormalizeContentType( + var requestContentType = LlmTestServerProviderHttpTracePayloads.NormalizeContentType( context.Request.ContentType ); - var requestCaptureLimit = LlmTckProviderHttpTracePayloads.ShouldCaptureBody( + var requestCaptureLimit = LlmTestServerProviderHttpTracePayloads.ShouldCaptureBody( requestContentType ) ? _options.MaxPreviewBytes : 0; var originalRequestBody = context.Request.Body; var originalResponseBody = context.Response.Body; - var requestTee = new LlmTckBoundedReadTeeStream( + var requestTee = new LlmTestServerBoundedReadTeeStream( originalRequestBody, requestCaptureLimit ); - var responseTee = new LlmTckBoundedWriteTeeStream( + var responseTee = new LlmTestServerBoundedWriteTeeStream( originalResponseBody, _options.MaxPreviewBytes ); @@ -117,7 +135,7 @@ EndpointFilterDelegate next var route = (endpoint as RouteEndpoint)?.RoutePattern.RawText; var operationId = endpoint ?.Metadata - .GetMetadata() + .GetMetadata() ?.OperationId; try @@ -139,11 +157,15 @@ EndpointFilterDelegate next requestTee, responseTee ); - LlmTckProviderHttpTraceExtensions.SetLlmTckProviderHttpTraceId(context, requestId); + LlmTestServerProviderHttpTraceExtensions.SetLlmTestServerProviderHttpTraceId(context, requestId); traceStore.Start( - new LlmTckProviderHttpTrace + new LlmTestServerProviderHttpTrace { RequestId = requestId, + RetryGroupId = context.Request.Headers[LlmTestServerRequestHeaders.RetryGroup].FirstOrDefault(), + ConversationId = context.Request.Headers[ManagedCode.LlmTestServer.Conversations.LlmTestServerConversationHeaders.Id].FirstOrDefault(), + ConversationIdWasProvided = context.Request.Headers.ContainsKey(ManagedCode.LlmTestServer.Conversations.LlmTestServerConversationHeaders.Id), + SessionId = context.Request.Headers[ManagedCode.LlmTestServer.Sessions.LlmTestServerSessionHeaders.Id].ToString() is { Length: > 0 } sessionId ? sessionId : null, ProviderId = provider.ProviderId, ProviderNamespace = provider.Namespace, Method = context.Request.Method, @@ -151,7 +173,8 @@ EndpointFilterDelegate next Route = route, OperationId = operationId, StartedAt = session.StartedAt, - TransportState = LlmTckProviderHttpTraceTransportState.InProgress, + StartedMonotonicSeconds = session.StartedMonotonicSeconds, + TransportState = LlmTestServerProviderHttpTraceTransportState.InProgress, RequestContentType = requestContentType, RequestByteCount = context.Request.ContentLength, ModelId = ReadRouteModelId(context), @@ -170,16 +193,16 @@ EndpointFilterDelegate next } } - private IDisposable? TryBeginRuntimeScope(string requestId) + private IDisposable? TryBeginRuntimeScope(string requestId, string? conversationId, string providerNamespace) { - if (runtime is not ILlmTckRuntimeRequestScope runtimeRequestScope) + if (runtime is not ILlmTestServerRuntimeRequestScope runtimeRequestScope) { return null; } try { - return runtimeRequestScope.BeginRequest(requestId); + return runtimeRequestScope.BeginRequest(requestId, conversationId, providerNamespace); } catch (Exception exception) { @@ -221,7 +244,7 @@ private void LogFailure(Exception exception, string phase, string requestId) { logger.LogWarning( exception, - "LLM TCK provider HTTP tracing failed during {Phase} for request {RequestId}; provider behavior continues unchanged.", + "LLM Test Server provider HTTP tracing failed during {Phase} for request {RequestId}; provider behavior continues unchanged.", phase, requestId ); @@ -294,8 +317,8 @@ private static void TryDispose(IDisposable disposable) private sealed class TraceSession( HttpContext context, - ILlmTckProviderHttpTraceStore traceStore, - ILlmTckRuntime runtime, + ILlmTestServerProviderHttpTraceStore traceStore, + ILlmTestServerRuntime runtime, ILogger logger, string requestId, string? requestContentType, @@ -304,8 +327,8 @@ private sealed class TraceSession( string? operationId, Stream originalRequestBody, Stream originalResponseBody, - LlmTckBoundedReadTeeStream requestTee, - LlmTckBoundedWriteTeeStream responseTee + LlmTestServerBoundedReadTeeStream requestTee, + LlmTestServerBoundedWriteTeeStream responseTee ) { private readonly long _startedTimestamp = Stopwatch.GetTimestamp(); @@ -316,6 +339,8 @@ LlmTckBoundedWriteTeeStream responseTee public DateTimeOffset StartedAt { get; } = DateTimeOffset.UtcNow; + public double StartedMonotonicSeconds => (double)_startedTimestamp / Stopwatch.Frequency; + public void CaptureRequestBestEffort() { if (Interlocked.Exchange(ref _requestCaptured, 1) != 0) @@ -331,7 +356,7 @@ public void CaptureRequestBestEffort() var incomplete = expectedBytes is > 0 && observedBytes < expectedBytes; var capturedBytes = requestTee.GetCapturedBytes(); var preview = TryCreateMultipartPreview(expectedBytes ?? observedBytes) - ?? LlmTckProviderHttpTracePayloads.CreateRequestPreview( + ?? LlmTestServerProviderHttpTracePayloads.CreateRequestPreview( capturedBytes, requestContentType, expectedBytes ?? observedBytes, @@ -339,7 +364,7 @@ public void CaptureRequestBestEffort() ); traceStore.TryCaptureRequest( requestId, - new LlmTckProviderHttpTraceRequestCapture + new LlmTestServerProviderHttpTraceRequestCapture { ByteCount = preview.ByteCount, Preview = preview.Preview, @@ -363,7 +388,7 @@ public void CaptureRequestBestEffort() } } - private LlmTckProviderHttpPayloadPreview? TryCreateMultipartPreview(long? byteCount) + private LlmTestServerProviderHttpPayloadPreview? TryCreateMultipartPreview(long? byteCount) { if ( requestContentType?.StartsWith( @@ -380,7 +405,7 @@ public void CaptureRequestBestEffort() var form = context.Features.Get()?.Form; return form is null ? null - : LlmTckProviderHttpTracePayloads.CreateMultipartRequestPreview( + : LlmTestServerProviderHttpTracePayloads.CreateMultipartRequestPreview( form, requestContentType, byteCount, @@ -421,7 +446,7 @@ public Task CompleteBestEffortAsync(Exception? exception) RestoreResponseBodyBestEffort(); var transportState = ClassifyTransportState(context, exception); - int? statusCode = transportState == LlmTckProviderHttpTraceTransportState.Completed + int? statusCode = transportState == LlmTestServerProviderHttpTraceTransportState.Completed || context.Response.HasStarted ? context.Response.StatusCode : null; @@ -429,15 +454,27 @@ public Task CompleteBestEffortAsync(Exception? exception) try { - var responseContentType = LlmTckProviderHttpTracePayloads.NormalizeContentType( + var responseContentType = LlmTestServerProviderHttpTracePayloads.NormalizeContentType( context.Response.ContentType ); var responseBytes = responseTee.GetCapturedBytes(); + var responsePreview = LlmTestServerProviderHttpTracePayloads.CreateResponsePreview( + responseBytes, responseContentType, responseTee.BytesObserved, responseTee.IsTruncated); + if (statusCode >= 400) + { + EnrichFromErrorPayloadBestEffort(responsePreview); + } traceStore.TryComplete( requestId, - new LlmTckProviderHttpTraceCompletion + new LlmTestServerProviderHttpTraceCompletion { CompletedAt = DateTimeOffset.UtcNow, + CompletedMonotonicSeconds = (double)Stopwatch.GetTimestamp() / Stopwatch.Frequency, + RetryAfter = context.Response.Headers.RetryAfter.FirstOrDefault(), + TerminalMarkerObserved = context.HasLlmTestServerTerminalMarker(), + StreamDelivery = LlmTestServerStreamDeliveryEvidence.Read(context)?.Plan, + StreamDeliveryApplied = LlmTestServerStreamDeliveryEvidence.Read(context)?.Applied ?? false, + EmittedContentFrames = LlmTestServerStreamDeliveryEvidence.Read(context)?.EmittedContentFrames ?? 0, DurationMilliseconds = Stopwatch .GetElapsedTime(_startedTimestamp) .TotalMilliseconds, @@ -445,22 +482,17 @@ public Task CompleteBestEffortAsync(Exception? exception) TransportState = transportState, Route = route, OperationId = operationId, - IsStreaming = LlmTckProviderHttpTracePayloads.IsStreamingResponse( + IsStreaming = LlmTestServerProviderHttpTracePayloads.IsStreamingResponse( responseContentType, route, responseTee.SuccessfulFlushCount ), ResponseContentType = responseContentType, ResponseByteCount = responseTee.BytesObserved, - ResponsePreview = LlmTckProviderHttpTracePayloads.CreateResponsePreview( - responseBytes, - responseContentType, - responseTee.BytesObserved, - responseTee.IsTruncated - ), + ResponsePreview = responsePreview, ResponsePreviewTruncated = responseTee.IsTruncated, TransportError = exception?.GetType().Name - ?? (transportState == LlmTckProviderHttpTraceTransportState.Aborted + ?? (transportState == LlmTestServerProviderHttpTraceTransportState.Aborted ? "RequestAborted" : null), } @@ -484,11 +516,52 @@ public Task CompleteBestEffortAsync(Exception? exception) return Task.CompletedTask; } + private void EnrichFromErrorPayloadBestEffort(string? preview) + { + if (string.IsNullOrWhiteSpace(preview)) + { + return; + } + try + { + using var document = JsonDocument.Parse(preview); + var root = document.RootElement; + if (root.ValueKind != JsonValueKind.Object) + { + return; + } + var error = root; + if (root.TryGetProperty("error", out var native) && native.ValueKind == JsonValueKind.Object) + { + error = native; + } + else if (root.TryGetProperty("errors", out var errors) + && errors.ValueKind == JsonValueKind.Array && errors.GetArrayLength() > 0 + && errors[0].ValueKind == JsonValueKind.Object) + { + error = errors[0]; + } + var code = error.TryGetProperty("code", out var value) ? value + : error.TryGetProperty("type", out value) ? value : default; + traceStore.TryEnrich(requestId, new LlmTestServerProviderHttpTraceEnrichment + { + ErrorCode = code.ValueKind == JsonValueKind.String ? code.GetString() + : code.ValueKind == JsonValueKind.Number ? code.GetRawText() : null, + ErrorMessage = error.TryGetProperty("message", out var message) && message.ValueKind == JsonValueKind.String + ? message.GetString() : null, + }); + } + catch (JsonException) + { + // A truncated or non-JSON diagnostic preview must not affect the actual provider response. + } + } + private void EnrichFromRuntimeEventBestEffort() { try { - if (runtime is not ILlmTckRuntimeEventLookup lookup + if (runtime is not ILlmTestServerRuntimeEventLookup lookup || !lookup.TryGetEvent(requestId, out var runtimeEvent) || runtimeEvent is null) { @@ -497,15 +570,26 @@ private void EnrichFromRuntimeEventBestEffort() traceStore.TryEnrich( requestId, - new LlmTckProviderHttpTraceEnrichment + new LlmTestServerProviderHttpTraceEnrichment { + WireClockMode = runtimeEvent.WireMetadata?.ClockMode, + WireCreatedAt = runtimeEvent.WireMetadata?.CreatedAt, + WireSequence = runtimeEvent.WireMetadata?.Sequence, ModelId = runtimeEvent.ModelId, ScenarioId = runtimeEvent.ScenarioId, + ConversationId = runtimeEvent.ConversationId, + StepId = runtimeEvent.StepId, + TurnIndex = runtimeEvent.TurnIndex, + IsReplay = runtimeEvent.IsReplay, + OutputDisposition = runtimeEvent.OutputDisposition, RuntimeEventKind = runtimeEvent.Kind, RuntimeMessage = runtimeEvent.Message, RuntimeResponse = runtimeEvent.Response, Usage = runtimeEvent.Usage, Messages = runtimeEvent.Messages, + ResponseToolCalls = runtimeEvent.ResponseToolCalls, + NativeTools = runtimeEvent.NativeTools, + ResponseOutputItems = runtimeEvent.ResponseOutputItems, StreamChunks = runtimeEvent.StreamChunks, IsStreaming = runtimeEvent.IsStreaming, RuntimePayloadTruncated = runtimeEvent.PayloadTruncated, @@ -519,7 +603,7 @@ private void EnrichFromRuntimeEventBestEffort() } private void TryCompleteWithoutPayload( - LlmTckProviderHttpTraceTransportState transportState, + LlmTestServerProviderHttpTraceTransportState transportState, int? statusCode, Exception? exception ) @@ -528,9 +612,15 @@ private void TryCompleteWithoutPayload( { traceStore.TryComplete( requestId, - new LlmTckProviderHttpTraceCompletion + new LlmTestServerProviderHttpTraceCompletion { CompletedAt = DateTimeOffset.UtcNow, + CompletedMonotonicSeconds = (double)Stopwatch.GetTimestamp() / Stopwatch.Frequency, + RetryAfter = context.Response.Headers.RetryAfter.FirstOrDefault(), + TerminalMarkerObserved = context.HasLlmTestServerTerminalMarker(), + StreamDelivery = LlmTestServerStreamDeliveryEvidence.Read(context)?.Plan, + StreamDeliveryApplied = LlmTestServerStreamDeliveryEvidence.Read(context)?.Applied ?? false, + EmittedContentFrames = LlmTestServerStreamDeliveryEvidence.Read(context)?.EmittedContentFrames ?? 0, DurationMilliseconds = Stopwatch .GetElapsedTime(_startedTimestamp) .TotalMilliseconds, @@ -538,7 +628,7 @@ private void TryCompleteWithoutPayload( TransportState = transportState, Route = route, OperationId = operationId, - ResponseContentType = LlmTckProviderHttpTracePayloads.NormalizeContentType( + ResponseContentType = LlmTestServerProviderHttpTracePayloads.NormalizeContentType( context.Response.ContentType ), ResponseByteCount = responseTee.BytesObserved, @@ -592,7 +682,7 @@ private void DisposeBestEffort(IDisposable disposable, string phase) private void LogCaptureFailure(string phase) { logger.LogWarning( - "LLM TCK provider HTTP tracing could not retain all bytes during {Phase} for request {RequestId}; provider behavior was not affected.", + "LLM Test Server provider HTTP tracing could not retain all bytes during {Phase} for request {RequestId}; provider behavior was not affected.", phase, requestId ); @@ -602,13 +692,13 @@ private void LogFailure(Exception exception, string phase) { logger.LogWarning( exception, - "LLM TCK provider HTTP tracing failed during {Phase} for request {RequestId}; provider behavior continues unchanged.", + "LLM Test Server provider HTTP tracing failed during {Phase} for request {RequestId}; provider behavior continues unchanged.", phase, requestId ); } - private static LlmTckProviderHttpTraceTransportState ClassifyTransportState( + private static LlmTestServerProviderHttpTraceTransportState ClassifyTransportState( HttpContext context, Exception? exception ) @@ -618,21 +708,21 @@ private static LlmTckProviderHttpTraceTransportState ClassifyTransportState( || exception is ConnectionAbortedException ) { - return LlmTckProviderHttpTraceTransportState.Aborted; + return LlmTestServerProviderHttpTraceTransportState.Aborted; } if (exception is OperationCanceledException) { - return LlmTckProviderHttpTraceTransportState.Cancelled; + return LlmTestServerProviderHttpTraceTransportState.Cancelled; } return exception is null - ? LlmTckProviderHttpTraceTransportState.Completed - : LlmTckProviderHttpTraceTransportState.Faulted; + ? LlmTestServerProviderHttpTraceTransportState.Completed + : LlmTestServerProviderHttpTraceTransportState.Faulted; } } - private sealed class LlmTckProviderHttpTraceResult( + private sealed class LlmTestServerProviderHttpTraceResult( IResult result, TraceSession session ) : IResult diff --git a/src/ManagedCode.LlmTck.Hosting/LlmTckProviderHttpTracePayloads.cs b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerProviderHttpTracePayloads.cs similarity index 98% rename from src/ManagedCode.LlmTck.Hosting/LlmTckProviderHttpTracePayloads.cs rename to src/ManagedCode.LlmTestServer.Hosting/LlmTestServerProviderHttpTracePayloads.cs index 9df7158..3d00824 100644 --- a/src/ManagedCode.LlmTck.Hosting/LlmTckProviderHttpTracePayloads.cs +++ b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerProviderHttpTracePayloads.cs @@ -4,13 +4,13 @@ using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.WebUtilities; -namespace ManagedCode.LlmTck.Hosting; +namespace ManagedCode.LlmTestServer.Hosting; -internal static class LlmTckProviderHttpTracePayloads +internal static class LlmTestServerProviderHttpTracePayloads { private const string _redactedValue = "[redacted]"; - public static LlmTckProviderHttpPayloadPreview CreateRequestPreview( + public static LlmTestServerProviderHttpPayloadPreview CreateRequestPreview( ReadOnlyMemory body, string? contentType, long? byteCount, @@ -75,7 +75,7 @@ bool truncated : CreateOmittedBodySummary("binary", contentType, byteCount); } - public static LlmTckProviderHttpPayloadPreview CreateMultipartRequestPreview( + public static LlmTestServerProviderHttpPayloadPreview CreateMultipartRequestPreview( IFormCollection form, string? contentType, long? byteCount, @@ -576,7 +576,7 @@ _ when char.IsHighSurrogate(character) } } -internal readonly record struct LlmTckProviderHttpPayloadPreview( +internal readonly record struct LlmTestServerProviderHttpPayloadPreview( string? Preview, bool IsTruncated, long? ByteCount, diff --git a/src/ManagedCode.LlmTck.Hosting/LlmTckProviderHttpTraceStore.cs b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerProviderHttpTraceStore.cs similarity index 52% rename from src/ManagedCode.LlmTck.Hosting/LlmTckProviderHttpTraceStore.cs rename to src/ManagedCode.LlmTestServer.Hosting/LlmTestServerProviderHttpTraceStore.cs index 8d50b4a..0ce5d63 100644 --- a/src/ManagedCode.LlmTck.Hosting/LlmTckProviderHttpTraceStore.cs +++ b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerProviderHttpTraceStore.cs @@ -1,17 +1,19 @@ using System.Text; -using ManagedCode.LlmTck.Scenarios; +using System.Text.Json; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Scenarios; using Microsoft.Extensions.Options; -namespace ManagedCode.LlmTck.Hosting; +namespace ManagedCode.LlmTestServer.Hosting; -public sealed class LlmTckProviderHttpTraceStore : ILlmTckProviderHttpTraceStore +public sealed class LlmTestServerProviderHttpTraceStore : ILlmTestServerProviderHttpTraceStore { private const int _traceOverheadBytes = 512; private const int _messageOverheadBytes = 64; private const int _chunkOverheadBytes = 32; private const int _maxMetadataBytes = 4 * 1024; private readonly object _gate = new(); - private readonly Dictionary _traces = new( + private readonly Dictionary _traces = new( StringComparer.Ordinal ); private readonly LinkedList _order = []; @@ -19,8 +21,11 @@ public sealed class LlmTckProviderHttpTraceStore : ILlmTckProviderHttpTraceStore private readonly int _maxPreviewBytes; private readonly int _maxRetainedBytes; private long _retainedByteCount; + private long _totalObserved; + private long _evictedCount; + private bool _retentionTruncated; - public LlmTckProviderHttpTraceStore(IOptions options) + public LlmTestServerProviderHttpTraceStore(IOptions options) { ArgumentNullException.ThrowIfNull(options); _maxEntries = options.Value.MaxEntries; @@ -42,22 +47,48 @@ public long RetainedByteCount } } - public IReadOnlyList GetSnapshot() + public IReadOnlyList GetSnapshot() { lock (_gate) { - var snapshot = new LlmTckProviderHttpTrace[_order.Count]; + var snapshot = new LlmTestServerProviderHttpTrace[_order.Count]; var index = 0; for (var node = _order.Last; node is not null; node = node.Previous) { - snapshot[index++] = _traces[node.Value]; + var trace = _traces[node.Value]; + snapshot[index++] = trace with + { + NativeTools = Array.AsReadOnly(trace.NativeTools.Select(CloneNativeTool).ToArray()), + ResponseOutputItems = Array.AsReadOnly(trace.ResponseOutputItems.Select(CloneOutputItem).ToArray()), + Messages = Array.AsReadOnly(trace.Messages.Select(message => message with + { OutputItems = message.OutputItems.Select(CloneOutputItem).ToList() }).ToArray()), + }; } return snapshot; } } - public void Start(LlmTckProviderHttpTrace trace) + public LlmTestServerRequestSnapshot GetEvidenceSnapshot() + { + lock (_gate) + { + var entries = GetSnapshot(); + return new() + { + Requests = entries, + TotalMatched = entries.Count, + TotalObserved = _totalObserved, + EvictedCount = _evictedCount, + RetentionTruncated = _retentionTruncated, + RetainedByteCount = _retainedByteCount, + FirstSequence = entries.Count == 0 ? null : entries.Min(entry => entry.Sequence), + LastSequence = entries.Count == 0 ? null : entries.Max(entry => entry.Sequence) + }; + } + } + + public void Start(LlmTestServerProviderHttpTrace trace) { ArgumentNullException.ThrowIfNull(trace); ArgumentException.ThrowIfNullOrWhiteSpace(trace.RequestId); @@ -68,9 +99,11 @@ public void Start(LlmTckProviderHttpTrace trace) { _retainedByteCount -= GetEntryRetainedByteCount(trace.RequestId, existing); _order.Remove(trace.RequestId); + _evictedCount++; } - var retained = BoundTrace(trace); + var retained = BoundTrace(trace with { Sequence = ++_totalObserved }); + _retentionTruncated |= retained.RetentionTruncated || retained.RequestPreviewTruncated || retained.ResponsePreviewTruncated || retained.RuntimePayloadTruncated; _traces[trace.RequestId] = retained; _order.AddLast(trace.RequestId); _retainedByteCount += GetEntryRetainedByteCount(trace.RequestId, retained); @@ -78,7 +111,7 @@ public void Start(LlmTckProviderHttpTrace trace) } } - public bool TryComplete(string requestId, LlmTckProviderHttpTraceCompletion completion) + public bool TryComplete(string requestId, LlmTestServerProviderHttpTraceCompletion completion) { ArgumentException.ThrowIfNullOrWhiteSpace(requestId); ArgumentNullException.ThrowIfNull(completion); @@ -95,6 +128,12 @@ public bool TryComplete(string requestId, LlmTckProviderHttpTraceCompletion comp trace with { CompletedAt = completion.CompletedAt, + CompletedMonotonicSeconds = completion.CompletedMonotonicSeconds, + RetryAfter = completion.RetryAfter, + TerminalMarkerObserved = completion.TerminalMarkerObserved, + StreamDelivery = completion.StreamDelivery, + StreamDeliveryApplied = completion.StreamDeliveryApplied, + EmittedContentFrames = completion.EmittedContentFrames, DurationMilliseconds = completion.DurationMilliseconds, StatusCode = completion.StatusCode, IsCompleted = true, @@ -115,7 +154,7 @@ trace with public bool TryCaptureRequest( string requestId, - LlmTckProviderHttpTraceRequestCapture capture + LlmTestServerProviderHttpTraceRequestCapture capture ) { ArgumentException.ThrowIfNullOrWhiteSpace(requestId); @@ -142,7 +181,7 @@ trace with } } - public bool TryEnrich(string requestId, LlmTckProviderHttpTraceEnrichment enrichment) + public bool TryEnrich(string requestId, LlmTestServerProviderHttpTraceEnrichment enrichment) { ArgumentException.ThrowIfNullOrWhiteSpace(requestId); ArgumentNullException.ThrowIfNull(enrichment); @@ -158,9 +197,17 @@ public bool TryEnrich(string requestId, LlmTckProviderHttpTraceEnrichment enrich requestId, trace with { + WireClockMode = enrichment.WireClockMode ?? trace.WireClockMode, + WireCreatedAt = enrichment.WireCreatedAt ?? trace.WireCreatedAt, + WireSequence = enrichment.WireSequence ?? trace.WireSequence, OperationId = enrichment.OperationId ?? trace.OperationId, ModelId = enrichment.ModelId ?? trace.ModelId, ScenarioId = enrichment.ScenarioId ?? trace.ScenarioId, + ConversationId = enrichment.ConversationId ?? trace.ConversationId, + StepId = enrichment.StepId ?? trace.StepId, + TurnIndex = enrichment.TurnIndex ?? trace.TurnIndex, + IsReplay = enrichment.IsReplay ?? trace.IsReplay, + OutputDisposition = enrichment.OutputDisposition ?? trace.OutputDisposition, RuntimeEventKind = enrichment.RuntimeEventKind ?? trace.RuntimeEventKind, RuntimeMessage = enrichment.RuntimeMessage ?? trace.RuntimeMessage, RuntimeResponse = enrichment.RuntimeResponse ?? trace.RuntimeResponse, @@ -168,6 +215,9 @@ trace with ErrorMessage = enrichment.ErrorMessage ?? trace.ErrorMessage, Usage = enrichment.Usage ?? trace.Usage, Messages = enrichment.Messages ?? trace.Messages, + ResponseToolCalls = enrichment.ResponseToolCalls ?? trace.ResponseToolCalls, + NativeTools = enrichment.NativeTools ?? trace.NativeTools, + ResponseOutputItems = enrichment.ResponseOutputItems ?? trace.ResponseOutputItems, StreamChunks = enrichment.StreamChunks ?? trace.StreamChunks, IsStreaming = enrichment.IsStreaming ?? trace.IsStreaming, RuntimePayloadTruncated = @@ -185,23 +235,33 @@ public void Reset() _traces.Clear(); _order.Clear(); _retainedByteCount = 0; + _totalObserved = 0; + _evictedCount = 0; + _retentionTruncated = false; } } - private void ReplaceTrace(string requestId, LlmTckProviderHttpTrace trace) + private void ReplaceTrace(string requestId, LlmTestServerProviderHttpTrace trace) { var previous = _traces[requestId]; _retainedByteCount -= GetEntryRetainedByteCount(requestId, previous); var retained = BoundTrace(trace); + _retentionTruncated |= retained.RetentionTruncated || retained.RequestPreviewTruncated || retained.ResponsePreviewTruncated || retained.RuntimePayloadTruncated; _traces[requestId] = retained; _retainedByteCount += GetEntryRetainedByteCount(requestId, retained); TrimOldestEntries(); } - private LlmTckProviderHttpTrace BoundTrace(LlmTckProviderHttpTrace trace) + private LlmTestServerProviderHttpTrace BoundTrace(LlmTestServerProviderHttpTrace trace) { var remaining = _maxRetainedBytes - _traceOverheadBytes; var retentionTruncated = trace.RetentionTruncated; + var streamDelivery = trace.StreamDelivery; + if (streamDelivery is not null) + { + if (remaining >= 64) { remaining -= 64; } + else { streamDelivery = null; retentionTruncated = true; } + } var requestId = RetainRequiredText( trace.RequestId, _maxMetadataBytes, @@ -262,6 +322,9 @@ ref retentionTruncated ref remaining, ref retentionTruncated ); + var sessionId = RetainText(trace.SessionId, _maxMetadataBytes, ref remaining, ref retentionTruncated); + var conversationId = RetainText(trace.ConversationId, _maxMetadataBytes, ref remaining, ref retentionTruncated); + var stepId = RetainText(trace.StepId, _maxMetadataBytes, ref remaining, ref retentionTruncated); var scenarioId = RetainText( trace.ScenarioId, _maxMetadataBytes, @@ -305,6 +368,7 @@ ref retentionTruncated ref remaining, ref retentionTruncated ); + var outputDisposition = RetainOutputDisposition(trace.OutputDisposition, ref remaining, ref retentionTruncated); var streamChunks = RetainChunks( trace.StreamChunks, ref remaining, @@ -317,6 +381,12 @@ ref retentionTruncated ref retentionTruncated, out var requestPreviewTruncated ); + var wireClockMode = RetainText(trace.WireClockMode, _maxMetadataBytes, ref remaining, ref retentionTruncated); + var retryGroupId = RetainText(trace.RetryGroupId, _maxMetadataBytes, ref remaining, ref retentionTruncated); + var retryAfter = RetainText(trace.RetryAfter, _maxMetadataBytes, ref remaining, ref retentionTruncated); + var responseToolCalls = RetainToolCalls(trace.ResponseToolCalls, ref remaining, ref retentionTruncated); + var nativeTools = RetainNativeTools(trace.NativeTools, ref remaining, ref retentionTruncated); + var responseOutputItems = RetainOutputItems(trace.ResponseOutputItems, ref remaining, ref retentionTruncated); var messages = RetainMessages( trace.Messages, ref remaining, @@ -342,20 +412,31 @@ ref retentionTruncated trace.ResponsePreviewTruncated || responsePreviewTruncated, ModelId = modelId, ScenarioId = scenarioId, + SessionId = sessionId, + ConversationId = conversationId, + StepId = stepId, RuntimeMessage = runtimeMessage, RuntimeResponse = runtimeResponse, ErrorCode = errorCode, ErrorMessage = errorMessage, TransportError = transportError, Messages = messages, + ResponseToolCalls = responseToolCalls, + NativeTools = nativeTools, + ResponseOutputItems = responseOutputItems, + OutputDisposition = outputDisposition, + StreamDelivery = streamDelivery, + RetryGroupId = retryGroupId, + WireClockMode = wireClockMode, + RetryAfter = retryAfter, StreamChunks = streamChunks, RetentionTruncated = retentionTruncated, RetainedByteCount = _maxRetainedBytes - remaining, }; } - private IReadOnlyList RetainMessages( - IReadOnlyList source, + private IReadOnlyList RetainMessages( + IReadOnlyList source, ref int remaining, ref bool truncated ) @@ -366,7 +447,7 @@ ref bool truncated } var maximumItemCount = remaining / _messageOverheadBytes; - var retained = new List(Math.Min(source.Count, maximumItemCount)); + var retained = new List(Math.Min(source.Count, maximumItemCount)); for (var index = 0; index < source.Count; index++) { if (remaining < _messageOverheadBytes) @@ -389,7 +470,11 @@ ref truncated ref remaining, ref truncated ); - retained.Add(new LlmTckMessage { Role = role, Content = content }); + var callId = RetainText(message.ToolCallId, _maxMetadataBytes, ref remaining, ref truncated); + var calls = RetainToolCalls(message.ToolCalls, ref remaining, ref truncated); + var outputItems = RetainOutputItems(message.OutputItems, ref remaining, ref truncated); + var outputDisposition = RetainOutputDisposition(message.OutputDisposition, ref remaining, ref truncated); + retained.Add(new LlmTestServerMessage { Role = role, Content = content, OutputDisposition = outputDisposition, ToolCallId = callId, ToolCalls = [.. calls], OutputItems = [.. outputItems] }); } if (retained.Count < source.Count) @@ -400,6 +485,152 @@ ref truncated return retained.Count == 0 ? [] : Array.AsReadOnly(retained.ToArray()); } + private LlmTestServerOutputDisposition? RetainOutputDisposition(LlmTestServerOutputDisposition? disposition, ref int remaining, ref bool truncated) + { + if (disposition is null) { return null; } + if (remaining < _chunkOverheadBytes) { truncated = true; return null; } + remaining -= _chunkOverheadBytes; + return disposition with + { + Category = RetainText(disposition.Category, _maxMetadataBytes, ref remaining, ref truncated), + Explanation = RetainText(disposition.Explanation, _maxPreviewBytes, ref remaining, ref truncated), + }; + } + + private IReadOnlyList RetainToolCalls(IReadOnlyList source, ref int remaining, ref bool truncated) + { + var retained = new List(); + foreach (var call in source) + { + if (remaining < _messageOverheadBytes) { truncated = true; break; } + remaining -= _messageOverheadBytes; + var id = RetainRequiredText(call.Id, _maxMetadataBytes, ref remaining, ref truncated); + var name = RetainRequiredText(call.Name, _maxMetadataBytes, ref remaining, ref truncated); + var arguments = RetainRequiredText(call.ArgumentsJson, _maxPreviewBytes, ref remaining, ref truncated); + var continuation = RetainText(call.ContinuationToken, _maxPreviewBytes, ref remaining, ref truncated); + retained.Add(new() { Id = id, Name = name, ArgumentsJson = arguments, ContinuationToken = continuation }); + } + return Array.AsReadOnly(retained.ToArray()); + } + + private IReadOnlyList RetainNativeTools( + IReadOnlyList source, ref int remaining, ref bool truncated) + { + var retained = new List(); + foreach (var tool in source) + { + var payloadBytes = JsonSerializer.SerializeToUtf8Bytes(tool).LongLength; + var nestedOverhead = tool.WebSearchOptions is { } options + ? (long)options.AllowedDomains.Count * _chunkOverheadBytes + + (options.UserLocation is null ? 0 : _messageOverheadBytes) + : 0; + if (tool.FileSearchOptions is { } file) + { nestedOverhead += (long)file.VectorStoreIds.Count * _chunkOverheadBytes + _messageOverheadBytes; } + if (tool.CodeInterpreterOptions is { } code) + { nestedOverhead += (long)code.FileIds.Count * _chunkOverheadBytes + _messageOverheadBytes; } + if (tool.AnthropicWebSearchOptions is { } anthropic) + { + nestedOverhead += (long)((anthropic.AllowedDomains?.Count ?? 0) + (anthropic.BlockedDomains?.Count ?? 0)) * _chunkOverheadBytes + + (anthropic.UserLocation is null ? 0 : _messageOverheadBytes); + } + var cost = _messageOverheadBytes + payloadBytes + nestedOverhead; + if (payloadBytes > _maxPreviewBytes || cost > remaining) { truncated = true; break; } + remaining -= (int)cost; + retained.Add(CloneNativeTool(tool)); + } + return Array.AsReadOnly(retained.ToArray()); + } + + private IReadOnlyList RetainOutputItems( + IReadOnlyList source, ref int remaining, ref bool truncated) + { + var retained = new List(); + foreach (var item in source) + { + var payloadBytes = JsonSerializer.SerializeToUtf8Bytes(item).LongLength; + var nestedOverhead = item switch + { + LlmTestServerResponseMessageItem message => (long)message.StreamChunks.Count * _chunkOverheadBytes + + (long)message.Citations.Count * _messageOverheadBytes, + LlmTestServerResponseWebSearchItem web => (long)web.Queries.Count * _chunkOverheadBytes + + (long)web.Sources.Count * _messageOverheadBytes, + LlmTestServerResponseFileSearchItem file => (long)(file.Queries.Count + file.VectorStoreIds.Count) * _chunkOverheadBytes + + file.Results.Sum(result => (long)_messageOverheadBytes + (long)(result.Attributes?.Count ?? 0) * _messageOverheadBytes), + LlmTestServerResponseCodeInterpreterItem code => (long)(code.InputFileIds.Count + code.GeneratedFileIds.Count + code.CodeChunks.Count) * _chunkOverheadBytes + + (long)(code.Outputs?.Count ?? 0) * _messageOverheadBytes, + LlmTestServerResponseServerToolUseItem => _messageOverheadBytes, + LlmTestServerResponseServerToolResultItem returned => _messageOverheadBytes + + (returned.Result is LlmTestServerWebSearchResults found ? (long)found.Results.Count * _messageOverheadBytes : 0), + LlmTestServerResponseFunctionCallItem => _messageOverheadBytes, + _ => 0, + }; + var cost = _messageOverheadBytes + payloadBytes + nestedOverhead; + if (payloadBytes > _maxPreviewBytes || cost > remaining) { truncated = true; break; } + remaining -= (int)cost; + retained.Add(CloneOutputItem(item)); + } + return Array.AsReadOnly(retained.ToArray()); + } + + private static LlmTestServerNativeToolDeclaration CloneNativeTool(LlmTestServerNativeToolDeclaration tool) + { + return tool with + { + WebSearchOptions = tool.WebSearchOptions is { } options ? options with + { + AllowedDomains = [.. options.AllowedDomains], + UserLocation = options.UserLocation is { } location ? location with { } : null, + } : null, + FileSearchOptions = tool.FileSearchOptions is { } file ? file with + { VectorStoreIds = [.. file.VectorStoreIds], Filters = file.Filters?.Clone() } : null, + CodeInterpreterOptions = tool.CodeInterpreterOptions is { } code ? code with + { FileIds = [.. code.FileIds] } : null, + AnthropicWebSearchOptions = tool.AnthropicWebSearchOptions is { } anthropic ? anthropic with + { + AllowedDomains = anthropic.AllowedDomains is { } allowed ? [.. allowed] : null, + BlockedDomains = anthropic.BlockedDomains is { } blocked ? [.. blocked] : null, + UserLocation = anthropic.UserLocation is { } anthropicLocation ? anthropicLocation with { } : null, + } : null, + }; + } + + private static LlmTestServerResponseItem CloneOutputItem(LlmTestServerResponseItem item) + { + return item switch + { + LlmTestServerResponseMessageItem message => message with + { StreamChunks = [.. message.StreamChunks], Citations = message.Citations.Select(citation => citation with { }).ToList() }, + LlmTestServerResponseFunctionCallItem function => function with { Call = function.Call with { } }, + LlmTestServerResponseWebSearchItem web => web with + { Queries = [.. web.Queries], Sources = web.Sources.Select(source => source with { }).ToList() }, + LlmTestServerResponseFileSearchItem file => file with + { + VectorStoreIds = [.. file.VectorStoreIds], + Queries = [.. file.Queries], + Results = file.Results.Select(result => result with + { Attributes = result.Attributes?.ToDictionary(pair => pair.Key, pair => pair.Value.Clone(), StringComparer.Ordinal) }).ToList(), + }, + LlmTestServerResponseCodeInterpreterItem code => code with + { + InputFileIds = [.. code.InputFileIds], + GeneratedFileIds = [.. code.GeneratedFileIds], + CodeChunks = [.. code.CodeChunks], + Outputs = code.Outputs?.Select(output => output with { }).ToList(), + }, + LlmTestServerResponseServerToolUseItem call => call with { Input = call.Input with { } }, + LlmTestServerResponseServerToolResultItem returned => returned with + { + Result = returned.Result switch + { + LlmTestServerWebSearchResults found => found with { Results = found.Results.Select(result => result with { }).ToList() }, + LlmTestServerWebSearchErrorResult error => error with { }, + _ => throw new InvalidOperationException("Unsupported server tool result."), + }, + }, + _ => throw new InvalidOperationException("Unsupported structured response item."), + }; + } + private IReadOnlyList RetainChunks( IReadOnlyList source, ref int remaining, @@ -569,7 +800,7 @@ _ when char.IsHighSurrogate(character) private static long GetEntryRetainedByteCount( string storageKey, - LlmTckProviderHttpTrace trace + LlmTestServerProviderHttpTrace trace ) { var storageKeyBytes = string.Equals(storageKey, trace.RequestId, StringComparison.Ordinal) @@ -593,6 +824,7 @@ private void TrimOldestEntries() if (_traces.Remove(oldest.Value, out var trace)) { _retainedByteCount -= GetEntryRetainedByteCount(oldest.Value, trace); + _evictedCount++; } } diff --git a/src/ManagedCode.LlmTck.Hosting/LlmTckRequestPolicy.cs b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerRequestPolicy.cs similarity index 93% rename from src/ManagedCode.LlmTck.Hosting/LlmTckRequestPolicy.cs rename to src/ManagedCode.LlmTestServer.Hosting/LlmTestServerRequestPolicy.cs index 3556488..6cb0734 100644 --- a/src/ManagedCode.LlmTck.Hosting/LlmTckRequestPolicy.cs +++ b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerRequestPolicy.cs @@ -1,15 +1,15 @@ using System.Text.Json; -using ManagedCode.LlmTck.Anthropic; -using ManagedCode.LlmTck.Bedrock; -using ManagedCode.LlmTck.Cohere; -using ManagedCode.LlmTck.Gemini; -using ManagedCode.LlmTck.Ollama; -using ManagedCode.LlmTck.OpenAI; +using ManagedCode.LlmTestServer.Anthropic; +using ManagedCode.LlmTestServer.Bedrock; +using ManagedCode.LlmTestServer.Cohere; +using ManagedCode.LlmTestServer.Gemini; +using ManagedCode.LlmTestServer.Ollama; +using ManagedCode.LlmTestServer.OpenAI; using Microsoft.AspNetCore.Http; -namespace ManagedCode.LlmTck.Hosting; +namespace ManagedCode.LlmTestServer.Hosting; -internal static class LlmTckRequestPolicy +internal static class LlmTestServerRequestPolicy { public static IResult? ValidateApiVersion(HttpContext context) { diff --git a/src/ManagedCode.LlmTck.Hosting/LlmTckServiceDefaultsExtensions.cs b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerServiceDefaultsExtensions.cs similarity index 91% rename from src/ManagedCode.LlmTck.Hosting/LlmTckServiceDefaultsExtensions.cs rename to src/ManagedCode.LlmTestServer.Hosting/LlmTestServerServiceDefaultsExtensions.cs index 451e651..b21dae3 100644 --- a/src/ManagedCode.LlmTck.Hosting/LlmTckServiceDefaultsExtensions.cs +++ b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerServiceDefaultsExtensions.cs @@ -4,9 +4,9 @@ using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Hosting; -namespace ManagedCode.LlmTck.Hosting; +namespace ManagedCode.LlmTestServer.Hosting; -public static class LlmTckServiceDefaultsExtensions +public static class LlmTestServerServiceDefaultsExtensions { public static TBuilder AddServiceDefaults(this TBuilder builder) where TBuilder : IHostApplicationBuilder diff --git a/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerSessionOptions.cs b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerSessionOptions.cs new file mode 100644 index 0000000..5d0f6e5 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerSessionOptions.cs @@ -0,0 +1,8 @@ +namespace ManagedCode.LlmTestServer.Hosting; + +public sealed class LlmTestServerSessionOptions +{ + public int MaxSessions { get; set; } = 32; + public long MaxReservedStateBytes { get; set; } = 512L * 1024 * 1024; + public int MaxConfigurationBytes { get; set; } = 1024 * 1024; +} diff --git a/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerSessionRegistry.cs b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerSessionRegistry.cs new file mode 100644 index 0000000..730ca41 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerSessionRegistry.cs @@ -0,0 +1,112 @@ +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using ManagedCode.LlmTestServer.Configuration; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Sessions; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Options; + +namespace ManagedCode.LlmTestServer.Hosting; + +internal sealed class LlmTestServerSessionRegistry( + LlmTestServerConfiguration template, + LlmTestServerProviderHttpTraceStore defaultTraceStore, + IOptions traceOptions, + IOptions options) +{ + private readonly object _gate = new(); + private readonly Dictionary _sessions = new(StringComparer.Ordinal); + private readonly LlmTestServerSessionOptions _options = options.Value; + private long _reservedBytes; + + internal State Default { get; } = new(new LlmTestServerRuntime(template), defaultTraceStore, [], 0); + + internal State? Resolve(HttpContext? context) + { + if (context is null || !context.Request.Headers.ContainsKey(LlmTestServerSessionHeaders.Id) + && !context.Request.Headers.ContainsKey(LlmTestServerSessionHeaders.AccessKey)) { return Default; } + var ids = context.Request.Headers[LlmTestServerSessionHeaders.Id]; + var keys = context.Request.Headers[LlmTestServerSessionHeaders.AccessKey]; + if (ids.Count != 1 || keys.Count != 1 || ids[0] is not { Length: <= 128 } id || keys[0] is not { Length: <= 128 } key) { return null; } + lock (_gate) + { + return _sessions.TryGetValue(id, out var state) + && CryptographicOperations.FixedTimeEquals(SHA256.HashData(Encoding.UTF8.GetBytes(key)), state.KeyHash) + ? state : null; + } + } + + internal LlmTestServerSession? Create(LlmTestServerConfiguration? configuration) + { + configuration ??= template; + var runtime = new LlmTestServerRuntime(configuration); + var reservation = Reservation(configuration); + lock (_gate) + { + if (_sessions.Count >= _options.MaxSessions || reservation > _options.MaxReservedStateBytes - _reservedBytes) { return null; } + var id = "session_" + Guid.NewGuid().ToString("N"); + var key = Convert.ToHexString(RandomNumberGenerator.GetBytes(32)); + _sessions.Add(id, new State(runtime, new LlmTestServerProviderHttpTraceStore(traceOptions), SHA256.HashData(Encoding.UTF8.GetBytes(key)), reservation)); + _reservedBytes += reservation; + return new LlmTestServerSession { Id = id, AccessKey = key, ReservedStateBytes = reservation }; + } + } + + internal Task? ConfigureAsync(State state, LlmTestServerConfiguration configuration) + { + var replacement = new LlmTestServerRuntime(configuration); + var reservation = Reservation(configuration); + lock (_gate) + { + if (!_sessions.ContainsValue(state)) { return null; } + if (reservation > _options.MaxReservedStateBytes - _reservedBytes + state.ReservedBytes) + { throw new InvalidOperationException("Session state capacity exceeded."); } + var retired = state.Runtime.RetireAsync(); + state.Runtime = replacement; + _reservedBytes += reservation - state.ReservedBytes; + state.ReservedBytes = reservation; + return retired; + } + } + + internal Task? DeleteAsync(string id, State state) + { + lock (_gate) + { + if (!_sessions.TryGetValue(id, out var found) || !ReferenceEquals(found, state)) { return null; } + _sessions.Remove(id); + _reservedBytes -= state.ReservedBytes; + state.TraceStore.Reset(); + return state.Runtime.RetireAsync(); + } + } + + private long Reservation(LlmTestServerConfiguration configuration) + { + var configurationBytes = JsonSerializer.SerializeToUtf8Bytes(configuration).LongLength; + if (configurationBytes > _options.MaxConfigurationBytes) { throw new ArgumentException("Session configuration is too large.", nameof(configuration)); } + try + { + return checked(16L * 1024 * 1024 + configurationBytes * 2 + traceOptions.Value.MaxRetainedBytes + + configuration.MaxStoredChatResponseBytes + configuration.MaxVideoBytes + + configuration.MaxConversationBytes + configuration.MaxConversations * 512L + + configuration.MaxPromptCacheEntries * 256L + configuration.MaxVideoJobs * 512L + + configuration.MaxStoredChatResponses * 512L); + } + catch (OverflowException exception) { throw new ArgumentException("Session capacity is too large.", nameof(configuration), exception); } + } + + internal sealed class State(LlmTestServerRuntime runtime, LlmTestServerProviderHttpTraceStore traceStore, byte[] keyHash, long reservedBytes) + { + internal LlmTestServerRuntime Runtime { get; set; } = runtime; + internal LlmTestServerProviderHttpTraceStore TraceStore { get; } = traceStore; + internal byte[] KeyHash { get; } = keyHash; + internal long ReservedBytes { get; set; } = reservedBytes; + } +} + +internal sealed class LlmTestServerSessionRequestScope(IHttpContextAccessor accessor, LlmTestServerSessionRegistry registry) +{ + internal LlmTestServerSessionRegistry.State State { get; } = registry.Resolve(accessor.HttpContext) ?? registry.Default; +} diff --git a/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerStreamDeliveryEvidence.cs b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerStreamDeliveryEvidence.cs new file mode 100644 index 0000000..ff2bfbd --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Hosting/LlmTestServerStreamDeliveryEvidence.cs @@ -0,0 +1,47 @@ +using ManagedCode.LlmTestServer.Scenarios; +using Microsoft.AspNetCore.Connections; +using Microsoft.AspNetCore.Http; + +namespace ManagedCode.LlmTestServer.Hosting; + +internal sealed record LlmTestServerStreamDeliveryEvidence(LlmTestServerStreamDelivery Plan, int EmittedContentFrames, bool Applied) +{ + private const string _key = "ManagedCode.LlmTestServer.StreamDelivery"; + + internal static LlmTestServerStreamDeliveryEvidence? Read(HttpContext context) + { + return context.Items.TryGetValue(_key, out var value) ? value as LlmTestServerStreamDeliveryEvidence : null; + } + + internal static void Record(HttpContext context, LlmTestServerStreamDelivery plan, int frames, bool applied) + { + context.Items[_key] = new LlmTestServerStreamDeliveryEvidence(plan, frames, applied); + } + internal static void Begin(HttpContext context, LlmTestServerStreamDelivery? plan) + { + if (plan is not null) { Record(context, plan, 0, false); } + } + + internal static async Task AfterContentFrameAsync(HttpContext context, string content, CancellationToken cancellationToken) + { + if (content.Length == 0 || Read(context) is not { } evidence) { return true; } + var frames = evidence.EmittedContentFrames + 1; + var atBoundary = !evidence.Applied && frames == evidence.Plan.AfterContentFrames; + Record(context, evidence.Plan, frames, evidence.Applied || atBoundary); + if (!atBoundary) { return true; } + switch (evidence.Plan.Mode) + { + case LlmTestServerStreamDeliveryMode.EarlyCleanEnd: + return false; + case LlmTestServerStreamDeliveryMode.DelayThenContinue: + await Task.Delay(evidence.Plan.DelayMilliseconds, cancellationToken).ConfigureAwait(false); + return true; + case LlmTestServerStreamDeliveryMode.AbortConnection: + context.Abort(); + throw new ConnectionAbortedException("Configured stream delivery aborted the connection after a content frame."); + default: + throw new InvalidOperationException("The admitted stream delivery mode is not supported."); + } + } + +} diff --git a/src/ManagedCode.LlmTestServer.Hosting/ManagedCode.LlmTestServer.Hosting.csproj b/src/ManagedCode.LlmTestServer.Hosting/ManagedCode.LlmTestServer.Hosting.csproj new file mode 100644 index 0000000..b73e8d2 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Hosting/ManagedCode.LlmTestServer.Hosting.csproj @@ -0,0 +1,25 @@ + + + ManagedCode.LlmTestServer.Hosting + ASP.NET Core endpoint mapping for ManagedCode.LlmTestServer provider emulation and control APIs. + + + + + + + + + + + + + + + + + + + + + diff --git a/src/ManagedCode.LlmTck.Hosting/TypeSafeMultipartRequestReader.cs b/src/ManagedCode.LlmTestServer.Hosting/TypeSafeMultipartRequestReader.cs similarity index 97% rename from src/ManagedCode.LlmTck.Hosting/TypeSafeMultipartRequestReader.cs rename to src/ManagedCode.LlmTestServer.Hosting/TypeSafeMultipartRequestReader.cs index e1cae83..06f8867 100644 --- a/src/ManagedCode.LlmTck.Hosting/TypeSafeMultipartRequestReader.cs +++ b/src/ManagedCode.LlmTestServer.Hosting/TypeSafeMultipartRequestReader.cs @@ -1,10 +1,10 @@ using System.Text.Json; -using ManagedCode.LlmTck.SystemOne; +using ManagedCode.LlmTestServer.SystemOne; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.WebUtilities; using Microsoft.Net.Http.Headers; -namespace ManagedCode.LlmTck.Hosting; +namespace ManagedCode.LlmTestServer.Hosting; internal static class TypeSafeMultipartRequestReader { @@ -29,7 +29,7 @@ internal static class TypeSafeMultipartRequestReader if (first is null || PartName(first) != _requestPart || !IsJson(first.ContentType)) { return Failure("The first multipart part must be named request with application/json content."); } var json = await JsonSerializer.DeserializeAsync(first.Body, cancellationToken: cancellationToken).ConfigureAwait(false); - var shape = SystemOneDecisionMapper.Validate(json, Decisions.LlmTckDecisionProvider.TypeSafe); + var shape = SystemOneDecisionMapper.Validate(json, Decisions.LlmTestServerDecisionProvider.TypeSafe); if (!shape.IsValid) { return Failure(shape.Error!, StatusCodes.Status422UnprocessableEntity); } if (json.TryGetProperty("assets", out var declared) && declared.ValueKind == JsonValueKind.Object && declared.EnumerateObject().Select(entry => entry.Name).Distinct(StringComparer.Ordinal).Count() != declared.EnumerateObject().Count()) diff --git a/src/ManagedCode.LlmTestServer.Mistral/AGENTS.md b/src/ManagedCode.LlmTestServer.Mistral/AGENTS.md new file mode 100644 index 0000000..a24f0b5 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Mistral/AGENTS.md @@ -0,0 +1,5 @@ +# ManagedCode.LlmTestServer.Mistral + +This package owns Mistral compatibility constants, provider profiles, and Mistral wire contracts. + +Keep deterministic runtime behavior in `ManagedCode.LlmTestServer`; keep HTTP hosting in `ManagedCode.LlmTestServer.Hosting`. diff --git a/src/ManagedCode.LlmTestServer.Mistral/ManagedCode.LlmTestServer.Mistral.csproj b/src/ManagedCode.LlmTestServer.Mistral/ManagedCode.LlmTestServer.Mistral.csproj new file mode 100644 index 0000000..4101746 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Mistral/ManagedCode.LlmTestServer.Mistral.csproj @@ -0,0 +1,10 @@ + + + ManagedCode.LlmTestServer.Mistral + Mistral compatibility profile and adapters for ManagedCode.LlmTestServer. + + + + + + diff --git a/src/ManagedCode.LlmTck.Mistral/MistralCompatibility.cs b/src/ManagedCode.LlmTestServer.Mistral/MistralCompatibility.cs similarity index 52% rename from src/ManagedCode.LlmTck.Mistral/MistralCompatibility.cs rename to src/ManagedCode.LlmTestServer.Mistral/MistralCompatibility.cs index df59e23..f5c7caa 100644 --- a/src/ManagedCode.LlmTck.Mistral/MistralCompatibility.cs +++ b/src/ManagedCode.LlmTestServer.Mistral/MistralCompatibility.cs @@ -1,27 +1,27 @@ -using ManagedCode.LlmTck.Providers; +using ManagedCode.LlmTestServer.Providers; -namespace ManagedCode.LlmTck.Mistral; +namespace ManagedCode.LlmTestServer.Mistral; public static class MistralCompatibility { - public const string ProviderId = LlmTckCompatibilityTags.Mistral; + public const string ProviderId = LlmTestServerCompatibilityTags.Mistral; - public static LlmTckProviderProfile Profile { get; } = + public static LlmTestServerProviderProfile Profile { get; } = new() { Id = ProviderId, DisplayName = "Mistral", - Protocol = LlmTckProtocolFamily.Mistral, + Protocol = LlmTestServerProtocolFamily.Mistral, DefaultEndpointPath = "/mistral/v1/chat/completions", Capabilities = [ - LlmTckProviderCapability.Chat, - LlmTckProviderCapability.StreamingChat, - LlmTckProviderCapability.Tools, - LlmTckProviderCapability.StructuredOutput, - LlmTckProviderCapability.Embeddings, + LlmTestServerProviderCapability.Chat, + LlmTestServerProviderCapability.StreamingChat, + LlmTestServerProviderCapability.Tools, + LlmTestServerProviderCapability.StructuredOutput, + LlmTestServerProviderCapability.Embeddings, ], - CompatibilityTags = [ProviderId, LlmTckCompatibilityTags.OpenAICompatible], + CompatibilityTags = [ProviderId, LlmTestServerCompatibilityTags.OpenAICompatible], ApiContract = new() { DocumentationUrl = "https://docs.mistral.ai/api/endpoint/chat", @@ -31,7 +31,7 @@ public static class MistralCompatibility [ new() { - Id = LlmTckProviderOperationIds.Mistral.ChatComplete, + Id = LlmTestServerProviderOperationIds.Mistral.ChatComplete, Method = "POST", Path = "/mistral/v1/chat/completions", DocumentationUrl = "https://docs.mistral.ai/api/endpoint/chat", @@ -39,20 +39,20 @@ public static class MistralCompatibility ImplementedByHosting = true, Capabilities = [ - LlmTckProviderCapability.Chat, - LlmTckProviderCapability.StreamingChat, - LlmTckProviderCapability.Tools, - LlmTckProviderCapability.StructuredOutput, + LlmTestServerProviderCapability.Chat, + LlmTestServerProviderCapability.StreamingChat, + LlmTestServerProviderCapability.Tools, + LlmTestServerProviderCapability.StructuredOutput, ], }, new() { - Id = LlmTckProviderOperationIds.Mistral.EmbeddingsCreate, + Id = LlmTestServerProviderOperationIds.Mistral.EmbeddingsCreate, Method = "POST", Path = "/mistral/v1/embeddings", DocumentationUrl = "https://docs.mistral.ai/api/endpoint/embeddings", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Embeddings], + Capabilities = [LlmTestServerProviderCapability.Embeddings], }, ], }, diff --git a/src/ManagedCode.LlmTestServer.Ollama/AGENTS.md b/src/ManagedCode.LlmTestServer.Ollama/AGENTS.md new file mode 100644 index 0000000..7e9c114 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Ollama/AGENTS.md @@ -0,0 +1,5 @@ +# ManagedCode.LlmTestServer.Ollama + +This package owns Ollama compatibility constants, provider profiles, and Ollama wire contracts. + +Keep deterministic runtime behavior in `ManagedCode.LlmTestServer`; keep HTTP hosting in `ManagedCode.LlmTestServer.Hosting`. diff --git a/src/ManagedCode.LlmTestServer.Ollama/ManagedCode.LlmTestServer.Ollama.csproj b/src/ManagedCode.LlmTestServer.Ollama/ManagedCode.LlmTestServer.Ollama.csproj new file mode 100644 index 0000000..fc4c7b5 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Ollama/ManagedCode.LlmTestServer.Ollama.csproj @@ -0,0 +1,13 @@ + + + ManagedCode.LlmTestServer.Ollama + Ollama compatibility profile and adapters for ManagedCode.LlmTestServer. + + + + + + + + + diff --git a/src/ManagedCode.LlmTck.Ollama/OllamaCompatibility.cs b/src/ManagedCode.LlmTestServer.Ollama/OllamaCompatibility.cs similarity index 53% rename from src/ManagedCode.LlmTck.Ollama/OllamaCompatibility.cs rename to src/ManagedCode.LlmTestServer.Ollama/OllamaCompatibility.cs index a16d61d..fe84666 100644 --- a/src/ManagedCode.LlmTck.Ollama/OllamaCompatibility.cs +++ b/src/ManagedCode.LlmTestServer.Ollama/OllamaCompatibility.cs @@ -1,31 +1,31 @@ -using ManagedCode.LlmTck.Providers; +using ManagedCode.LlmTestServer.Providers; -namespace ManagedCode.LlmTck.Ollama; +namespace ManagedCode.LlmTestServer.Ollama; public static class OllamaCompatibility { - public const string ProviderId = LlmTckCompatibilityTags.Ollama; + public const string ProviderId = LlmTestServerCompatibilityTags.Ollama; - public static LlmTckProviderProfile Profile { get; } = + public static LlmTestServerProviderProfile Profile { get; } = new() { Id = ProviderId, DisplayName = "Ollama", - Protocol = LlmTckProtocolFamily.Ollama, + Protocol = LlmTestServerProtocolFamily.Ollama, DefaultEndpointPath = "/ollama/api/chat", Capabilities = [ - LlmTckProviderCapability.Chat, - LlmTckProviderCapability.StreamingChat, - LlmTckProviderCapability.Tools, - LlmTckProviderCapability.StructuredOutput, - LlmTckProviderCapability.Embeddings, + LlmTestServerProviderCapability.Chat, + LlmTestServerProviderCapability.StreamingChat, + LlmTestServerProviderCapability.Tools, + LlmTestServerProviderCapability.StructuredOutput, + LlmTestServerProviderCapability.Embeddings, ], CompatibilityTags = [ ProviderId, - LlmTckCompatibilityTags.LocalLlm, - LlmTckCompatibilityTags.OpenAICompatible, + LlmTestServerCompatibilityTags.LocalLlm, + LlmTestServerCompatibilityTags.OpenAICompatible, ], ApiContract = new() { @@ -36,7 +36,7 @@ public static class OllamaCompatibility [ new() { - Id = LlmTckProviderOperationIds.Ollama.ChatCreate, + Id = LlmTestServerProviderOperationIds.Ollama.ChatCreate, Method = "POST", Path = "/ollama/api/chat", DocumentationUrl = "https://docs.ollama.com/api/chat", @@ -44,20 +44,20 @@ public static class OllamaCompatibility ImplementedByHosting = true, Capabilities = [ - LlmTckProviderCapability.Chat, - LlmTckProviderCapability.StreamingChat, - LlmTckProviderCapability.Tools, - LlmTckProviderCapability.StructuredOutput, + LlmTestServerProviderCapability.Chat, + LlmTestServerProviderCapability.StreamingChat, + LlmTestServerProviderCapability.Tools, + LlmTestServerProviderCapability.StructuredOutput, ], }, new() { - Id = LlmTckProviderOperationIds.Ollama.EmbeddingsCreate, + Id = LlmTestServerProviderOperationIds.Ollama.EmbeddingsCreate, Method = "POST", Path = "/ollama/api/embed", DocumentationUrl = "https://docs.ollama.com/api/embed", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Embeddings], + Capabilities = [LlmTestServerProviderCapability.Embeddings], }, ], }, diff --git a/src/ManagedCode.LlmTck.Ollama/OllamaModels.cs b/src/ManagedCode.LlmTestServer.Ollama/OllamaModels.cs similarity index 99% rename from src/ManagedCode.LlmTck.Ollama/OllamaModels.cs rename to src/ManagedCode.LlmTestServer.Ollama/OllamaModels.cs index c84dab7..191cb71 100644 --- a/src/ManagedCode.LlmTck.Ollama/OllamaModels.cs +++ b/src/ManagedCode.LlmTestServer.Ollama/OllamaModels.cs @@ -1,7 +1,7 @@ using System.Text.Json; using System.Text.Json.Serialization; -namespace ManagedCode.LlmTck.Ollama; +namespace ManagedCode.LlmTestServer.Ollama; public sealed record OllamaChatRequest { diff --git a/src/ManagedCode.LlmTck.Ollama/OllamaRequestValidation.cs b/src/ManagedCode.LlmTestServer.Ollama/OllamaRequestValidation.cs similarity index 82% rename from src/ManagedCode.LlmTck.Ollama/OllamaRequestValidation.cs rename to src/ManagedCode.LlmTestServer.Ollama/OllamaRequestValidation.cs index cc0b8c0..6c05828 100644 --- a/src/ManagedCode.LlmTck.Ollama/OllamaRequestValidation.cs +++ b/src/ManagedCode.LlmTestServer.Ollama/OllamaRequestValidation.cs @@ -1,12 +1,12 @@ using System.Text.Json; -using ManagedCode.LlmTck.Providers; -using static ManagedCode.LlmTck.ProviderValidation.LlmTckJsonValidation; +using ManagedCode.LlmTestServer.Providers; +using static ManagedCode.LlmTestServer.ProviderValidation.LlmTestServerJsonValidation; -namespace ManagedCode.LlmTck.Ollama; +namespace ManagedCode.LlmTestServer.Ollama; public static class OllamaRequestValidation { - public static LlmTckRequestValidationResult Validate(JsonElement body) + public static LlmTestServerRequestValidationResult Validate(JsonElement body) { return Check(body, ValidateBody); } diff --git a/src/ManagedCode.LlmTck.Ollama/OllamaWireMapper.cs b/src/ManagedCode.LlmTestServer.Ollama/OllamaWireMapper.cs similarity index 72% rename from src/ManagedCode.LlmTck.Ollama/OllamaWireMapper.cs rename to src/ManagedCode.LlmTestServer.Ollama/OllamaWireMapper.cs index 0fce4fc..0f786e1 100644 --- a/src/ManagedCode.LlmTck.Ollama/OllamaWireMapper.cs +++ b/src/ManagedCode.LlmTestServer.Ollama/OllamaWireMapper.cs @@ -1,39 +1,41 @@ using System.Text.Json; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Scenarios; +using ManagedCode.LlmTestServer.Determinism; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Scenarios; -namespace ManagedCode.LlmTck.Ollama; +namespace ManagedCode.LlmTestServer.Ollama; public static class OllamaWireMapper { - public static LlmTckChatRequest ToRuntimeRequest(OllamaChatRequest request) + public static LlmTestServerChatRequest ToRuntimeRequest(OllamaChatRequest request) { return new() { - Tools = request.Tools.Select(tool => new LlmTckToolDefinition { Name = tool.Function.Name, Description = tool.Function.Description, ParametersJson = tool.Function.Parameters.GetRawText() }).ToList(), + Tools = request.Tools.Select(tool => new LlmTestServerToolDefinition { Name = tool.Function.Name, Description = tool.Function.Description, ParametersJson = tool.Function.Parameters.GetRawText() }).ToList(), RequireJson = request.Format.ValueKind == JsonValueKind.Object || request.Format.ValueKind == JsonValueKind.String && request.Format.GetString() == "json", ResponseSchemaJson = request.Format.ValueKind == JsonValueKind.Object ? request.Format.GetRawText() : null, ModelId = request.Model, Stream = request.Stream != false, - PromptCachePolicy = LlmTckPromptCachePolicy.Ollama, + PromptCachePolicy = LlmTestServerPromptCachePolicy.Ollama, Messages = request .Messages - .Select(message => new LlmTckMessage + .Select(message => new LlmTestServerMessage { Role = message.Role, Content = message.TextContent, ToolCallId = message.ToolName, - ToolCalls = message.ToolCalls?.Select((call, index) => new LlmTckToolCall { Id = call.Function.Name + "_" + index, Name = call.Function.Name, ArgumentsJson = call.Function.Arguments.GetRawText() }).ToList() ?? [], + ToolCalls = message.ToolCalls?.Select((call, index) => new LlmTestServerToolCall { Id = call.Function.Name + "_" + index, Name = call.Function.Name, ArgumentsJson = call.Function.Arguments.GetRawText() }).ToList() ?? [], }) .ToList(), }; } - public static OllamaChatResponse ToChatResponse(LlmTckChatResult result) + public static OllamaChatResponse ToChatResponse(LlmTestServerChatResult result) { return new() { Model = result.ModelId, + CreatedAt = (result.WireMetadata ?? LlmTestServerResponseMetadata.Live()).CreatedAt, Message = new OllamaChatMessageResponse { Content = result.Content, ToolCalls = MapTools(result) }, Done = true, DoneReason = "stop", @@ -44,7 +46,7 @@ public static OllamaChatResponse ToChatResponse(LlmTckChatResult result) } public static OllamaChatResponse ToChatChunk( - LlmTckChatResult result, + LlmTestServerChatResult result, string content, bool done ) @@ -52,6 +54,7 @@ bool done return new() { Model = result.ModelId, + CreatedAt = (result.WireMetadata ?? LlmTestServerResponseMetadata.Live()).CreatedAt, Message = new OllamaChatMessageResponse { Content = content, ToolCalls = done ? MapTools(result) : null }, Done = done, DoneReason = done ? "stop" : null, @@ -61,7 +64,7 @@ bool done }; } - private static List? MapTools(LlmTckChatResult result) + private static List? MapTools(LlmTestServerChatResult result) { return result.ToolCalls.Count == 0 ? null : result.ToolCalls.Select(call => new OllamaToolCall { Function = new() { Name = call.Name, Arguments = JsonSerializer.Deserialize(call.ArgumentsJson) } }).ToList(); diff --git a/src/ManagedCode.LlmTck.OpenAI/AGENTS.md b/src/ManagedCode.LlmTestServer.OpenAI/AGENTS.md similarity index 83% rename from src/ManagedCode.LlmTck.OpenAI/AGENTS.md rename to src/ManagedCode.LlmTestServer.OpenAI/AGENTS.md index 4f9aa71..c9c2ef4 100644 --- a/src/ManagedCode.LlmTck.OpenAI/AGENTS.md +++ b/src/ManagedCode.LlmTestServer.OpenAI/AGENTS.md @@ -1,4 +1,4 @@ -# ManagedCode.LlmTck.OpenAI +# ManagedCode.LlmTestServer.OpenAI This package owns OpenAI-compatible wire contracts and mapping only. diff --git a/src/ManagedCode.LlmTestServer.OpenAI/ManagedCode.LlmTestServer.OpenAI.csproj b/src/ManagedCode.LlmTestServer.OpenAI/ManagedCode.LlmTestServer.OpenAI.csproj new file mode 100644 index 0000000..469c8ab --- /dev/null +++ b/src/ManagedCode.LlmTestServer.OpenAI/ManagedCode.LlmTestServer.OpenAI.csproj @@ -0,0 +1,13 @@ + + + ManagedCode.LlmTestServer.OpenAI + OpenAI-compatible wire models and adapters for ManagedCode.LlmTestServer. + + + + + + + + + diff --git a/src/ManagedCode.LlmTck.OpenAI/OpenAiAudioModels.cs b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiAudioModels.cs similarity index 94% rename from src/ManagedCode.LlmTck.OpenAI/OpenAiAudioModels.cs rename to src/ManagedCode.LlmTestServer.OpenAI/OpenAiAudioModels.cs index 294d43a..f5cd8e3 100644 --- a/src/ManagedCode.LlmTck.OpenAI/OpenAiAudioModels.cs +++ b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiAudioModels.cs @@ -1,12 +1,12 @@ using System.Text.Json.Serialization; -using ManagedCode.LlmTck.Models; +using ManagedCode.LlmTestServer.Models; -namespace ManagedCode.LlmTck.OpenAI; +namespace ManagedCode.LlmTestServer.OpenAI; public sealed record OpenAiAudioSpeechRequest { [JsonPropertyName("model")] - public string Model { get; init; } = LlmTckKnownModelIds.Gpt4OMiniTts; + public string Model { get; init; } = LlmTestServerKnownModelIds.Gpt4OMiniTts; [JsonPropertyName("input")] public string Input { get; init; } = string.Empty; diff --git a/src/ManagedCode.LlmTck.OpenAI/OpenAiChatModels.cs b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiChatModels.cs similarity index 86% rename from src/ManagedCode.LlmTck.OpenAI/OpenAiChatModels.cs rename to src/ManagedCode.LlmTestServer.OpenAI/OpenAiChatModels.cs index b6a7a9c..7c2bef7 100644 --- a/src/ManagedCode.LlmTck.OpenAI/OpenAiChatModels.cs +++ b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiChatModels.cs @@ -1,7 +1,7 @@ using System.Text.Json; using System.Text.Json.Serialization; -namespace ManagedCode.LlmTck.OpenAI; +namespace ManagedCode.LlmTestServer.OpenAI; public sealed record OpenAiChatCompletionRequest { @@ -57,6 +57,23 @@ public sealed record OpenAiChatMessage [JsonPropertyName("content")] public JsonElement Content { get; init; } = JsonSerializer.SerializeToElement(string.Empty); + [JsonPropertyName("refusal")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Refusal { get; init; } + + [JsonIgnore] + public string? RefusalTextContent + { + get + { + if (Refusal is not null) { return Refusal; } + if (Content.ValueKind != JsonValueKind.Array) { return null; } + var parts = Content.EnumerateArray().Where(part => part.ValueKind == JsonValueKind.Object + && part.TryGetProperty("type", out var type) && type.ValueKind == JsonValueKind.String && type.GetString() == "refusal").ToArray(); + return parts.Length == 0 ? null : string.Concat(parts.Select(part => part.GetProperty("refusal").GetString())); + } + } + [JsonIgnore] public string TextContent => ReadTextContent(Content); @@ -75,6 +92,7 @@ private static string ReadTextContent(JsonElement content) { JsonValueKind.String => content.GetString() ?? string.Empty, JsonValueKind.Array => string.Concat(content.EnumerateArray().Select(ReadTextContent)), + JsonValueKind.Object when content.TryGetProperty("type", out var type) && type.ValueKind == JsonValueKind.String && type.GetString() == "refusal" => string.Empty, JsonValueKind.Object when content.TryGetProperty("text", out var text) => ReadTextContent(text), JsonValueKind.Object when content.TryGetProperty("content", out var nested) => ReadTextContent(nested), JsonValueKind.Undefined or JsonValueKind.Null => string.Empty, diff --git a/src/ManagedCode.LlmTck.OpenAI/OpenAiCompatibility.cs b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiCompatibility.cs similarity index 63% rename from src/ManagedCode.LlmTck.OpenAI/OpenAiCompatibility.cs rename to src/ManagedCode.LlmTestServer.OpenAI/OpenAiCompatibility.cs index d2c529b..52221f3 100644 --- a/src/ManagedCode.LlmTck.OpenAI/OpenAiCompatibility.cs +++ b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiCompatibility.cs @@ -1,34 +1,34 @@ -using ManagedCode.LlmTck.Providers; +using ManagedCode.LlmTestServer.Providers; -namespace ManagedCode.LlmTck.OpenAI; +namespace ManagedCode.LlmTestServer.OpenAI; public static class OpenAiCompatibility { - public const string ProviderId = LlmTckCompatibilityTags.OpenAI; + public const string ProviderId = LlmTestServerCompatibilityTags.OpenAI; - public static LlmTckProviderProfile Profile { get; } = + public static LlmTestServerProviderProfile Profile { get; } = new() { Id = ProviderId, DisplayName = "OpenAI", - Protocol = LlmTckProtocolFamily.OpenAI, + Protocol = LlmTestServerProtocolFamily.OpenAI, DefaultEndpointPath = "/openai/v1", Capabilities = [ - LlmTckProviderCapability.Decisions, - LlmTckProviderCapability.Models, - LlmTckProviderCapability.Chat, - LlmTckProviderCapability.StreamingChat, - LlmTckProviderCapability.Tools, - LlmTckProviderCapability.StructuredOutput, - LlmTckProviderCapability.Embeddings, - LlmTckProviderCapability.Images, - LlmTckProviderCapability.StreamingImages, - LlmTckProviderCapability.Video, - LlmTckProviderCapability.Audio, - LlmTckProviderCapability.StreamingAudio, + LlmTestServerProviderCapability.Decisions, + LlmTestServerProviderCapability.Models, + LlmTestServerProviderCapability.Chat, + LlmTestServerProviderCapability.StreamingChat, + LlmTestServerProviderCapability.Tools, + LlmTestServerProviderCapability.StructuredOutput, + LlmTestServerProviderCapability.Embeddings, + LlmTestServerProviderCapability.Images, + LlmTestServerProviderCapability.StreamingImages, + LlmTestServerProviderCapability.Video, + LlmTestServerProviderCapability.Audio, + LlmTestServerProviderCapability.StreamingAudio, ], - CompatibilityTags = [ProviderId, LlmTckCompatibilityTags.OpenAICompatible], + CompatibilityTags = [ProviderId, LlmTestServerCompatibilityTags.OpenAICompatible], ApiContract = new() { DocumentationUrl = "https://developers.openai.com/api/reference/", @@ -36,22 +36,22 @@ public static class OpenAiCompatibility DocumentationVersion = "v1", Operations = [ - new() { Id = LlmTckProviderOperationIds.OpenAI.DecisionsCreate, Method = "POST", Path = "/openai/v1/decisions", + new() { Id = LlmTestServerProviderOperationIds.OpenAI.DecisionsCreate, Method = "POST", Path = "/openai/v1/decisions", DocumentationUrl = "https://developers.openai.com/api/reference/resources/decisions/methods/create/", - ImplementedByHosting = true, Capabilities = [LlmTckProviderCapability.Decisions] }, + ImplementedByHosting = true, Capabilities = [LlmTestServerProviderCapability.Decisions] }, new() { - Id = LlmTckProviderOperationIds.OpenAI.ModelsList, + Id = LlmTestServerProviderOperationIds.OpenAI.ModelsList, Method = "GET", Path = "/openai/v1/models", DocumentationUrl = "https://developers.openai.com/api/reference/resources/models/methods/list/", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Models], + Capabilities = [LlmTestServerProviderCapability.Models], }, new() { - Id = LlmTckProviderOperationIds.OpenAI.ChatCompletionsCreate, + Id = LlmTestServerProviderOperationIds.OpenAI.ChatCompletionsCreate, Method = "POST", Path = "/openai/v1/chat/completions", DocumentationUrl = @@ -60,15 +60,15 @@ public static class OpenAiCompatibility ImplementedByHosting = true, Capabilities = [ - LlmTckProviderCapability.Chat, - LlmTckProviderCapability.StreamingChat, - LlmTckProviderCapability.Tools, - LlmTckProviderCapability.StructuredOutput, + LlmTestServerProviderCapability.Chat, + LlmTestServerProviderCapability.StreamingChat, + LlmTestServerProviderCapability.Tools, + LlmTestServerProviderCapability.StructuredOutput, ], }, new() { - Id = LlmTckProviderOperationIds.OpenAI.ResponsesCreate, + Id = LlmTestServerProviderOperationIds.OpenAI.ResponsesCreate, Method = "POST", Path = "/openai/v1/responses", DocumentationUrl = @@ -77,26 +77,26 @@ public static class OpenAiCompatibility ImplementedByHosting = true, Capabilities = [ - LlmTckProviderCapability.Chat, - LlmTckProviderCapability.StreamingChat, - LlmTckProviderCapability.Tools, - LlmTckProviderCapability.StructuredOutput, - LlmTckProviderCapability.Images, + LlmTestServerProviderCapability.Chat, + LlmTestServerProviderCapability.StreamingChat, + LlmTestServerProviderCapability.Tools, + LlmTestServerProviderCapability.StructuredOutput, + LlmTestServerProviderCapability.Images, ], }, new() { - Id = LlmTckProviderOperationIds.OpenAI.EmbeddingsCreate, + Id = LlmTestServerProviderOperationIds.OpenAI.EmbeddingsCreate, Method = "POST", Path = "/openai/v1/embeddings", DocumentationUrl = "https://developers.openai.com/api/reference/resources/embeddings/methods/create/", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Embeddings], + Capabilities = [LlmTestServerProviderCapability.Embeddings], }, new() { - Id = LlmTckProviderOperationIds.OpenAI.ImagesCreate, + Id = LlmTestServerProviderOperationIds.OpenAI.ImagesCreate, Method = "POST", Path = "/openai/v1/images/generations", DocumentationUrl = @@ -105,13 +105,13 @@ public static class OpenAiCompatibility ImplementedByHosting = true, Capabilities = [ - LlmTckProviderCapability.Images, - LlmTckProviderCapability.StreamingImages, + LlmTestServerProviderCapability.Images, + LlmTestServerProviderCapability.StreamingImages, ], }, new() { - Id = LlmTckProviderOperationIds.OpenAI.ImagesEditsCreate, + Id = LlmTestServerProviderOperationIds.OpenAI.ImagesEditsCreate, Method = "POST", Path = "/openai/v1/images/edits", DocumentationUrl = @@ -120,33 +120,33 @@ public static class OpenAiCompatibility ImplementedByHosting = true, Capabilities = [ - LlmTckProviderCapability.Images, - LlmTckProviderCapability.StreamingImages, + LlmTestServerProviderCapability.Images, + LlmTestServerProviderCapability.StreamingImages, ], }, new() { - Id = LlmTckProviderOperationIds.OpenAI.ImagesVariationsCreate, + Id = LlmTestServerProviderOperationIds.OpenAI.ImagesVariationsCreate, Method = "POST", Path = "/openai/v1/images/variations", DocumentationUrl = "https://developers.openai.com/api/reference/resources/images/methods/create_variation/", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Images], + Capabilities = [LlmTestServerProviderCapability.Images], }, new() { - Id = LlmTckProviderOperationIds.OpenAI.AudioSpeechCreate, + Id = LlmTestServerProviderOperationIds.OpenAI.AudioSpeechCreate, Method = "POST", Path = "/openai/v1/audio/speech", DocumentationUrl = "https://developers.openai.com/api/reference/resources/audio/subresources/speech/methods/create/", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Audio], + Capabilities = [LlmTestServerProviderCapability.Audio], }, new() { - Id = LlmTckProviderOperationIds.OpenAI.AudioTranscriptionsCreate, + Id = LlmTestServerProviderOperationIds.OpenAI.AudioTranscriptionsCreate, Method = "POST", Path = "/openai/v1/audio/transcriptions", DocumentationUrl = @@ -155,119 +155,119 @@ public static class OpenAiCompatibility ImplementedByHosting = true, Capabilities = [ - LlmTckProviderCapability.Audio, - LlmTckProviderCapability.StreamingAudio, + LlmTestServerProviderCapability.Audio, + LlmTestServerProviderCapability.StreamingAudio, ], }, new() { - Id = LlmTckProviderOperationIds.OpenAI.AudioTranslationsCreate, + Id = LlmTestServerProviderOperationIds.OpenAI.AudioTranslationsCreate, Method = "POST", Path = "/openai/v1/audio/translations", DocumentationUrl = "https://developers.openai.com/api/reference/resources/audio/subresources/translations/methods/create/", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Audio], + Capabilities = [LlmTestServerProviderCapability.Audio], }, new() { - Id = LlmTckProviderOperationIds.OpenAI.VideosCreate, + Id = LlmTestServerProviderOperationIds.OpenAI.VideosCreate, Method = "POST", Path = "/openai/v1/videos", DocumentationUrl = "https://developers.openai.com/api/reference/resources/videos/methods/create/", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Video], + Capabilities = [LlmTestServerProviderCapability.Video], }, new() { - Id = LlmTckProviderOperationIds.OpenAI.VideosList, + Id = LlmTestServerProviderOperationIds.OpenAI.VideosList, Method = "GET", Path = "/openai/v1/videos", DocumentationUrl = "https://developers.openai.com/api/reference/resources/videos/methods/list/", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Video], + Capabilities = [LlmTestServerProviderCapability.Video], }, new() { - Id = LlmTckProviderOperationIds.OpenAI.VideosRetrieve, + Id = LlmTestServerProviderOperationIds.OpenAI.VideosRetrieve, Method = "GET", Path = "/openai/v1/videos/{videoId}", DocumentationUrl = "https://developers.openai.com/api/reference/resources/videos/methods/retrieve/", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Video], + Capabilities = [LlmTestServerProviderCapability.Video], }, new() { - Id = LlmTckProviderOperationIds.OpenAI.VideosDelete, + Id = LlmTestServerProviderOperationIds.OpenAI.VideosDelete, Method = "DELETE", Path = "/openai/v1/videos/{videoId}", DocumentationUrl = "https://developers.openai.com/api/reference/resources/videos/methods/delete/", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Video], + Capabilities = [LlmTestServerProviderCapability.Video], }, new() { - Id = LlmTckProviderOperationIds.OpenAI.VideosContentRetrieve, + Id = LlmTestServerProviderOperationIds.OpenAI.VideosContentRetrieve, Method = "GET", Path = "/openai/v1/videos/{videoId}/content", DocumentationUrl = "https://developers.openai.com/api/reference/resources/videos/methods/download_content/", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Video], + Capabilities = [LlmTestServerProviderCapability.Video], }, new() { - Id = LlmTckProviderOperationIds.OpenAI.VideosEditsCreate, + Id = LlmTestServerProviderOperationIds.OpenAI.VideosEditsCreate, Method = "POST", Path = "/openai/v1/videos/edits", DocumentationUrl = "https://developers.openai.com/api/reference/resources/videos/methods/edit/", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Video], + Capabilities = [LlmTestServerProviderCapability.Video], }, new() { - Id = LlmTckProviderOperationIds.OpenAI.VideosExtensionsCreate, + Id = LlmTestServerProviderOperationIds.OpenAI.VideosExtensionsCreate, Method = "POST", Path = "/openai/v1/videos/extensions", DocumentationUrl = "https://developers.openai.com/api/reference/resources/videos/methods/extend/", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Video], + Capabilities = [LlmTestServerProviderCapability.Video], }, new() { - Id = LlmTckProviderOperationIds.OpenAI.VideosRemix, + Id = LlmTestServerProviderOperationIds.OpenAI.VideosRemix, Method = "POST", Path = "/openai/v1/videos/{videoId}/remix", DocumentationUrl = "https://developers.openai.com/api/reference/resources/videos/methods/remix/", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Video], + Capabilities = [LlmTestServerProviderCapability.Video], }, new() { - Id = LlmTckProviderOperationIds.OpenAI.VideosCharactersCreate, + Id = LlmTestServerProviderOperationIds.OpenAI.VideosCharactersCreate, Method = "POST", Path = "/openai/v1/videos/characters", DocumentationUrl = "https://developers.openai.com/api/reference/resources/videos/methods/create_character/", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Video], + Capabilities = [LlmTestServerProviderCapability.Video], }, new() { - Id = LlmTckProviderOperationIds.OpenAI.VideosCharactersRetrieve, + Id = LlmTestServerProviderOperationIds.OpenAI.VideosCharactersRetrieve, Method = "GET", Path = "/openai/v1/videos/characters/{characterId}", DocumentationUrl = "https://developers.openai.com/api/reference/resources/videos/methods/get_character/", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Video], + Capabilities = [LlmTestServerProviderCapability.Video], }, ], }, diff --git a/src/ManagedCode.LlmTck.OpenAI/OpenAiDecisionFixtureIds.cs b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiDecisionFixtureIds.cs similarity index 95% rename from src/ManagedCode.LlmTck.OpenAI/OpenAiDecisionFixtureIds.cs rename to src/ManagedCode.LlmTestServer.OpenAI/OpenAiDecisionFixtureIds.cs index 07262de..1c334fd 100644 --- a/src/ManagedCode.LlmTck.OpenAI/OpenAiDecisionFixtureIds.cs +++ b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiDecisionFixtureIds.cs @@ -1,7 +1,7 @@ using System.Globalization; using System.Text.Json; -namespace ManagedCode.LlmTck.OpenAI; +namespace ManagedCode.LlmTestServer.OpenAI; /// Creates typed native OpenAI choice identities for neutral decision fixtures. public static class OpenAiDecisionFixtureIds diff --git a/src/ManagedCode.LlmTck.OpenAI/OpenAiDecisionInputValidation.cs b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiDecisionInputValidation.cs similarity index 98% rename from src/ManagedCode.LlmTck.OpenAI/OpenAiDecisionInputValidation.cs rename to src/ManagedCode.LlmTestServer.OpenAI/OpenAiDecisionInputValidation.cs index 991553e..a05cb98 100644 --- a/src/ManagedCode.LlmTck.OpenAI/OpenAiDecisionInputValidation.cs +++ b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiDecisionInputValidation.cs @@ -1,7 +1,7 @@ using System.Buffers.Text; using System.Text.Json; -namespace ManagedCode.LlmTck.OpenAI; +namespace ManagedCode.LlmTestServer.OpenAI; public static class OpenAiDecisionInputValidation { diff --git a/src/ManagedCode.LlmTck.OpenAI/OpenAiDecisionMapper.cs b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiDecisionMapper.cs similarity index 90% rename from src/ManagedCode.LlmTck.OpenAI/OpenAiDecisionMapper.cs rename to src/ManagedCode.LlmTestServer.OpenAI/OpenAiDecisionMapper.cs index 09ebe8b..9523baf 100644 --- a/src/ManagedCode.LlmTck.OpenAI/OpenAiDecisionMapper.cs +++ b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiDecisionMapper.cs @@ -1,9 +1,9 @@ using System.Globalization; using System.Text.Json; -using ManagedCode.LlmTck.Decisions; -using ManagedCode.LlmTck.Providers; +using ManagedCode.LlmTestServer.Decisions; +using ManagedCode.LlmTestServer.Providers; -namespace ManagedCode.LlmTck.OpenAI; +namespace ManagedCode.LlmTestServer.OpenAI; public static class OpenAiDecisionMapper { @@ -12,7 +12,7 @@ public static class OpenAiDecisionMapper private const int _minChoices = 2; private const int _maxSafetyIdentifierLength = 128; - public static LlmTckRequestValidationResult Validate(JsonElement body) + public static LlmTestServerRequestValidationResult Validate(JsonElement body) { if (body.ValueKind != JsonValueKind.Object || !body.TryGetProperty("model", out var model) || !ValidString(model) || string.IsNullOrWhiteSpace(model.GetString()) @@ -55,23 +55,23 @@ private static bool ValidString(JsonElement value, int maxLength = MaxQuestionTe return value.ValueKind == JsonValueKind.String && value.GetString()!.Length <= maxLength; } - public static LlmTckDecisionRequest ToRequest(OpenAiDecisionRequest request) + public static LlmTestServerDecisionRequest ToRequest(OpenAiDecisionRequest request) { return new() { - Provider = LlmTckDecisionProvider.OpenAI, + Provider = LlmTestServerDecisionProvider.OpenAI, ModelId = request.Model, Input = request.Input.ValueKind == JsonValueKind.String ? request.Input.GetString()! : JsonSerializer.Serialize(request.Input), - Questions = request.Questions.Select((question, index) => new LlmTckDecisionQuestion + Questions = request.Questions.Select((question, index) => new LlmTestServerDecisionQuestion { Name = OpenAiDecisionFixtureIds.ForQuestionIndex(index), - Kind = question.Type switch { OpenAiDecisionTypes.Predicate => LlmTckDecisionKind.Predicate, OpenAiDecisionTypes.Choice => LlmTckDecisionKind.Choice, _ => LlmTckDecisionKind.Score }, + Kind = question.Type switch { OpenAiDecisionTypes.Predicate => LlmTestServerDecisionKind.Predicate, OpenAiDecisionTypes.Choice => LlmTestServerDecisionKind.Choice, _ => LlmTestServerDecisionKind.Score }, Options = question.Type == OpenAiDecisionTypes.Score ? question.Levels.Select(option => option.Label).ToList() : question.Choices.Select(option => ChoiceId(option.Value)).ToList(), }).ToList(), }; } - public static OpenAiDecisionResponse ToResponse(OpenAiDecisionRequest request, LlmTckDecisionResult result) + public static OpenAiDecisionResponse ToResponse(OpenAiDecisionRequest request, LlmTestServerDecisionResult result) { return new() { @@ -88,7 +88,7 @@ public static OpenAiDecisionResponse ToResponse(OpenAiDecisionRequest request, L }; } - private static OpenAiDecisionAnswer ToAnswer(OpenAiDecisionQuestion question, LlmTckDecisionAnswer answer) + private static OpenAiDecisionAnswer ToAnswer(OpenAiDecisionQuestion question, LlmTestServerDecisionAnswer answer) { return new() { diff --git a/src/ManagedCode.LlmTck.OpenAI/OpenAiDecisionModels.cs b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiDecisionModels.cs similarity index 98% rename from src/ManagedCode.LlmTck.OpenAI/OpenAiDecisionModels.cs rename to src/ManagedCode.LlmTestServer.OpenAI/OpenAiDecisionModels.cs index a61a1b0..af758eb 100644 --- a/src/ManagedCode.LlmTck.OpenAI/OpenAiDecisionModels.cs +++ b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiDecisionModels.cs @@ -1,7 +1,7 @@ using System.Text.Json; using System.Text.Json.Serialization; -namespace ManagedCode.LlmTck.OpenAI; +namespace ManagedCode.LlmTestServer.OpenAI; public static class OpenAiDecisionTypes { diff --git a/src/ManagedCode.LlmTck.OpenAI/OpenAiEmbeddingModels.cs b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiEmbeddingModels.cs similarity index 92% rename from src/ManagedCode.LlmTck.OpenAI/OpenAiEmbeddingModels.cs rename to src/ManagedCode.LlmTestServer.OpenAI/OpenAiEmbeddingModels.cs index 0b4a652..ee7d752 100644 --- a/src/ManagedCode.LlmTck.OpenAI/OpenAiEmbeddingModels.cs +++ b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiEmbeddingModels.cs @@ -1,13 +1,13 @@ using System.Text.Json; using System.Text.Json.Serialization; -using ManagedCode.LlmTck.Models; +using ManagedCode.LlmTestServer.Models; -namespace ManagedCode.LlmTck.OpenAI; +namespace ManagedCode.LlmTestServer.OpenAI; public sealed record OpenAiEmbeddingRequest { [JsonPropertyName("model")] - public string Model { get; init; } = LlmTckKnownModelIds.TextEmbedding3Small; + public string Model { get; init; } = LlmTestServerKnownModelIds.TextEmbedding3Small; [JsonPropertyName("input")] public JsonElement Input { get; init; } @@ -29,7 +29,7 @@ public sealed record OpenAiEmbeddingResponse public string ObjectType { get; init; } = "list"; [JsonPropertyName("model")] - public string Model { get; init; } = LlmTckKnownModelIds.TextEmbedding3Small; + public string Model { get; init; } = LlmTestServerKnownModelIds.TextEmbedding3Small; [JsonPropertyName("data")] public List Data { get; init; } = []; diff --git a/src/ManagedCode.LlmTck.OpenAI/OpenAiErrorResponse.cs b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiErrorResponse.cs similarity index 91% rename from src/ManagedCode.LlmTck.OpenAI/OpenAiErrorResponse.cs rename to src/ManagedCode.LlmTestServer.OpenAI/OpenAiErrorResponse.cs index 2a37e6a..1855897 100644 --- a/src/ManagedCode.LlmTck.OpenAI/OpenAiErrorResponse.cs +++ b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiErrorResponse.cs @@ -1,6 +1,6 @@ using System.Text.Json.Serialization; -namespace ManagedCode.LlmTck.OpenAI; +namespace ManagedCode.LlmTestServer.OpenAI; public sealed record OpenAiErrorResponse { diff --git a/src/ManagedCode.LlmTck.OpenAI/OpenAiImageModels.cs b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiImageModels.cs similarity index 92% rename from src/ManagedCode.LlmTck.OpenAI/OpenAiImageModels.cs rename to src/ManagedCode.LlmTestServer.OpenAI/OpenAiImageModels.cs index 532f533..26654a4 100644 --- a/src/ManagedCode.LlmTck.OpenAI/OpenAiImageModels.cs +++ b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiImageModels.cs @@ -1,13 +1,13 @@ using System.Text.Json; using System.Text.Json.Serialization; -using ManagedCode.LlmTck.Models; +using ManagedCode.LlmTestServer.Models; -namespace ManagedCode.LlmTck.OpenAI; +namespace ManagedCode.LlmTestServer.OpenAI; public sealed record OpenAiImageGenerationRequest { [JsonPropertyName("model")] - public string Model { get; init; } = LlmTckKnownModelIds.GptImage1; + public string Model { get; init; } = LlmTestServerKnownModelIds.GptImage1; [JsonPropertyName("prompt")] public string Prompt { get; init; } = string.Empty; @@ -79,7 +79,7 @@ public sealed record OpenAiImageReference public sealed record OpenAiImageEditRequest { [JsonPropertyName("model")] - public string Model { get; init; } = LlmTckKnownModelIds.GptImage1; + public string Model { get; init; } = LlmTestServerKnownModelIds.GptImage1; [JsonPropertyName("prompt")] public string Prompt { get; init; } = string.Empty; diff --git a/src/ManagedCode.LlmTestServer.OpenAI/OpenAiNativeFileCodeModels.cs b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiNativeFileCodeModels.cs new file mode 100644 index 0000000..ad8b56e --- /dev/null +++ b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiNativeFileCodeModels.cs @@ -0,0 +1,67 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace ManagedCode.LlmTestServer.OpenAI; + +public sealed record OpenAiFileSearchRankingOptions +{ + [JsonPropertyName("ranker")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? Ranker { get; init; } + [JsonPropertyName("score_threshold")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public double? ScoreThreshold { get; init; } +} + +public sealed record OpenAiResponseFileSearchCall : OpenAiResponseOutputItem +{ + [JsonIgnore] public override string Type => "file_search_call"; + [JsonPropertyName("queries")] public List Queries { get; init; } = []; + [JsonPropertyName("results")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public List? Results { get; init; } +} + +public sealed record OpenAiFileSearchResult +{ + [JsonPropertyName("file_id")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? FileId { get; init; } + [JsonPropertyName("filename")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? Filename { get; init; } + [JsonPropertyName("text")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? Text { get; init; } + [JsonPropertyName("score")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public double? Score { get; init; } + [JsonPropertyName("attributes")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public Dictionary? Attributes { get; init; } +} + +public sealed record OpenAiResponseCodeInterpreterCall : OpenAiResponseOutputItem +{ + [JsonIgnore] public override string Type => "code_interpreter_call"; + [JsonPropertyName("container_id")] public string ContainerId { get; init; } = string.Empty; + [JsonPropertyName("code")] public string? Code { get; init; } + [JsonPropertyName("outputs")] public List? Outputs { get; init; } +} + +[JsonPolymorphic(TypeDiscriminatorPropertyName = "type")] +[JsonDerivedType(typeof(OpenAiCodeInterpreterLogsOutput), "logs")] +[JsonDerivedType(typeof(OpenAiCodeInterpreterImageOutput), "image")] +public abstract record OpenAiCodeInterpreterOutput; + +public sealed record OpenAiCodeInterpreterLogsOutput : OpenAiCodeInterpreterOutput +{ + [JsonPropertyName("logs")] public string Logs { get; init; } = string.Empty; +} + +public sealed record OpenAiCodeInterpreterImageOutput : OpenAiCodeInterpreterOutput +{ + [JsonPropertyName("url")] public string Url { get; init; } = string.Empty; +} + +public sealed record OpenAiFileCitation +{ + [JsonPropertyName("type")] public string Type { get; init; } = "file_citation"; + [JsonPropertyName("file_id")] public string FileId { get; init; } = string.Empty; + [JsonPropertyName("filename")] public string Filename { get; init; } = string.Empty; + [JsonPropertyName("index")] public int Index { get; init; } +} + +public sealed record OpenAiContainerFileCitation +{ + [JsonPropertyName("type")] public string Type { get; init; } = "container_file_citation"; + [JsonPropertyName("container_id")] public string ContainerId { get; init; } = string.Empty; + [JsonPropertyName("file_id")] public string FileId { get; init; } = string.Empty; + [JsonPropertyName("filename")] public string Filename { get; init; } = string.Empty; + [JsonPropertyName("start_index")] public int StartIndex { get; init; } + [JsonPropertyName("end_index")] public int EndIndex { get; init; } +} diff --git a/src/ManagedCode.LlmTestServer.OpenAI/OpenAiNativeFileCodeValidation.cs b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiNativeFileCodeValidation.cs new file mode 100644 index 0000000..18d9f59 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiNativeFileCodeValidation.cs @@ -0,0 +1,187 @@ +using System.Text.Json; +using static ManagedCode.LlmTestServer.OpenAI.OpenAiNativeWebSearchValidation; +using static ManagedCode.LlmTestServer.ProviderValidation.LlmTestServerJsonValidation; + +namespace ManagedCode.LlmTestServer.OpenAI; + +internal static class OpenAiNativeFileCodeValidation +{ + internal const string _fileType = "file_search"; + internal const string _codeType = "code_interpreter"; + internal const string _resultsInclude = "file_search_call.results"; + internal const string _outputsInclude = "code_interpreter_call.outputs"; + + internal static void ValidateTool(JsonElement tool) + { + if (tool.GetProperty("type").GetString() == _fileType) + { + Fields(tool, "type", "vector_store_ids", "max_num_results", "ranking_options", "filters"); + IdArray(tool.GetProperty("vector_store_ids")); + if (tool.TryGetProperty("max_num_results", out var maximum) + && (!maximum.TryGetInt32(out var count) || count is < 1 or > 50)) + { throw new JsonException("File search result limit must be an integer from one to 50."); } + if (tool.TryGetProperty("ranking_options", out var ranking)) + { + Fields(ranking, "ranker", "score_threshold"); + if (ranking.TryGetProperty("ranker", out var ranker)) { Choice(ranker, "auto", "default-2024-11-15"); } + if (ranking.TryGetProperty("score_threshold", out var score)) { Score(score); } + } + if (tool.TryGetProperty("filters", out var filters) && filters.ValueKind != JsonValueKind.Null) + { + var nodes = 0; + ValidateFilter(filters, 1, ref nodes); + } + return; + } + Fields(tool, "type", "container"); + var container = tool.GetProperty("container"); + if (container.ValueKind == JsonValueKind.String) { Id(container); return; } + Fields(container, "type", "file_ids"); + Choice(container.GetProperty("type"), "auto"); + if (container.TryGetProperty("file_ids", out var files)) { IdArray(files); } + } + + internal static void ValidateHistory(JsonElement item) + { + var file = item.GetProperty("type").GetString() == "file_search_call"; + if (file) { Fields(item, "type", "id", "status", "queries", "results"); } + else { Fields(item, "type", "id", "status", "container_id", "code", "outputs"); } + Id(item.GetProperty("id")); + Choice(item.GetProperty("status"), "completed"); + if (file) + { + var queries = item.GetProperty("queries"); + Array(queries); + if (queries.GetArrayLength() is < 1 or > 128) { throw new JsonException("File queries exceed the fixture budget."); } + foreach (var query in queries.EnumerateArray()) { NonEmpty(query); } + if (item.TryGetProperty("results", out var results) && results.ValueKind != JsonValueKind.Null) + { + Array(results); + if (results.GetArrayLength() > 128) { throw new JsonException("Too many file results."); } + foreach (var result in results.EnumerateArray()) { ValidateResult(result); } + } + return; + } + Id(item.GetProperty("container_id")); + if (item.TryGetProperty("code", out var code) && code.ValueKind != JsonValueKind.Null) { String(code); } + if (item.TryGetProperty("outputs", out var outputs) && outputs.ValueKind != JsonValueKind.Null) + { + Array(outputs); + if (outputs.GetArrayLength() > 128) { throw new JsonException("Too many code outputs."); } + foreach (var output in outputs.EnumerateArray()) + { + Choice(output.GetProperty("type"), "logs", "image"); + if (output.GetProperty("type").GetString() == "logs") + { Fields(output, "type", "logs"); String(output.GetProperty("logs")); } + else { Fields(output, "type", "url"); Url(output.GetProperty("url")); } + } + } + } + + internal static string ValidateCitation(JsonElement citation, int textLength) + { + if (citation.GetProperty("type").GetString() == "file_citation") + { + Fields(citation, "type", "file_id", "filename", "index"); + Id(citation.GetProperty("file_id")); + NonEmpty(citation.GetProperty("filename")); + if (!citation.GetProperty("index").TryGetInt32(out var index) || index < 0 || index > textLength) + { throw new JsonException("Invalid file citation insertion index."); } + return _fileType; + } + Fields(citation, "type", "container_id", "file_id", "filename", "start_index", "end_index"); + Choice(citation.GetProperty("type"), "container_file_citation"); + Id(citation.GetProperty("container_id")); + Id(citation.GetProperty("file_id")); + NonEmpty(citation.GetProperty("filename")); + if (!citation.GetProperty("start_index").TryGetInt32(out var start) + || !citation.GetProperty("end_index").TryGetInt32(out var end) + || start < 0 || end < start || end > textLength) + { throw new JsonException("Invalid container citation range."); } + return _codeType; + } + + private static void ValidateResult(JsonElement result) + { + Fields(result, "file_id", "filename", "text", "score", "attributes"); + if (result.TryGetProperty("file_id", out var file)) { Id(file); } + if (result.TryGetProperty("filename", out var name)) { NonEmpty(name); } + if (result.TryGetProperty("text", out var text)) { String(text); } + if (result.TryGetProperty("score", out var score)) { Score(score); } + if (result.TryGetProperty("attributes", out var attributes) && attributes.ValueKind != JsonValueKind.Null) + { + Object(attributes); + var keys = new HashSet(StringComparer.Ordinal); + foreach (var property in attributes.EnumerateObject()) + { + if (keys.Count >= 16 || !keys.Add(property.Name) || string.IsNullOrWhiteSpace(property.Name) || property.Name.Length > 64) + { throw new JsonException("Invalid file attribute key budget."); } + Scalar(property.Value); + if (property.Value.ValueKind == JsonValueKind.String && property.Value.GetString()!.Length > 512) + { throw new JsonException("File attribute text exceeds the fixture budget."); } + } + } + } + + private static void ValidateFilter(JsonElement filter, int depth, ref int nodes) + { + if (depth > 8 || ++nodes > 128) { throw new JsonException("File filter exceeds the fixture depth or node budget."); } + var type = filter.GetProperty("type"); + Choice(type, "eq", "ne", "gt", "gte", "lt", "lte", "in", "nin", "and", "or"); + if (type.GetString() is "and" or "or") + { + Fields(filter, "type", "filters"); + var children = filter.GetProperty("filters"); + Array(children); + if (children.GetArrayLength() is < 1 or > 128) { throw new JsonException("Invalid compound filter size."); } + foreach (var child in children.EnumerateArray()) { ValidateFilter(child, depth + 1, ref nodes); } + return; + } + Fields(filter, "type", "key", "value"); + NonEmpty(filter.GetProperty("key")); + if (filter.GetProperty("key").GetString()!.Length > 64) { throw new JsonException("Filter attribute key exceeds the fixture budget."); } + var value = filter.GetProperty("value"); + if (type.GetString() is "in" or "nin") + { + Array(value); + if (value.GetArrayLength() is < 1 or > 128) { throw new JsonException("Invalid filter comparison list size."); } + var kind = value[0].ValueKind; + if (kind is not (JsonValueKind.String or JsonValueKind.Number)) { throw new JsonException("Comparison arrays require strings or numbers."); } + foreach (var entry in value.EnumerateArray()) + { + if (entry.ValueKind != kind) { throw new JsonException("Comparison arrays must have one scalar type."); } + Scalar(entry); + } + return; + } + Scalar(value); + if (type.GetString() is not ("eq" or "ne") && value.ValueKind is not (JsonValueKind.String or JsonValueKind.Number)) + { throw new JsonException("Ordering comparisons require strings or numbers."); } + } + + private static void Scalar(JsonElement value) + { + if (value.ValueKind is JsonValueKind.String or JsonValueKind.True or JsonValueKind.False) { return; } + if (value.ValueKind == JsonValueKind.Number && value.TryGetDouble(out var number) && double.IsFinite(number)) { return; } + throw new JsonException("A finite scalar attribute value is required."); + } + + private static void Score(JsonElement value) + { + if (value.ValueKind != JsonValueKind.Number || !value.TryGetDouble(out var score) || !double.IsFinite(score) || score is < 0 or > 1) + { throw new JsonException("A score from zero to one is required."); } + } + + private static void IdArray(JsonElement value) + { + Array(value); + if (value.GetArrayLength() > 128) { throw new JsonException("Resource IDs exceed the fixture budget."); } + foreach (var id in value.EnumerateArray()) { Id(id); } + } + + private static void Id(JsonElement value) + { + NonEmpty(value); + ValidateItemId(value.GetString()!); + } +} diff --git a/src/ManagedCode.LlmTestServer.OpenAI/OpenAiNativeWebSearchModels.cs b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiNativeWebSearchModels.cs new file mode 100644 index 0000000..58a635d --- /dev/null +++ b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiNativeWebSearchModels.cs @@ -0,0 +1,48 @@ +using System.Text.Json.Serialization; + +namespace ManagedCode.LlmTestServer.OpenAI; + +public sealed record OpenAiWebSearchLocation +{ + [JsonPropertyName("type")] public string Type { get; init; } = "approximate"; + [JsonPropertyName("city")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? City { get; init; } + [JsonPropertyName("region")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? Region { get; init; } + [JsonPropertyName("country")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? Country { get; init; } + [JsonPropertyName("timezone")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? Timezone { get; init; } +} + +public sealed record OpenAiWebSearchFilters +{ + [JsonPropertyName("allowed_domains")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public List? AllowedDomains { get; init; } +} + +public sealed record OpenAiResponseWebSearchCall : OpenAiResponseOutputItem +{ + [JsonIgnore] public override string Type => "web_search_call"; + [JsonPropertyName("action")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public OpenAiWebSearchAction? Action { get; init; } +} + +public sealed record OpenAiWebSearchAction +{ + [JsonPropertyName("type")] public string Type { get; init; } = "search"; + [JsonPropertyName("query")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? Query { get; init; } + [JsonPropertyName("queries")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public List? Queries { get; init; } + [JsonPropertyName("url")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? Url { get; init; } + [JsonPropertyName("pattern")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? Pattern { get; init; } + [JsonPropertyName("sources")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public List? Sources { get; init; } +} + +public sealed record OpenAiWebSearchSource +{ + [JsonPropertyName("type")] public string Type { get; init; } = "url"; + [JsonPropertyName("url")] public string Url { get; init; } = string.Empty; +} + +public sealed record OpenAiUrlCitation +{ + [JsonPropertyName("type")] public string Type { get; init; } = "url_citation"; + [JsonPropertyName("start_index")] public int StartIndex { get; init; } + [JsonPropertyName("end_index")] public int EndIndex { get; init; } + [JsonPropertyName("url")] public string Url { get; init; } = string.Empty; + [JsonPropertyName("title")] public string Title { get; init; } = string.Empty; +} diff --git a/src/ManagedCode.LlmTestServer.OpenAI/OpenAiNativeWebSearchValidation.cs b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiNativeWebSearchValidation.cs new file mode 100644 index 0000000..531498d --- /dev/null +++ b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiNativeWebSearchValidation.cs @@ -0,0 +1,307 @@ +using System.Globalization; +using System.Text.Json; +using ManagedCode.LlmTestServer.Models; +using static ManagedCode.LlmTestServer.ProviderValidation.LlmTestServerJsonValidation; + +namespace ManagedCode.LlmTestServer.OpenAI; + +internal static class OpenAiNativeWebSearchValidation +{ + internal const string _toolType = "web_search"; + internal const string _sourcesInclude = "web_search_call.action.sources"; + + internal static void ValidateTool(JsonElement tool) + { + Fields(tool, "type", "search_context_size", "user_location", "filters"); + Choice(tool.GetProperty("type"), _toolType); + if (tool.TryGetProperty("search_context_size", out var context)) { Choice(context, "low", "medium", "high"); } + if (tool.TryGetProperty("user_location", out var location) && location.ValueKind != JsonValueKind.Null) { ValidateLocation(location); } + if (tool.TryGetProperty("filters", out var filters) && filters.ValueKind != JsonValueKind.Null) + { + Fields(filters, "allowed_domains"); + if (!filters.TryGetProperty("allowed_domains", out var domains) || domains.ValueKind == JsonValueKind.Null) { return; } + Array(domains); + if (domains.GetArrayLength() > 100) { throw new JsonException("A bounded domain allowlist is required."); } + foreach (var entry in domains.EnumerateArray()) + { + NonEmpty(entry); + var domain = entry.GetString()!; + if (domain.Length > 253 || Uri.CheckHostName(domain) != UriHostNameType.Dns + || domain.Split('.').Any(label => label.Length is < 1 or > 63 + || label[0] == '-' || label[^1] == '-' + || label.Any(character => !char.IsAsciiLetterOrDigit(character) && character != '-'))) + { throw new JsonException("A bare domain name is required."); } + } + } + } + + internal static bool IsNativeToolType(string? type) + { + return type is _toolType or OpenAiNativeFileCodeValidation._fileType or OpenAiNativeFileCodeValidation._codeType; + } + + internal static bool IsNativeHistoryType(string? type) + { + return type is "web_search_call" or "file_search_call" or "code_interpreter_call"; + } + + internal static void ValidateResponseOptions(JsonElement body, bool enabled) + { + var tools = OptionalArray(body, "tools").ToArray(); + var contracts = new HashSet(StringComparer.Ordinal); + foreach (var tool in tools) + { + var type = tool.GetProperty("type").GetString(); + if (IsNativeToolType(type)) { contracts.Add(type!); } + } + if (body.TryGetProperty("include", out var include)) + { + Array(include); + foreach (var entry in include.EnumerateArray()) + { + Choice(entry, _sourcesInclude, OpenAiNativeFileCodeValidation._resultsInclude, OpenAiNativeFileCodeValidation._outputsInclude); + if (!enabled) { throw new JsonException("Unsupported include."); } + } + } + if (body.TryGetProperty("tool_choice", out var choice) && choice.ValueKind == JsonValueKind.Object + && IsNativeToolType(choice.GetProperty("type").GetString())) + { + Fields(choice, "type"); + var selected = choice.GetProperty("type").GetString()!; + if (!enabled || !tools.Any(tool => tool.GetProperty("type").GetString() == selected)) + { throw new JsonException("Native tool choice requires its declaration."); } + contracts.Add(selected); + } + if (body.TryGetProperty("input", out var input) && input.ValueKind == JsonValueKind.Object) + { + if ((input.TryGetProperty("type", out var objectType) && IsNativeHistoryType(objectType.GetString())) + || ValidateMessageCitations(input, enabled)) + { throw new JsonException("Native history must use an input array."); } + } + if (input.ValueKind == JsonValueKind.Array) + { + foreach (var item in input.EnumerateArray()) + { + if (item.ValueKind != JsonValueKind.Object) { continue; } + var itemType = item.TryGetProperty("type", out var type) ? type.GetString() : null; + if (IsNativeHistoryType(itemType)) + { + if (!enabled) { throw new JsonException("Native history is unsupported on this provider."); } + if (itemType == "web_search_call") { ValidateHistoryCall(item); } + else + { + OpenAiNativeFileCodeValidation.ValidateHistory(item); + } + } + if (HasCitations(item) && itemType is not (null or "message")) + { throw new JsonException("Unsupported annotated native message type."); } + if (itemType is null or "message") { ValidateMessageCitations(item, enabled); } + } + ValidateNativeGroups(input); + } + foreach (var contract in contracts) + { + var model = contract switch + { + OpenAiNativeFileCodeValidation._fileType => LlmTestServerKnownModelIds.Gpt4OMini, + OpenAiNativeFileCodeValidation._codeType => LlmTestServerKnownModelIds.Gpt41, + _ => LlmTestServerKnownModelIds.Gpt41Mini, + }; + if (!enabled || body.GetProperty("model").GetString() != model) + { throw new JsonException("Native tool requires a supported Responses model."); } + } + } + + private static void ValidateHistoryCall(JsonElement item) + { + Fields(item, "type", "id", "status", "action"); + NonEmpty(item.GetProperty("id")); + ValidateItemId(item.GetProperty("id").GetString()!); + Choice(item.GetProperty("status"), "completed"); + var action = item.GetProperty("action"); + Fields(action, "type", "queries", "sources"); + Choice(action.GetProperty("type"), "search"); + var queries = action.GetProperty("queries"); + Array(queries); + if (queries.GetArrayLength() is < 1 or > 32) { throw new JsonException("Search history requires one to 32 queries."); } + foreach (var query in queries.EnumerateArray()) { NonEmpty(query); } + if (action.TryGetProperty("sources", out var sources)) + { + Array(sources); + if (sources.GetArrayLength() > 128) { throw new JsonException("Too many search sources."); } + foreach (var source in sources.EnumerateArray()) + { + Fields(source, "type", "url"); + Choice(source.GetProperty("type"), "url"); + Url(source.GetProperty("url")); + } + } + } + + internal static bool ValidateMessageCitations(JsonElement item, bool enabled) + { + if (item.TryGetProperty("role", out var role) && role.ValueKind == JsonValueKind.String + && role.GetString() == "assistant" && item.TryGetProperty("id", out var providedId) + && providedId.ValueKind is not (JsonValueKind.String or JsonValueKind.Null)) + { throw new JsonException("An assistant output ID must be a string or null."); } + if (!item.TryGetProperty("content", out var content) || content.ValueKind != JsonValueKind.Array) { return false; } + var annotated = false; + foreach (var part in content.EnumerateArray()) + { + if (part.ValueKind != JsonValueKind.Object || !part.TryGetProperty("annotations", out var annotations)) { continue; } + Array(annotations); + if (annotations.GetArrayLength() == 0) { continue; } + if (!enabled || content.GetArrayLength() != 1) { throw new JsonException("Unsupported annotated native message."); } + if (annotations.GetArrayLength() > 128) { throw new JsonException("Too many URL citations."); } + if (item.TryGetProperty("id", out var messageId)) + { + String(messageId); + ValidateItemId(messageId.GetString()!); + } + Choice(item.GetProperty("role"), "assistant"); + Fields(part, "type", "text", "annotations"); + Choice(part.GetProperty("type"), "output_text"); + String(part.GetProperty("text")); + var text = part.GetProperty("text").GetString()!; + if (text.Any(character => character > 127)) { throw new JsonException("Native citation text must be ASCII in this contract."); } + foreach (var citation in annotations.EnumerateArray()) + { + if (citation.GetProperty("type").GetString() != "url_citation") + { + _ = OpenAiNativeFileCodeValidation.ValidateCitation(citation, text.Length); + continue; + } + Fields(citation, "type", "start_index", "end_index", "url", "title"); + Choice(citation.GetProperty("type"), "url_citation"); + if (!citation.GetProperty("start_index").TryGetInt32(out var start) + || !citation.GetProperty("end_index").TryGetInt32(out var end) + || start < 0 || end <= start || end > text.Length) + { throw new JsonException("Invalid native citation range."); } + Url(citation.GetProperty("url")); + String(citation.GetProperty("title")); + } + annotated = true; + } + return annotated; + } + + private static void ValidateNativeGroups(JsonElement input) + { + var group = new List(); + var count = 0; + var native = false; + void FinishGroup() + { + if (native) + { + if (count > 128) { throw new JsonException("Too many native assistant output items."); } + var ids = new HashSet(StringComparer.Ordinal); + foreach (var item in group) + { + var type = item.TryGetProperty("type", out var discriminator) ? discriminator.GetString() : null; + string? id = null; + if (type == "function_call") + { + id = item.TryGetProperty("id", out var functionId) + ? functionId.GetString() : "fc_" + item.GetProperty("call_id").GetString(); + if (string.IsNullOrWhiteSpace(id)) { throw new JsonException("A native function output ID is required."); } + } + else if (IsNativeHistoryType(type) || HasCitations(item) || item.TryGetProperty("id", out _)) + { + if (item.TryGetProperty("id", out var outputId)) + { + String(outputId); + id = outputId.GetString(); + } + else { id = string.Empty; } + } + if (id is null) { continue; } + ValidateItemId(id); + if (id.Length > 0 && !ids.Add(id)) { throw new JsonException("Duplicate native assistant output ID."); } + } + } + group.Clear(); + count = 0; + native = false; + } + foreach (var item in input.EnumerateArray()) + { + if (item.ValueKind != JsonValueKind.Object) + { + if (!string.IsNullOrWhiteSpace(OpenAiWireMapper.ReadResponseInputContent(item))) { FinishGroup(); } + continue; + } + var type = item.TryGetProperty("type", out var discriminator) ? discriminator.GetString() : null; + var assistant = IsNativeHistoryType(type) || type == "function_call" + || type != "function_call_output" && item.TryGetProperty("role", out var role) && role.GetString() == "assistant"; + var citations = HasCitations(item); + var retained = IsNativeHistoryType(type) || type is "function_call" or "function_call_output" || citations + || assistant && item.TryGetProperty("id", out _) + || !string.IsNullOrWhiteSpace(item.TryGetProperty("content", out var content) ? OpenAiWireMapper.ReadResponseInputContent(content) + : item.TryGetProperty("text", out var text) ? OpenAiWireMapper.ReadResponseInputContent(text) : item.ToString()); + if (!retained) { continue; } + if (!assistant) { FinishGroup(); continue; } + count++; + native |= IsNativeHistoryType(type) || citations; + if (group.Count < 129) { group.Add(item); } + if (native && count > 128) { throw new JsonException("Too many native assistant output items."); } + } + FinishGroup(); + } + + private static bool HasCitations(JsonElement item) + { + return item.TryGetProperty("content", out var content) && content.ValueKind == JsonValueKind.Array + && content.EnumerateArray().Any(part => part.ValueKind == JsonValueKind.Object + && part.TryGetProperty("annotations", out var annotations) && annotations.ValueKind == JsonValueKind.Array + && annotations.GetArrayLength() > 0); + } + + internal static void ValidateItemId(string id) + { + if (id.Length > 128 || id.Length > 0 && string.IsNullOrWhiteSpace(id)) + { throw new JsonException("Native output IDs must be bounded nonblank labels."); } + } + + internal static void Url(JsonElement value) + { + NonEmpty(value); + if (value.GetString()!.Any(char.IsWhiteSpace) || !Uri.TryCreate(value.GetString(), UriKind.Absolute, out var uri) || uri.Scheme is not ("http" or "https")) + { throw new JsonException("An absolute HTTP URL is required."); } + } + + private static void ValidateLocation(JsonElement location) + { + Fields(location, "type", "city", "region", "country", "timezone"); + if (location.TryGetProperty("type", out var approximation)) { Choice(approximation, "approximate"); } + foreach (var field in new[] { "city", "region", "country", "timezone" }) + { + if (!location.TryGetProperty(field, out var value) || value.ValueKind == JsonValueKind.Null) { continue; } + String(value); + var text = value.GetString()!; + if (field == "country") + { + if (text.Length != 2 || text.Any(character => !char.IsAsciiLetter(character))) + { throw new JsonException("An ISO country code is required."); } + try + { + if (!string.Equals(new RegionInfo(text).TwoLetterISORegionName, text, StringComparison.OrdinalIgnoreCase)) { throw new JsonException("Unknown country code."); } + } + catch (ArgumentException) { throw new JsonException("Unknown country code."); } + } + if (field == "timezone" && (text.Any(char.IsWhiteSpace) || !TimeZoneInfo.TryFindSystemTimeZoneById(text, out _))) + { throw new JsonException("An IANA timezone is required."); } + } + } + + internal static void Fields(JsonElement value, params string[] allowed) + { + Object(value); + var seen = new HashSet(StringComparer.Ordinal); + foreach (var property in value.EnumerateObject()) + { + if (!allowed.Contains(property.Name, StringComparer.Ordinal) || !seen.Add(property.Name)) + { throw new JsonException("Unsupported or duplicate native tool field."); } + } + } +} diff --git a/src/ManagedCode.LlmTestServer.OpenAI/OpenAiRequestValidation.cs b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiRequestValidation.cs new file mode 100644 index 0000000..fb590d3 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiRequestValidation.cs @@ -0,0 +1,154 @@ +using System.Text.Json; +using ManagedCode.LlmTestServer.Providers; +using static ManagedCode.LlmTestServer.ProviderValidation.LlmTestServerJsonValidation; + +namespace ManagedCode.LlmTestServer.OpenAI; + +public static class OpenAiRequestValidation +{ + public static LlmTestServerRequestValidationResult ValidateChat(JsonElement body) + { + return Check(body, value => Validate(value, false)); + } + + public static LlmTestServerRequestValidationResult ValidateResponse(JsonElement body) + { + return Check(body, value => Validate(value, true)); + } + + public static LlmTestServerRequestValidationResult ValidateNativeResponse(JsonElement body) + { + return Check(body, value => Validate(value, true, nativeTools: true)); + } + + private static void Validate(JsonElement body, bool responses, bool nativeTools = false) + { + if (body.TryGetProperty("functions", out _) || body.TryGetProperty("function_call", out _)) + { + throw new JsonException("Use tools and tool_choice."); + } + foreach (var tool in OptionalArray(body, "tools")) + { + if (nativeTools && OpenAiNativeWebSearchValidation.IsNativeToolType(tool.GetProperty("type").GetString())) + { + if (tool.GetProperty("type").GetString() == OpenAiNativeWebSearchValidation._toolType) + { OpenAiNativeWebSearchValidation.ValidateTool(tool); } + else { OpenAiNativeFileCodeValidation.ValidateTool(tool); } + continue; + } + Choice(tool.GetProperty("type"), "function"); + var function = responses ? tool : tool.GetProperty("function"); + Name(function); OptionalSchema(function, "parameters"); + } + if (body.TryGetProperty("tool_choice", out var choice)) + { + if (choice.ValueKind == JsonValueKind.Object) + { + if (nativeTools && OpenAiNativeWebSearchValidation.IsNativeToolType(choice.GetProperty("type").GetString())) + { + // Exact choice fields and declaration applicability are validated below. + } + else + { + Choice(choice.GetProperty("type"), "function"); + Name(responses ? choice : choice.GetProperty("function")); + } + } + else { Choice(choice, "auto", "none", "required"); } + } + if (body.TryGetProperty("parallel_tool_calls", out var parallel)) { _ = parallel.GetBoolean(); } + if (body.TryGetProperty("response_format", out var format) && format.ValueKind != JsonValueKind.Null) + { + ValidateFormat(format, false); + } + if (responses && body.TryGetProperty("text", out var text) && text.ValueKind != JsonValueKind.Null && text.TryGetProperty("format", out format)) + { + ValidateFormat(format, true); + } + if (responses) { OpenAiNativeWebSearchValidation.ValidateResponseOptions(body, nativeTools); } + var field = responses ? "input" : "messages"; + if (!body.TryGetProperty(field, out var messages)) { return; } + if (messages.ValueKind == JsonValueKind.Object) { ValidateRefusalHistory(messages, responses); return; } + if (messages.ValueKind != JsonValueKind.Array) { return; } + foreach (var message in messages.EnumerateArray()) + { + if (message.ValueKind != JsonValueKind.Object) { continue; } + ValidateRefusalHistory(message, responses); + if (!responses) + { + if (message.TryGetProperty("type", out var nativeType) + && OpenAiNativeWebSearchValidation.IsNativeHistoryType(nativeType.GetString())) + { throw new JsonException("Native history is unsupported on this operation."); } + _ = OpenAiNativeWebSearchValidation.ValidateMessageCitations(message, enabled: false); + } + foreach (var call in OptionalArray(message, "tool_calls", allowNull: true)) + { + Name(call.GetProperty("function")); NonEmpty(call.GetProperty("id")); String(call.GetProperty("function").GetProperty("arguments")); + } + if (responses && message.TryGetProperty("type", out var type)) + { + if (type.GetString() == "function_call") + { + Name(message); NonEmpty(message.GetProperty("call_id")); String(message.GetProperty("arguments")); + if (message.TryGetProperty("id", out var outputId) + && outputId.ValueKind is not (JsonValueKind.String or JsonValueKind.Null)) + { throw new JsonException("A function output ID must be a string or null."); } + } + if (type.GetString() == "function_call_output") { NonEmpty(message.GetProperty("call_id")); _ = message.GetProperty("output"); } + } + } + } + + private static void ValidateRefusalHistory(JsonElement message, bool responses) + { + var hasField = message.TryGetProperty("refusal", out var refusal) && refusal.ValueKind != JsonValueKind.Null; + var hasContent = message.TryGetProperty("content", out var content); + if (hasContent && content.ValueKind == JsonValueKind.Object + && content.TryGetProperty("type", out var objectType) && objectType.ValueKind == JsonValueKind.String + && objectType.GetString() == "refusal") + { throw new JsonException("Refusal content requires an array of typed refusal parts."); } + var hasParts = hasContent && content.ValueKind == JsonValueKind.Array + && content.EnumerateArray().Any(part => part.ValueKind == JsonValueKind.Object + && part.TryGetProperty("type", out var type) && type.ValueKind == JsonValueKind.String && type.GetString() == "refusal"); + if (!hasField && !hasParts) { return; } + if (!message.TryGetProperty("role", out var role) || role.ValueKind != JsonValueKind.String || role.GetString() != "assistant") + { throw new JsonException("Refusal history requires an assistant message."); } + if (!responses) + { + if (hasParts) + { + if (hasField) { throw new JsonException("Refusal history cannot mix the refusal field and refusal content parts."); } + foreach (var part in content.EnumerateArray()) + { + if (part.ValueKind != JsonValueKind.Object + || part.EnumerateObject().Any(property => property.Name is not ("type" or "refusal"))) + { throw new JsonException("This Chat history fixture requires pure refusal content parts."); } + Choice(part.GetProperty("type"), "refusal"); + String(part.GetProperty("refusal")); + } + return; + } + String(refusal); + if (message.TryGetProperty("content", out content) && content.ValueKind != JsonValueKind.Null + && !string.IsNullOrEmpty(OpenAiWireMapper.ReadResponseInputContent(content))) + { throw new JsonException("This refusal history fixture cannot mix refusal with ordinary content."); } + return; + } + if (hasField || content.GetArrayLength() != 1) + { throw new JsonException("Responses refusal history requires one typed refusal content part."); } + var responsePart = content[0]; + if (responsePart.EnumerateObject().Any(property => property.Name is not ("type" or "refusal"))) + { throw new JsonException("Unsupported refusal history content fields."); } + String(responsePart.GetProperty("refusal")); + } + + private static void ValidateFormat(JsonElement format, bool responses) + { + Choice(format.GetProperty("type"), "text", "json_object", "json_schema"); + if (format.GetProperty("type").GetString() == "json_schema") + { + Schema(responses ? format.GetProperty("schema") : format.GetProperty("json_schema").GetProperty("schema")); + } + OptionalSchema(format, "schema"); + } +} diff --git a/src/ManagedCode.LlmTestServer.OpenAI/OpenAiResponseModels.cs b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiResponseModels.cs new file mode 100644 index 0000000..0304224 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiResponseModels.cs @@ -0,0 +1,192 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace ManagedCode.LlmTestServer.OpenAI; + +public sealed record OpenAiResponseRequest +{ + [JsonPropertyName("include")] + public List Include { get; init; } = []; + + [JsonPropertyName("parallel_tool_calls")] + public bool ParallelToolCalls { get; init; } = true; + + [JsonPropertyName("tools")] public List Tools { get; init; } = []; + [JsonPropertyName("tool_choice")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public JsonElement ToolChoice { get; init; } + [JsonPropertyName("text")] public OpenAiResponseTextOptions? Text { get; init; } + + [JsonPropertyName("model")] + public string Model { get; init; } = string.Empty; + + [JsonPropertyName("input")] + public JsonElement Input { get; init; } + + [JsonPropertyName("instructions")] + public string? Instructions { get; init; } + + [JsonPropertyName("stream")] + public bool Stream { get; init; } + + [JsonPropertyName("store")] + public bool? Store { get; init; } + + [JsonPropertyName("previous_response_id")] + public string? PreviousResponseId { get; init; } + + [JsonPropertyName("prompt_cache_key")] + public string? PromptCacheKey { get; init; } + + [JsonPropertyName("session_id")] + public string? SessionId { get; init; } +} + +public sealed record OpenAiResponse +{ + [JsonPropertyName("id")] + public string Id { get; init; } = string.Empty; + + [JsonPropertyName("object")] + public string ObjectType { get; init; } = "response"; + + [JsonPropertyName("created_at")] + public long CreatedAt { get; init; } + + [JsonPropertyName("model")] + public string Model { get; init; } = string.Empty; + + [JsonPropertyName("output")] + public List Output { get; init; } = []; + + [JsonPropertyName("usage")] + public OpenAiResponseUsage Usage { get; init; } = new(); + + [JsonPropertyName("incomplete_details")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public OpenAiResponseIncompleteDetails? IncompleteDetails { get; init; } + + [JsonPropertyName("status")] + public string Status { get; init; } = "completed"; +} + +[JsonPolymorphic(TypeDiscriminatorPropertyName = "type")] +[JsonDerivedType(typeof(OpenAiResponseOutputMessage), "message")] +[JsonDerivedType(typeof(OpenAiResponseFunctionCall), "function_call")] +[JsonDerivedType(typeof(OpenAiResponseWebSearchCall), "web_search_call")] +[JsonDerivedType(typeof(OpenAiResponseFileSearchCall), "file_search_call")] +[JsonDerivedType(typeof(OpenAiResponseCodeInterpreterCall), "code_interpreter_call")] +public abstract record OpenAiResponseOutputItem +{ + [JsonIgnore] public abstract string Type { get; } + [JsonPropertyName("id")] public string Id { get; init; } = string.Empty; + [JsonPropertyName("status")] public string Status { get; init; } = "completed"; +} + +public sealed record OpenAiResponseIncompleteDetails +{ + [JsonPropertyName("reason")] public string Reason { get; init; } = "content_filter"; +} + +public sealed record OpenAiResponseOutputMessage : OpenAiResponseOutputItem +{ + [JsonIgnore] public override string Type => "message"; + [JsonPropertyName("role")] public string Role { get; init; } = "assistant"; + [JsonPropertyName("content")] public List Content { get; init; } = []; +} + +public sealed record OpenAiResponseFunctionCall : OpenAiResponseOutputItem +{ + [JsonIgnore] public override string Type => "function_call"; + [JsonPropertyName("call_id")] public string CallId { get; init; } = string.Empty; + [JsonPropertyName("name")] public string Name { get; init; } = string.Empty; + [JsonPropertyName("arguments")] public string Arguments { get; init; } = "{}"; +} + +[JsonPolymorphic(TypeDiscriminatorPropertyName = "type")] +[JsonDerivedType(typeof(OpenAiResponseOutputText), "output_text")] +[JsonDerivedType(typeof(OpenAiResponseRefusal), "refusal")] +public abstract record OpenAiResponseContentPart +{ + [JsonIgnore] public abstract string Type { get; } +} + +public sealed record OpenAiResponseRefusal : OpenAiResponseContentPart +{ + [JsonIgnore] public override string Type => "refusal"; + [JsonPropertyName("refusal")] public string Refusal { get; init; } = string.Empty; +} + +public sealed record OpenAiResponseOutputText : OpenAiResponseContentPart +{ + [JsonIgnore] public override string Type => "output_text"; + + [JsonPropertyName("text")] + public string Text { get; init; } = string.Empty; + + [JsonPropertyName("annotations")] + public List Annotations { get; init; } = []; +} + +public sealed record OpenAiResponseUsage +{ + [JsonPropertyName("input_tokens")] + public int InputTokens { get; init; } + + [JsonPropertyName("output_tokens")] + public int OutputTokens { get; init; } + + [JsonPropertyName("total_tokens")] + public int TotalTokens { get; init; } + + [JsonPropertyName("input_tokens_details")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public OpenAiInputTokensDetails? InputTokensDetails { get; init; } + + [JsonPropertyName("output_tokens_details")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public OpenAiOutputTokensDetails? OutputTokensDetails { get; init; } +} + +public sealed record OpenAiInputTokensDetails +{ + [JsonPropertyName("cached_tokens")] + public int CachedTokens { get; init; } + + [JsonPropertyName("cache_write_tokens")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? CacheWriteTokens { get; init; } +} + +public sealed record OpenAiOutputTokensDetails +{ + [JsonPropertyName("reasoning_tokens")] + public int ReasoningTokens { get; init; } +} + +public sealed record OpenAiResponseTool +{ + [JsonPropertyName("search_context_size")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? SearchContextSize { get; init; } + [JsonPropertyName("user_location")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public OpenAiWebSearchLocation? UserLocation { get; init; } + [JsonPropertyName("filters")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public JsonElement? Filters { get; init; } + [JsonIgnore] + public OpenAiWebSearchFilters? WebSearchFilters => Type == "web_search" && Filters is { ValueKind: JsonValueKind.Object } filters + ? filters.Deserialize() : null; + [JsonPropertyName("vector_store_ids")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public List? VectorStoreIds { get; init; } + [JsonPropertyName("max_num_results")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public int? MaxNumResults { get; init; } + [JsonPropertyName("ranking_options")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public OpenAiFileSearchRankingOptions? RankingOptions { get; init; } + [JsonPropertyName("container")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public JsonElement Container { get; init; } + [JsonPropertyName("type")] public string Type { get; init; } = "function"; + [JsonPropertyName("name")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? Name { get; init; } + [JsonPropertyName("description")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? Description { get; init; } + [JsonPropertyName("parameters")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public JsonElement Parameters { get; init; } +} +public sealed record OpenAiResponseTextOptions +{ + [JsonPropertyName("format")] public OpenAiResponseTextFormat? Format { get; init; } +} +public sealed record OpenAiResponseTextFormat +{ + [JsonPropertyName("type")] public string Type { get; init; } = "text"; + [JsonPropertyName("name")] public string? Name { get; init; } + [JsonPropertyName("schema")][JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public JsonElement Schema { get; init; } + [JsonPropertyName("strict")] public bool? Strict { get; init; } +} diff --git a/src/ManagedCode.LlmTck.OpenAI/OpenAiToolModels.cs b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiToolModels.cs similarity index 77% rename from src/ManagedCode.LlmTck.OpenAI/OpenAiToolModels.cs rename to src/ManagedCode.LlmTestServer.OpenAI/OpenAiToolModels.cs index c57eba9..d839b82 100644 --- a/src/ManagedCode.LlmTck.OpenAI/OpenAiToolModels.cs +++ b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiToolModels.cs @@ -1,8 +1,8 @@ using System.Text.Json; using System.Text.Json.Serialization; -using ManagedCode.LlmTck.Scenarios; +using ManagedCode.LlmTestServer.Scenarios; -namespace ManagedCode.LlmTck.OpenAI; +namespace ManagedCode.LlmTestServer.OpenAI; public sealed record OpenAiTool { @@ -41,21 +41,21 @@ public sealed record OpenAiJsonSchema } public static class OpenAiToolMapper { - public static LlmTckChatRequest Apply(LlmTckChatRequest request, List? tools, JsonElement choice, OpenAiResponseFormat? format) + public static LlmTestServerChatRequest Apply(LlmTestServerChatRequest request, List? tools, JsonElement choice, OpenAiResponseFormat? format) { var named = choice.ValueKind == JsonValueKind.Object && choice.TryGetProperty("function", out var function) ? function.GetProperty("name").GetString() : null; return request with { - Tools = tools?.Select(tool => new LlmTckToolDefinition { Name = tool.Function.Name, Description = tool.Function.Description, ParametersJson = tool.Function.Parameters.GetRawText() }).ToList() ?? [], + Tools = tools?.Select(tool => new LlmTestServerToolDefinition { Name = tool.Function.Name, Description = tool.Function.Description, ParametersJson = tool.Function.Parameters.GetRawText() }).ToList() ?? [], ToolChoice = named is not null || choice.ValueKind == JsonValueKind.String && choice.GetString() == "required" - ? LlmTckToolChoice.Required : choice.ValueKind == JsonValueKind.String && choice.GetString() == "none" ? LlmTckToolChoice.None : LlmTckToolChoice.Auto, + ? LlmTestServerToolChoice.Required : choice.ValueKind == JsonValueKind.String && choice.GetString() == "none" ? LlmTestServerToolChoice.None : LlmTestServerToolChoice.Auto, RequiredToolName = named, RequireJson = format?.Type is "json_object" or "json_schema", ResponseSchemaJson = format?.JsonSchema?.Schema.GetRawText(), }; } - public static OpenAiToolCall ToWire(LlmTckToolCall call, int? index = null) + public static OpenAiToolCall ToWire(LlmTestServerToolCall call, int? index = null) { return new() { @@ -65,7 +65,7 @@ public static OpenAiToolCall ToWire(LlmTckToolCall call, int? index = null) }; } - public static LlmTckToolCall ToRuntime(OpenAiToolCall call) + public static LlmTestServerToolCall ToRuntime(OpenAiToolCall call) { return new() { diff --git a/src/ManagedCode.LlmTck.OpenAI/OpenAiVideoModels.cs b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiVideoModels.cs similarity index 95% rename from src/ManagedCode.LlmTck.OpenAI/OpenAiVideoModels.cs rename to src/ManagedCode.LlmTestServer.OpenAI/OpenAiVideoModels.cs index c7e23ef..da06d45 100644 --- a/src/ManagedCode.LlmTck.OpenAI/OpenAiVideoModels.cs +++ b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiVideoModels.cs @@ -1,12 +1,12 @@ using System.Text.Json.Serialization; -using ManagedCode.LlmTck.Models; +using ManagedCode.LlmTestServer.Models; -namespace ManagedCode.LlmTck.OpenAI; +namespace ManagedCode.LlmTestServer.OpenAI; public sealed record OpenAiVideoCreateRequest { [JsonPropertyName("model")] - public string Model { get; init; } = LlmTckKnownModelIds.Sora2; + public string Model { get; init; } = LlmTestServerKnownModelIds.Sora2; [JsonPropertyName("prompt")] public string Prompt { get; init; } = string.Empty; @@ -138,7 +138,7 @@ public sealed record OpenAiVideoRemixRequest public sealed record OpenAiVideoCharacterResponse { [JsonPropertyName("id")] - public string Id { get; init; } = "char_llm_tck"; + public string Id { get; init; } = "char_llm_test_server"; [JsonPropertyName("created_at")] public long CreatedAt { get; init; } @@ -150,7 +150,7 @@ public sealed record OpenAiVideoCharacterResponse public sealed record AzureVideoGenerationJobRequest { [JsonPropertyName("model")] - public string Model { get; init; } = LlmTckKnownModelIds.Sora2; + public string Model { get; init; } = LlmTestServerKnownModelIds.Sora2; [JsonPropertyName("prompt")] public string Prompt { get; init; } = string.Empty; diff --git a/src/ManagedCode.LlmTestServer.OpenAI/OpenAiWireMapper.NativeFileCode.cs b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiWireMapper.NativeFileCode.cs new file mode 100644 index 0000000..e52ae1c --- /dev/null +++ b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiWireMapper.NativeFileCode.cs @@ -0,0 +1,247 @@ +using System.Text.Json; +using ManagedCode.LlmTestServer.Scenarios; + +namespace ManagedCode.LlmTestServer.OpenAI; + +public static partial class OpenAiWireMapper +{ + private static LlmTestServerNativeToolContract? NativeContract(string? type) + { + return type switch + { + "web_search" => LlmTestServerNativeToolContract.OpenAiResponsesWebSearch, + "file_search" => LlmTestServerNativeToolContract.OpenAiResponsesFileSearch, + "code_interpreter" => LlmTestServerNativeToolContract.OpenAiResponsesCodeInterpreter, + _ => null, + }; + } + + private static LlmTestServerNativeToolInclude? NativeInclude(string property) + { + return property switch + { + _webSearchSources => LlmTestServerNativeToolInclude.WebSearchSources, + _fileSearchResults => LlmTestServerNativeToolInclude.FileSearchResults, + _codeInterpreterOutputs => LlmTestServerNativeToolInclude.CodeInterpreterOutputs, + _ => null, + }; + } + + private static bool Includes(IReadOnlyCollection? properties, string property) + { + return properties?.Contains(property, StringComparer.Ordinal) == true; + } + + private static bool IsNativeItem(LlmTestServerResponseItem item) + { + return item is LlmTestServerResponseWebSearchItem or LlmTestServerResponseFileSearchItem + or LlmTestServerResponseCodeInterpreterItem; + } + + private static LlmTestServerNativeToolDeclaration CreateNativeDeclaration(OpenAiResponseTool tool) + { + var contract = NativeContract(tool.Type)!.Value; + return contract switch + { + LlmTestServerNativeToolContract.OpenAiResponsesWebSearch => new() + { + Contract = contract, + WebSearchOptions = new() + { + SearchContextSize = tool.SearchContextSize, + AllowedDomains = tool.WebSearchFilters?.AllowedDomains is { } domains ? [.. domains] : [], + UserLocation = tool.UserLocation is { } location ? new() + { + Type = location.Type, + Country = location.Country, + Region = location.Region, + City = location.City, + Timezone = location.Timezone, + } : null, + }, + }, + LlmTestServerNativeToolContract.OpenAiResponsesFileSearch => new() + { + Contract = contract, + FileSearchOptions = new() + { + VectorStoreIds = tool.VectorStoreIds is { } stores ? [.. stores] : [], + MaxNumResults = tool.MaxNumResults, + Ranker = tool.RankingOptions?.Ranker, + ScoreThreshold = tool.RankingOptions?.ScoreThreshold, + Filters = tool.Filters is { ValueKind: not (JsonValueKind.Null or JsonValueKind.Undefined) } filters + ? filters.Clone() : null, + }, + }, + LlmTestServerNativeToolContract.OpenAiResponsesCodeInterpreter => new() + { + Contract = contract, + CodeInterpreterOptions = tool.Container.ValueKind == JsonValueKind.String + ? new() { ContainerId = tool.Container.GetString() } + : new() + { + AutomaticContainer = true, + FileIds = tool.Container.TryGetProperty("file_ids", out var files) + ? files.EnumerateArray().Select(file => file.GetString()!).ToList() : [], + }, + }, + _ => throw new InvalidOperationException("Unsupported native declaration."), + }; + } + + private static object MapCitation(LlmTestServerResponseCitation citation) + { + return citation switch + { + LlmTestServerUrlCitation url => new OpenAiUrlCitation + { + StartIndex = url.StartIndex, + EndIndex = url.EndIndex, + Url = url.Url, + Title = url.Title, + }, + LlmTestServerFileCitation file => new OpenAiFileCitation + { + FileId = file.FileId, + Filename = file.Filename, + Index = file.Index, + }, + LlmTestServerContainerFileCitation file => new OpenAiContainerFileCitation + { + ContainerId = file.ContainerId, + FileId = file.FileId, + Filename = file.Filename, + StartIndex = file.StartIndex, + EndIndex = file.EndIndex, + }, + _ => throw new InvalidOperationException("Unsupported Responses citation."), + }; + } + + private static OpenAiResponseFileSearchCall MapFileSearchItem(LlmTestServerResponseFileSearchItem item, + IReadOnlyCollection? includedProperties) + { + return new() + { + Id = item.Id, + Queries = [.. item.Queries], + Results = Includes(includedProperties, _fileSearchResults) ? item.Results.Select(result => new OpenAiFileSearchResult + { + FileId = result.FileId, + Filename = result.Filename, + Text = result.Text, + Score = result.Score, + Attributes = result.Attributes?.ToDictionary(pair => pair.Key, pair => pair.Value.Clone(), StringComparer.Ordinal), + }).ToList() : null, + }; + } + + private static OpenAiResponseCodeInterpreterCall MapCodeInterpreterItem(LlmTestServerResponseCodeInterpreterItem item, + IReadOnlyCollection? includedProperties) + { + return new() + { + Id = item.Id, + ContainerId = item.ContainerId, + Code = item.Code, + Outputs = Includes(includedProperties, _codeInterpreterOutputs) + ? item.Outputs?.Select(MapCodeOutput).ToList() : null, + }; + } + + private static OpenAiCodeInterpreterOutput MapCodeOutput(LlmTestServerCodeInterpreterOutput output) + { + return output switch + { + LlmTestServerCodeInterpreterLogsOutput logs => new OpenAiCodeInterpreterLogsOutput { Logs = logs.Logs }, + LlmTestServerCodeInterpreterImageOutput image => new OpenAiCodeInterpreterImageOutput { Url = image.Url }, + _ => throw new InvalidOperationException("Unsupported code interpreter output."), + }; + } + + private static LlmTestServerResponseItem ReadNativeToolItem(JsonElement item) + { + return item.GetProperty("type").GetString() switch + { + "web_search_call" => ReadNativeWebSearchItem(item), + "file_search_call" => ReadFileSearchItem(item), + "code_interpreter_call" => ReadCodeInterpreterItem(item), + _ => throw new InvalidOperationException("Unsupported native history item."), + }; + } + + private static LlmTestServerResponseFileSearchItem ReadFileSearchItem(JsonElement item) + { + return new() + { + Id = item.GetProperty("id").GetString()!, + Queries = item.GetProperty("queries").EnumerateArray().Select(query => query.GetString()!).ToList(), + Results = item.TryGetProperty("results", out var results) && results.ValueKind == JsonValueKind.Array + ? results.EnumerateArray().Select(result => new LlmTestServerFileSearchResult + { + FileId = OptionalString(result, "file_id"), + Filename = OptionalString(result, "filename"), + Text = OptionalString(result, "text"), + Score = result.TryGetProperty("score", out var score) && score.ValueKind == JsonValueKind.Number ? score.GetDouble() : null, + Attributes = result.TryGetProperty("attributes", out var attributes) && attributes.ValueKind == JsonValueKind.Object + ? attributes.EnumerateObject().ToDictionary(pair => pair.Name, pair => pair.Value.Clone(), StringComparer.Ordinal) : null, + }).ToList() : [], + }; + } + + private static LlmTestServerResponseCodeInterpreterItem ReadCodeInterpreterItem(JsonElement item) + { + return new() + { + Id = item.GetProperty("id").GetString()!, + ContainerId = item.GetProperty("container_id").GetString()!, + Code = OptionalString(item, "code"), + Outputs = item.TryGetProperty("outputs", out var outputs) && outputs.ValueKind == JsonValueKind.Array + ? outputs.EnumerateArray().Select(ReadCodeOutput).ToList() : null, + }; + } + + private static LlmTestServerCodeInterpreterOutput ReadCodeOutput(JsonElement output) + { + return output.GetProperty("type").GetString() switch + { + "logs" => new LlmTestServerCodeInterpreterLogsOutput { Logs = output.GetProperty("logs").GetString()! }, + "image" => new LlmTestServerCodeInterpreterImageOutput { Url = output.GetProperty("url").GetString()! }, + _ => throw new InvalidOperationException("Unsupported code interpreter history output."), + }; + } + + private static LlmTestServerResponseCitation ReadCitation(JsonElement citation) + { + return citation.GetProperty("type").GetString() switch + { + "url_citation" => new LlmTestServerUrlCitation + { + StartIndex = citation.GetProperty("start_index").GetInt32(), + EndIndex = citation.GetProperty("end_index").GetInt32(), + Url = citation.GetProperty("url").GetString()!, + Title = citation.GetProperty("title").GetString()!, + }, + "file_citation" => new LlmTestServerFileCitation + { + FileId = citation.GetProperty("file_id").GetString()!, + Filename = citation.GetProperty("filename").GetString()!, + Index = citation.GetProperty("index").GetInt32(), + }, + "container_file_citation" => new LlmTestServerContainerFileCitation + { + ContainerId = citation.GetProperty("container_id").GetString()!, + FileId = citation.GetProperty("file_id").GetString()!, + Filename = citation.GetProperty("filename").GetString()!, + StartIndex = citation.GetProperty("start_index").GetInt32(), + EndIndex = citation.GetProperty("end_index").GetInt32(), + }, + _ => throw new InvalidOperationException("Unsupported Responses history citation."), + }; + } + + private static string? OptionalString(JsonElement item, string name) + { + return item.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.String ? value.GetString() : null; + } +} diff --git a/src/ManagedCode.LlmTestServer.OpenAI/OpenAiWireMapper.NativeResponses.cs b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiWireMapper.NativeResponses.cs new file mode 100644 index 0000000..093960f --- /dev/null +++ b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiWireMapper.NativeResponses.cs @@ -0,0 +1,262 @@ +using System.Text.Json; +using ManagedCode.LlmTestServer.Determinism; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Scenarios; + +namespace ManagedCode.LlmTestServer.OpenAI; + +public static partial class OpenAiWireMapper +{ + private const string _webSearchSources = "web_search_call.action.sources"; + private const string _fileSearchResults = "file_search_call.results"; + private const string _codeInterpreterOutputs = "code_interpreter_call.outputs"; + + private static JsonElement FunctionResponseChoice(JsonElement choice) + { + if (choice.ValueKind != JsonValueKind.Object) { return choice; } + return choice.GetProperty("type").GetString() == "function" + ? JsonSerializer.SerializeToElement(new { function = choice }) + : JsonSerializer.SerializeToElement("required"); + } + + private static LlmTestServerChatRequest ApplyNativeResponseRequest(OpenAiResponseRequest source, + LlmTestServerChatRequest request, bool allowNativeTools) + { + return request with + { + NativeToolSurface = allowNativeTools ? LlmTestServerNativeToolSurface.OpenAiResponses : null, + NativeTools = source.Tools.Where(tool => NativeContract(tool.Type) is not null) + .Select(CreateNativeDeclaration).ToList(), + RequiredNativeTool = source.ToolChoice.ValueKind == JsonValueKind.Object + ? NativeContract(source.ToolChoice.GetProperty("type").GetString()) : null, + NativeIncludes = source.Include.Select(NativeInclude).Where(include => include is not null) + .Select(include => include!.Value).ToList(), + }; + } + + private static List CreateResponseOutput(LlmTestServerChatResult result, + IReadOnlyCollection? includedProperties) + { + var output = new List(); + if (result.OutputItems.Count > 0) + { + for (var index = 0; index < result.OutputItems.Count; index++) + { + output.Add(MapResponseItem(result.OutputItems[index], index, result.WireMetadata, includedProperties)); + } + return output; + } + if (result.Content.Length > 0 || result.ToolCalls.Count == 0) + { + var message = CreateOutputMessage(result.Content, completed: true, result.WireMetadata); + if (result.OutputDisposition?.Kind == LlmTestServerOutputDispositionKind.Refused) + { message = message with { Content = [new OpenAiResponseRefusal { Refusal = result.Content }] }; } + if (result.OutputDisposition?.Kind == LlmTestServerOutputDispositionKind.Filtered) + { message = message with { Status = "incomplete" }; } + output.Add(message); + } + output.AddRange(result.ToolCalls.Select(call => new OpenAiResponseFunctionCall + { Id = "fc_" + call.Id, CallId = call.Id, Name = call.Name, Arguments = call.ArgumentsJson })); + return output; + } + + private static OpenAiResponseOutputItem MapResponseItem(LlmTestServerResponseItem item, int index, + LlmTestServerResponseMetadata? metadata, IReadOnlyCollection? includedProperties) + { + return item switch + { + LlmTestServerResponseMessageItem message => new OpenAiResponseOutputMessage + { + Id = string.IsNullOrEmpty(message.Id) + ? "msg_" + (metadata ?? LlmTestServerResponseMetadata.Live()).IdFor(index == 0 ? "message" : $"message/{index}") : message.Id, + Content = [new OpenAiResponseOutputText + { + Text = message.Content, + Annotations = message.Citations.Select(MapCitation).ToList() + }] + }, + LlmTestServerResponseFunctionCallItem function => new OpenAiResponseFunctionCall + { + Id = string.IsNullOrEmpty(function.Id) ? "fc_" + function.Call.Id : function.Id, + CallId = function.Call.Id, + Name = function.Call.Name, + Arguments = function.Call.ArgumentsJson + }, + LlmTestServerResponseWebSearchItem web => new OpenAiResponseWebSearchCall + { + Id = web.Id, + Action = new OpenAiWebSearchAction + { + Type = "search", + Queries = [.. web.Queries], + Sources = Includes(includedProperties, _webSearchSources) ? web.Sources.Select(source => new OpenAiWebSearchSource { Url = source.Url }).ToList() : null + } + }, + LlmTestServerResponseFileSearchItem file => MapFileSearchItem(file, includedProperties), + LlmTestServerResponseCodeInterpreterItem code => MapCodeInterpreterItem(code, includedProperties), + _ => throw new InvalidOperationException("Unsupported Responses output item.") + }; + } + + public static IEnumerable ToResponseStreamEvents(LlmTestServerChatResult result, string responseId, + long created, OpenAiCacheUsageShape cacheUsageShape = OpenAiCacheUsageShape.None, + IReadOnlyCollection? includedProperties = null) + { + var response = ToResponse(result, responseId, created, cacheUsageShape, includedProperties); + var sequence = 0; + yield return new { type = "response.created", sequence_number = sequence++, response = response with { Status = "in_progress", IncompleteDetails = null, Output = [] } }; + yield return new { type = "response.in_progress", sequence_number = sequence++, response = response with { Status = "in_progress", IncompleteDetails = null, Output = [] } }; + for (var index = 0; index < response.Output.Count; index++) + { + var item = response.Output[index]; + OpenAiResponseOutputItem initial = item switch + { + OpenAiResponseFunctionCall function => function with { Status = "in_progress", Arguments = string.Empty }, + OpenAiResponseOutputMessage message => message with { Status = "in_progress", Content = [] }, + OpenAiResponseWebSearchCall web => web with { Status = "in_progress", Action = null }, + OpenAiResponseFileSearchCall file => file with { Status = "in_progress", Queries = [], Results = null }, + OpenAiResponseCodeInterpreterCall code => code with { Status = "in_progress", Code = code.Code is null ? null : string.Empty, Outputs = null }, + _ => throw new InvalidOperationException("Unsupported Responses output item.") + }; + yield return new { type = "response.output_item.added", sequence_number = sequence++, output_index = index, item = initial }; + if (item is OpenAiResponseFunctionCall functionItem) + { + yield return new { type = "response.function_call_arguments.delta", sequence_number = sequence++, item_id = item.Id, output_index = index, delta = functionItem.Arguments }; + yield return new { type = "response.function_call_arguments.done", sequence_number = sequence++, item_id = item.Id, output_index = index, arguments = functionItem.Arguments }; + } + else if (item is OpenAiResponseWebSearchCall) + { + yield return new { type = "response.web_search_call.in_progress", sequence_number = sequence++, item_id = item.Id, output_index = index }; + yield return new { type = "response.web_search_call.searching", sequence_number = sequence++, item_id = item.Id, output_index = index }; + yield return new { type = "response.web_search_call.completed", sequence_number = sequence++, item_id = item.Id, output_index = index }; + } + else if (item is OpenAiResponseFileSearchCall) + { + yield return new { type = "response.file_search_call.in_progress", sequence_number = sequence++, item_id = item.Id, output_index = index }; + yield return new { type = "response.file_search_call.searching", sequence_number = sequence++, item_id = item.Id, output_index = index }; + yield return new { type = "response.file_search_call.completed", sequence_number = sequence++, item_id = item.Id, output_index = index }; + } + else if (item is OpenAiResponseCodeInterpreterCall codeItem) + { + yield return new { type = "response.code_interpreter_call.in_progress", sequence_number = sequence++, item_id = item.Id, output_index = index }; + if (codeItem.Code is not null) + { + IReadOnlyList codeChunks = result.OutputItems.Count > index + && result.OutputItems[index] is LlmTestServerResponseCodeInterpreterItem { CodeChunks.Count: > 0 } codePlan + ? codePlan.CodeChunks : [codeItem.Code]; + foreach (var chunk in codeChunks) + { + yield return new { type = "response.code_interpreter_call_code.delta", sequence_number = sequence++, item_id = item.Id, output_index = index, delta = chunk }; + } + yield return new { type = "response.code_interpreter_call_code.done", sequence_number = sequence++, item_id = item.Id, output_index = index, code = codeItem.Code }; + } + yield return new { type = "response.code_interpreter_call.interpreting", sequence_number = sequence++, item_id = item.Id, output_index = index }; + yield return new { type = "response.code_interpreter_call.completed", sequence_number = sequence++, item_id = item.Id, output_index = index }; + } + else if (item is OpenAiResponseOutputMessage messageItem) + { + if (messageItem.Content[0] is OpenAiResponseRefusal refusal) + { + yield return new { type = "response.content_part.added", sequence_number = sequence++, item_id = item.Id, output_index = index, content_index = 0, part = (OpenAiResponseContentPart)new OpenAiResponseRefusal() }; + foreach (var chunk in result.StreamChunks.Count > 0 ? result.StreamChunks : [refusal.Refusal]) + { + yield return new { type = "response.refusal.delta", sequence_number = sequence++, item_id = item.Id, output_index = index, content_index = 0, delta = chunk }; + } + yield return new { type = "response.refusal.done", sequence_number = sequence++, item_id = item.Id, output_index = index, content_index = 0, refusal = refusal.Refusal }; + yield return new { type = "response.content_part.done", sequence_number = sequence++, item_id = item.Id, output_index = index, content_index = 0, part = (OpenAiResponseContentPart)refusal }; + } + else + { + var part = (OpenAiResponseOutputText)messageItem.Content[0]; + yield return new { type = "response.content_part.added", sequence_number = sequence++, item_id = item.Id, output_index = index, content_index = 0, part = (OpenAiResponseContentPart)new OpenAiResponseOutputText() }; + IReadOnlyList chunks = result.OutputItems.Count > index && result.OutputItems[index] is LlmTestServerResponseMessageItem messagePlan + ? messagePlan.StreamChunks.Count > 0 ? messagePlan.StreamChunks : [messagePlan.Content] + : result.StreamChunks.Count > 0 ? result.StreamChunks : [part.Text]; + foreach (var chunk in chunks) + { + yield return new { type = "response.output_text.delta", sequence_number = sequence++, item_id = item.Id, output_index = index, content_index = 0, delta = chunk, logprobs = Array.Empty() }; + } + for (var annotationIndex = 0; annotationIndex < part.Annotations.Count; annotationIndex++) + { + yield return new { type = "response.output_text.annotation.added", sequence_number = sequence++, item_id = item.Id, output_index = index, content_index = 0, annotation_index = annotationIndex, annotation = part.Annotations[annotationIndex] }; + } + yield return new { type = "response.output_text.done", sequence_number = sequence++, item_id = item.Id, output_index = index, content_index = 0, text = part.Text, logprobs = Array.Empty() }; + yield return new { type = "response.content_part.done", sequence_number = sequence++, item_id = item.Id, output_index = index, content_index = 0, part = (OpenAiResponseContentPart)part }; + } + } + yield return new { type = "response.output_item.done", sequence_number = sequence++, output_index = index, item }; + } + yield return new { type = response.Status == "incomplete" ? "response.incomplete" : "response.completed", sequence_number = sequence, response }; + } + + private static LlmTestServerResponseWebSearchItem ReadNativeWebSearchItem(JsonElement item) + { + var action = item.GetProperty("action"); + return new() + { + Id = item.GetProperty("id").GetString()!, + Queries = action.TryGetProperty("queries", out var queries) + ? queries.EnumerateArray().Select(query => query.GetString()!).ToList() : [], + Sources = action.TryGetProperty("sources", out var sources) + ? sources.EnumerateArray().Select(source => new LlmTestServerWebSearchSource { Url = source.GetProperty("url").GetString()! }).ToList() : [] + }; + } + + private static List ReadNativeMessageItems(JsonElement item, JsonElement content) + { + if (HasRefusalPart(content)) { return []; } + var citations = new List(); + var parts = content.ValueKind == JsonValueKind.Array ? content.EnumerateArray().ToArray() : []; + foreach (var part in parts) + { + if (part.ValueKind != JsonValueKind.Object || !part.TryGetProperty("annotations", out var annotations)) { continue; } + foreach (var annotation in annotations.EnumerateArray()) + { + citations.Add(ReadCitation(annotation)); + } + } + var outputId = item.TryGetProperty("id", out var id) ? id.GetString()! : string.Empty; + if (citations.Count == 0 && string.IsNullOrEmpty(outputId)) { return []; } + return [new LlmTestServerResponseMessageItem + { + Id = outputId, + Content = ReadResponseInputContent(content), Citations = citations + }]; + } + + private static List CoalesceNativeAssistantGroups(List messages) + { + var result = new List(); + for (var index = 0; index < messages.Count;) + { + var end = index; + while (end < messages.Count && messages[end].Role == "assistant") { end++; } + if (end == index) { result.Add(messages[index++]); continue; } + var group = messages.GetRange(index, end - index); + if (!group.Any(message => message.OutputItems.Any(item => IsNativeItem(item) + || item is LlmTestServerResponseMessageItem { Citations.Count: > 0 }))) + { + result.AddRange(group.Select(message => message with { OutputItems = [] })); + index = end; + continue; + } + var items = new List(); + foreach (var message in group) + { + if (message.OutputItems.Count > 0) { items.AddRange(message.OutputItems); continue; } + if (message.Content.Length > 0) { items.Add(new LlmTestServerResponseMessageItem { Content = message.Content }); } + items.AddRange(message.ToolCalls.Select(call => new LlmTestServerResponseFunctionCallItem { Call = call })); + } + result.Add(new LlmTestServerMessage + { + Role = "assistant", + OutputDisposition = group.Select(message => message.OutputDisposition).FirstOrDefault(disposition => disposition is not null), + Content = string.Concat(group.Select(message => message.Content)), + ToolCalls = group.SelectMany(message => message.ToolCalls).ToList(), + OutputItems = items + }); + index = end; + } + return result; + } +} diff --git a/src/ManagedCode.LlmTck.OpenAI/OpenAiWireMapper.cs b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiWireMapper.cs similarity index 70% rename from src/ManagedCode.LlmTck.OpenAI/OpenAiWireMapper.cs rename to src/ManagedCode.LlmTestServer.OpenAI/OpenAiWireMapper.cs index 4a29c9c..ae09ef2 100644 --- a/src/ManagedCode.LlmTck.OpenAI/OpenAiWireMapper.cs +++ b/src/ManagedCode.LlmTestServer.OpenAI/OpenAiWireMapper.cs @@ -1,9 +1,10 @@ using System.Text.Json; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Scenarios; +using ManagedCode.LlmTestServer.Determinism; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Scenarios; -namespace ManagedCode.LlmTck.OpenAI; +namespace ManagedCode.LlmTestServer.OpenAI; public enum OpenAiCacheUsageShape { @@ -13,11 +14,11 @@ public enum OpenAiCacheUsageShape DeepSeekPromptCache, } -public static class OpenAiWireMapper +public static partial class OpenAiWireMapper { - public static LlmTckChatRequest ToRuntimeRequest( + public static LlmTestServerChatRequest ToRuntimeRequest( OpenAiChatCompletionRequest request, - LlmTckPromptCachePolicy promptCachePolicy = LlmTckPromptCachePolicy.None + LlmTestServerPromptCachePolicy promptCachePolicy = LlmTestServerPromptCachePolicy.None ) { return OpenAiToolMapper.Apply(new() @@ -33,10 +34,11 @@ public static LlmTckChatRequest ToRuntimeRequest( ), Messages = request .Messages - .Select(message => new LlmTckMessage + .Select(message => new LlmTestServerMessage { Role = message.Role, - Content = message.TextContent, + Content = message.RefusalTextContent ?? message.TextContent, + OutputDisposition = message.RefusalTextContent is null ? null : new() { Kind = LlmTestServerOutputDispositionKind.Refused }, ToolCallId = message.ToolCallId, ToolCalls = message.ToolCalls?.Select(OpenAiToolMapper.ToRuntime).ToList() ?? [], }) @@ -44,9 +46,10 @@ public static LlmTckChatRequest ToRuntimeRequest( }, request.Tools, request.ToolChoice, request.ResponseFormat); } - public static LlmTckChatRequest ToRuntimeRequest( + public static LlmTestServerChatRequest ToRuntimeRequest( OpenAiResponseRequest request, - LlmTckPromptCachePolicy promptCachePolicy = LlmTckPromptCachePolicy.None + LlmTestServerPromptCachePolicy promptCachePolicy = LlmTestServerPromptCachePolicy.None, + bool allowNativeTools = false ) { var messages = ReadResponseInputMessages(request.Input); @@ -54,7 +57,7 @@ public static LlmTckChatRequest ToRuntimeRequest( { messages.Insert( 0, - new LlmTckMessage + new LlmTestServerMessage { Role = "system", Content = request.Instructions, @@ -62,7 +65,7 @@ public static LlmTckChatRequest ToRuntimeRequest( ); } - return OpenAiToolMapper.Apply(new() + var normalized = OpenAiToolMapper.Apply(new() { ModelId = request.Model, AllowParallelToolCalls = request.ParallelToolCalls, @@ -74,21 +77,22 @@ public static LlmTckChatRequest ToRuntimeRequest( promptCachePolicy ), Messages = messages, - }, request.Tools.Select(tool => new OpenAiTool { Type = tool.Type, Function = new() { Name = tool.Name, Description = tool.Description, Parameters = tool.Parameters } }).ToList(), - request.ToolChoice.ValueKind == JsonValueKind.Object ? JsonSerializer.SerializeToElement(new { function = request.ToolChoice }) : request.ToolChoice, + }, request.Tools.Where(tool => tool.Type == "function").Select(tool => new OpenAiTool { Type = tool.Type, Function = new() { Name = tool.Name ?? string.Empty, Description = tool.Description, Parameters = tool.Parameters.ValueKind == JsonValueKind.Undefined ? JsonSerializer.SerializeToElement(new { type = "object" }) : tool.Parameters } }).ToList(), + FunctionResponseChoice(request.ToolChoice), request.Text?.Format is { } format ? new() { Type = format.Type, JsonSchema = format.Type == "json_schema" ? new() { Schema = format.Schema } : null } : null); + return ApplyNativeResponseRequest(request, normalized, allowNativeTools); } public static OpenAiChatCompletionResponse ToChatResponse( - LlmTckChatResult result, + LlmTestServerChatResult result, OpenAiCacheUsageShape cacheUsageShape = OpenAiCacheUsageShape.None ) { - var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); + var now = (result.WireMetadata ?? LlmTestServerResponseMetadata.Live()).CreatedAt.ToUnixTimeSeconds(); return new OpenAiChatCompletionResponse { - Id = CreateResponseId(), + Id = CreateResponseId(result.WireMetadata), Created = now, Model = result.ModelId, Choices = @@ -96,8 +100,10 @@ public static OpenAiChatCompletionResponse ToChatResponse( new OpenAiChatChoice { Index = 0, - Message = OpenAiChatMessage.FromText("assistant", result.Content) with { ToolCalls = result.ToolCalls.Count > 0 ? result.ToolCalls.Select(call => OpenAiToolMapper.ToWire(call)).ToList() : null }, - FinishReason = result.ToolCalls.Count > 0 ? "tool_calls" : "stop", + Message = result.OutputDisposition?.Kind == LlmTestServerOutputDispositionKind.Refused + ? new OpenAiChatMessage { Role = "assistant", Content = JsonSerializer.SerializeToElement(null), Refusal = result.Content } + : OpenAiChatMessage.FromText("assistant", result.Content) with { ToolCalls = result.ToolCalls.Count > 0 ? result.ToolCalls.Select(call => OpenAiToolMapper.ToWire(call)).ToList() : null }, + FinishReason = result.OutputDisposition?.Kind == LlmTestServerOutputDispositionKind.Filtered ? "content_filter" : result.ToolCalls.Count > 0 ? "tool_calls" : "stop", }, ], Usage = CreateUsage(result.Usage, cacheUsageShape), @@ -105,7 +111,7 @@ public static OpenAiChatCompletionResponse ToChatResponse( } public static OpenAiChatCompletionChunk ToChatChunk( - LlmTckChatResult result, + LlmTestServerChatResult result, string content, string responseId, long created, @@ -124,20 +130,24 @@ public static OpenAiChatCompletionChunk ToChatChunk( Index = 0, Delta = OpenAiChatMessage.FromText("assistant", content) with { + Content = result.OutputDisposition?.Kind == LlmTestServerOutputDispositionKind.Refused + ? JsonSerializer.SerializeToElement(null) : JsonSerializer.SerializeToElement(content), + Refusal = result.OutputDisposition?.Kind == LlmTestServerOutputDispositionKind.Refused && finishReason is null ? content : null, ToolCalls = finishReason is null && result.ToolCalls.Count > 0 ? result.ToolCalls.Select((call, index) => OpenAiToolMapper.ToWire(call, index)).ToList() : null, }, - FinishReason = finishReason, + FinishReason = finishReason is not null && result.OutputDisposition?.Kind == LlmTestServerOutputDispositionKind.Filtered ? "content_filter" : finishReason, }, ], }; } public static OpenAiResponse ToResponse( - LlmTckChatResult result, + LlmTestServerChatResult result, string responseId, long created, - OpenAiCacheUsageShape cacheUsageShape = OpenAiCacheUsageShape.None + OpenAiCacheUsageShape cacheUsageShape = OpenAiCacheUsageShape.None, + IReadOnlyCollection? includedProperties = null ) { return new() @@ -145,7 +155,9 @@ public static OpenAiResponse ToResponse( Id = responseId, CreatedAt = created, Model = result.ModelId, - Output = CreateResponseOutput(result), + Output = CreateResponseOutput(result, includedProperties), + Status = result.OutputDisposition?.Kind == LlmTestServerOutputDispositionKind.Filtered ? "incomplete" : "completed", + IncompleteDetails = result.OutputDisposition?.Kind == LlmTestServerOutputDispositionKind.Filtered ? new() : null, Usage = new OpenAiResponseUsage { InputTokens = result.Usage.InputTokens, @@ -165,55 +177,8 @@ public static OpenAiResponse ToResponse( }; } - private static List CreateResponseOutput(LlmTckChatResult result) - { - var output = new List(); - if (result.Content.Length > 0 || result.ToolCalls.Count == 0) - { - output.Add(CreateOutputMessage(result.Content, completed: true)); - } - - output.AddRange(result.ToolCalls.Select(call => new OpenAiResponseOutputMessage - { Type = "function_call", Id = "fc_" + call.Id, CallId = call.Id, Name = call.Name, Arguments = call.ArgumentsJson })); - return output; - } - - public static IEnumerable ToResponseStreamEvents( - LlmTckChatResult result, string responseId, long created, - OpenAiCacheUsageShape cacheUsageShape = OpenAiCacheUsageShape.None) - { - var response = ToResponse(result, responseId, created, cacheUsageShape); - var sequence = 0; - yield return new { type = "response.created", sequence_number = sequence++, response = response with { Status = "in_progress", Output = [] } }; - yield return new { type = "response.in_progress", sequence_number = sequence++, response = response with { Status = "in_progress", Output = [] } }; - for (var index = 0; index < response.Output.Count; index++) - { - var item = response.Output[index]; - yield return new { type = "response.output_item.added", sequence_number = sequence++, output_index = index, item = item with { Status = "in_progress", Content = [], Arguments = item.Arguments is null ? null : "" } }; - if (item.Type == "function_call") - { - yield return new { type = "response.function_call_arguments.delta", sequence_number = sequence++, item_id = item.Id, output_index = index, delta = item.Arguments }; - yield return new { type = "response.function_call_arguments.done", sequence_number = sequence++, item_id = item.Id, output_index = index, arguments = item.Arguments }; - } - else - { - yield return new { type = "response.content_part.added", sequence_number = sequence++, item_id = item.Id, output_index = index, content_index = 0, part = new OpenAiResponseOutputText() }; - IReadOnlyList chunks = result.StreamChunks.Count > 0 ? result.StreamChunks : [result.Content]; - foreach (var chunk in chunks) - { - yield return new { type = "response.output_text.delta", sequence_number = sequence++, item_id = item.Id, output_index = index, content_index = 0, delta = chunk, logprobs = Array.Empty() }; - } - - yield return new { type = "response.output_text.done", sequence_number = sequence++, item_id = item.Id, output_index = index, content_index = 0, text = result.Content, logprobs = Array.Empty() }; - yield return new { type = "response.content_part.done", sequence_number = sequence++, item_id = item.Id, output_index = index, content_index = 0, part = item.Content[0] }; - } - yield return new { type = "response.output_item.done", sequence_number = sequence++, output_index = index, item }; - } - yield return new { type = "response.completed", sequence_number = sequence, response }; - } - public static object ToResponseCreatedEvent( - LlmTckChatResult result, + LlmTestServerChatResult result, string responseId, long created ) @@ -232,14 +197,14 @@ long created }; } - public static object ToResponseOutputItemAddedEvent(string responseId) + public static object ToResponseOutputItemAddedEvent(string responseId, LlmTestServerResponseMetadata? metadata = null) { return new { type = "response.output_item.added", response_id = responseId, output_index = 0, - item = CreateOutputMessage(string.Empty, completed: false), + item = (OpenAiResponseOutputItem)CreateOutputMessage(string.Empty, completed: false, metadata), }; } @@ -251,7 +216,7 @@ public static object ToResponseContentPartAddedEvent(string responseId) response_id = responseId, output_index = 0, content_index = 0, - part = new OpenAiResponseOutputText(), + part = (OpenAiResponseContentPart)new OpenAiResponseOutputText(), }; } @@ -267,19 +232,19 @@ public static object ToResponseContentPartDeltaEvent(string responseId, string d }; } - public static object ToResponseOutputItemDoneEvent(string responseId, LlmTckChatResult result) + public static object ToResponseOutputItemDoneEvent(string responseId, LlmTestServerChatResult result) { return new { type = "response.output_item.done", response_id = responseId, output_index = 0, - item = CreateOutputMessage(result.Content, completed: true), + item = (OpenAiResponseOutputItem)CreateOutputMessage(result.Content, completed: true, result.WireMetadata), }; } public static object ToResponseDoneEvent( - LlmTckChatResult result, + LlmTestServerChatResult result, string responseId, long created, OpenAiCacheUsageShape cacheUsageShape = OpenAiCacheUsageShape.None @@ -287,12 +252,12 @@ public static object ToResponseDoneEvent( { return new { - type = "response.completed", + type = result.OutputDisposition?.Kind == LlmTestServerOutputDispositionKind.Filtered ? "response.incomplete" : "response.completed", response = ToResponse(result, responseId, created, cacheUsageShape), }; } - public static object ToModelsResponse(IEnumerable models) + public static object ToModelsResponse(IEnumerable models) { return new { @@ -310,12 +275,13 @@ public static object ToModelsResponse(IEnumerable models) public static OpenAiEmbeddingResponse ToEmbeddingResponse( string model, IReadOnlyList> vectors, - bool encodeAsBase64 = false + bool encodeAsBase64 = false, + LlmTestServerResponseMetadata? metadata = null ) { return new() { - Id = CreateResponseId(), + Id = CreateResponseId(metadata), Model = model, Data = vectors .Select((vector, index) => OpenAiEmbeddingData.FromVector(index, vector, encodeAsBase64)) @@ -333,12 +299,13 @@ public static OpenAiImageGenerationResponse ToImageResponse( string? background = null, string? outputFormat = null, string? quality = null, - string? size = null + string? size = null, + LlmTestServerResponseMetadata? metadata = null ) { return new OpenAiImageGenerationResponse { - Created = DateTimeOffset.UtcNow.ToUnixTimeSeconds(), + Created = (metadata ?? LlmTestServerResponseMetadata.Live()).CreatedAt.ToUnixTimeSeconds(), Background = background, OutputFormat = outputFormat, Quality = quality, @@ -354,14 +321,15 @@ public static object ToImageStreamingEvent( string? background, string? outputFormat, string? quality, - string? size + string? size, + LlmTestServerResponseMetadata? metadata = null ) { var payload = new Dictionary { ["type"] = eventType, ["b64_json"] = ReadBase64Payload(dataUri), - ["created_at"] = DateTimeOffset.UtcNow.ToUnixTimeSeconds(), + ["created_at"] = (metadata ?? LlmTestServerResponseMetadata.Live()).CreatedAt.ToUnixTimeSeconds(), ["size"] = size ?? "1024x1024", ["quality"] = quality ?? "auto", ["background"] = background ?? "auto", @@ -390,7 +358,7 @@ public static object ToImageStreamingEvent( return payload; } - public static OpenAiVideoResponse ToVideoResponse(LlmTckVideoResult result) + public static OpenAiVideoResponse ToVideoResponse(LlmTestServerVideoResult result) { return new() { @@ -408,7 +376,7 @@ public static OpenAiVideoResponse ToVideoResponse(LlmTckVideoResult result) }; } - public static OpenAiVideoListResponse ToVideoListResponse(LlmTckVideoResult result) + public static OpenAiVideoListResponse ToVideoListResponse(LlmTestServerVideoResult result) { var video = ToVideoResponse(result); return new() @@ -439,7 +407,7 @@ long createdAt } public static AzureVideoGenerationJobResponse ToAzureVideoGenerationJobResponse( - LlmTckVideoResult result + LlmTestServerVideoResult result ) { var (width, height) = ReadVideoSize(result.Size); @@ -460,7 +428,7 @@ LlmTckVideoResult result } public static AzureVideoGenerationResponse ToAzureVideoGenerationResponse( - LlmTckVideoResult result + LlmTestServerVideoResult result ) { var (width, height) = ReadVideoSize(result.Size); @@ -477,7 +445,7 @@ LlmTckVideoResult result } public static AzureVideoGenerationJobListResponse ToAzureVideoGenerationJobListResponse( - LlmTckVideoResult result + LlmTestServerVideoResult result ) { var job = ToAzureVideoGenerationJobResponse(result); @@ -490,25 +458,25 @@ LlmTckVideoResult result } public static OpenAiAudioTranscriptionResponse ToAudioTranscriptionResponse( - LlmTckTranscriptionResult result + LlmTestServerTranscriptionResult result ) { return new() { Text = result.Text, Usage = ToAudioTranscriptionUsage(result.Usage) }; } public static GroqAudioTranscriptionResponse ToGroqAudioTranscriptionResponse( - LlmTckTranscriptionResult result + LlmTestServerTranscriptionResult result ) { return new() { Text = result.Text, - XGroq = new GroqRequestMetadata { Id = $"req_{Guid.NewGuid():N}" }, + XGroq = new GroqRequestMetadata { Id = "req_" + (result.WireMetadata ?? LlmTestServerResponseMetadata.Live()).IdFor("groq-request") }, }; } public static OpenAiVerboseAudioTranscriptionResponse ToVerboseAudioTranscriptionResponse( - LlmTckTranscriptionResult result + LlmTestServerTranscriptionResult result ) { return new() @@ -519,7 +487,7 @@ LlmTckTranscriptionResult result }; } - private static OpenAiAudioTranscriptionUsage? ToAudioTranscriptionUsage(LlmTckTranscriptionUsage? usage) + private static OpenAiAudioTranscriptionUsage? ToAudioTranscriptionUsage(LlmTestServerTranscriptionUsage? usage) { if (usage is null) { @@ -552,7 +520,7 @@ public static object ToTranscriptTextDeltaEvent(string delta) }; } - public static object ToTranscriptTextDoneEvent(LlmTckTranscriptionResult result) + public static object ToTranscriptTextDoneEvent(LlmTestServerTranscriptionResult result) { return new { @@ -574,13 +542,13 @@ public static OpenAiErrorResponse ToError(string code, string message) }; } - public static string CreateResponseId() + public static string CreateResponseId(LlmTestServerResponseMetadata? metadata = null) { - return $"llmtck-{Guid.NewGuid():N}"; + return "llmtestserver-" + (metadata ?? LlmTestServerResponseMetadata.Live()).IdFor("chat"); } private static OpenAiUsage CreateUsage( - LlmTckTokenUsage usage, + LlmTestServerTokenUsage usage, OpenAiCacheUsageShape cacheUsageShape = OpenAiCacheUsageShape.None ) { @@ -606,7 +574,7 @@ private static OpenAiUsage CreateUsage( } private static OpenAiPromptTokensDetails? CreatePromptTokensDetails( - LlmTckTokenUsage usage, + LlmTestServerTokenUsage usage, OpenAiCacheUsageShape cacheUsageShape ) { @@ -626,7 +594,7 @@ OpenAiCacheUsageShape cacheUsageShape } private static OpenAiInputTokensDetails? CreateInputTokensDetails( - LlmTckTokenUsage usage, + LlmTestServerTokenUsage usage, OpenAiCacheUsageShape cacheUsageShape ) { @@ -648,10 +616,10 @@ OpenAiCacheUsageShape cacheUsageShape private static string? ReadPromptCacheKey( string? promptCacheKey, string? sessionId, - LlmTckPromptCachePolicy promptCachePolicy + LlmTestServerPromptCachePolicy promptCachePolicy ) { - if (promptCachePolicy == LlmTckPromptCachePolicy.OpenRouter + if (promptCachePolicy == LlmTestServerPromptCachePolicy.OpenRouter && !string.IsNullOrWhiteSpace(sessionId)) { return sessionId; @@ -678,24 +646,25 @@ private static string ReadBase64Payload(string dataUri) private static OpenAiResponseOutputMessage CreateOutputMessage( string content, - bool completed + bool completed, + LlmTestServerResponseMetadata? metadata = null ) { return new() { - Id = $"msg_{Guid.NewGuid():N}", + Id = "msg_" + (metadata ?? LlmTestServerResponseMetadata.Live()).IdFor("message"), Status = completed ? "completed" : "in_progress", Content = [new OpenAiResponseOutputText { Text = content }], }; } - private static List ReadResponseInputMessages(JsonElement input) + private static List ReadResponseInputMessages(JsonElement input) { - return input.ValueKind switch + var messages = input.ValueKind switch { JsonValueKind.String => [ - new LlmTckMessage + new LlmTestServerMessage { Role = "user", Content = input.GetString() ?? string.Empty, @@ -704,7 +673,7 @@ private static List ReadResponseInputMessages(JsonElement input) JsonValueKind.Array => input .EnumerateArray() .Select(ReadResponseInputMessage) - .Where(message => !string.IsNullOrWhiteSpace(message.Content) || message.ToolCalls.Count > 0 || message.ToolCallId is not null) + .Where(message => !string.IsNullOrWhiteSpace(message.Content) || message.ToolCalls.Count > 0 || message.ToolCallId is not null || message.OutputItems.Count > 0 || message.OutputDisposition is not null) .ToList(), JsonValueKind.Object => [ @@ -712,13 +681,14 @@ private static List ReadResponseInputMessages(JsonElement input) ], _ => [], }; + return CoalesceNativeAssistantGroups(messages); } - private static LlmTckMessage ReadResponseInputMessage(JsonElement item) + private static LlmTestServerMessage ReadResponseInputMessage(JsonElement item) { if (item.ValueKind != JsonValueKind.Object) { - return new LlmTckMessage + return new LlmTestServerMessage { Role = "user", Content = item.ValueKind == JsonValueKind.String ? item.GetString() ?? string.Empty : item.ToString(), @@ -727,9 +697,29 @@ private static LlmTckMessage ReadResponseInputMessage(JsonElement item) if (item.TryGetProperty("type", out var itemType)) { + if (itemType.GetString() is "web_search_call" or "file_search_call" or "code_interpreter_call") + { + return new() { Role = "assistant", OutputItems = [ReadNativeToolItem(item)] }; + } if (itemType.GetString() == "function_call") { - return new() { Role = "assistant", ToolCalls = [new() { Id = item.GetProperty("call_id").GetString()!, Name = item.GetProperty("name").GetString()!, ArgumentsJson = item.GetProperty("arguments").GetString()! }] }; + var call = new LlmTestServerToolCall + { + Id = item.GetProperty("call_id").GetString()!, + Name = item.GetProperty("name").GetString()!, + ArgumentsJson = item.GetProperty("arguments").GetString()! + }; + return new() + { + Role = "assistant", + ToolCalls = [call], + OutputItems = [new LlmTestServerResponseFunctionCallItem + { + Id = item.TryGetProperty("id", out var outputId) && outputId.ValueKind == JsonValueKind.String + ? outputId.GetString()! : "fc_" + call.Id, + Call = call + }] + }; } if (itemType.GetString() == "function_call_output") @@ -743,35 +733,45 @@ private static LlmTckMessage ReadResponseInputMessage(JsonElement item) if (item.TryGetProperty("content", out var content)) { - return new LlmTckMessage + return new LlmTestServerMessage { Role = role, Content = ReadResponseInputContent(content), + OutputDisposition = HasRefusalPart(content) ? new() { Kind = LlmTestServerOutputDispositionKind.Refused } : null, + OutputItems = role == "assistant" ? ReadNativeMessageItems(item, content) : [], }; } if (item.TryGetProperty("text", out var text)) { - return new LlmTckMessage + return new LlmTestServerMessage { Role = role, Content = ReadResponseInputContent(text), }; } - return new LlmTckMessage + return new LlmTestServerMessage { Role = role, Content = item.ToString(), }; } - private static string ReadResponseInputContent(JsonElement content) + private static bool HasRefusalPart(JsonElement content) + { + return content.ValueKind == JsonValueKind.Array && content.EnumerateArray().Any(part => part.ValueKind == JsonValueKind.Object + && part.TryGetProperty("type", out var type) && type.ValueKind == JsonValueKind.String && type.GetString() == "refusal"); + } + + internal static string ReadResponseInputContent(JsonElement content) { return content.ValueKind switch { JsonValueKind.String => content.GetString() ?? string.Empty, JsonValueKind.Array => string.Concat(content.EnumerateArray().Select(ReadResponseInputContent)), + JsonValueKind.Object when content.TryGetProperty("type", out var partType) && partType.ValueKind == JsonValueKind.String && partType.GetString() == "refusal" + && content.TryGetProperty("refusal", out var refusal) => refusal.GetString() ?? string.Empty, JsonValueKind.Object when content.TryGetProperty("text", out var text) => ReadResponseInputContent(text), JsonValueKind.Object when content.TryGetProperty("content", out var nested) => diff --git a/src/ManagedCode.LlmTestServer.OpenRouter/AGENTS.md b/src/ManagedCode.LlmTestServer.OpenRouter/AGENTS.md new file mode 100644 index 0000000..1012e00 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.OpenRouter/AGENTS.md @@ -0,0 +1,5 @@ +# ManagedCode.LlmTestServer.OpenRouter + +This package owns OpenRouter compatibility constants, provider profiles, and OpenRouter wire contracts. + +Keep deterministic runtime behavior in `ManagedCode.LlmTestServer`; keep HTTP hosting in `ManagedCode.LlmTestServer.Hosting`. diff --git a/src/ManagedCode.LlmTestServer.OpenRouter/ManagedCode.LlmTestServer.OpenRouter.csproj b/src/ManagedCode.LlmTestServer.OpenRouter/ManagedCode.LlmTestServer.OpenRouter.csproj new file mode 100644 index 0000000..347ab92 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.OpenRouter/ManagedCode.LlmTestServer.OpenRouter.csproj @@ -0,0 +1,10 @@ + + + ManagedCode.LlmTestServer.OpenRouter + OpenRouter compatibility profile and adapters for ManagedCode.LlmTestServer. + + + + + + diff --git a/src/ManagedCode.LlmTck.OpenRouter/OpenRouterCompatibility.cs b/src/ManagedCode.LlmTestServer.OpenRouter/OpenRouterCompatibility.cs similarity index 55% rename from src/ManagedCode.LlmTck.OpenRouter/OpenRouterCompatibility.cs rename to src/ManagedCode.LlmTestServer.OpenRouter/OpenRouterCompatibility.cs index 1e0ad0a..61a218c 100644 --- a/src/ManagedCode.LlmTck.OpenRouter/OpenRouterCompatibility.cs +++ b/src/ManagedCode.LlmTestServer.OpenRouter/OpenRouterCompatibility.cs @@ -1,27 +1,27 @@ -using ManagedCode.LlmTck.Providers; +using ManagedCode.LlmTestServer.Providers; -namespace ManagedCode.LlmTck.OpenRouter; +namespace ManagedCode.LlmTestServer.OpenRouter; public static class OpenRouterCompatibility { - public const string ProviderId = LlmTckCompatibilityTags.OpenRouter; + public const string ProviderId = LlmTestServerCompatibilityTags.OpenRouter; - public static LlmTckProviderProfile Profile { get; } = + public static LlmTestServerProviderProfile Profile { get; } = new() { Id = ProviderId, DisplayName = "OpenRouter", - Protocol = LlmTckProtocolFamily.OpenRouter, + Protocol = LlmTestServerProtocolFamily.OpenRouter, DefaultEndpointPath = "/openrouter/api/v1/chat/completions", Capabilities = [ - LlmTckProviderCapability.Models, - LlmTckProviderCapability.Chat, - LlmTckProviderCapability.StreamingChat, - LlmTckProviderCapability.Tools, - LlmTckProviderCapability.StructuredOutput, + LlmTestServerProviderCapability.Models, + LlmTestServerProviderCapability.Chat, + LlmTestServerProviderCapability.StreamingChat, + LlmTestServerProviderCapability.Tools, + LlmTestServerProviderCapability.StructuredOutput, ], - CompatibilityTags = [ProviderId, LlmTckCompatibilityTags.OpenAICompatible], + CompatibilityTags = [ProviderId, LlmTestServerCompatibilityTags.OpenAICompatible], ApiContract = new() { DocumentationUrl = "https://openrouter.ai/docs/api/reference/overview", @@ -31,7 +31,7 @@ public static class OpenRouterCompatibility [ new() { - Id = LlmTckProviderOperationIds.OpenRouter.ChatCompletionsCreate, + Id = LlmTestServerProviderOperationIds.OpenRouter.ChatCompletionsCreate, Method = "POST", Path = "/openrouter/api/v1/chat/completions", DocumentationUrl = "https://openrouter.ai/docs/api/reference/overview", @@ -39,15 +39,15 @@ public static class OpenRouterCompatibility ImplementedByHosting = true, Capabilities = [ - LlmTckProviderCapability.Chat, - LlmTckProviderCapability.StreamingChat, - LlmTckProviderCapability.Tools, - LlmTckProviderCapability.StructuredOutput, + LlmTestServerProviderCapability.Chat, + LlmTestServerProviderCapability.StreamingChat, + LlmTestServerProviderCapability.Tools, + LlmTestServerProviderCapability.StructuredOutput, ], }, new() { - Id = LlmTckProviderOperationIds.OpenRouter.ResponsesCreate, + Id = LlmTestServerProviderOperationIds.OpenRouter.ResponsesCreate, Method = "POST", Path = "/openrouter/api/v1/responses", DocumentationUrl = @@ -56,21 +56,21 @@ public static class OpenRouterCompatibility ImplementedByHosting = true, Capabilities = [ - LlmTckProviderCapability.Chat, - LlmTckProviderCapability.StreamingChat, - LlmTckProviderCapability.Tools, - LlmTckProviderCapability.StructuredOutput, + LlmTestServerProviderCapability.Chat, + LlmTestServerProviderCapability.StreamingChat, + LlmTestServerProviderCapability.Tools, + LlmTestServerProviderCapability.StructuredOutput, ], }, new() { - Id = LlmTckProviderOperationIds.OpenRouter.ModelsList, + Id = LlmTestServerProviderOperationIds.OpenRouter.ModelsList, Method = "GET", Path = "/openrouter/api/v1/models", DocumentationUrl = "https://openrouter.ai/docs/api/api-reference/models/list-all-models-and-their-properties", ImplementedByHosting = true, - Capabilities = [LlmTckProviderCapability.Models], + Capabilities = [LlmTestServerProviderCapability.Models], }, ], }, diff --git a/src/ManagedCode.LlmTestServer.Perplexity/AGENTS.md b/src/ManagedCode.LlmTestServer.Perplexity/AGENTS.md new file mode 100644 index 0000000..8fcb746 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Perplexity/AGENTS.md @@ -0,0 +1,5 @@ +# ManagedCode.LlmTestServer.Perplexity + +This package owns Perplexity compatibility constants, provider profiles, and Perplexity wire contracts. + +Keep deterministic runtime behavior in `ManagedCode.LlmTestServer`; keep HTTP hosting in `ManagedCode.LlmTestServer.Hosting`. diff --git a/src/ManagedCode.LlmTestServer.Perplexity/ManagedCode.LlmTestServer.Perplexity.csproj b/src/ManagedCode.LlmTestServer.Perplexity/ManagedCode.LlmTestServer.Perplexity.csproj new file mode 100644 index 0000000..42b0dba --- /dev/null +++ b/src/ManagedCode.LlmTestServer.Perplexity/ManagedCode.LlmTestServer.Perplexity.csproj @@ -0,0 +1,10 @@ + + + ManagedCode.LlmTestServer.Perplexity + Perplexity compatibility profile and adapters for ManagedCode.LlmTestServer. + + + + + + diff --git a/src/ManagedCode.LlmTck.Perplexity/PerplexityCompatibility.cs b/src/ManagedCode.LlmTestServer.Perplexity/PerplexityCompatibility.cs similarity index 54% rename from src/ManagedCode.LlmTck.Perplexity/PerplexityCompatibility.cs rename to src/ManagedCode.LlmTestServer.Perplexity/PerplexityCompatibility.cs index 418bee2..4618b60 100644 --- a/src/ManagedCode.LlmTck.Perplexity/PerplexityCompatibility.cs +++ b/src/ManagedCode.LlmTestServer.Perplexity/PerplexityCompatibility.cs @@ -1,25 +1,25 @@ -using ManagedCode.LlmTck.Providers; +using ManagedCode.LlmTestServer.Providers; -namespace ManagedCode.LlmTck.Perplexity; +namespace ManagedCode.LlmTestServer.Perplexity; public static class PerplexityCompatibility { - public const string ProviderId = LlmTckCompatibilityTags.Perplexity; + public const string ProviderId = LlmTestServerCompatibilityTags.Perplexity; - public static LlmTckProviderProfile Profile { get; } = + public static LlmTestServerProviderProfile Profile { get; } = new() { Id = ProviderId, DisplayName = "Perplexity", - Protocol = LlmTckProtocolFamily.Perplexity, + Protocol = LlmTestServerProtocolFamily.Perplexity, DefaultEndpointPath = "/perplexity/v1/sonar", Capabilities = [ - LlmTckProviderCapability.Chat, - LlmTckProviderCapability.StreamingChat, - LlmTckProviderCapability.StructuredOutput, + LlmTestServerProviderCapability.Chat, + LlmTestServerProviderCapability.StreamingChat, + LlmTestServerProviderCapability.StructuredOutput, ], - CompatibilityTags = [ProviderId, LlmTckCompatibilityTags.OpenAICompatible], + CompatibilityTags = [ProviderId, LlmTestServerCompatibilityTags.OpenAICompatible], ApiContract = new() { DocumentationUrl = "https://docs.perplexity.ai/api-reference/sonar-post", @@ -29,7 +29,7 @@ public static class PerplexityCompatibility [ new() { - Id = LlmTckProviderOperationIds.Perplexity.SonarCreate, + Id = LlmTestServerProviderOperationIds.Perplexity.SonarCreate, Method = "POST", Path = "/perplexity/v1/sonar", DocumentationUrl = "https://docs.perplexity.ai/api-reference/sonar-post", @@ -37,9 +37,9 @@ public static class PerplexityCompatibility ImplementedByHosting = true, Capabilities = [ - LlmTckProviderCapability.Chat, - LlmTckProviderCapability.StreamingChat, - LlmTckProviderCapability.StructuredOutput, + LlmTestServerProviderCapability.Chat, + LlmTestServerProviderCapability.StreamingChat, + LlmTestServerProviderCapability.StructuredOutput, ], }, ], diff --git a/src/ManagedCode.LlmTestServer.SystemOne/AGENTS.md b/src/ManagedCode.LlmTestServer.SystemOne/AGENTS.md new file mode 100644 index 0000000..e28bfc5 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.SystemOne/AGENTS.md @@ -0,0 +1,3 @@ +# ManagedCode.LlmTestServer.SystemOne + +Owns native SystemOne decision wire contracts and compatibility profiles. Keep deterministic fixtures in `ManagedCode.LlmTestServer` and routes in `ManagedCode.LlmTestServer.Hosting`. DTOs use explicit JSON names; preserve opaque choice IDs and native scores. diff --git a/src/ManagedCode.LlmTck.SystemOne/KevDecisionContent.cs b/src/ManagedCode.LlmTestServer.SystemOne/KevDecisionContent.cs similarity index 98% rename from src/ManagedCode.LlmTck.SystemOne/KevDecisionContent.cs rename to src/ManagedCode.LlmTestServer.SystemOne/KevDecisionContent.cs index 216d783..641178f 100644 --- a/src/ManagedCode.LlmTck.SystemOne/KevDecisionContent.cs +++ b/src/ManagedCode.LlmTestServer.SystemOne/KevDecisionContent.cs @@ -1,7 +1,7 @@ using System.Globalization; using System.Text.Json; -namespace ManagedCode.LlmTck.SystemOne; +namespace ManagedCode.LlmTestServer.SystemOne; public static class KevDecisionContent { diff --git a/src/ManagedCode.LlmTestServer.SystemOne/ManagedCode.LlmTestServer.SystemOne.csproj b/src/ManagedCode.LlmTestServer.SystemOne/ManagedCode.LlmTestServer.SystemOne.csproj new file mode 100644 index 0000000..4084370 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.SystemOne/ManagedCode.LlmTestServer.SystemOne.csproj @@ -0,0 +1,5 @@ + + ManagedCode.LlmTestServer.SystemOneNative SystemOne decision API compatibility for ManagedCode.LlmTestServer. + + + diff --git a/src/ManagedCode.LlmTestServer.SystemOne/SystemOneCompatibility.cs b/src/ManagedCode.LlmTestServer.SystemOne/SystemOneCompatibility.cs new file mode 100644 index 0000000..fdb71c7 --- /dev/null +++ b/src/ManagedCode.LlmTestServer.SystemOne/SystemOneCompatibility.cs @@ -0,0 +1,23 @@ +using ManagedCode.LlmTestServer.Providers; +namespace ManagedCode.LlmTestServer.SystemOne; + +public static class SystemOneCompatibility +{ + public const string ProviderId = LlmTestServerCompatibilityTags.SystemOne; + public static LlmTestServerProviderProfile Profile { get; } = new() + { + Id = ProviderId, + DisplayName = "SystemOne (Jev and Kev)", + Protocol = LlmTestServerProtocolFamily.SystemOne, + DefaultEndpointPath = "/systemone/v1/systemone", + Capabilities = [LlmTestServerProviderCapability.Decisions], + CompatibilityTags = [ProviderId], + ApiContract = new() + { + DocumentationUrl = "https://docs.typesafe.ai/api", + DocumentationRetrievedOn = "2026-10-20", + DocumentationVersion = "Jev 1.14 JSON/multipart decisions, images and box; Kev native decisions", + Operations = [new() { Id = LlmTestServerProviderOperationIds.SystemOne.DecisionsCreate, Method = "POST", Path = "/systemone/v1/systemone", DocumentationUrl = "https://docs.typesafe.ai/api", ImplementedByHosting = true, Capabilities = [LlmTestServerProviderCapability.Decisions] }] + }, + }; +} diff --git a/src/ManagedCode.LlmTck.SystemOne/SystemOneDecisionMapper.cs b/src/ManagedCode.LlmTestServer.SystemOne/SystemOneDecisionMapper.cs similarity index 71% rename from src/ManagedCode.LlmTck.SystemOne/SystemOneDecisionMapper.cs rename to src/ManagedCode.LlmTestServer.SystemOne/SystemOneDecisionMapper.cs index fe3c25a..d14cebe 100644 --- a/src/ManagedCode.LlmTck.SystemOne/SystemOneDecisionMapper.cs +++ b/src/ManagedCode.LlmTestServer.SystemOne/SystemOneDecisionMapper.cs @@ -1,9 +1,9 @@ using System.Globalization; using System.Text.Json; -using ManagedCode.LlmTck.Decisions; -using ManagedCode.LlmTck.Providers; +using ManagedCode.LlmTestServer.Decisions; +using ManagedCode.LlmTestServer.Providers; -namespace ManagedCode.LlmTck.SystemOne; +namespace ManagedCode.LlmTestServer.SystemOne; public static class SystemOneDecisionMapper { @@ -13,23 +13,23 @@ public static class SystemOneDecisionMapper public const int KevMaxScoreLevels = 255; public const string KevDefaultModel = "kev-latest"; - public static LlmTckRequestValidationResult Validate(JsonElement body) + public static LlmTestServerRequestValidationResult Validate(JsonElement body) { return Validate(body, MaxChoices, KevMaxScoreLevels); } - public static LlmTckRequestValidationResult Validate(JsonElement body, int maxChoices, int maxScoreLevels) + public static LlmTestServerRequestValidationResult Validate(JsonElement body, int maxChoices, int maxScoreLevels) { return ValidateCore(body, maxChoices, maxScoreLevels, false, false); } - public static LlmTckRequestValidationResult Validate(JsonElement body, LlmTckDecisionProvider provider) + public static LlmTestServerRequestValidationResult Validate(JsonElement body, LlmTestServerDecisionProvider provider) { - return ValidateCore(body, MaxChoices, provider is LlmTckDecisionProvider.TypeSafe or LlmTckDecisionProvider.Cloudflare ? JevMaxScoreLevels : KevMaxScoreLevels, - provider == LlmTckDecisionProvider.Kev, provider == LlmTckDecisionProvider.Cloudflare, provider == LlmTckDecisionProvider.TypeSafe); + return ValidateCore(body, MaxChoices, provider is LlmTestServerDecisionProvider.TypeSafe or LlmTestServerDecisionProvider.Cloudflare ? JevMaxScoreLevels : KevMaxScoreLevels, + provider == LlmTestServerDecisionProvider.Kev, provider == LlmTestServerDecisionProvider.Cloudflare, provider == LlmTestServerDecisionProvider.TypeSafe); } - private static LlmTckRequestValidationResult ValidateCore(JsonElement body, int maxChoices, int maxScoreLevels, bool kev, bool allowScalarPredicateCriteria, bool typeSafe = false) + private static LlmTestServerRequestValidationResult ValidateCore(JsonElement body, int maxChoices, int maxScoreLevels, bool kev, bool allowScalarPredicateCriteria, bool typeSafe = false) { if (body.ValueKind != JsonValueKind.Object || (!body.TryGetProperty("model", out var model) ? !kev : model.ValueKind != JsonValueKind.String) || !body.TryGetProperty("state", out var state) || !IsContent(state, kev) || (!kev && state.ValueKind == JsonValueKind.Null) @@ -70,18 +70,18 @@ private static bool IsContent(JsonElement value, bool kev) || (kev && value.ValueKind is JsonValueKind.Number or JsonValueKind.True or JsonValueKind.False); } - public static LlmTckDecisionRequest ToRequest(SystemOneDecisionRequest request, LlmTckDecisionProvider provider = LlmTckDecisionProvider.TypeSafe) + public static LlmTestServerDecisionRequest ToRequest(SystemOneDecisionRequest request, LlmTestServerDecisionProvider provider = LlmTestServerDecisionProvider.TypeSafe) { return new() { Provider = provider, ModelId = request.Model ?? KevDefaultModel, Input = request.Assets is { Count: > 0 } ? JsonSerializer.Serialize(new { state = request.State, assets = request.Assets }) : JsonSerializer.Serialize(request.State), - Questions = request.Questions.Select(entry => new LlmTckDecisionQuestion + Questions = request.Questions.Select(entry => new LlmTestServerDecisionQuestion { Name = entry.Key, Kind = entry.Value.Type switch - { SystemOneDecisionTypes.Predicate => LlmTckDecisionKind.Predicate, SystemOneDecisionTypes.Choice => LlmTckDecisionKind.Choice, SystemOneDecisionTypes.Box => LlmTckDecisionKind.Box, _ => LlmTckDecisionKind.Score }, + { SystemOneDecisionTypes.Predicate => LlmTestServerDecisionKind.Predicate, SystemOneDecisionTypes.Choice => LlmTestServerDecisionKind.Choice, SystemOneDecisionTypes.Box => LlmTestServerDecisionKind.Box, _ => LlmTestServerDecisionKind.Score }, Options = entry.Value.Type is SystemOneDecisionTypes.Predicate or SystemOneDecisionTypes.Box ? [] : entry.Value.Type == SystemOneDecisionTypes.Score ? entry.Value.Criteria.EnumerateArray().Select(value => JsonSerializer.Serialize(value)).ToList() : entry.Value.Criteria.EnumerateObject().Select(value => value.Name).ToList(), @@ -89,10 +89,10 @@ public static LlmTckDecisionRequest ToRequest(SystemOneDecisionRequest request, }; } - public static SystemOneDecisionResponse ToResponse(SystemOneDecisionRequest request, LlmTckDecisionResult result, bool roundValues = false, - LlmTckDecisionProvider provider = LlmTckDecisionProvider.TypeSafe) + public static SystemOneDecisionResponse ToResponse(SystemOneDecisionRequest request, LlmTestServerDecisionResult result, bool roundValues = false, + LlmTestServerDecisionProvider provider = LlmTestServerDecisionProvider.TypeSafe) { - var kev = provider == LlmTckDecisionProvider.Kev; + var kev = provider == LlmTestServerDecisionProvider.Kev; return new() { Model = result.ModelId, @@ -109,7 +109,7 @@ public static SystemOneDecisionResponse ToResponse(SystemOneDecisionRequest requ }; } - private static LlmTckDecisionAnswer RoundedAnswer(bool roundValues, LlmTckDecisionAnswer answer) + private static LlmTestServerDecisionAnswer RoundedAnswer(bool roundValues, LlmTestServerDecisionAnswer answer) { if (!roundValues) { return answer; } return answer with @@ -126,18 +126,18 @@ private static LlmTckDecisionAnswer RoundedAnswer(bool roundValues, LlmTckDecisi return value.HasValue ? Math.Round(value.Value, _nativeProbabilityPrecision) : null; } - private static SystemOneDecisionAnswer ToAnswer(SystemOneDecisionQuestion question, LlmTckDecisionAnswer answer, bool kev) + private static SystemOneDecisionAnswer ToAnswer(SystemOneDecisionQuestion question, LlmTestServerDecisionAnswer answer, bool kev) { return new() { Type = question.Type, - Probability = answer.Kind == LlmTckDecisionKind.Predicate ? answer.Probability : null, - Choice = answer.Kind == LlmTckDecisionKind.Choice ? answer.Choice : null, - Score = answer.Kind == LlmTckDecisionKind.Score ? answer.Score : null, - Confidence = answer.Kind == LlmTckDecisionKind.Predicate ? null : answer.Confidence, - Probabilities = answer.Kind is LlmTckDecisionKind.Predicate or LlmTckDecisionKind.Box ? null : new(answer.Probabilities), + Probability = answer.Kind == LlmTestServerDecisionKind.Predicate ? answer.Probability : null, + Choice = answer.Kind == LlmTestServerDecisionKind.Choice ? answer.Choice : null, + Score = answer.Kind == LlmTestServerDecisionKind.Score ? answer.Score : null, + Confidence = answer.Kind == LlmTestServerDecisionKind.Predicate ? null : answer.Confidence, + Probabilities = answer.Kind is LlmTestServerDecisionKind.Predicate or LlmTestServerDecisionKind.Box ? null : new(answer.Probabilities), Box = answer.Box is { } box ? new() { Top = box.Top, Bottom = box.Bottom, Left = box.Left, Right = box.Right } : null, - Legend = answer.Kind == LlmTckDecisionKind.Score ? question.Criteria.EnumerateArray() + Legend = answer.Kind == LlmTestServerDecisionKind.Score ? question.Criteria.EnumerateArray() .Select((value, index) => (index, value)).ToDictionary(entry => entry.index.ToString(CultureInfo.InvariantCulture), entry => kev ? JsonSerializer.SerializeToElement(KevDecisionContent.Render(entry.value)) : entry.value.Clone()) : null, }; diff --git a/src/ManagedCode.LlmTck.SystemOne/SystemOneDecisionModels.cs b/src/ManagedCode.LlmTestServer.SystemOne/SystemOneDecisionModels.cs similarity index 98% rename from src/ManagedCode.LlmTck.SystemOne/SystemOneDecisionModels.cs rename to src/ManagedCode.LlmTestServer.SystemOne/SystemOneDecisionModels.cs index b89f7ed..4d50ea9 100644 --- a/src/ManagedCode.LlmTck.SystemOne/SystemOneDecisionModels.cs +++ b/src/ManagedCode.LlmTestServer.SystemOne/SystemOneDecisionModels.cs @@ -1,7 +1,7 @@ using System.Text.Json; using System.Text.Json.Serialization; -namespace ManagedCode.LlmTck.SystemOne; +namespace ManagedCode.LlmTestServer.SystemOne; public static class SystemOneDecisionTypes { diff --git a/src/ManagedCode.LlmTck.SystemOne/SystemOneNativeJson.cs b/src/ManagedCode.LlmTestServer.SystemOne/SystemOneNativeJson.cs similarity index 95% rename from src/ManagedCode.LlmTck.SystemOne/SystemOneNativeJson.cs rename to src/ManagedCode.LlmTestServer.SystemOne/SystemOneNativeJson.cs index 2a37c26..7ed6e42 100644 --- a/src/ManagedCode.LlmTck.SystemOne/SystemOneNativeJson.cs +++ b/src/ManagedCode.LlmTestServer.SystemOne/SystemOneNativeJson.cs @@ -1,7 +1,7 @@ using System.Text.Json; using System.Text.Json.Serialization.Metadata; -namespace ManagedCode.LlmTck.SystemOne; +namespace ManagedCode.LlmTestServer.SystemOne; /// Native SystemOne serialization, including required nullable Box confidence. public static class SystemOneNativeJson diff --git a/src/ManagedCode.LlmTck.SystemOne/TypeSafeImageValidation.cs b/src/ManagedCode.LlmTestServer.SystemOne/TypeSafeImageValidation.cs similarity index 98% rename from src/ManagedCode.LlmTck.SystemOne/TypeSafeImageValidation.cs rename to src/ManagedCode.LlmTestServer.SystemOne/TypeSafeImageValidation.cs index e15a66b..8e935e8 100644 --- a/src/ManagedCode.LlmTck.SystemOne/TypeSafeImageValidation.cs +++ b/src/ManagedCode.LlmTestServer.SystemOne/TypeSafeImageValidation.cs @@ -5,7 +5,7 @@ using MetadataExtractor.Formats.Jpeg; using MetadataExtractor.Formats.Png; -namespace ManagedCode.LlmTck.SystemOne; +namespace ManagedCode.LlmTestServer.SystemOne; internal static class TypeSafeImageValidation { diff --git a/src/ManagedCode.LlmTck.SystemOne/TypeSafeRequestValidation.cs b/src/ManagedCode.LlmTestServer.SystemOne/TypeSafeRequestValidation.cs similarity index 97% rename from src/ManagedCode.LlmTck.SystemOne/TypeSafeRequestValidation.cs rename to src/ManagedCode.LlmTestServer.SystemOne/TypeSafeRequestValidation.cs index f6a78dd..3d63973 100644 --- a/src/ManagedCode.LlmTck.SystemOne/TypeSafeRequestValidation.cs +++ b/src/ManagedCode.LlmTestServer.SystemOne/TypeSafeRequestValidation.cs @@ -1,7 +1,7 @@ using System.Text; using System.Text.Json; -namespace ManagedCode.LlmTck.SystemOne; +namespace ManagedCode.LlmTestServer.SystemOne; /// Validation for the claimed TypeSafe JSON API surface; no inference or token estimation. public static class TypeSafeRequestValidation @@ -19,7 +19,7 @@ public static TypeSafeValidationResult Validate(JsonElement body, bool imagesEna { if (!multipart && Encoding.UTF8.GetByteCount(body.GetRawText()) > MaxRequestBytes) { return new("TypeSafe request body exceeds 32 MiB.", 413); } - var shape = SystemOneDecisionMapper.Validate(body, Decisions.LlmTckDecisionProvider.TypeSafe); + var shape = SystemOneDecisionMapper.Validate(body, Decisions.LlmTestServerDecisionProvider.TypeSafe); if (!shape.IsValid) { return new(shape.Error!, 422); } var questions = body.GetProperty("questions"); var box = questions.EnumerateObject().Any(entry => entry.Value.GetProperty("type").GetString() == SystemOneDecisionTypes.Box); diff --git a/src/ManagedCode.LlmTck/AGENTS.md b/src/ManagedCode.LlmTestServer/AGENTS.md similarity index 90% rename from src/ManagedCode.LlmTck/AGENTS.md rename to src/ManagedCode.LlmTestServer/AGENTS.md index bc3ea8b..467740c 100644 --- a/src/ManagedCode.LlmTck/AGENTS.md +++ b/src/ManagedCode.LlmTestServer/AGENTS.md @@ -1,4 +1,4 @@ -# ManagedCode.LlmTck +# ManagedCode.LlmTestServer Keep this package provider-neutral. It owns deterministic scenarios, model metadata, runtime assertions, and modality fixtures. diff --git a/src/ManagedCode.LlmTck/Configuration/LlmTckConfiguration.cs b/src/ManagedCode.LlmTestServer/Configuration/LlmTestServerConfiguration.cs similarity index 59% rename from src/ManagedCode.LlmTck/Configuration/LlmTckConfiguration.cs rename to src/ManagedCode.LlmTestServer/Configuration/LlmTestServerConfiguration.cs index 7c4f247..d68044f 100644 --- a/src/ManagedCode.LlmTck/Configuration/LlmTckConfiguration.cs +++ b/src/ManagedCode.LlmTestServer/Configuration/LlmTestServerConfiguration.cs @@ -1,12 +1,18 @@ -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Scenarios; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Scenarios; -namespace ManagedCode.LlmTck.Configuration; +namespace ManagedCode.LlmTestServer.Configuration; -public sealed record LlmTckConfiguration +public sealed record LlmTestServerConfiguration { + public ManagedCode.LlmTestServer.Determinism.LlmTestServerDeterminism? Determinism { get; init; } + /// Maximum retained prompt-cache prefixes. Zero disables prompt caching. + public int MaxConversations { get; init; } = 256; + + public long MaxConversationBytes { get; init; } = 16 * 1024 * 1024; + public int MaxPromptCacheEntries { get; init; } = 4096; /// Maximum stored video jobs across providers. Zero rejects all new jobs. @@ -21,17 +27,17 @@ public sealed record LlmTckConfiguration /// Maximum total retained chat-history bytes including bounded record overhead. public long MaxStoredChatResponseBytes { get; init; } = 16 * 1024 * 1024; - public List Models { get; init; } = []; + public List Models { get; init; } = []; - public List DecisionScenarios { get; init; } = []; + public List DecisionScenarios { get; init; } = []; - public List ChatScenarios { get; init; } = []; + public List ChatScenarios { get; init; } = []; - public List Datasets { get; init; } = []; + public List Datasets { get; init; } = []; public string? RequiredBearerToken { get; init; } - public LlmTckFaultSimulation FaultSimulation { get; init; } = new(); + public LlmTestServerFaultSimulation FaultSimulation { get; init; } = new(); public List DefaultEmbeddingVector { get; init; } = [0.125f, 0.25f, 0.5f]; @@ -49,7 +55,7 @@ public sealed record LlmTckConfiguration /// Provider-reported usage returned for deterministic transcriptions, or /// to model providers that omit transcription usage. /// - public LlmTckTranscriptionUsage? DefaultTranscriptionUsage { get; init; } = new(); + public LlmTestServerTranscriptionUsage? DefaultTranscriptionUsage { get; init; } = new(); public string DefaultTranslationText { get; init; } = "translated audio fixture"; @@ -83,9 +89,9 @@ public sealed record LlmTckConfiguration public string DefaultVideoMediaType { get; init; } = "video/mp4"; - public string DefaultVideoId { get; init; } = "video_llm_tck"; + public string DefaultVideoId { get; init; } = "video_llm_test_server"; - public string DefaultVideoGenerationId { get; init; } = "gen_llm_tck"; + public string DefaultVideoGenerationId { get; init; } = "gen_llm_test_server"; public long DefaultVideoCreatedAtUnixTime { get; init; } = 1712697600; @@ -93,17 +99,17 @@ public sealed record LlmTckConfiguration public string DefaultVideoSeconds { get; init; } = "4"; - public static LlmTckConfiguration CreateDefault() + public static LlmTestServerConfiguration CreateDefault() { return new() { Models = [ - new LlmTckModel { Id = LlmTckKnownModelIds.Gpt41Mini, Kind = LlmTckModelKind.Chat }, - new LlmTckModel { Id = LlmTckKnownModelIds.TextEmbedding3Small, Kind = LlmTckModelKind.Embedding }, - new LlmTckModel { Id = LlmTckKnownModelIds.GptImage1, Kind = LlmTckModelKind.Image }, - new LlmTckModel { Id = LlmTckKnownModelIds.Gpt4OMiniTts, Kind = LlmTckModelKind.Audio }, - new LlmTckModel { Id = LlmTckKnownModelIds.Sora2, Kind = LlmTckModelKind.Video }, + new LlmTestServerModel { Id = LlmTestServerKnownModelIds.Gpt41Mini, Kind = LlmTestServerModelKind.Chat }, + new LlmTestServerModel { Id = LlmTestServerKnownModelIds.TextEmbedding3Small, Kind = LlmTestServerModelKind.Embedding }, + new LlmTestServerModel { Id = LlmTestServerKnownModelIds.GptImage1, Kind = LlmTestServerModelKind.Image }, + new LlmTestServerModel { Id = LlmTestServerKnownModelIds.Gpt4OMiniTts, Kind = LlmTestServerModelKind.Audio }, + new LlmTestServerModel { Id = LlmTestServerKnownModelIds.Sora2, Kind = LlmTestServerModelKind.Video }, ], }; } diff --git a/src/ManagedCode.LlmTestServer/Configuration/LlmTestServerConfigurationBuilder.Conversations.cs b/src/ManagedCode.LlmTestServer/Configuration/LlmTestServerConfigurationBuilder.Conversations.cs new file mode 100644 index 0000000..33a9472 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Configuration/LlmTestServerConfigurationBuilder.Conversations.cs @@ -0,0 +1,82 @@ +using ManagedCode.LlmTestServer.Conversations; +using ManagedCode.LlmTestServer.Scenarios; + +namespace ManagedCode.LlmTestServer.Configuration; + +public sealed partial class LlmTestServerConfigurationBuilder +{ + public LlmTestServerConfigurationBuilder AddConversation(string id, Action configure) + { + ArgumentException.ThrowIfNullOrWhiteSpace(id); + ArgumentNullException.ThrowIfNull(configure); + var builder = new LlmTestServerConversationBuilder(id); + configure(builder); + var scenario = builder.Build(); + _configuration.ChatScenarios.RemoveAll(existing => string.Equals(existing.Id, id, StringComparison.OrdinalIgnoreCase)); + _configuration.ChatScenarios.Add(scenario); + return this; + } + + public LlmTestServerConfigurationBuilder WithConversationCapacity(int maximumConversations, long maximumBytes) + { + ArgumentOutOfRangeException.ThrowIfNegative(maximumConversations); + ArgumentOutOfRangeException.ThrowIfNegative(maximumBytes); + _configuration = _configuration with { MaxConversations = maximumConversations, MaxConversationBytes = maximumBytes }; + return this; + } + + private static LlmTestServerConversation? SnapshotConversation(LlmTestServerScenario scenario) + { + var graph = scenario.Conversation; + if (graph is null) { return null; } + ValidateIdentifier(scenario.Id); + ArgumentNullException.ThrowIfNull(graph.Steps); + if (scenario.Responses.Count != 0 || graph.Steps.Count is < 1 or > 256 || graph.MaxSteps is < 1 or > 4096 + || !Enum.IsDefined(graph.Exhaustion)) { throw new ArgumentException("Invalid conversation shape or limits.", nameof(scenario)); } + var ids = new HashSet(StringComparer.Ordinal); + foreach (var step in graph.Steps) + { + ArgumentNullException.ThrowIfNull(step); + ValidateIdentifier(step.Id); + if (!ids.Add(step.Id)) { throw new ArgumentException("Duplicate conversation step ID.", nameof(scenario)); } + } + if (!ids.Contains(graph.StartStepId)) { throw new ArgumentException("Unknown conversation start step.", nameof(scenario)); } + return graph with + { + Steps = graph.Steps.Select(step => + { + ArgumentNullException.ThrowIfNull(step.Match); + ArgumentNullException.ThrowIfNull(step.Match.JsonFields); + ArgumentNullException.ThrowIfNull(step.NextSteps); + if (step.Terminal == (step.NextSteps.Count > 0) || step.NextSteps.Any(next => !ids.Contains(next)) + || step.NextSteps.Distinct(StringComparer.Ordinal).Count() != step.NextSteps.Count) + { throw new ArgumentException("A step must terminate or transition to distinct known steps.", nameof(scenario)); } + if (step.Match.JsonFields.Count > 32) { throw new ArgumentException("A conversation step supports at most 32 JSON predicates.", nameof(scenario)); } + var match = step.Match; + if (match.TurnIndex is < 1 || match.LatestRole is not (null or "user" or "assistant" or "tool" or "system" or "developer")) + { throw new ArgumentException("Invalid conversation match predicate.", nameof(scenario)); } + if (match.ToolName is not null) { ValidateIdentifier(match.ToolName); } + if (match.ToolCallId is not null) { ValidateIdentifier(match.ToolCallId); } + if (match.ToolArgumentsJson is not null) { LlmTestServerJsonSchema.ValidateJson(match.ToolArgumentsJson); } + foreach (var field in match.JsonFields) + { + ArgumentNullException.ThrowIfNull(field); + LlmTestServerJsonFieldSelector.Parse(field.Path); + LlmTestServerJsonSchema.ValidateJson(field.EqualsJson); + } + return step with + { + Response = SnapshotResponse(step.Response), + NextSteps = [.. step.NextSteps], + Match = match with { JsonFields = [.. match.JsonFields.Select(field => field with { })] } + }; + }).ToList(), + }; + } + + private static void ValidateIdentifier(string value) + { + ArgumentException.ThrowIfNullOrWhiteSpace(value); + if (value.Length > 128) { throw new ArgumentException("Conversation identifiers are limited to 128 characters.", nameof(value)); } + } +} diff --git a/src/ManagedCode.LlmTestServer/Configuration/LlmTestServerConfigurationBuilder.Determinism.cs b/src/ManagedCode.LlmTestServer/Configuration/LlmTestServerConfigurationBuilder.Determinism.cs new file mode 100644 index 0000000..008a3d5 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Configuration/LlmTestServerConfigurationBuilder.Determinism.cs @@ -0,0 +1,13 @@ +namespace ManagedCode.LlmTestServer.Configuration; + +public sealed partial class LlmTestServerConfigurationBuilder +{ + public LlmTestServerConfigurationBuilder WithDeterminism(string seed, DateTimeOffset fixedUtc) + { + ArgumentException.ThrowIfNullOrWhiteSpace(seed); + if (seed.Length > 256) { throw new ArgumentException("Deterministic seeds are limited to 256 characters.", nameof(seed)); } + _configuration = _configuration with { Determinism = new() { Seed = seed, FixedUtc = fixedUtc.ToUniversalTime() } }; + return this; + } + +} diff --git a/src/ManagedCode.LlmTck/Configuration/LlmTckConfigurationBuilder.cs b/src/ManagedCode.LlmTestServer/Configuration/LlmTestServerConfigurationBuilder.cs similarity index 54% rename from src/ManagedCode.LlmTck/Configuration/LlmTckConfigurationBuilder.cs rename to src/ManagedCode.LlmTestServer/Configuration/LlmTestServerConfigurationBuilder.cs index 653ea21..c22fd96 100644 --- a/src/ManagedCode.LlmTck/Configuration/LlmTckConfigurationBuilder.cs +++ b/src/ManagedCode.LlmTestServer/Configuration/LlmTestServerConfigurationBuilder.cs @@ -1,16 +1,16 @@ -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Scenarios; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Scenarios; -namespace ManagedCode.LlmTck.Configuration; +namespace ManagedCode.LlmTestServer.Configuration; -public sealed class LlmTckConfigurationBuilder +public sealed partial class LlmTestServerConfigurationBuilder { - private LlmTckConfiguration _configuration = LlmTckConfiguration.CreateDefault(); + private LlmTestServerConfiguration _configuration = LlmTestServerConfiguration.CreateDefault(); - public LlmTckConfigurationBuilder AddModel( + public LlmTestServerConfigurationBuilder AddModel( string id, - LlmTckModelKind kind, + LlmTestServerModelKind kind, int reasoningTokens = 0 ) { @@ -27,18 +27,18 @@ public LlmTckConfigurationBuilder AddModel( string.Equals(model.Id, id, StringComparison.OrdinalIgnoreCase) ); _configuration.Models.Add( - new LlmTckModel + new LlmTestServerModel { Id = id, Kind = kind, - ReasoningTokens = kind == LlmTckModelKind.Chat ? reasoningTokens : 0, + ReasoningTokens = kind == LlmTestServerModelKind.Chat ? reasoningTokens : 0, } ); return this; } - public LlmTckConfigurationBuilder AddReasoningChatModel(string id, int reasoningTokens) + public LlmTestServerConfigurationBuilder AddReasoningChatModel(string id, int reasoningTokens) { if (reasoningTokens <= 0) { @@ -48,135 +48,135 @@ public LlmTckConfigurationBuilder AddReasoningChatModel(string id, int reasoning ); } - return AddModel(id, LlmTckModelKind.Chat, reasoningTokens); + return AddModel(id, LlmTestServerModelKind.Chat, reasoningTokens); } - public LlmTckConfigurationBuilder AddGpt55(int reasoningTokens = 0) + public LlmTestServerConfigurationBuilder AddGpt55(int reasoningTokens = 0) { - return AddModel(LlmTckKnownModelIds.Gpt55, LlmTckModelKind.Chat, reasoningTokens); + return AddModel(LlmTestServerKnownModelIds.Gpt55, LlmTestServerModelKind.Chat, reasoningTokens); } - public LlmTckConfigurationBuilder AddGpt55Pro(int reasoningTokens = 0) + public LlmTestServerConfigurationBuilder AddGpt55Pro(int reasoningTokens = 0) { - return AddModel(LlmTckKnownModelIds.Gpt55Pro, LlmTckModelKind.Chat, reasoningTokens); + return AddModel(LlmTestServerKnownModelIds.Gpt55Pro, LlmTestServerModelKind.Chat, reasoningTokens); } - public LlmTckConfigurationBuilder AddGpt54(int reasoningTokens = 0) + public LlmTestServerConfigurationBuilder AddGpt54(int reasoningTokens = 0) { - return AddModel(LlmTckKnownModelIds.Gpt54, LlmTckModelKind.Chat, reasoningTokens); + return AddModel(LlmTestServerKnownModelIds.Gpt54, LlmTestServerModelKind.Chat, reasoningTokens); } - public LlmTckConfigurationBuilder AddGpt54Pro(int reasoningTokens = 0) + public LlmTestServerConfigurationBuilder AddGpt54Pro(int reasoningTokens = 0) { - return AddModel(LlmTckKnownModelIds.Gpt54Pro, LlmTckModelKind.Chat, reasoningTokens); + return AddModel(LlmTestServerKnownModelIds.Gpt54Pro, LlmTestServerModelKind.Chat, reasoningTokens); } - public LlmTckConfigurationBuilder AddGpt54Mini(int reasoningTokens = 0) + public LlmTestServerConfigurationBuilder AddGpt54Mini(int reasoningTokens = 0) { - return AddModel(LlmTckKnownModelIds.Gpt54Mini, LlmTckModelKind.Chat, reasoningTokens); + return AddModel(LlmTestServerKnownModelIds.Gpt54Mini, LlmTestServerModelKind.Chat, reasoningTokens); } - public LlmTckConfigurationBuilder AddGpt54Nano(int reasoningTokens = 0) + public LlmTestServerConfigurationBuilder AddGpt54Nano(int reasoningTokens = 0) { - return AddModel(LlmTckKnownModelIds.Gpt54Nano, LlmTckModelKind.Chat, reasoningTokens); + return AddModel(LlmTestServerKnownModelIds.Gpt54Nano, LlmTestServerModelKind.Chat, reasoningTokens); } - public LlmTckConfigurationBuilder AddGpt5(int reasoningTokens = 0) + public LlmTestServerConfigurationBuilder AddGpt5(int reasoningTokens = 0) { - return AddModel(LlmTckKnownModelIds.Gpt5, LlmTckModelKind.Chat, reasoningTokens); + return AddModel(LlmTestServerKnownModelIds.Gpt5, LlmTestServerModelKind.Chat, reasoningTokens); } - public LlmTckConfigurationBuilder AddGpt5Pro(int reasoningTokens = 0) + public LlmTestServerConfigurationBuilder AddGpt5Pro(int reasoningTokens = 0) { - return AddModel(LlmTckKnownModelIds.Gpt5Pro, LlmTckModelKind.Chat, reasoningTokens); + return AddModel(LlmTestServerKnownModelIds.Gpt5Pro, LlmTestServerModelKind.Chat, reasoningTokens); } - public LlmTckConfigurationBuilder AddGpt5Mini(int reasoningTokens = 0) + public LlmTestServerConfigurationBuilder AddGpt5Mini(int reasoningTokens = 0) { - return AddModel(LlmTckKnownModelIds.Gpt5Mini, LlmTckModelKind.Chat, reasoningTokens); + return AddModel(LlmTestServerKnownModelIds.Gpt5Mini, LlmTestServerModelKind.Chat, reasoningTokens); } - public LlmTckConfigurationBuilder AddGpt5Nano(int reasoningTokens = 0) + public LlmTestServerConfigurationBuilder AddGpt5Nano(int reasoningTokens = 0) { - return AddModel(LlmTckKnownModelIds.Gpt5Nano, LlmTckModelKind.Chat, reasoningTokens); + return AddModel(LlmTestServerKnownModelIds.Gpt5Nano, LlmTestServerModelKind.Chat, reasoningTokens); } - public LlmTckConfigurationBuilder AddGpt41() + public LlmTestServerConfigurationBuilder AddGpt41() { - return AddModel(LlmTckKnownModelIds.Gpt41, LlmTckModelKind.Chat); + return AddModel(LlmTestServerKnownModelIds.Gpt41, LlmTestServerModelKind.Chat); } - public LlmTckConfigurationBuilder AddGpt41Mini() + public LlmTestServerConfigurationBuilder AddGpt41Mini() { - return AddModel(LlmTckKnownModelIds.Gpt41Mini, LlmTckModelKind.Chat); + return AddModel(LlmTestServerKnownModelIds.Gpt41Mini, LlmTestServerModelKind.Chat); } - public LlmTckConfigurationBuilder AddGpt4OMini() + public LlmTestServerConfigurationBuilder AddGpt4OMini() { - return AddModel(LlmTckKnownModelIds.Gpt4OMini, LlmTckModelKind.Chat); + return AddModel(LlmTestServerKnownModelIds.Gpt4OMini, LlmTestServerModelKind.Chat); } - public LlmTckConfigurationBuilder AddTextEmbedding3Small() + public LlmTestServerConfigurationBuilder AddTextEmbedding3Small() { - return AddModel(LlmTckKnownModelIds.TextEmbedding3Small, LlmTckModelKind.Embedding); + return AddModel(LlmTestServerKnownModelIds.TextEmbedding3Small, LlmTestServerModelKind.Embedding); } - public LlmTckConfigurationBuilder AddTextEmbedding3Large() + public LlmTestServerConfigurationBuilder AddTextEmbedding3Large() { - return AddModel(LlmTckKnownModelIds.TextEmbedding3Large, LlmTckModelKind.Embedding); + return AddModel(LlmTestServerKnownModelIds.TextEmbedding3Large, LlmTestServerModelKind.Embedding); } - public LlmTckConfigurationBuilder AddTextEmbeddingAda002() + public LlmTestServerConfigurationBuilder AddTextEmbeddingAda002() { - return AddModel(LlmTckKnownModelIds.TextEmbeddingAda002, LlmTckModelKind.Embedding); + return AddModel(LlmTestServerKnownModelIds.TextEmbeddingAda002, LlmTestServerModelKind.Embedding); } - public LlmTckConfigurationBuilder AddGptImage2() + public LlmTestServerConfigurationBuilder AddGptImage2() { - return AddModel(LlmTckKnownModelIds.GptImage2, LlmTckModelKind.Image); + return AddModel(LlmTestServerKnownModelIds.GptImage2, LlmTestServerModelKind.Image); } - public LlmTckConfigurationBuilder AddGptImage15() + public LlmTestServerConfigurationBuilder AddGptImage15() { - return AddModel(LlmTckKnownModelIds.GptImage15, LlmTckModelKind.Image); + return AddModel(LlmTestServerKnownModelIds.GptImage15, LlmTestServerModelKind.Image); } - public LlmTckConfigurationBuilder AddGptImage1() + public LlmTestServerConfigurationBuilder AddGptImage1() { - return AddModel(LlmTckKnownModelIds.GptImage1, LlmTckModelKind.Image); + return AddModel(LlmTestServerKnownModelIds.GptImage1, LlmTestServerModelKind.Image); } - public LlmTckConfigurationBuilder AddGptImage1Mini() + public LlmTestServerConfigurationBuilder AddGptImage1Mini() { - return AddModel(LlmTckKnownModelIds.GptImage1Mini, LlmTckModelKind.Image); + return AddModel(LlmTestServerKnownModelIds.GptImage1Mini, LlmTestServerModelKind.Image); } - public LlmTckConfigurationBuilder AddGpt4OMiniTts() + public LlmTestServerConfigurationBuilder AddGpt4OMiniTts() { - return AddModel(LlmTckKnownModelIds.Gpt4OMiniTts, LlmTckModelKind.Audio); + return AddModel(LlmTestServerKnownModelIds.Gpt4OMiniTts, LlmTestServerModelKind.Audio); } - public LlmTckConfigurationBuilder AddTts1() + public LlmTestServerConfigurationBuilder AddTts1() { - return AddModel(LlmTckKnownModelIds.Tts1, LlmTckModelKind.Audio); + return AddModel(LlmTestServerKnownModelIds.Tts1, LlmTestServerModelKind.Audio); } - public LlmTckConfigurationBuilder AddTts1Hd() + public LlmTestServerConfigurationBuilder AddTts1Hd() { - return AddModel(LlmTckKnownModelIds.Tts1Hd, LlmTckModelKind.Audio); + return AddModel(LlmTestServerKnownModelIds.Tts1Hd, LlmTestServerModelKind.Audio); } - public LlmTckConfigurationBuilder AddSora2() + public LlmTestServerConfigurationBuilder AddSora2() { - return AddModel(LlmTckKnownModelIds.Sora2, LlmTckModelKind.Video); + return AddModel(LlmTestServerKnownModelIds.Sora2, LlmTestServerModelKind.Video); } - public LlmTckConfigurationBuilder AddSora2Pro() + public LlmTestServerConfigurationBuilder AddSora2Pro() { - return AddModel(LlmTckKnownModelIds.Sora2Pro, LlmTckModelKind.Video); + return AddModel(LlmTestServerKnownModelIds.Sora2Pro, LlmTestServerModelKind.Video); } - public LlmTckConfigurationBuilder AddDefaultOpenAiModels() + public LlmTestServerConfigurationBuilder AddDefaultOpenAiModels() { return AddGpt41Mini() .AddTextEmbedding3Small() @@ -185,7 +185,7 @@ public LlmTckConfigurationBuilder AddDefaultOpenAiModels() .AddSora2(); } - public LlmTckConfigurationBuilder AddCurrentOpenAiChatModels(int reasoningTokens = 0) + public LlmTestServerConfigurationBuilder AddCurrentOpenAiChatModels(int reasoningTokens = 0) { return AddGpt55(reasoningTokens) .AddGpt55Pro(reasoningTokens) @@ -202,14 +202,14 @@ public LlmTckConfigurationBuilder AddCurrentOpenAiChatModels(int reasoningTokens .AddGpt4OMini(); } - public LlmTckConfigurationBuilder AddOpenAiEmbeddingModels() + public LlmTestServerConfigurationBuilder AddOpenAiEmbeddingModels() { return AddTextEmbedding3Small() .AddTextEmbedding3Large() .AddTextEmbeddingAda002(); } - public LlmTckConfigurationBuilder AddOpenAiImageModels() + public LlmTestServerConfigurationBuilder AddOpenAiImageModels() { return AddGptImage2() .AddGptImage15() @@ -217,20 +217,20 @@ public LlmTckConfigurationBuilder AddOpenAiImageModels() .AddGptImage1Mini(); } - public LlmTckConfigurationBuilder AddOpenAiAudioModels() + public LlmTestServerConfigurationBuilder AddOpenAiAudioModels() { return AddGpt4OMiniTts() .AddTts1() .AddTts1Hd(); } - public LlmTckConfigurationBuilder AddOpenAiVideoModels() + public LlmTestServerConfigurationBuilder AddOpenAiVideoModels() { return AddSora2() .AddSora2Pro(); } - public LlmTckConfigurationBuilder AddKnownOpenAiModels(int reasoningTokens = 0) + public LlmTestServerConfigurationBuilder AddKnownOpenAiModels(int reasoningTokens = 0) { return AddCurrentOpenAiChatModels(reasoningTokens) .AddOpenAiEmbeddingModels() @@ -239,7 +239,7 @@ public LlmTckConfigurationBuilder AddKnownOpenAiModels(int reasoningTokens = 0) .AddOpenAiVideoModels(); } - public LlmTckConfigurationBuilder RequireBearerToken(string token) + public LlmTestServerConfigurationBuilder RequireBearerToken(string token) { ArgumentException.ThrowIfNullOrWhiteSpace(token); @@ -247,7 +247,7 @@ public LlmTckConfigurationBuilder RequireBearerToken(string token) return this; } - public LlmTckConfigurationBuilder SimulateRateLimitAfter(int allowedRequests) + public LlmTestServerConfigurationBuilder SimulateRateLimitAfter(int allowedRequests) { if (allowedRequests < 0) { @@ -267,7 +267,7 @@ public LlmTckConfigurationBuilder SimulateRateLimitAfter(int allowedRequests) return this; } - public LlmTckConfigurationBuilder SimulateContentFilter(params string[] blockedTerms) + public LlmTestServerConfigurationBuilder SimulateContentFilter(params string[] blockedTerms) { ArgumentNullException.ThrowIfNull(blockedTerms); if (blockedTerms.Length == 0) @@ -297,15 +297,15 @@ public LlmTckConfigurationBuilder SimulateContentFilter(params string[] blockedT return this; } - public LlmTckConfigurationBuilder AddChatScenario( + public LlmTestServerConfigurationBuilder AddChatScenario( string id, - Action configure + Action configure ) { ArgumentException.ThrowIfNullOrWhiteSpace(id); ArgumentNullException.ThrowIfNull(configure); - var builder = new LlmTckScenarioBuilder(id); + var builder = new LlmTestServerScenarioBuilder(id); configure(builder); var scenario = builder.Build(); @@ -317,41 +317,41 @@ Action configure return this; } - public LlmTckConfigurationBuilder AddDecisionModel(string id, ManagedCode.LlmTck.Decisions.LlmTckDecisionProvider provider) + public LlmTestServerConfigurationBuilder AddDecisionModel(string id, ManagedCode.LlmTestServer.Decisions.LlmTestServerDecisionProvider provider) { if (!Enum.IsDefined(provider)) { throw new ArgumentOutOfRangeException(nameof(provider)); } - AddModel(id, LlmTckModelKind.Decision); + AddModel(id, LlmTestServerModelKind.Decision); var index = _configuration.Models.FindIndex(model => model.Id == id); _configuration.Models[index] = _configuration.Models[index] with { DecisionProvider = provider }; return this; } - public LlmTckConfigurationBuilder AddJevLatest() + public LlmTestServerConfigurationBuilder AddJevLatest() { - return AddDecisionModel(LlmTckKnownModelIds.JevLatest, ManagedCode.LlmTck.Decisions.LlmTckDecisionProvider.TypeSafe); + return AddDecisionModel(ManagedCode.LlmTestServer.Decisions.LlmTestServerDecisionModelIds.TypeSafeJev, ManagedCode.LlmTestServer.Decisions.LlmTestServerDecisionProvider.TypeSafe); } - public LlmTckConfigurationBuilder AddKevLatest() + public LlmTestServerConfigurationBuilder AddKevLatest() { - return AddDecisionModel(LlmTckKnownModelIds.KevLatest, ManagedCode.LlmTck.Decisions.LlmTckDecisionProvider.Kev); + return AddDecisionModel(ManagedCode.LlmTestServer.Decisions.LlmTestServerDecisionModelIds.Kev, ManagedCode.LlmTestServer.Decisions.LlmTestServerDecisionProvider.Kev); } - public LlmTckConfigurationBuilder AddClef() + public LlmTestServerConfigurationBuilder AddClef() { - return AddDecisionModel(LlmTckKnownModelIds.Clef, ManagedCode.LlmTck.Decisions.LlmTckDecisionProvider.Cloudflare); + return AddDecisionModel(ManagedCode.LlmTestServer.Decisions.LlmTestServerDecisionModelIds.CloudflareClef, ManagedCode.LlmTestServer.Decisions.LlmTestServerDecisionProvider.Cloudflare); } - public LlmTckConfigurationBuilder AddClefFlash() + public LlmTestServerConfigurationBuilder AddClefFlash() { - return AddDecisionModel(LlmTckKnownModelIds.ClefFlash, ManagedCode.LlmTck.Decisions.LlmTckDecisionProvider.Cloudflare); + return AddDecisionModel(ManagedCode.LlmTestServer.Decisions.LlmTestServerDecisionModelIds.CloudflareClefFlash, ManagedCode.LlmTestServer.Decisions.LlmTestServerDecisionProvider.Cloudflare); } - public LlmTckConfigurationBuilder AddOpenAiDecisionModel(string id) + public LlmTestServerConfigurationBuilder AddOpenAiDecisionModel(string id) { - return AddDecisionModel(id, ManagedCode.LlmTck.Decisions.LlmTckDecisionProvider.OpenAI); + return AddDecisionModel(id, ManagedCode.LlmTestServer.Decisions.LlmTestServerDecisionProvider.OpenAI); } - public LlmTckConfigurationBuilder AddDecisionScenario(ManagedCode.LlmTck.Decisions.LlmTckDecisionScenario scenario) + public LlmTestServerConfigurationBuilder AddDecisionScenario(ManagedCode.LlmTestServer.Decisions.LlmTestServerDecisionScenario scenario) { var snapshot = SnapshotDecisionScenario(scenario); _configuration.DecisionScenarios.RemoveAll(existing => existing.Id == snapshot.Id); @@ -359,15 +359,15 @@ public LlmTckConfigurationBuilder AddDecisionScenario(ManagedCode.LlmTck.Decisio return this; } - public LlmTckConfigurationBuilder AddDataset( + public LlmTestServerConfigurationBuilder AddDataset( string id, - Action configure + Action configure ) { ArgumentException.ThrowIfNullOrWhiteSpace(id); ArgumentNullException.ThrowIfNull(configure); - var builder = new LlmTckScenarioDatasetBuilder(id); + var builder = new LlmTestServerScenarioDatasetBuilder(id); configure(builder); var dataset = builder.Build(); @@ -379,7 +379,7 @@ Action configure return this; } - public LlmTckConfigurationBuilder WithDefaultEmbeddingVector(params float[] values) + public LlmTestServerConfigurationBuilder WithDefaultEmbeddingVector(params float[] values) { ArgumentNullException.ThrowIfNull(values); if (values.Length == 0) @@ -393,7 +393,7 @@ public LlmTckConfigurationBuilder WithDefaultEmbeddingVector(params float[] valu return this; } - public LlmTckConfigurationBuilder WithDefaultImageDataUri(string dataUri) + public LlmTestServerConfigurationBuilder WithDefaultImageDataUri(string dataUri) { ArgumentException.ThrowIfNullOrWhiteSpace(dataUri); if (!dataUri.StartsWith("data:", StringComparison.OrdinalIgnoreCase)) @@ -405,7 +405,7 @@ public LlmTckConfigurationBuilder WithDefaultImageDataUri(string dataUri) return this; } - public LlmTckConfigurationBuilder WithDefaultAudio(byte[] bytes, string mediaType) + public LlmTestServerConfigurationBuilder WithDefaultAudio(byte[] bytes, string mediaType) { ArgumentNullException.ThrowIfNull(bytes); ArgumentException.ThrowIfNullOrWhiteSpace(mediaType); @@ -423,7 +423,7 @@ public LlmTckConfigurationBuilder WithDefaultAudio(byte[] bytes, string mediaTyp return this; } - public LlmTckConfigurationBuilder WithDefaultTranscriptionText(string text) + public LlmTestServerConfigurationBuilder WithDefaultTranscriptionText(string text) { ArgumentException.ThrowIfNullOrWhiteSpace(text); @@ -431,7 +431,7 @@ public LlmTckConfigurationBuilder WithDefaultTranscriptionText(string text) return this; } - public LlmTckConfigurationBuilder WithDefaultTranscriptionUsage(LlmTckTranscriptionUsage usage) + public LlmTestServerConfigurationBuilder WithDefaultTranscriptionUsage(LlmTestServerTranscriptionUsage usage) { ArgumentNullException.ThrowIfNull(usage); usage.Validate(); @@ -439,13 +439,13 @@ public LlmTckConfigurationBuilder WithDefaultTranscriptionUsage(LlmTckTranscript return this; } - public LlmTckConfigurationBuilder WithoutDefaultTranscriptionUsage() + public LlmTestServerConfigurationBuilder WithoutDefaultTranscriptionUsage() { _configuration = _configuration with { DefaultTranscriptionUsage = null }; return this; } - public LlmTckConfigurationBuilder WithDefaultTranslationText(string text) + public LlmTestServerConfigurationBuilder WithDefaultTranslationText(string text) { ArgumentException.ThrowIfNullOrWhiteSpace(text); @@ -453,7 +453,7 @@ public LlmTckConfigurationBuilder WithDefaultTranslationText(string text) return this; } - public LlmTckConfigurationBuilder WithDefaultVideo(byte[] bytes, string mediaType) + public LlmTestServerConfigurationBuilder WithDefaultVideo(byte[] bytes, string mediaType) { ArgumentNullException.ThrowIfNull(bytes); ArgumentException.ThrowIfNullOrWhiteSpace(mediaType); @@ -471,14 +471,14 @@ public LlmTckConfigurationBuilder WithDefaultVideo(byte[] bytes, string mediaTyp return this; } - public LlmTckConfigurationBuilder WithPromptCacheCapacity(int capacity) + public LlmTestServerConfigurationBuilder WithPromptCacheCapacity(int capacity) { ArgumentOutOfRangeException.ThrowIfNegative(capacity); _configuration = _configuration with { MaxPromptCacheEntries = capacity }; return this; } - public LlmTckConfigurationBuilder WithVideoCapacity(int maxJobs, long maxBytes) + public LlmTestServerConfigurationBuilder WithVideoCapacity(int maxJobs, long maxBytes) { ArgumentOutOfRangeException.ThrowIfNegative(maxJobs); ArgumentOutOfRangeException.ThrowIfNegative(maxBytes); @@ -486,7 +486,7 @@ public LlmTckConfigurationBuilder WithVideoCapacity(int maxJobs, long maxBytes) return this; } - public LlmTckConfigurationBuilder WithChatResponseCapacity(int maxResponses, long maxBytes) + public LlmTestServerConfigurationBuilder WithChatResponseCapacity(int maxResponses, long maxBytes) { ArgumentOutOfRangeException.ThrowIfNegative(maxResponses); ArgumentOutOfRangeException.ThrowIfNegative(maxBytes); @@ -494,15 +494,22 @@ public LlmTckConfigurationBuilder WithChatResponseCapacity(int maxResponses, lon return this; } - public LlmTckConfiguration Build() + public LlmTestServerConfiguration Build() { return Snapshot(_configuration); } - internal static LlmTckConfiguration Snapshot(LlmTckConfiguration configuration) + internal static LlmTestServerConfiguration Snapshot(LlmTestServerConfiguration configuration) { ArgumentNullException.ThrowIfNull(configuration); + if (configuration.Determinism is { } determinism) + { + ArgumentException.ThrowIfNullOrWhiteSpace(determinism.Seed); + if (determinism.Seed.Length > 256) { throw new ArgumentException("Deterministic seeds are limited to 256 characters.", nameof(configuration)); } + } ArgumentOutOfRangeException.ThrowIfNegative(configuration.MaxPromptCacheEntries); + ArgumentOutOfRangeException.ThrowIfNegative(configuration.MaxConversations); + ArgumentOutOfRangeException.ThrowIfNegative(configuration.MaxConversationBytes); ArgumentOutOfRangeException.ThrowIfNegative(configuration.MaxVideoJobs); ArgumentOutOfRangeException.ThrowIfNegative(configuration.MaxVideoBytes); ArgumentOutOfRangeException.ThrowIfNegative(configuration.MaxStoredChatResponses); @@ -529,7 +536,7 @@ internal static LlmTckConfiguration Snapshot(LlmTckConfiguration configuration) }; } - private static ManagedCode.LlmTck.Decisions.LlmTckDecisionScenario SnapshotDecisionScenario(ManagedCode.LlmTck.Decisions.LlmTckDecisionScenario scenario) + private static ManagedCode.LlmTestServer.Decisions.LlmTestServerDecisionScenario SnapshotDecisionScenario(ManagedCode.LlmTestServer.Decisions.LlmTestServerDecisionScenario scenario) { ArgumentNullException.ThrowIfNull(scenario); ArgumentException.ThrowIfNullOrWhiteSpace(scenario.Id); @@ -548,10 +555,10 @@ private static ManagedCode.LlmTck.Decisions.LlmTckDecisionScenario SnapshotDecis } var valid = answer.Kind switch { - ManagedCode.LlmTck.Decisions.LlmTckDecisionKind.Predicate => answer.Probability.HasValue, - ManagedCode.LlmTck.Decisions.LlmTckDecisionKind.Choice => answer.Choice is not null, - ManagedCode.LlmTck.Decisions.LlmTckDecisionKind.Score => answer.Score.HasValue, - ManagedCode.LlmTck.Decisions.LlmTckDecisionKind.Box => answer.Box is not null + ManagedCode.LlmTestServer.Decisions.LlmTestServerDecisionKind.Predicate => answer.Probability.HasValue, + ManagedCode.LlmTestServer.Decisions.LlmTestServerDecisionKind.Choice => answer.Choice is not null, + ManagedCode.LlmTestServer.Decisions.LlmTestServerDecisionKind.Score => answer.Score.HasValue, + ManagedCode.LlmTestServer.Decisions.LlmTestServerDecisionKind.Box => answer.Box is not null && (scenario.AllowMalformedResponse || answer.Box.IsValid), _ => false, }; @@ -564,7 +571,7 @@ private static ManagedCode.LlmTck.Decisions.LlmTckDecisionScenario SnapshotDecis }; } - private static LlmTckModel SnapshotModel(LlmTckModel model) + private static LlmTestServerModel SnapshotModel(LlmTestServerModel model) { ArgumentNullException.ThrowIfNull(model); ArgumentException.ThrowIfNullOrWhiteSpace(model.Id); @@ -578,11 +585,11 @@ private static LlmTckModel SnapshotModel(LlmTckModel model) return model with { - ReasoningTokens = model.Kind == LlmTckModelKind.Chat ? model.ReasoningTokens : 0, + ReasoningTokens = model.Kind == LlmTestServerModelKind.Chat ? model.ReasoningTokens : 0, }; } - internal static LlmTckScenario SnapshotScenario(LlmTckScenario scenario) + internal static LlmTestServerScenario SnapshotScenario(LlmTestServerScenario scenario) { ArgumentNullException.ThrowIfNull(scenario); ArgumentNullException.ThrowIfNull(scenario.Match); @@ -591,6 +598,7 @@ internal static LlmTckScenario SnapshotScenario(LlmTckScenario scenario) return scenario with { + Conversation = SnapshotConversation(scenario), Match = scenario.Match with { Messages = scenario.Match.Messages.Select(SnapshotMessage).ToList() }, Responses = [ @@ -599,24 +607,33 @@ .. scenario.Responses.Select(SnapshotResponse), }; } - private static LlmTckMessage SnapshotMessage(LlmTckMessage message) + private static LlmTestServerMessage SnapshotMessage(LlmTestServerMessage message) { ArgumentNullException.ThrowIfNull(message); ArgumentNullException.ThrowIfNull(message.ToolCalls); + if (message.OutputDisposition is { } disposition) { LlmTestServerOutputDisposition.Validate(disposition); } foreach (var call in message.ToolCalls) { ArgumentNullException.ThrowIfNull(call); ArgumentException.ThrowIfNullOrWhiteSpace(call.Id); ArgumentException.ThrowIfNullOrWhiteSpace(call.Name); - LlmTckJsonSchema.ValidateJson(call.ArgumentsJson); + LlmTestServerJsonSchema.ValidateJson(call.ArgumentsJson); } - return message with { ToolCalls = [.. message.ToolCalls] }; + return message with { OutputDisposition = LlmTestServerOutputDisposition.ForHistory(message.OutputDisposition), ToolCalls = [.. message.ToolCalls], OutputItems = LlmTestServerResponseItems.Snapshot(message.OutputItems) }; } - private static LlmTckScenarioResponse SnapshotResponse(LlmTckScenarioResponse response) + private static LlmTestServerScenarioResponse SnapshotResponse(LlmTestServerScenarioResponse response) { ArgumentNullException.ThrowIfNull(response); + ArgumentNullException.ThrowIfNull(response.Content); ArgumentNullException.ThrowIfNull(response.StreamChunks); + foreach (var chunk in response.StreamChunks) { ArgumentNullException.ThrowIfNull(chunk); } + if (response.Error is { } error) + { + if (error.StatusCode is < 400 or > 599 || error.RetryAfterSeconds is < 0 or > 86400) { throw new ArgumentOutOfRangeException(nameof(response)); } + ArgumentException.ThrowIfNullOrWhiteSpace(error.Code); + ArgumentNullException.ThrowIfNull(error.Message); + } if (response.DelayMilliseconds < 0) { throw new ArgumentOutOfRangeException(nameof(response)); @@ -628,12 +645,15 @@ private static LlmTckScenarioResponse SnapshotResponse(LlmTckScenarioResponse re ArgumentNullException.ThrowIfNull(call); ArgumentException.ThrowIfNullOrWhiteSpace(call.Id); ArgumentException.ThrowIfNullOrWhiteSpace(call.Name); - LlmTckJsonSchema.ValidateJson(call.ArgumentsJson); + LlmTestServerJsonSchema.ValidateJson(call.ArgumentsJson); } - return response with { StreamChunks = [.. response.StreamChunks], ToolCalls = [.. response.ToolCalls] }; + ArgumentNullException.ThrowIfNull(response.OutputItems); + if (response.OutputItems.Count > 0 && (response.Content.Length > 0 || response.ToolCalls.Count > 0 || response.StreamChunks.Count > 0 || response.Error is not null)) + { throw new ArgumentException("An ordered output plan cannot also specify legacy output fields or an error.", nameof(response)); } + return response with { OutputDisposition = LlmTestServerOutputDisposition.Snapshot(response), StreamDelivery = LlmTestServerStreamDelivery.Snapshot(response), StreamChunks = [.. response.StreamChunks], ToolCalls = [.. response.ToolCalls], OutputItems = LlmTestServerResponseItems.Snapshot(response.OutputItems, validateChunks: true) }; } - private static LlmTckScenarioDataset SnapshotDataset(LlmTckScenarioDataset dataset) + private static LlmTestServerScenarioDataset SnapshotDataset(LlmTestServerScenarioDataset dataset) { ArgumentNullException.ThrowIfNull(dataset); ArgumentNullException.ThrowIfNull(dataset.ChatScenarios); @@ -644,8 +664,8 @@ private static LlmTckScenarioDataset SnapshotDataset(LlmTckScenarioDataset datas }; } - private static LlmTckFaultSimulation SnapshotFaultSimulation( - LlmTckFaultSimulation faultSimulation + private static LlmTestServerFaultSimulation SnapshotFaultSimulation( + LlmTestServerFaultSimulation faultSimulation ) { ArgumentNullException.ThrowIfNull(faultSimulation); diff --git a/src/ManagedCode.LlmTck/Configuration/LlmTckFaultSimulation.cs b/src/ManagedCode.LlmTestServer/Configuration/LlmTestServerFaultSimulation.cs similarity index 55% rename from src/ManagedCode.LlmTck/Configuration/LlmTckFaultSimulation.cs rename to src/ManagedCode.LlmTestServer/Configuration/LlmTestServerFaultSimulation.cs index a987675..f28e57b 100644 --- a/src/ManagedCode.LlmTck/Configuration/LlmTckFaultSimulation.cs +++ b/src/ManagedCode.LlmTestServer/Configuration/LlmTestServerFaultSimulation.cs @@ -1,6 +1,6 @@ -namespace ManagedCode.LlmTck.Configuration; +namespace ManagedCode.LlmTestServer.Configuration; -public sealed record LlmTckFaultSimulation +public sealed record LlmTestServerFaultSimulation { public int? MaxRequestsBeforeRateLimit { get; init; } diff --git a/src/ManagedCode.LlmTestServer/Control/LlmTestServerControlRoutes.cs b/src/ManagedCode.LlmTestServer/Control/LlmTestServerControlRoutes.cs new file mode 100644 index 0000000..03d60e3 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Control/LlmTestServerControlRoutes.cs @@ -0,0 +1,16 @@ +namespace ManagedCode.LlmTestServer.Control; + +public static class LlmTestServerControlRoutes +{ + public const string Verify = "/admin-api/verify"; + + public const string Admin = "/"; + public const string Models = "/admin-api/models"; + public const string Assertions = "/admin-api/assertions"; + public const string Requests = "/admin-api/requests"; + public const string Configure = "/admin-api/configure"; + public const string Conversations = "/admin-api/conversations"; + public const string ExplainMatch = "/admin-api/match/explain"; + public const string Sessions = "/admin-api/sessions"; + public const string Reset = "/admin-api/reset"; +} diff --git a/src/ManagedCode.LlmTestServer/Conversations/LlmTestServerConversation.cs b/src/ManagedCode.LlmTestServer/Conversations/LlmTestServerConversation.cs new file mode 100644 index 0000000..da1fdf2 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Conversations/LlmTestServerConversation.cs @@ -0,0 +1,46 @@ +using ManagedCode.LlmTestServer.Scenarios; + +namespace ManagedCode.LlmTestServer.Conversations; + +public static class LlmTestServerConversationHeaders +{ + public const string Id = "X-Llm-Test-Server-Conversation"; +} + +public enum LlmTestServerConversationExhaustion { Reject, RepeatLastResponse, Restart } + +public sealed record LlmTestServerConversation +{ + public string StartStepId { get; init; } = string.Empty; + public int MaxSteps { get; init; } = 64; + public bool ReplaySuccessfulRequests { get; init; } = true; + public LlmTestServerConversationExhaustion Exhaustion { get; init; } + public List Steps { get; init; } = []; +} + +public sealed record LlmTestServerConversationStep +{ + public string Id { get; init; } = string.Empty; + public LlmTestServerConversationMatch Match { get; init; } = new(); + public LlmTestServerScenarioResponse Response { get; init; } = new(); + public List NextSteps { get; init; } = []; + public bool Terminal { get; init; } +} + +public sealed record LlmTestServerConversationMatch +{ + public string? LatestRole { get; init; } + public string? TextEquals { get; init; } + public string? TextContains { get; init; } + public int? TurnIndex { get; init; } + public string? ToolName { get; init; } + public string? ToolCallId { get; init; } + public string? ToolArgumentsJson { get; init; } + public List JsonFields { get; init; } = []; +} + +public sealed record LlmTestServerJsonFieldMatch +{ + public string Path { get; init; } = "$"; + public string EqualsJson { get; init; } = "null"; +} diff --git a/src/ManagedCode.LlmTestServer/Conversations/LlmTestServerConversationBuilder.cs b/src/ManagedCode.LlmTestServer/Conversations/LlmTestServerConversationBuilder.cs new file mode 100644 index 0000000..4f8965c --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Conversations/LlmTestServerConversationBuilder.cs @@ -0,0 +1,35 @@ +using ManagedCode.LlmTestServer.Configuration; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Scenarios; + +namespace ManagedCode.LlmTestServer.Conversations; + +public sealed class LlmTestServerConversationBuilder(string id) +{ + private string _modelId = LlmTestServerKnownModelIds.Gpt41Mini; + private string? _token; + private LlmTestServerConversation _conversation = new(); + + public LlmTestServerConversationBuilder ForModel(string modelId) { ArgumentException.ThrowIfNullOrWhiteSpace(modelId); _modelId = modelId; return this; } + public LlmTestServerConversationBuilder RequireBearerToken(string token) { ArgumentException.ThrowIfNullOrWhiteSpace(token); _token = token; return this; } + public LlmTestServerConversationBuilder StartAt(string stepId) { ArgumentException.ThrowIfNullOrWhiteSpace(stepId); _conversation = _conversation with { StartStepId = stepId }; return this; } + public LlmTestServerConversationBuilder WithMaxSteps(int maximum) { _conversation = _conversation with { MaxSteps = maximum }; return this; } + public LlmTestServerConversationBuilder ReplaySuccessfulRequests(bool enabled = true) { _conversation = _conversation with { ReplaySuccessfulRequests = enabled }; return this; } + public LlmTestServerConversationBuilder OnExhaustion(LlmTestServerConversationExhaustion policy) { _conversation = _conversation with { Exhaustion = policy }; return this; } + + public LlmTestServerConversationBuilder Step(string stepId, Action configure) + { + ArgumentNullException.ThrowIfNull(configure); + var step = new LlmTestServerConversationStepBuilder(stepId); + configure(step); + _conversation.Steps.Add(step.Build()); + return this; + } + + public LlmTestServerScenario Build() + { + var graph = _conversation with { StartStepId = _conversation.StartStepId.Length > 0 ? _conversation.StartStepId : _conversation.Steps.FirstOrDefault()?.Id ?? string.Empty }; + return LlmTestServerConfigurationBuilder.SnapshotScenario(new LlmTestServerScenario + { Id = id, ModelId = _modelId, RequiredBearerToken = _token, Conversation = graph }); + } +} diff --git a/src/ManagedCode.LlmTestServer/Conversations/LlmTestServerConversationDiagnostics.cs b/src/ManagedCode.LlmTestServer/Conversations/LlmTestServerConversationDiagnostics.cs new file mode 100644 index 0000000..b08f72b --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Conversations/LlmTestServerConversationDiagnostics.cs @@ -0,0 +1,51 @@ +using ManagedCode.LlmTestServer.Providers; +using ManagedCode.LlmTestServer.Scenarios; + +namespace ManagedCode.LlmTestServer.Conversations; + +public sealed record LlmTestServerMatchExplanationRequest +{ + public LlmTestServerChatRequest Request { get; init; } = new(); + public string ProviderNamespace { get; init; } = LlmTestServerProviderRouteNamespaces.OpenAI; +} + +public sealed record LlmTestServerMatchExplanation +{ + public string? ErrorCode { get; init; } + public string ConversationId { get; init; } = "default"; + public int TurnIndex { get; init; } + public string? SelectedScenarioId { get; init; } + public string? SelectedStepId { get; init; } + public List Candidates { get; init; } = []; +} + +public sealed record LlmTestServerMatchCandidate +{ + public string ScenarioId { get; init; } = string.Empty; + public string? StepId { get; init; } + public bool IsMatch { get; init; } + public bool IsSelected { get; init; } + public bool IsShadowed { get; init; } + public List Mismatches { get; init; } = []; +} + +public sealed record LlmTestServerConversationStatus +{ + public string ScenarioId { get; init; } = string.Empty; + public string ProviderNamespace { get; init; } = string.Empty; + public string ConversationId { get; init; } = "default"; + public bool IsBusy { get; init; } + public bool IsTerminal { get; init; } + public int ExecutedSteps { get; init; } + public int LastTurnIndex { get; init; } + public List NextSteps { get; init; } = []; + public List VisitedSteps { get; init; } = []; + public List UnvisitedSteps { get; init; } = []; +} + +public interface ILlmTestServerConversationDiagnostics +{ + LlmTestServerMatchExplanation Explain(LlmTestServerChatRequest request, string providerNamespace, + string? conversationId = null, string? bearerToken = null); + IReadOnlyList GetConversations(); +} diff --git a/src/ManagedCode.LlmTestServer/Conversations/LlmTestServerConversationStepBuilder.cs b/src/ManagedCode.LlmTestServer/Conversations/LlmTestServerConversationStepBuilder.cs new file mode 100644 index 0000000..19400c1 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Conversations/LlmTestServerConversationStepBuilder.cs @@ -0,0 +1,41 @@ +using System.Text.Json; +using ManagedCode.LlmTestServer.Scenarios; + +namespace ManagedCode.LlmTestServer.Conversations; + +public sealed class LlmTestServerConversationStepBuilder(string id) +{ + private readonly LlmTestServerScenarioBuilder _response = new(id); + private LlmTestServerConversationMatch _match = new(); + private List _next = []; + private bool _terminal; + + public LlmTestServerConversationStepBuilder WhenLatestRole(string role) { _match = _match with { LatestRole = role }; return this; } + public LlmTestServerConversationStepBuilder WhenLatestUserContains(string text) { return WhenLatestRole("user").WhenLatestTextContains(text); } + public LlmTestServerConversationStepBuilder WhenLatestUserEquals(string text) { _match = _match with { LatestRole = "user", TextEquals = text }; return this; } + public LlmTestServerConversationStepBuilder WhenLatestTextContains(string text) { _match = _match with { TextContains = text }; return this; } + public LlmTestServerConversationStepBuilder WhenTurn(int index) { _match = _match with { TurnIndex = index }; return this; } + public LlmTestServerConversationStepBuilder WhenToolResult(string name, string? callId = null) { _match = _match with { LatestRole = "tool", ToolName = name, ToolCallId = callId }; return this; } + public LlmTestServerConversationStepBuilder WithToolArgumentsJson(string json) { _match = _match with { ToolArgumentsJson = json }; return this; } + public LlmTestServerConversationStepBuilder WhenLatestJson(string path, string expectedJson) { _match.JsonFields.Add(new() { Path = path, EqualsJson = expectedJson }); return this; } + public LlmTestServerConversationStepBuilder WhenLatestJsonValue(string path, T value) { return WhenLatestJson(path, JsonSerializer.Serialize(value)); } + public LlmTestServerConversationStepBuilder RespondsWithItems(params LlmTestServerResponseItem[] items) { _response.RespondsWithItems(items); return this; } + public LlmTestServerConversationStepBuilder Responds(string text, params string[] chunks) { _response.Responds(text, chunks); return this; } + public LlmTestServerConversationStepBuilder RespondsJson(string json) { _response.RespondsJson(json); return this; } + public LlmTestServerConversationStepBuilder CallsTool(string name, string argumentsJson, string callId = "call_fixture") { _response.CallsTool(name, argumentsJson, callId); return this; } + public LlmTestServerConversationStepBuilder CallsTools(params LlmTestServerToolCall[] calls) { _response.CallsTools(calls); return this; } + public LlmTestServerConversationStepBuilder Fails(int status, string code, string message) { _response.Fails(status, code, message); return this; } + public LlmTestServerConversationStepBuilder RetryAfter(int seconds) { _response.RetryAfter(seconds); return this; } + public LlmTestServerConversationStepBuilder DelaysBy(int milliseconds) { _response.DelaysBy(milliseconds); return this; } + public LlmTestServerConversationStepBuilder WithOutputDisposition(LlmTestServerOutputDisposition disposition) { _response.WithOutputDisposition(disposition); return this; } + public LlmTestServerConversationStepBuilder WithStreamDelivery(LlmTestServerStreamDelivery delivery) { _response.WithStreamDelivery(delivery); return this; } + public LlmTestServerConversationStepBuilder Then(params string[] stepIds) { _next = [.. stepIds]; _terminal = false; return this; } + public LlmTestServerConversationStepBuilder Complete() { _next = []; _terminal = true; return this; } + + internal LlmTestServerConversationStep Build() + { + var responses = _response.Build().Responses; + if (responses.Count != 1) { throw new ArgumentException("A conversation step requires exactly one response.", nameof(id)); } + return new() { Id = id, Match = _match, Response = responses[0], NextSteps = _next, Terminal = _terminal }; + } +} diff --git a/src/ManagedCode.LlmTestServer/Conversations/LlmTestServerJsonFieldSelector.cs b/src/ManagedCode.LlmTestServer/Conversations/LlmTestServerJsonFieldSelector.cs new file mode 100644 index 0000000..e79106f --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Conversations/LlmTestServerJsonFieldSelector.cs @@ -0,0 +1,66 @@ +using System.Globalization; +using System.Text.Json; + +namespace ManagedCode.LlmTestServer.Conversations; + +internal static class LlmTestServerJsonFieldSelector +{ + internal static string[] Parse(string path) + { + ArgumentNullException.ThrowIfNull(path); + if (path.Length > 256) { throw new ArgumentException("JSON field paths are limited to 256 characters.", nameof(path)); } + if (path == "$" || path.Length == 0) { return []; } + if (path[0] == '/') + { + var parts = path[1..].Split('/'); + foreach (var part in parts) + { + for (var index = 0; index < part.Length; index++) + { + if (part[index] == '~' && (++index == part.Length || part[index] is not ('0' or '1'))) + { throw new ArgumentException("Invalid JSON pointer escape.", nameof(path)); } + } + } + return parts.Select(part => part.Replace("~1", "/", StringComparison.Ordinal).Replace("~0", "~", StringComparison.Ordinal)).ToArray(); + } + if (path[0] != '$') { throw new ArgumentException("Use a JSON pointer or a JSON path starting with $.", nameof(path)); } + var result = new List(); + for (var index = 1; index < path.Length;) + { + if (path[index] == '.') + { + var start = ++index; + while (index < path.Length && (char.IsAsciiLetterOrDigit(path[index]) || path[index] == '_')) { index++; } + if (start == index) { throw new ArgumentException("Invalid JSON property selector.", nameof(path)); } + result.Add(path[start..index]); + } + else if (path[index] == '[') + { + var start = ++index; + while (index < path.Length && char.IsAsciiDigit(path[index])) { index++; } + if (start == index || index >= path.Length || path[index++] != ']') { throw new ArgumentException("Invalid JSON array selector.", nameof(path)); } + result.Add(path[start..(index - 1)]); + } + else { throw new ArgumentException("Unsupported JSON field selector.", nameof(path)); } + } + return [.. result]; + } + + internal static bool Matches(string json, string path, string expected) + { + try + { + using var document = JsonDocument.Parse(json); + using var value = JsonDocument.Parse(expected); + var current = document.RootElement; + foreach (var part in Parse(path)) + { + if (current.ValueKind == JsonValueKind.Object && current.TryGetProperty(part, out var property)) { current = property; } + else if (current.ValueKind == JsonValueKind.Array && int.TryParse(part, NumberStyles.None, CultureInfo.InvariantCulture, out var index) && index >= 0 && index < current.GetArrayLength()) { current = current[index]; } + else { return false; } + } + return JsonElement.DeepEquals(current, value.RootElement); + } + catch (JsonException) { return false; } + } +} diff --git a/src/ManagedCode.LlmTestServer/Decisions/LlmTestServerDecisionFixtureValidation.cs b/src/ManagedCode.LlmTestServer/Decisions/LlmTestServerDecisionFixtureValidation.cs new file mode 100644 index 0000000..071bb84 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Decisions/LlmTestServerDecisionFixtureValidation.cs @@ -0,0 +1,27 @@ +using System.Globalization; + +namespace ManagedCode.LlmTestServer.Decisions; + +internal static class LlmTestServerDecisionFixtureValidation +{ + private const double _probabilityRoundingError = 0.00005; + private const double _floatingPointTolerance = 0.000001; + + public static bool IsConformant(LlmTestServerDecisionProvider provider, LlmTestServerDecisionQuestion question, LlmTestServerDecisionAnswer answer) + { + if (provider == LlmTestServerDecisionProvider.Unspecified || answer.Kind == LlmTestServerDecisionKind.Predicate) { return true; } + if (answer.Kind == LlmTestServerDecisionKind.Box) + { return provider == LlmTestServerDecisionProvider.TypeSafe && answer.Box is { IsValid: true } && answer.Probabilities.Count == 0; } + var domain = question.Kind == LlmTestServerDecisionKind.Score + ? Enumerable.Range(0, question.Options.Count).Select(index => index.ToString(CultureInfo.InvariantCulture)).ToArray() + : question.Options.ToArray(); + if (!answer.Confidence.HasValue || answer.Probabilities.Count != domain.Length + || !domain.All(answer.Probabilities.ContainsKey)) { return false; } + var tolerance = provider is LlmTestServerDecisionProvider.Kev or LlmTestServerDecisionProvider.Cloudflare + ? _probabilityRoundingError * domain.Length + _floatingPointTolerance : _floatingPointTolerance; + if (Math.Abs(answer.Probabilities.Values.Sum() - 1) > tolerance) { return false; } + if (provider != LlmTestServerDecisionProvider.OpenAI && answer.Kind == LlmTestServerDecisionKind.Choice && answer.Probabilities[answer.Choice!] + _floatingPointTolerance < answer.Probabilities.Values.Max()) { return false; } + return provider != LlmTestServerDecisionProvider.Cloudflare + || Math.Abs(answer.Confidence.Value - answer.Probabilities.Values.Max()) <= _probabilityRoundingError + _floatingPointTolerance; + } +} diff --git a/src/ManagedCode.LlmTestServer/Decisions/LlmTestServerDecisionModelIds.cs b/src/ManagedCode.LlmTestServer/Decisions/LlmTestServerDecisionModelIds.cs new file mode 100644 index 0000000..50f3376 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Decisions/LlmTestServerDecisionModelIds.cs @@ -0,0 +1,14 @@ +namespace ManagedCode.LlmTestServer.Decisions; + +/// Explicit native decision fixture contracts, independently documented by their providers. +public static class LlmTestServerDecisionModelIds +{ + /// TypeSafe native alias: https://api.typesafe.ai/openapi.json . + public const string TypeSafeJev = "jev-latest"; + /// Original Kev request default: https://github.com/jaredpalmer/kev/blob/main/kev/api.py . + public const string Kev = "kev-latest"; + /// Cloudflare native body model: https://developers.cloudflare.com/workers-ai/models/clef/ . + public const string CloudflareClef = "clef"; + /// Cloudflare native body model: https://developers.cloudflare.com/workers-ai/models/clef-flash/ . + public const string CloudflareClefFlash = "clef-flash"; +} diff --git a/src/ManagedCode.LlmTck/Decisions/LlmTckDecisionModels.cs b/src/ManagedCode.LlmTestServer/Decisions/LlmTestServerDecisionModels.cs similarity index 52% rename from src/ManagedCode.LlmTck/Decisions/LlmTckDecisionModels.cs rename to src/ManagedCode.LlmTestServer/Decisions/LlmTestServerDecisionModels.cs index 54b24c5..cbf4664 100644 --- a/src/ManagedCode.LlmTck/Decisions/LlmTckDecisionModels.cs +++ b/src/ManagedCode.LlmTestServer/Decisions/LlmTestServerDecisionModels.cs @@ -1,40 +1,40 @@ -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Scenarios; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Scenarios; -namespace ManagedCode.LlmTck.Decisions; +namespace ManagedCode.LlmTestServer.Decisions; -public enum LlmTckDecisionProvider { Unspecified, TypeSafe, Kev, Cloudflare, OpenAI } +public enum LlmTestServerDecisionProvider { Unspecified, TypeSafe, Kev, Cloudflare, OpenAI } -public enum LlmTckDecisionKind { Predicate, Choice, Score, Box } +public enum LlmTestServerDecisionKind { Predicate, Choice, Score, Box } -public sealed record LlmTckDecisionQuestion +public sealed record LlmTestServerDecisionQuestion { public string Name { get; init; } = string.Empty; - public LlmTckDecisionKind Kind { get; init; } + public LlmTestServerDecisionKind Kind { get; init; } public List Options { get; init; } = []; } -public sealed record LlmTckDecisionRequest +public sealed record LlmTestServerDecisionRequest { - public LlmTckDecisionProvider Provider { get; init; } + public LlmTestServerDecisionProvider Provider { get; init; } public string ModelId { get; init; } = string.Empty; public string Input { get; init; } = string.Empty; - public List Questions { get; init; } = []; + public List Questions { get; init; } = []; } -public sealed record LlmTckDecisionAnswer +public sealed record LlmTestServerDecisionAnswer { - public LlmTckDecisionKind Kind { get; init; } + public LlmTestServerDecisionKind Kind { get; init; } public bool Refused { get; init; } public double? Probability { get; init; } public string? Choice { get; init; } public double? Score { get; init; } public double? Confidence { get; init; } public Dictionary Probabilities { get; init; } = []; - public LlmTckBoundingBox? Box { get; init; } + public LlmTestServerBoundingBox? Box { get; init; } } -public sealed record LlmTckBoundingBox +public sealed record LlmTestServerBoundingBox { public required double Top { get; init; } public required double Bottom { get; init; } @@ -45,33 +45,33 @@ public sealed record LlmTckBoundingBox && Top < Bottom && Left < Right; } -public sealed record LlmTckDecisionScenario +public sealed record LlmTestServerDecisionScenario { public string Id { get; init; } = string.Empty; public string ModelId { get; init; } = string.Empty; public string? ResolvedModelId { get; init; } public string? ExpectedInput { get; init; } - public Dictionary Answers { get; init; } = []; - public LlmTckTokenUsage Usage { get; init; } = new(); - public LlmTckScenarioError? Error { get; init; } + public Dictionary Answers { get; init; } = []; + public LlmTestServerTokenUsage Usage { get; init; } = new(); + public LlmTestServerScenarioError? Error { get; init; } public bool AllowMalformedResponse { get; init; } - public LlmTckDecisionMetadata Metadata { get; init; } = new(); + public LlmTestServerDecisionMetadata Metadata { get; init; } = new(); } -public sealed record LlmTckDecisionResult +public sealed record LlmTestServerDecisionResult { public bool IsSuccess { get; init; } public string ModelId { get; init; } = string.Empty; public string? ScenarioId { get; init; } - public Dictionary Answers { get; init; } = []; - public LlmTckTokenUsage Usage { get; init; } = new(); + public Dictionary Answers { get; init; } = []; + public LlmTestServerTokenUsage Usage { get; init; } = new(); public int StatusCode { get; init; } = 200; public string? ErrorCode { get; init; } public string? ErrorMessage { get; init; } - public LlmTckDecisionMetadata Metadata { get; init; } = new(); + public LlmTestServerDecisionMetadata Metadata { get; init; } = new(); } -public sealed record LlmTckDecisionMetadata +public sealed record LlmTestServerDecisionMetadata { public double LatencyMilliseconds { get; init; } public bool? Truncated { get; init; } diff --git a/src/ManagedCode.LlmTestServer/Determinism/LlmTestServerResponseMetadata.cs b/src/ManagedCode.LlmTestServer/Determinism/LlmTestServerResponseMetadata.cs new file mode 100644 index 0000000..22c2dc6 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Determinism/LlmTestServerResponseMetadata.cs @@ -0,0 +1,42 @@ +using System.Globalization; +using System.Security.Cryptography; +using System.Text; + +namespace ManagedCode.LlmTestServer.Determinism; + +public sealed record LlmTestServerDeterminism +{ + public string Seed { get; init; } = string.Empty; + public DateTimeOffset FixedUtc { get; init; } +} + +public sealed record LlmTestServerResponseMetadata +{ + public string Identity { get; init; } = Guid.NewGuid().ToString("N"); + public DateTimeOffset CreatedAt { get; init; } = DateTimeOffset.UtcNow; + public string ClockMode { get; init; } = "live"; + public long Sequence { get; init; } + + public string IdFor(string domain) + { + ArgumentException.ThrowIfNullOrWhiteSpace(domain); + return Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(Identity + ":" + domain)))[..32]; + } + + public static LlmTestServerResponseMetadata Live() + { + return new(); + } + + internal static LlmTestServerResponseMetadata Create(LlmTestServerDeterminism? settings, string provider, long sequence) + { + return settings is null ? new() { Sequence = sequence } : new() + { + Identity = Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes( + settings.Seed + "\n" + provider + "\n" + sequence.ToString(CultureInfo.InvariantCulture)))), + CreatedAt = settings.FixedUtc.ToUniversalTime(), + ClockMode = "fixed", + Sequence = sequence, + }; + } +} diff --git a/src/ManagedCode.LlmTestServer/Diagnostics/LlmTestServerProviderHttpTrace.cs b/src/ManagedCode.LlmTestServer/Diagnostics/LlmTestServerProviderHttpTrace.cs new file mode 100644 index 0000000..2745b28 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Diagnostics/LlmTestServerProviderHttpTrace.cs @@ -0,0 +1,123 @@ +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Scenarios; + +namespace ManagedCode.LlmTestServer.Diagnostics; + +public sealed record LlmTestServerProviderHttpTrace +{ + public string? WireClockMode { get; init; } + public DateTimeOffset? WireCreatedAt { get; init; } + public long? WireSequence { get; init; } + public long Sequence { get; init; } + public string? RetryGroupId { get; init; } + public string? RetryAfter { get; init; } + public LlmTestServerOutputDisposition? OutputDisposition { get; init; } + public LlmTestServerStreamDelivery? StreamDelivery { get; init; } + public bool StreamDeliveryApplied { get; init; } + public int EmittedContentFrames { get; init; } + + public bool? TerminalMarkerObserved { get; init; } + public IReadOnlyList NativeTools { get; init; } = []; + public IReadOnlyList ResponseOutputItems { get; init; } = []; + public IReadOnlyList ResponseToolCalls { get; init; } = []; + public string? ConversationId { get; init; } + public bool ConversationIdWasProvided { get; init; } + public string? StepId { get; init; } + public int? TurnIndex { get; init; } + public bool IsReplay { get; init; } + + public string? SessionId { get; init; } + + public string RequestId { get; init; } = string.Empty; + + public string ProviderId { get; init; } = string.Empty; + + public string ProviderNamespace { get; init; } = string.Empty; + + public string Method { get; init; } = string.Empty; + + public string Path { get; init; } = string.Empty; + + public string? Route { get; init; } + + public string? OperationId { get; init; } + + public double? StartedMonotonicSeconds { get; init; } + + public double? CompletedMonotonicSeconds { get; init; } + + public DateTimeOffset StartedAt { get; init; } + + public DateTimeOffset? CompletedAt { get; init; } + + public double? DurationMilliseconds { get; init; } + + public int? StatusCode { get; init; } + + public bool IsCompleted { get; init; } + + public LlmTestServerProviderHttpTraceTransportState TransportState { get; init; } + + public bool? IsSuccess => !IsCompleted + ? null + : TransportState != LlmTestServerProviderHttpTraceTransportState.Completed + ? false + : StatusCode is null + ? null + : StatusCode < 400; + + public bool IsStreaming { get; init; } + + public string? RequestContentType { get; init; } + + public long? RequestByteCount { get; init; } + + public string? RequestPreview { get; init; } + + public bool RequestPreviewTruncated { get; init; } + + public string? ResponseContentType { get; init; } + + public long ResponseByteCount { get; init; } + + public string? ResponsePreview { get; init; } + + public bool ResponsePreviewTruncated { get; init; } + + public string? ModelId { get; init; } + + public string? ScenarioId { get; init; } + + public LlmTestServerEventKind? RuntimeEventKind { get; init; } + + public string? RuntimeMessage { get; init; } + + public string? RuntimeResponse { get; init; } + + public string? ErrorCode { get; init; } + + public string? ErrorMessage { get; init; } + + public string? TransportError { get; init; } + + public LlmTestServerTokenUsage? Usage { get; init; } + + public IReadOnlyList Messages { get; init; } = []; + + public IReadOnlyList StreamChunks { get; init; } = []; + + public bool RuntimePayloadTruncated { get; init; } + + public bool RetentionTruncated { get; init; } + + public int RetainedByteCount { get; init; } +} + +public enum LlmTestServerProviderHttpTraceTransportState +{ + InProgress, + Completed, + Cancelled, + Aborted, + Faulted, +} diff --git a/src/ManagedCode.LlmTestServer/Diagnostics/LlmTestServerRequestHeaders.cs b/src/ManagedCode.LlmTestServer/Diagnostics/LlmTestServerRequestHeaders.cs new file mode 100644 index 0000000..65267c3 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Diagnostics/LlmTestServerRequestHeaders.cs @@ -0,0 +1,6 @@ +namespace ManagedCode.LlmTestServer.Diagnostics; + +public static class LlmTestServerRequestHeaders +{ + public const string RetryGroup = "X-Llm-Test-Server-Retry-Group"; +} diff --git a/src/ManagedCode.LlmTestServer/Diagnostics/LlmTestServerRequestQuery.cs b/src/ManagedCode.LlmTestServer/Diagnostics/LlmTestServerRequestQuery.cs new file mode 100644 index 0000000..ddda9d7 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Diagnostics/LlmTestServerRequestQuery.cs @@ -0,0 +1,74 @@ +namespace ManagedCode.LlmTestServer.Diagnostics; + +public enum LlmTestServerRequestOrder { Descending, Ascending } + +public sealed record LlmTestServerRequestQuery +{ + public string? Provider { get; init; } + public string? Model { get; init; } + public string? RequestId { get; init; } + public int? Status { get; init; } + public string? Operation { get; init; } + public string? Scenario { get; init; } + public string? Conversation { get; init; } + public string? Step { get; init; } + public bool? Streaming { get; init; } + public bool? Completed { get; init; } + public bool? Success { get; init; } + public string? ErrorCode { get; init; } + public DateTimeOffset? From { get; init; } + public DateTimeOffset? To { get; init; } + public LlmTestServerRequestOrder Order { get; init; } + public int Offset { get; init; } + public int Limit { get; init; } = 500; +} + +public sealed record LlmTestServerRequestSnapshot +{ + public IReadOnlyList Requests { get; init; } = []; + public int TotalMatched { get; init; } + public long TotalObserved { get; init; } + public long EvictedCount { get; init; } + public bool RetentionTruncated { get; init; } + public long RetainedByteCount { get; init; } + public long? FirstSequence { get; init; } + public long? LastSequence { get; init; } +} + +public sealed record LlmTestServerRequestExport +{ + public const int CurrentSchemaVersion = 1; + public int SchemaVersion { get; init; } = CurrentSchemaVersion; + public LlmTestServerRequestQuery Query { get; init; } = new(); + public LlmTestServerRequestSnapshot Evidence { get; init; } = new(); +} + +public static class LlmTestServerRequestQueries +{ + public static LlmTestServerRequestSnapshot Apply(LlmTestServerRequestSnapshot snapshot, LlmTestServerRequestQuery query) + { + ArgumentNullException.ThrowIfNull(snapshot); + ArgumentNullException.ThrowIfNull(query); + if (query.Offset < 0 || query.Limit is < 1 or > 500 || query.Status is < 100 or > 599 + || !Enum.IsDefined(query.Order) || query.From > query.To) + { throw new ArgumentException("Invalid request query range, order or pagination.", nameof(query)); } + var matches = snapshot.Requests.Where(entry => + (query.Provider is null || entry.ProviderId == query.Provider) + && (query.Model is null || entry.ModelId == query.Model) + && (query.RequestId is null || entry.RequestId == query.RequestId) + && (query.Status is null || entry.StatusCode == query.Status) + && (query.Operation is null || entry.OperationId == query.Operation) + && (query.Scenario is null || entry.ScenarioId == query.Scenario) + && (query.Conversation is null || entry.ConversationId == query.Conversation) + && (query.Step is null || entry.StepId == query.Step) + && (query.Streaming is null || entry.IsStreaming == query.Streaming) + && (query.Completed is null || entry.IsCompleted == query.Completed) + && (query.Success is null || entry.IsSuccess == query.Success) + && (query.ErrorCode is null || entry.ErrorCode == query.ErrorCode) + && (query.From is null || entry.StartedAt >= query.From) + && (query.To is null || entry.StartedAt <= query.To)); + var ordered = (query.Order == LlmTestServerRequestOrder.Ascending + ? matches.OrderBy(entry => entry.Sequence) : matches.OrderByDescending(entry => entry.Sequence)).ToArray(); + return snapshot with { Requests = Array.AsReadOnly(ordered.Skip(query.Offset).Take(query.Limit).ToArray()), TotalMatched = ordered.Length }; + } +} diff --git a/src/ManagedCode.LlmTck/ManagedCode.LlmTck.csproj b/src/ManagedCode.LlmTestServer/ManagedCode.LlmTestServer.csproj similarity index 89% rename from src/ManagedCode.LlmTck/ManagedCode.LlmTck.csproj rename to src/ManagedCode.LlmTestServer/ManagedCode.LlmTestServer.csproj index 470c9c4..a83735f 100644 --- a/src/ManagedCode.LlmTck/ManagedCode.LlmTck.csproj +++ b/src/ManagedCode.LlmTestServer/ManagedCode.LlmTestServer.csproj @@ -1,6 +1,6 @@ - ManagedCode.LlmTck + ManagedCode.LlmTestServer Deterministic scenario runtime for LLM API compatibility tests. diff --git a/src/ManagedCode.LlmTck/Models/LlmTckKnownModelIds.cs b/src/ManagedCode.LlmTestServer/Models/LlmTestServerKnownModelIds.cs similarity index 83% rename from src/ManagedCode.LlmTck/Models/LlmTckKnownModelIds.cs rename to src/ManagedCode.LlmTestServer/Models/LlmTestServerKnownModelIds.cs index 1c8c6bb..5a00557 100644 --- a/src/ManagedCode.LlmTck/Models/LlmTckKnownModelIds.cs +++ b/src/ManagedCode.LlmTestServer/Models/LlmTestServerKnownModelIds.cs @@ -1,11 +1,8 @@ -namespace ManagedCode.LlmTck.Models; +namespace ManagedCode.LlmTestServer.Models; -public static class LlmTckKnownModelIds +public static class LlmTestServerKnownModelIds { - public const string JevLatest = "jev-latest"; - public const string KevLatest = "kev-latest"; - public const string Clef = "clef"; - public const string ClefFlash = "clef-flash"; + public const string ClaudeSonnet55 = "claude-sonnet-5-5"; public const string Gpt55 = "gpt-5.5"; public const string Gpt55Pro = "gpt-5.5-pro"; diff --git a/src/ManagedCode.LlmTestServer/Models/LlmTestServerModel.cs b/src/ManagedCode.LlmTestServer/Models/LlmTestServerModel.cs new file mode 100644 index 0000000..62b9985 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Models/LlmTestServerModel.cs @@ -0,0 +1,14 @@ +namespace ManagedCode.LlmTestServer.Models; + +public sealed record LlmTestServerModel +{ + public string Id { get; init; } = LlmTestServerKnownModelIds.Gpt41Mini; + + public LlmTestServerModelKind Kind { get; init; } = LlmTestServerModelKind.Chat; + + public string OwnedBy { get; init; } = "llm-test-server"; + + public ManagedCode.LlmTestServer.Decisions.LlmTestServerDecisionProvider DecisionProvider { get; init; } + + public int ReasoningTokens { get; init; } +} diff --git a/src/ManagedCode.LlmTestServer/Models/LlmTestServerModelKind.cs b/src/ManagedCode.LlmTestServer/Models/LlmTestServerModelKind.cs new file mode 100644 index 0000000..7e6166f --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Models/LlmTestServerModelKind.cs @@ -0,0 +1,11 @@ +namespace ManagedCode.LlmTestServer.Models; + +public enum LlmTestServerModelKind +{ + Chat, + Embedding, + Image, + Audio, + Video, + Decision, +} diff --git a/src/ManagedCode.LlmTck/Providers/LlmTckCompatibilityTags.cs b/src/ManagedCode.LlmTestServer/Providers/LlmTestServerCompatibilityTags.cs similarity index 92% rename from src/ManagedCode.LlmTck/Providers/LlmTckCompatibilityTags.cs rename to src/ManagedCode.LlmTestServer/Providers/LlmTestServerCompatibilityTags.cs index a332db5..787232b 100644 --- a/src/ManagedCode.LlmTck/Providers/LlmTckCompatibilityTags.cs +++ b/src/ManagedCode.LlmTestServer/Providers/LlmTestServerCompatibilityTags.cs @@ -1,6 +1,6 @@ -namespace ManagedCode.LlmTck.Providers; +namespace ManagedCode.LlmTestServer.Providers; -public static class LlmTckCompatibilityTags +public static class LlmTestServerCompatibilityTags { public const string SystemOne = "systemone"; public const string Cloudflare = "cloudflare"; diff --git a/src/ManagedCode.LlmTck/Providers/LlmTckProtocolFamily.cs b/src/ManagedCode.LlmTestServer/Providers/LlmTestServerProtocolFamily.cs similarity index 74% rename from src/ManagedCode.LlmTck/Providers/LlmTckProtocolFamily.cs rename to src/ManagedCode.LlmTestServer/Providers/LlmTestServerProtocolFamily.cs index b0cebca..35bd0ea 100644 --- a/src/ManagedCode.LlmTck/Providers/LlmTckProtocolFamily.cs +++ b/src/ManagedCode.LlmTestServer/Providers/LlmTestServerProtocolFamily.cs @@ -1,6 +1,6 @@ -namespace ManagedCode.LlmTck.Providers; +namespace ManagedCode.LlmTestServer.Providers; -public enum LlmTckProtocolFamily +public enum LlmTestServerProtocolFamily { OpenAI, AzureOpenAI, diff --git a/src/ManagedCode.LlmTestServer/Providers/LlmTestServerProviderApiContract.cs b/src/ManagedCode.LlmTestServer/Providers/LlmTestServerProviderApiContract.cs new file mode 100644 index 0000000..a54bf51 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Providers/LlmTestServerProviderApiContract.cs @@ -0,0 +1,12 @@ +namespace ManagedCode.LlmTestServer.Providers; + +public sealed record LlmTestServerProviderApiContract +{ + public required string DocumentationUrl { get; init; } + + public required string DocumentationRetrievedOn { get; init; } + + public string? DocumentationVersion { get; init; } + + public IReadOnlyList Operations { get; init; } = []; +} diff --git a/src/ManagedCode.LlmTck/Providers/LlmTckProviderCapability.cs b/src/ManagedCode.LlmTestServer/Providers/LlmTestServerProviderCapability.cs similarity index 67% rename from src/ManagedCode.LlmTck/Providers/LlmTckProviderCapability.cs rename to src/ManagedCode.LlmTestServer/Providers/LlmTestServerProviderCapability.cs index 8543c21..9ea3d7c 100644 --- a/src/ManagedCode.LlmTck/Providers/LlmTckProviderCapability.cs +++ b/src/ManagedCode.LlmTestServer/Providers/LlmTestServerProviderCapability.cs @@ -1,6 +1,6 @@ -namespace ManagedCode.LlmTck.Providers; +namespace ManagedCode.LlmTestServer.Providers; -public enum LlmTckProviderCapability +public enum LlmTestServerProviderCapability { Models, Chat, diff --git a/src/ManagedCode.LlmTck/Providers/LlmTckProviderOperation.cs b/src/ManagedCode.LlmTestServer/Providers/LlmTestServerProviderOperation.cs similarity index 68% rename from src/ManagedCode.LlmTck/Providers/LlmTckProviderOperation.cs rename to src/ManagedCode.LlmTestServer/Providers/LlmTestServerProviderOperation.cs index 00deb17..9a10655 100644 --- a/src/ManagedCode.LlmTck/Providers/LlmTckProviderOperation.cs +++ b/src/ManagedCode.LlmTestServer/Providers/LlmTestServerProviderOperation.cs @@ -1,6 +1,6 @@ -namespace ManagedCode.LlmTck.Providers; +namespace ManagedCode.LlmTestServer.Providers; -public sealed record LlmTckProviderOperation +public sealed record LlmTestServerProviderOperation { public required string Id { get; init; } @@ -18,5 +18,5 @@ public sealed record LlmTckProviderOperation public bool ImplementedByHosting { get; init; } - public IReadOnlyList Capabilities { get; init; } = []; + public IReadOnlyList Capabilities { get; init; } = []; } diff --git a/src/ManagedCode.LlmTck/Providers/LlmTckProviderOperationIds.cs b/src/ManagedCode.LlmTestServer/Providers/LlmTestServerProviderOperationIds.cs similarity index 97% rename from src/ManagedCode.LlmTck/Providers/LlmTckProviderOperationIds.cs rename to src/ManagedCode.LlmTestServer/Providers/LlmTestServerProviderOperationIds.cs index 1c3c288..7d0e115 100644 --- a/src/ManagedCode.LlmTck/Providers/LlmTckProviderOperationIds.cs +++ b/src/ManagedCode.LlmTestServer/Providers/LlmTestServerProviderOperationIds.cs @@ -1,6 +1,6 @@ -namespace ManagedCode.LlmTck.Providers; +namespace ManagedCode.LlmTestServer.Providers; -public static class LlmTckProviderOperationIds +public static class LlmTestServerProviderOperationIds { public static class SystemOne { @@ -83,6 +83,7 @@ public static class Gemini public const string ModelsGenerateContent = "models.generateContent"; public const string ModelsStreamGenerateContent = "models.streamGenerateContent"; public const string ModelsEmbedContent = "models.embedContent"; + public const string ModelsBatchEmbedContents = "models.batchEmbedContents"; public const string ModelsPredictLongRunningVideo = "models.predictLongRunning.video"; public const string ModelsOperationsGetVideo = "models.operations.get.video"; public const string FilesGetGeneratedVideo = "files.get.generatedVideo"; diff --git a/src/ManagedCode.LlmTestServer/Providers/LlmTestServerProviderProfile.cs b/src/ManagedCode.LlmTestServer/Providers/LlmTestServerProviderProfile.cs new file mode 100644 index 0000000..49363db --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Providers/LlmTestServerProviderProfile.cs @@ -0,0 +1,18 @@ +namespace ManagedCode.LlmTestServer.Providers; + +public sealed record LlmTestServerProviderProfile +{ + public required string Id { get; init; } + + public required string DisplayName { get; init; } + + public required LlmTestServerProtocolFamily Protocol { get; init; } + + public string DefaultEndpointPath { get; init; } = "/"; + + public IReadOnlyList Capabilities { get; init; } = []; + + public IReadOnlyList CompatibilityTags { get; init; } = []; + + public required LlmTestServerProviderApiContract ApiContract { get; init; } +} diff --git a/src/ManagedCode.LlmTck/Providers/LlmTckProviderRouteNamespaces.cs b/src/ManagedCode.LlmTestServer/Providers/LlmTestServerProviderRouteNamespaces.cs similarity index 93% rename from src/ManagedCode.LlmTck/Providers/LlmTckProviderRouteNamespaces.cs rename to src/ManagedCode.LlmTestServer/Providers/LlmTestServerProviderRouteNamespaces.cs index 078453a..63a3c36 100644 --- a/src/ManagedCode.LlmTck/Providers/LlmTckProviderRouteNamespaces.cs +++ b/src/ManagedCode.LlmTestServer/Providers/LlmTestServerProviderRouteNamespaces.cs @@ -1,6 +1,6 @@ -namespace ManagedCode.LlmTck.Providers; +namespace ManagedCode.LlmTestServer.Providers; -public static class LlmTckProviderRouteNamespaces +public static class LlmTestServerProviderRouteNamespaces { public const string SystemOne = "/systemone"; public const string Cloudflare = "/cloudflare"; diff --git a/src/ManagedCode.LlmTck/Providers/LlmTckRequestValidationResult.cs b/src/ManagedCode.LlmTestServer/Providers/LlmTestServerRequestValidationResult.cs similarity index 53% rename from src/ManagedCode.LlmTck/Providers/LlmTckRequestValidationResult.cs rename to src/ManagedCode.LlmTestServer/Providers/LlmTestServerRequestValidationResult.cs index d466761..70e3ee7 100644 --- a/src/ManagedCode.LlmTck/Providers/LlmTckRequestValidationResult.cs +++ b/src/ManagedCode.LlmTestServer/Providers/LlmTestServerRequestValidationResult.cs @@ -1,7 +1,7 @@ -namespace ManagedCode.LlmTck.Providers; +namespace ManagedCode.LlmTestServer.Providers; /// Provider-neutral result of validating a provider request before mapping it. -public sealed record LlmTckRequestValidationResult(string? Error = null) +public sealed record LlmTestServerRequestValidationResult(string? Error = null) { public bool IsValid => Error is null; } diff --git a/src/ManagedCode.LlmTck/Runtime/ILlmTckRuntime.cs b/src/ManagedCode.LlmTestServer/Runtime/ILlmTestServerRuntime.cs similarity index 53% rename from src/ManagedCode.LlmTck/Runtime/ILlmTckRuntime.cs rename to src/ManagedCode.LlmTestServer/Runtime/ILlmTestServerRuntime.cs index 6c31b83..5ed74a4 100644 --- a/src/ManagedCode.LlmTck/Runtime/ILlmTckRuntime.cs +++ b/src/ManagedCode.LlmTestServer/Runtime/ILlmTestServerRuntime.cs @@ -1,47 +1,48 @@ -using ManagedCode.LlmTck.Configuration; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.Scenarios; +using ManagedCode.LlmTestServer.Configuration; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Scenarios; -namespace ManagedCode.LlmTck.Runtime; +namespace ManagedCode.LlmTestServer.Runtime; -public interface ILlmTckRuntime +public interface ILlmTestServerRuntime { - Task ConfigureAsync(LlmTckConfiguration configuration, CancellationToken cancellationToken = default); + Task ConfigureAsync(LlmTestServerConfiguration configuration, CancellationToken cancellationToken = default); Task ResetAsync(CancellationToken cancellationToken = default); bool IsBearerTokenAccepted(string? bearerToken); - IReadOnlyList GetModels(); + IReadOnlyList GetModels(); - Task CompleteChatAsync( - LlmTckChatRequest request, + Task CompleteChatAsync( + LlmTestServerChatRequest request, string? bearerToken = null, CancellationToken cancellationToken = default ); - Task CreateEmbeddingAsync( + Task CreateEmbeddingAsync( string modelId, IReadOnlyList inputs, string? bearerToken = null, + IReadOnlyList? outputDimensions = null, CancellationToken cancellationToken = default ); - Task GenerateImageAsync( + Task GenerateImageAsync( string modelId, string prompt, string? bearerToken = null, CancellationToken cancellationToken = default ); - Task GenerateAudioAsync( + Task GenerateAudioAsync( string modelId, string input, string? bearerToken = null, CancellationToken cancellationToken = default ); - Task TranscribeAudioAsync( + Task TranscribeAudioAsync( string modelId, string fileName, string? prompt = null, @@ -49,7 +50,7 @@ Task TranscribeAudioAsync( CancellationToken cancellationToken = default ); - Task TranslateAudioAsync( + Task TranslateAudioAsync( string modelId, string fileName, string? prompt = null, @@ -57,17 +58,17 @@ Task TranslateAudioAsync( CancellationToken cancellationToken = default ); - Task GenerateVideoAsync( + Task GenerateVideoAsync( string modelId, string prompt, string? bearerToken = null, CancellationToken cancellationToken = default ); - Task DecideAsync( - ManagedCode.LlmTck.Decisions.LlmTckDecisionRequest request, + Task DecideAsync( + ManagedCode.LlmTestServer.Decisions.LlmTestServerDecisionRequest request, string? bearerToken = null, CancellationToken cancellationToken = default); - LlmTckAssertionSummary GetAssertionSummary(); + LlmTestServerAssertionSummary GetAssertionSummary(); } diff --git a/src/ManagedCode.LlmTck/Runtime/ILlmTckRuntimeEventLookup.cs b/src/ManagedCode.LlmTestServer/Runtime/ILlmTestServerRuntimeEventLookup.cs similarity index 52% rename from src/ManagedCode.LlmTck/Runtime/ILlmTckRuntimeEventLookup.cs rename to src/ManagedCode.LlmTestServer/Runtime/ILlmTestServerRuntimeEventLookup.cs index e422495..ad7fc71 100644 --- a/src/ManagedCode.LlmTck/Runtime/ILlmTckRuntimeEventLookup.cs +++ b/src/ManagedCode.LlmTestServer/Runtime/ILlmTestServerRuntimeEventLookup.cs @@ -1,10 +1,10 @@ -namespace ManagedCode.LlmTck.Runtime; +namespace ManagedCode.LlmTestServer.Runtime; /// /// Optional fast lookup surface for hosts that correlate a completed request with its runtime /// outcome without materializing the aggregate assertion summary. /// -public interface ILlmTckRuntimeEventLookup +public interface ILlmTestServerRuntimeEventLookup { - bool TryGetEvent(string requestId, out LlmTckRuntimeEvent? runtimeEvent); + bool TryGetEvent(string requestId, out LlmTestServerRuntimeEvent? runtimeEvent); } diff --git a/src/ManagedCode.LlmTestServer/Runtime/ILlmTestServerRuntimeRequestScope.cs b/src/ManagedCode.LlmTestServer/Runtime/ILlmTestServerRuntimeRequestScope.cs new file mode 100644 index 0000000..c21d4b2 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Runtime/ILlmTestServerRuntimeRequestScope.cs @@ -0,0 +1,10 @@ +namespace ManagedCode.LlmTestServer.Runtime; + +/// +/// Optional provider-neutral correlation scope used by hosts to associate runtime events with an +/// external request without changing the operation methods on . +/// +public interface ILlmTestServerRuntimeRequestScope +{ + IDisposable BeginRequest(string requestId, string? conversationId = null, string? providerNamespace = null); +} diff --git a/src/ManagedCode.LlmTestServer/Runtime/ILlmTestServerVideoStore.cs b/src/ManagedCode.LlmTestServer/Runtime/ILlmTestServerVideoStore.cs new file mode 100644 index 0000000..c20d558 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Runtime/ILlmTestServerVideoStore.cs @@ -0,0 +1,9 @@ +namespace ManagedCode.LlmTestServer.Runtime; + +public interface ILlmTestServerVideoStore +{ + LlmTestServerVideoResult StoreVideo(string provider, LlmTestServerVideoResult video); + IReadOnlyList ListVideos(string provider); + LlmTestServerVideoResult? FindVideo(string provider, string id, bool generation = false); + bool DeleteVideo(string provider, string id); +} diff --git a/src/ManagedCode.LlmTck/Runtime/LlmTckAssertionSummary.cs b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerAssertionSummary.cs similarity index 77% rename from src/ManagedCode.LlmTck/Runtime/LlmTckAssertionSummary.cs rename to src/ManagedCode.LlmTestServer/Runtime/LlmTestServerAssertionSummary.cs index 23fa12b..d346301 100644 --- a/src/ManagedCode.LlmTck/Runtime/LlmTckAssertionSummary.cs +++ b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerAssertionSummary.cs @@ -1,6 +1,6 @@ -namespace ManagedCode.LlmTck.Runtime; +namespace ManagedCode.LlmTestServer.Runtime; -public sealed record LlmTckAssertionSummary +public sealed record LlmTestServerAssertionSummary { public int TotalEvents { get; init; } @@ -28,5 +28,5 @@ public sealed record LlmTckAssertionSummary public int TotalTokens { get; init; } - public List Events { get; init; } = []; + public List Events { get; init; } = []; } diff --git a/src/ManagedCode.LlmTck/Runtime/LlmTckEventKind.cs b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerEventKind.cs similarity index 57% rename from src/ManagedCode.LlmTck/Runtime/LlmTckEventKind.cs rename to src/ManagedCode.LlmTestServer/Runtime/LlmTestServerEventKind.cs index bfba4e5..13fe6b0 100644 --- a/src/ManagedCode.LlmTck/Runtime/LlmTckEventKind.cs +++ b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerEventKind.cs @@ -1,6 +1,6 @@ -namespace ManagedCode.LlmTck.Runtime; +namespace ManagedCode.LlmTestServer.Runtime; -public enum LlmTckEventKind +public enum LlmTestServerEventKind { Matched, Unmatched, diff --git a/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerInputTokenComposition.cs b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerInputTokenComposition.cs new file mode 100644 index 0000000..1f6f19c --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerInputTokenComposition.cs @@ -0,0 +1,21 @@ +namespace ManagedCode.LlmTestServer.Runtime; + +public sealed record LlmTestServerInputTokenComposition +{ + public int SystemTokens { get; init; } + + public int UserTokens { get; init; } + + public int AssistantTokens { get; init; } + + public int ToolCallTokens { get; init; } + + public int ToolResultTokens { get; init; } + + public int ToolDefinitionTokens { get; init; } + + public int OtherTokens { get; init; } + + public int TotalTokens => SystemTokens + UserTokens + AssistantTokens + ToolCallTokens + ToolResultTokens + + ToolDefinitionTokens + OtherTokens; +} diff --git a/src/ManagedCode.LlmTck/Runtime/LlmTckResults.cs b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerResults.cs similarity index 54% rename from src/ManagedCode.LlmTck/Runtime/LlmTckResults.cs rename to src/ManagedCode.LlmTestServer/Runtime/LlmTestServerResults.cs index 1b95f04..43a1c6f 100644 --- a/src/ManagedCode.LlmTck/Runtime/LlmTckResults.cs +++ b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerResults.cs @@ -1,38 +1,47 @@ -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.Scenarios; +using ManagedCode.LlmTestServer.Determinism; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Scenarios; -namespace ManagedCode.LlmTck.Runtime; +namespace ManagedCode.LlmTestServer.Runtime; -public sealed record LlmTckChatResult +public sealed record LlmTestServerChatResult { + public LlmTestServerOutputDisposition? OutputDisposition { get; init; } + + public LlmTestServerStreamDelivery? StreamDelivery { get; init; } + + public List OutputItems { get; init; } = []; + + public LlmTestServerResponseMetadata? WireMetadata { get; init; } + public bool IsSuccess { get; init; } public int StatusCode { get; init; } = 200; - public string ModelId { get; init; } = LlmTckKnownModelIds.Gpt41Mini; + public string ModelId { get; init; } = LlmTestServerKnownModelIds.Gpt41Mini; public string? ScenarioId { get; init; } public string? ResponseId { get; init; } - public List ToolCalls { get; init; } = []; + public List ToolCalls { get; init; } = []; public string Content { get; init; } = string.Empty; public List StreamChunks { get; init; } = []; - public LlmTckTokenUsage Usage { get; init; } = new(); + public LlmTestServerTokenUsage Usage { get; init; } = new(); public string? ErrorCode { get; init; } public string? ErrorMessage { get; init; } - public static LlmTckChatResult Success( + public static LlmTestServerChatResult Success( string modelId, string scenarioId, string content, IEnumerable streamChunks, - LlmTckTokenUsage? usage = null + LlmTestServerTokenUsage? usage = null ) { return new() @@ -42,17 +51,17 @@ public static LlmTckChatResult Success( ScenarioId = scenarioId, Content = content, StreamChunks = [.. streamChunks], - Usage = usage ?? new LlmTckTokenUsage(), + Usage = usage ?? new LlmTestServerTokenUsage(), }; } - public static LlmTckChatResult Failure( + public static LlmTestServerChatResult Failure( string modelId, int statusCode, string errorCode, string errorMessage, string? scenarioId = null, - LlmTckTokenUsage? usage = null + LlmTestServerTokenUsage? usage = null ) { return new() @@ -63,18 +72,20 @@ public static LlmTckChatResult Failure( StatusCode = statusCode, ErrorCode = errorCode, ErrorMessage = errorMessage, - Usage = usage ?? new LlmTckTokenUsage(), + Usage = usage ?? new LlmTestServerTokenUsage(), }; } } -public sealed record LlmTckEmbeddingResult +public sealed record LlmTestServerEmbeddingResult { + public LlmTestServerResponseMetadata? WireMetadata { get; init; } + public bool IsSuccess { get; init; } public int StatusCode { get; init; } = 200; - public string ModelId { get; init; } = LlmTckKnownModelIds.TextEmbedding3Small; + public string ModelId { get; init; } = LlmTestServerKnownModelIds.TextEmbedding3Small; public List> Vectors { get; init; } = []; @@ -83,13 +94,15 @@ public sealed record LlmTckEmbeddingResult public string? ErrorMessage { get; init; } } -public sealed record LlmTckImageResult +public sealed record LlmTestServerImageResult { + public LlmTestServerResponseMetadata? WireMetadata { get; init; } + public bool IsSuccess { get; init; } public int StatusCode { get; init; } = 200; - public string ModelId { get; init; } = LlmTckKnownModelIds.GptImage1; + public string ModelId { get; init; } = LlmTestServerKnownModelIds.GptImage1; public string DataUri { get; init; } = string.Empty; @@ -98,13 +111,15 @@ public sealed record LlmTckImageResult public string? ErrorMessage { get; init; } } -public sealed record LlmTckAudioResult +public sealed record LlmTestServerAudioResult { + public LlmTestServerResponseMetadata? WireMetadata { get; init; } + public bool IsSuccess { get; init; } public int StatusCode { get; init; } = 200; - public string ModelId { get; init; } = LlmTckKnownModelIds.Gpt4OMiniTts; + public string ModelId { get; init; } = LlmTestServerKnownModelIds.Gpt4OMiniTts; public byte[] Bytes { get; init; } = []; @@ -115,15 +130,17 @@ public sealed record LlmTckAudioResult public string? ErrorMessage { get; init; } } -public sealed record LlmTckTranscriptionResult +public sealed record LlmTestServerTranscriptionResult { - public LlmTckTranscriptionUsage? Usage { get; init; } + public LlmTestServerResponseMetadata? WireMetadata { get; init; } + + public LlmTestServerTranscriptionUsage? Usage { get; init; } public bool IsSuccess { get; init; } public int StatusCode { get; init; } = 200; - public string ModelId { get; init; } = LlmTckKnownModelIds.Gpt4OMiniTts; + public string ModelId { get; init; } = LlmTestServerKnownModelIds.Gpt4OMiniTts; public string Text { get; init; } = string.Empty; @@ -132,19 +149,21 @@ public sealed record LlmTckTranscriptionResult public string? ErrorMessage { get; init; } } -public sealed record LlmTckVideoResult +public sealed record LlmTestServerVideoResult { + public LlmTestServerResponseMetadata? WireMetadata { get; init; } + public bool IsSuccess { get; init; } public int StatusCode { get; init; } = 200; - public string ModelId { get; init; } = LlmTckKnownModelIds.Sora2; + public string ModelId { get; init; } = LlmTestServerKnownModelIds.Sora2; public string Prompt { get; init; } = string.Empty; - public string VideoId { get; init; } = "video_llm_tck"; + public string VideoId { get; init; } = "video_llm_test_server"; - public string GenerationId { get; init; } = "gen_llm_tck"; + public string GenerationId { get; init; } = "gen_llm_test_server"; public string? RemixedFromVideoId { get; init; } @@ -158,7 +177,7 @@ public sealed record LlmTckVideoResult public string Seconds { get; init; } = "4"; - public LlmTckTokenUsage Usage { get; init; } = new(); + public LlmTestServerTokenUsage Usage { get; init; } = new(); public string? ErrorCode { get; init; } diff --git a/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.AnthropicNativeFixtures.cs b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.AnthropicNativeFixtures.cs new file mode 100644 index 0000000..4811119 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.AnthropicNativeFixtures.cs @@ -0,0 +1,60 @@ +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Scenarios; + +namespace ManagedCode.LlmTestServer.Runtime; + +public sealed partial class LlmTestServerRuntime +{ + private static string? ValidateAnthropicNativeFixture(LlmTestServerChatRequest request, LlmTestServerScenarioResponse response, + IReadOnlyList declarations, IReadOnlyList calls, + IReadOnlyList citations) + { + const LlmTestServerNativeToolContract contract = LlmTestServerNativeToolContract.AnthropicMessagesWebSearch20250305; + if (declarations.Count > 1 || declarations.Any(declaration => declaration.Contract != contract) + || request.NativeIncludes.Count > 0) + { return "The Anthropic Messages surface requires its supported native web search declaration and no Responses includes."; } + if (declarations.Count > 0 && !string.Equals(request.ModelId, LlmTestServerKnownModelIds.ClaudeSonnet55, StringComparison.Ordinal)) + { return "Native Anthropic web search is unsupported for this fixture model."; } + var allItems = response.OutputItems.Concat(request.Messages.SelectMany(message => message.OutputItems)).ToList(); + if (allItems.Any(item => LlmTestServerResponseItems.IsNative(item) && LlmTestServerResponseItems.Contract(item) != contract) + || allItems.OfType().SelectMany(message => message.Citations) + .Any(citation => citation is not LlmTestServerAnthropicWebSearchCitation)) + { return "Native outputs, history and citations must match the Anthropic Messages surface."; } + if (calls.Count > 0 && declarations.Count == 0) + { return "The fixture calls a native tool that was not declared."; } + if (request.RequiredToolName is not null && calls.Count > 0) + { return "A named function choice cannot select a native tool."; } + if (request.RequiredNativeTool is { } required + && (required != contract || calls.Count == 0 || response.ToolCalls.Count > 0)) + { return "The selected native tool must match every fixture call."; } + var options = declarations.Count > 0 ? declarations[0].AnthropicWebSearchOptions : null; + var attempts = 0; + var sources = new List(); + var results = response.OutputItems.OfType() + .ToDictionary(result => result.ToolUseId, result => result.Result, StringComparer.Ordinal); + foreach (var call in calls.OfType()) + { + var result = results[call.Id]; + if (result is LlmTestServerWebSearchErrorResult { ErrorCode: "max_uses_exceeded" }) + { + if (options?.MaxUses is not { } limit || attempts < limit) + { return "A max_uses_exceeded fixture requires an exhausted explicit search budget."; } + continue; + } + if (options?.MaxUses is { } maximum && attempts >= maximum) + { return "The fixture exceeds the declared native search budget."; } + attempts++; + if (result is LlmTestServerWebSearchResults web) { sources.AddRange(web.Results); } + } + if (sources.Any(source => !LlmTestServerResponseItems.AllowsAnthropicUrl(options, source.Url))) + { return "Native web search results must respect the declared domain filters."; } + foreach (var citation in citations) + { + if (citation is not LlmTestServerAnthropicWebSearchCitation web || declarations.Count == 0 + || !sources.Any(source => string.Equals(source.Url, web.Url, StringComparison.Ordinal)) + || !LlmTestServerResponseItems.AllowsAnthropicUrl(options, web.Url)) + { return "Anthropic web citations must reference a configured result under the declared domain filters."; } + } + return null; + } +} diff --git a/src/ManagedCode.LlmTck/Runtime/LlmTckRuntime.ChatHistory.cs b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.ChatHistory.cs similarity index 52% rename from src/ManagedCode.LlmTck/Runtime/LlmTckRuntime.ChatHistory.cs rename to src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.ChatHistory.cs index 7f2cb54..d7bbd27 100644 --- a/src/ManagedCode.LlmTck/Runtime/LlmTckRuntime.ChatHistory.cs +++ b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.ChatHistory.cs @@ -1,27 +1,27 @@ using System.Security.Cryptography; using System.Text; -using ManagedCode.LlmTck.Scenarios; +using ManagedCode.LlmTestServer.Scenarios; -namespace ManagedCode.LlmTck.Runtime; +namespace ManagedCode.LlmTestServer.Runtime; -public sealed partial class LlmTckRuntime +public sealed partial class LlmTestServerRuntime { private readonly Dictionary _chatResponses = new(StringComparer.Ordinal); private long _chatResponseBytes; - private LlmTckChatResult? RestoreChatResponse(ref LlmTckChatRequest request, string? token) + private LlmTestServerChatResult? RestoreChatResponse(ref LlmTestServerChatRequest request, string? token) { if (!request.StoreResponse && request.PreviousResponseId is null) { return null; } if (string.IsNullOrWhiteSpace(request.HistoryNamespace)) { - return LlmTckChatResult.Failure(request.ModelId, 400, "invalid_history_namespace", "Response history requires an exact namespace."); + return LlmTestServerChatResult.Failure(request.ModelId, 400, "invalid_history_namespace", "Response history requires an exact namespace."); } if (request.PreviousResponseId is null) { return null; } if (!_chatResponses.TryGetValue(request.PreviousResponseId, out var previous) || previous.Namespace != request.HistoryNamespace || previous.TokenHash != TokenHash(token)) { - return LlmTckChatResult.Failure(request.ModelId, 404, "response_not_found", "The previous response is unavailable in this scope."); + return LlmTestServerChatResult.Failure(request.ModelId, 404, "response_not_found", "The previous response is unavailable in this scope."); } request = request with { @@ -30,28 +30,28 @@ public sealed partial class LlmTckRuntime return null; } - private LlmTckChatResult RetainChatResponse(LlmTckChatRequest request, LlmTckChatResult result, string? token) + private LlmTestServerChatResult RetainChatResponse(LlmTestServerChatRequest request, LlmTestServerChatResult result, string? token) { if (!request.StoreResponse) { return result; } var messages = SnapshotHistory(request.Messages); - messages.Add(new LlmTckMessage { Role = "assistant", Content = result.Content, ToolCalls = [.. result.ToolCalls] }); + messages.Add(new LlmTestServerMessage { Role = "assistant", Content = result.Content, OutputDisposition = LlmTestServerOutputDisposition.ForHistory(result.OutputDisposition), ToolCalls = [.. result.ToolCalls], OutputItems = LlmTestServerResponseItems.Snapshot(result.OutputItems) }); var bytes = HistoryBytes(messages) + Encoding.UTF8.GetByteCount(request.HistoryNamespace!); if (_chatResponses.Count >= _configuration.MaxStoredChatResponses || bytes > _configuration.MaxStoredChatResponseBytes - _chatResponseBytes) { - return LlmTckChatResult.Failure(request.ModelId, 409, "llm_tck_response_capacity_exceeded", "Response history capacity exceeded. Reset the runtime or increase its configured capacity."); + return LlmTestServerChatResult.Failure(request.ModelId, 409, "llm_test_server_response_capacity_exceeded", "Response history capacity exceeded. Reset the runtime or increase its configured capacity."); } - var id = $"resp_{Guid.NewGuid():N}"; + var id = "resp_" + (result.WireMetadata ?? GetResponseMetadata()).IdFor("response"); _chatResponses.Add(id, new StoredChatResponse(request.HistoryNamespace!, TokenHash(token), messages)); _chatResponseBytes += bytes; return result with { ResponseId = id }; } - private static List SnapshotHistory(IEnumerable messages) + private static List SnapshotHistory(IEnumerable messages) { - return messages.Where(message => !IsInstruction(message)).Select(message => message with { ToolCalls = [.. message.ToolCalls] }).ToList(); + return messages.Where(message => !IsInstruction(message)).Select(message => message with { OutputDisposition = LlmTestServerOutputDisposition.ForHistory(message.OutputDisposition), ToolCalls = [.. message.ToolCalls], OutputItems = LlmTestServerResponseItems.Snapshot(message.OutputItems) }).ToList(); } - private static bool IsInstruction(LlmTckMessage message) + private static bool IsInstruction(LlmTestServerMessage message) { return message.Role is "system" or "developer"; } @@ -61,14 +61,16 @@ private static string TokenHash(string? token) return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(token ?? string.Empty))); } - private static long HistoryBytes(IEnumerable messages) + private static long HistoryBytes(IEnumerable messages) { return 256L + messages.Sum(message => 256L + Encoding.UTF8.GetByteCount(message.Role) + Encoding.UTF8.GetByteCount(message.Content) + Encoding.UTF8.GetByteCount(message.ToolCallId ?? string.Empty) + + OutputDispositionBytes(message.OutputDisposition) + + LlmTestServerResponseItems.Bytes(message.OutputItems) + message.ToolCalls.Sum(call => 128L + Encoding.UTF8.GetByteCount(call.Id) - + Encoding.UTF8.GetByteCount(call.Name) + Encoding.UTF8.GetByteCount(call.ArgumentsJson))); + + Encoding.UTF8.GetByteCount(call.Name) + Encoding.UTF8.GetByteCount(call.ArgumentsJson) + Encoding.UTF8.GetByteCount(call.ContinuationToken ?? string.Empty))); } - private sealed record StoredChatResponse(string Namespace, string TokenHash, List Messages); + private sealed record StoredChatResponse(string Namespace, string TokenHash, List Messages); } diff --git a/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.ConversationDiagnostics.cs b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.ConversationDiagnostics.cs new file mode 100644 index 0000000..a7527ff --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.ConversationDiagnostics.cs @@ -0,0 +1,148 @@ +using System.Security.Cryptography; +using System.Text.Json; +using ManagedCode.LlmTestServer.Conversations; +using ManagedCode.LlmTestServer.Scenarios; + +namespace ManagedCode.LlmTestServer.Runtime; + +public sealed partial class LlmTestServerRuntime +{ + public LlmTestServerMatchExplanation Explain(LlmTestServerChatRequest request, string providerNamespace, + string? conversationId = null, string? bearerToken = null) + { + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(request.Messages); + ArgumentNullException.ThrowIfNull(request.Tools); + foreach (var tool in request.Tools) + { + ArgumentNullException.ThrowIfNull(tool); + ArgumentException.ThrowIfNullOrWhiteSpace(tool.Name); + } + ArgumentException.ThrowIfNullOrWhiteSpace(providerNamespace); + foreach (var message in request.Messages) + { + ArgumentNullException.ThrowIfNull(message); + ArgumentNullException.ThrowIfNull(message.Content); + ArgumentNullException.ThrowIfNull(message.ToolCalls); + foreach (var call in message.ToolCalls) + { + ArgumentNullException.ThrowIfNull(call); + ArgumentException.ThrowIfNullOrWhiteSpace(call.Id); + ArgumentException.ThrowIfNullOrWhiteSpace(call.Name); + ArgumentNullException.ThrowIfNull(call.ArgumentsJson); + } + } + lock (_gate) + { + var key = new ConversationKey(providerNamespace, conversationId ?? "default", TokenHash(bearerToken)); + var historyError = RestoreChatResponse(ref request, bearerToken); + if (historyError is not null) + { + return new() { ConversationId = key.ConversationId, TurnIndex = LogicalTurn(request.Messages), ErrorCode = historyError.ErrorCode }; + } + var selected = SelectChatScenario(request, bearerToken, key); + var candidates = new List(); + var selectedScenario = selected?.Id; + string? selectedStep = null; + foreach (var scenario in GetChatScenarios(_configuration)) + { + var baseFailures = new List(); + if (!string.Equals(scenario.ModelId, request.ModelId, StringComparison.OrdinalIgnoreCase)) { baseFailures.Add("model"); } + else if (!ScenarioMatches(scenario, request)) { baseFailures.Add("history.messages"); } + if (!HasRequiredToken(scenario.RequiredBearerToken, bearerToken)) { baseFailures.Add("authorization"); } + if (scenario.Conversation is not { } graph) + { + var matches = baseFailures.Count == 0; + candidates.Add(new() + { + ScenarioId = scenario.Id, + IsMatch = matches, + IsSelected = ReferenceEquals(selected, scenario), + IsShadowed = matches && !ReferenceEquals(selected, scenario), + Mismatches = baseFailures + }); + continue; + } + var state = FindConversation(scenario, key); + var fingerprint = ConversationFingerprint(request); + var replay = state?.LastResult is { IsSuccess: true } && + (graph.ReplaySuccessfulRequests && state.Fingerprint == fingerprint + || state.Terminal && graph.Exhaustion == LlmTestServerConversationExhaustion.RepeatLastResponse); + var restart = state?.Terminal == true && graph.Exhaustion == LlmTestServerConversationExhaustion.Restart && !replay; + var stepIds = replay ? new[] { state!.LastStep!.Id } + : state is null || restart ? new[] { graph.StartStepId } : state.NextSteps.Count == 0 && state.LastStep is not null ? new[] { state.LastStep.Id } : state.NextSteps; + var selectedTargetId = replay ? state!.LastStep!.Id : stepIds.FirstOrDefault(id => + MatchStep(graph.Steps.First(step => step.Id == id).Match, request).Count == 0); + foreach (var id in stepIds) + { + var step = graph.Steps.First(item => item.Id == id); + var failures = new List(baseFailures); + if (state?.Busy == true) { failures.Add("state.busy"); } + if (!replay) + { + failures.AddRange(MatchStep(step.Match, request)); + if (!restart && state?.Terminal == true) { failures.Add("state.completed"); } + if (!restart && state is not null && state.Executions >= graph.MaxSteps) { failures.Add("state.executionLimit"); } + if (!restart && state?.LastResult is { IsSuccess: true } && LogicalTurn(request.Messages) <= state.LastTurn) + { failures.Add("turn.notAdvanced"); } + } + if (!request.PopulateCacheOnly && ValidateFixture(request, step.Response) is not null) { failures.Add("fixture.mismatch"); } + var matches = failures.Count == 0; + var select = ReferenceEquals(selected, scenario) && id == selectedTargetId && matches; + candidates.Add(new() + { + ScenarioId = scenario.Id, + StepId = id, + IsMatch = matches, + IsSelected = select, + IsShadowed = matches && !select, + Mismatches = failures + }); + if (select) { selectedScenario = scenario.Id; selectedStep = id; } + } + } + return new() + { + ConversationId = key.ConversationId, + TurnIndex = LogicalTurn(request.Messages), + SelectedScenarioId = selectedScenario, + SelectedStepId = selectedStep, + Candidates = candidates + }; + } + } + + public IReadOnlyList GetConversations() + { + lock (_gate) + { + return _conversations.SelectMany(pair => pair.Value.Select(entry => new LlmTestServerConversationStatus + { + ScenarioId = pair.Key.Id, + ProviderNamespace = entry.Key.Provider, + ConversationId = entry.Key.ConversationId, + IsBusy = entry.Value.Busy, + IsTerminal = entry.Value.Terminal, + ExecutedSteps = entry.Value.Executions, + LastTurnIndex = entry.Value.LastTurn, + NextSteps = [.. entry.Value.NextSteps], + VisitedSteps = pair.Key.Conversation!.Steps.Where(step => entry.Value.Visited.Contains(step.Id)).Select(step => step.Id).ToList(), + UnvisitedSteps = pair.Key.Conversation!.Steps.Where(step => !entry.Value.Visited.Contains(step.Id)).Select(step => step.Id).ToList(), + })).ToArray(); + } + } + + private static string ConversationFingerprint(LlmTestServerChatRequest request) + { + return Convert.ToHexString( + SHA256.HashData(JsonSerializer.SerializeToUtf8Bytes(request with + { + RequestId = null, + Stream = false, + StreamDeliverySurface = null, + OutputDispositionSurface = null, + PreviousResponseId = null, + Messages = request.Messages.Select(message => message with { OutputDisposition = LlmTestServerOutputDisposition.ForHistory(message.OutputDisposition) }).ToList(), + }))); + } +} diff --git a/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.Conversations.cs b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.Conversations.cs new file mode 100644 index 0000000..fb81183 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.Conversations.cs @@ -0,0 +1,220 @@ +using System.Text; +using ManagedCode.LlmTestServer.Conversations; +using ManagedCode.LlmTestServer.Scenarios; + +namespace ManagedCode.LlmTestServer.Runtime; + +public sealed partial class LlmTestServerRuntime : ILlmTestServerConversationDiagnostics +{ + private readonly Dictionary> _conversations = new(ReferenceEqualityComparer.Instance); + private int _conversationCount; + private long _conversationBytes; + + private ConversationKey ConversationIdentity(string? token) + { + return new( + _activeRequest.Value?.ProviderNamespace ?? string.Empty, + _activeRequest.Value?.ConversationId ?? "default", TokenHash(token)); + } + + private ConversationState? FindConversation(LlmTestServerScenario scenario, ConversationKey key) + { + return _conversations.TryGetValue(scenario, out var states) && states.TryGetValue(key, out var state) ? state : null; + } + + private LlmTestServerScenario? SelectChatScenario(LlmTestServerChatRequest request, string? token, ConversationKey? explicitKey = null) + { + var key = explicitKey ?? ConversationIdentity(token); + return GetChatScenarios(_configuration).FirstOrDefault(scenario => ScenarioMatches(scenario, request) + && (scenario.Conversation is null || FindConversation(scenario, key) is not null + || MatchStep(scenario.Conversation.Steps.First(step => step.Id == scenario.Conversation.StartStepId).Match, request).Count == 0)); + } + + private LlmTestServerChatResult? PrepareConversation(LlmTestServerScenario scenario, LlmTestServerChatRequest request, + string? token, out ConversationLease? lease) + { + lease = null; + var graph = scenario.Conversation!; + var key = ConversationIdentity(token); + var state = FindConversation(scenario, key); + var turn = LogicalTurn(request.Messages); + var fingerprint = ConversationFingerprint(request); + LlmTestServerChatResult Failure(string code, string message, string? stepId = null) + { + var usage = CreateChatUsage(request); + AddEvent(LlmTestServerEventKind.ErrorReturned, scenario.Id, request.ModelId, message, FormatChatRequest(request), message, + usage, chatRequest: request, stepId: stepId, logicalTurnIndex: turn); + return LlmTestServerChatResult.Failure(request.ModelId, 409, code, message, scenario.Id, usage); + } + if (state?.LastResult is { } priorResult && ValidateToolContinuationHistory(request, priorResult) is { } continuationError) + { return Failure("llm_test_server_fixture_mismatch", continuationError); } + LlmTestServerChatResult Replay() + { + var error = ValidateFixture(request, state!.LastStep!.Response); + if (error is not null) { return Failure("llm_test_server_fixture_mismatch", error, state.LastStep.Id); } + var result = CopyConversationResult(state.LastResult!); + AddEvent(LlmTestServerEventKind.Matched, scenario.Id, request.ModelId, "Conversation response replayed.", + FormatChatRequest(request), result.Content, result.Usage, request, request.Stream ? result.StreamChunks : null, + state.LastStep.Id, turn, true, result.ToolCalls, wireMetadata: result.WireMetadata, responseOutputItems: result.OutputItems, streamDelivery: result.StreamDelivery, outputDisposition: result.OutputDisposition); + return result; + } + if (state?.Busy == true) { return Failure("conversation_busy", "A conversation step is already in flight."); } + if (state?.LastResult is { IsSuccess: true } && graph.ReplaySuccessfulRequests && state.Fingerprint == fingerprint) + { return Replay(); } + var restart = false; + if (state?.Terminal == true) + { + if (graph.Exhaustion == LlmTestServerConversationExhaustion.RepeatLastResponse && state.LastResult is { IsSuccess: true }) { return Replay(); } + if (graph.Exhaustion != LlmTestServerConversationExhaustion.Restart) + { return Failure("conversation_completed", "The conversation is complete.", state.LastStep?.Id); } + restart = true; + } + if (!restart && state is not null && state.Executions >= graph.MaxSteps) + { return Failure("conversation_exhausted", "The conversation execution limit was reached."); } + if (!restart && state?.LastResult is { IsSuccess: true } && turn <= state.LastTurn) + { return Failure("conversation_turn_not_advanced", "Changed or stale input cannot advance an already successful logical turn."); } + var next = state is null || restart ? new[] { graph.StartStepId } : state.NextSteps; + var step = next.Select(id => graph.Steps.First(candidate => candidate.Id == id)).FirstOrDefault(candidate => MatchStep(candidate.Match, request).Count == 0); + if (step is null) { return Failure("conversation_step_mismatch", "No active conversation step matched the latest input."); } + if (request.PopulateCacheOnly) + { + var cached = CreateChatUsage(request, updatePromptCache: true); + var usage = cached with { OutputTokens = 0, ReasoningTokens = 0, TotalTokens = cached.InputTokens }; + AddEvent(LlmTestServerEventKind.Matched, scenario.Id, request.ModelId, "Conversation prompt cache populated.", FormatChatRequest(request), + string.Empty, usage, request, stepId: step.Id, logicalTurnIndex: turn); + return LlmTestServerChatResult.Success(request.ModelId, scenario.Id, string.Empty, [], usage); + } + var fixtureError = ValidateFixture(request, step.Response); + if (fixtureError is not null) { return Failure("llm_test_server_fixture_mismatch", fixtureError, step.Id); } + var newState = state is null; + var baseBytes = state?.BaseBytes ?? 1024L + Bytes(key.Provider) + Bytes(key.ConversationId) + graph.Steps.Count * 128L; + var resultBytes = ResponseStateBytes(scenario, step.Response); + var previousBytes = state?.Bytes ?? 0; + var reservation = baseBytes + Math.Max(state?.ResultBytes ?? 0, resultBytes); + if (newState && _conversationCount >= _configuration.MaxConversations + || reservation - previousBytes > _configuration.MaxConversationBytes - _conversationBytes) + { return Failure("conversation_capacity_exceeded", "The conversation state capacity was exceeded.", step.Id); } + if (state is null) + { + state = new() { BaseBytes = baseBytes }; + if (!_conversations.TryGetValue(scenario, out var states)) { states = []; _conversations.Add(scenario, states); } + states.Add(key, state); + _conversationCount++; + } + _conversationBytes += reservation - previousBytes; + state.Bytes = reservation; + state.Busy = true; + lease = new(scenario, key, state, step, fingerprint, turn, newState, restart, previousBytes, resultBytes); + return null; + } + + private void CommitConversation(ConversationLease lease, LlmTestServerChatResult result) + { + var state = lease.State; + if (lease.Restart) { state.Executions = 0; state.Visited.Clear(); } + state.Executions++; + state.Visited.Add(lease.Step.Id); + state.LastStep = lease.Step; + state.LastTurn = lease.Turn; + state.LastResult = CopyConversationResult(result); + state.Fingerprint = lease.Fingerprint; + state.NextSteps = lease.Step.NextSteps; + state.Terminal = lease.Step.Terminal; + state.Busy = false; + _conversationBytes += state.BaseBytes + lease.ResultBytes - state.Bytes; + state.ResultBytes = lease.ResultBytes; + state.Bytes = state.BaseBytes + lease.ResultBytes; + } + + private void RollBackResponse(LlmTestServerScenario scenario, int position, long generation, ConversationLease? lease) + { + if (lease is null) { RollBackReservedResponse(scenario, position, generation); return; } + lock (_gate) + { + if (_retired || generation != _generation) { return; } + _conversationBytes += lease.PreviousBytes - lease.State.Bytes; + if (lease.NewState) { _conversations[scenario].Remove(lease.Key); _conversationCount--; } + lease.State.Bytes = lease.PreviousBytes; + lease.State.Busy = false; + } + } + + private void ClearConversations() { _conversations.Clear(); _conversationBytes = 0; _conversationCount = 0; } + private static long Bytes(string value) + { + return Encoding.UTF8.GetByteCount(value); + } + + private static long ResponseStateBytes(LlmTestServerScenario scenario, LlmTestServerScenarioResponse response) + { + response = LlmTestServerResponseItems.Project(response); + return 1024L + (response.StreamDelivery is null ? 0 : 64) + + OutputDispositionBytes(response.OutputDisposition) + + Bytes(scenario.Id) + Bytes(scenario.ModelId) + Bytes(response.Content) + response.StreamChunks.Sum(chunk => Bytes(chunk)) + + response.ToolCalls.Sum(call => Bytes(call.Id) + Bytes(call.Name) + Bytes(call.ArgumentsJson) + Bytes(call.ContinuationToken ?? string.Empty) + 128) + + LlmTestServerResponseItems.Bytes(response.OutputItems) + + (response.Error is { } error ? Bytes(error.Code) + Bytes(error.Message) : 0); + } + + private static long OutputDispositionBytes(LlmTestServerOutputDisposition? disposition) + { + return disposition is null ? 0 + : 64L + Bytes(disposition.Category ?? string.Empty) + Bytes(disposition.Explanation ?? string.Empty); + } + + private static LlmTestServerChatResult CopyConversationResult(LlmTestServerChatResult result) + { + return result with { ToolCalls = [.. result.ToolCalls], StreamChunks = [.. result.StreamChunks], OutputItems = LlmTestServerResponseItems.Snapshot(result.OutputItems) }; + } + + private static int LogicalTurn(IEnumerable messages) + { + var turns = 1; + string? previous = null; + foreach (var message in messages) + { + if (message.Role is "system" or "developer") { continue; } + if (message.Role == "assistant" && previous != "assistant") { turns++; } + previous = message.Role; + } + return turns; + } + + private static List MatchStep(LlmTestServerConversationMatch match, LlmTestServerChatRequest request) + { + var failures = new List(); + var latest = request.Messages.LastOrDefault(); + if (match.LatestRole is not null && latest?.Role != match.LatestRole) { failures.Add("latest.role"); } + if (match.TextEquals is not null && latest?.Content != match.TextEquals) { failures.Add("latest.text.equals"); } + if (match.TextContains is not null && (latest is null || !latest.Content.Contains(match.TextContains, StringComparison.Ordinal))) { failures.Add("latest.text.contains"); } + if (match.TurnIndex is not null && match.TurnIndex != LogicalTurn(request.Messages)) { failures.Add("turn.index"); } + var call = latest?.Role == "tool" + ? request.Messages.SelectMany(message => message.ToolCalls).LastOrDefault(candidate => candidate.Id == latest.ToolCallId) + : latest?.ToolCalls.LastOrDefault(); + if (match.ToolName is not null && call?.Name != match.ToolName) { failures.Add("tool.name"); } + if (match.ToolCallId is not null && (latest?.Role == "tool" ? latest.ToolCallId : call?.Id) != match.ToolCallId) { failures.Add("tool.callId"); } + if (match.ToolArgumentsJson is not null && (call is null || !LlmTestServerJsonFieldSelector.Matches(call.ArgumentsJson, "$", match.ToolArgumentsJson))) { failures.Add("tool.arguments"); } + foreach (var field in match.JsonFields) + { if (latest is null || !LlmTestServerJsonFieldSelector.Matches(latest.Content, field.Path, field.EqualsJson)) { failures.Add("latest.json:" + field.Path); } } + return failures; + } + + private readonly record struct ConversationKey(string Provider, string ConversationId, string CredentialHash); + private sealed class ConversationState + { + internal long BaseBytes { get; init; } + internal long Bytes { get; set; } + internal long ResultBytes { get; set; } + internal bool Busy { get; set; } + internal bool Terminal { get; set; } + internal int Executions { get; set; } + internal int LastTurn { get; set; } + internal string? Fingerprint { get; set; } + internal LlmTestServerConversationStep? LastStep { get; set; } + internal LlmTestServerChatResult? LastResult { get; set; } + internal IReadOnlyList NextSteps { get; set; } = []; + internal HashSet Visited { get; } = new(StringComparer.Ordinal); + } + private sealed record ConversationLease(LlmTestServerScenario Scenario, ConversationKey Key, ConversationState State, + LlmTestServerConversationStep Step, string Fingerprint, int Turn, bool NewState, bool Restart, long PreviousBytes, long ResultBytes); +} diff --git a/src/ManagedCode.LlmTck/Runtime/LlmTckRuntime.Decisions.cs b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.Decisions.cs similarity index 65% rename from src/ManagedCode.LlmTck/Runtime/LlmTckRuntime.Decisions.cs rename to src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.Decisions.cs index 9f9442e..2632256 100644 --- a/src/ManagedCode.LlmTck/Runtime/LlmTckRuntime.Decisions.cs +++ b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.Decisions.cs @@ -1,13 +1,13 @@ using System.Globalization; using System.Text.Json; -using ManagedCode.LlmTck.Decisions; -using ManagedCode.LlmTck.Models; +using ManagedCode.LlmTestServer.Decisions; +using ManagedCode.LlmTestServer.Models; -namespace ManagedCode.LlmTck.Runtime; +namespace ManagedCode.LlmTestServer.Runtime; -public sealed partial class LlmTckRuntime +public sealed partial class LlmTestServerRuntime { - public Task DecideAsync(LlmTckDecisionRequest request, + public Task DecideAsync(LlmTestServerDecisionRequest request, string? bearerToken = null, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(request); @@ -20,7 +20,7 @@ public Task DecideAsync(LlmTckDecisionRequest request, } } - private LlmTckDecisionResult DecideCore(LlmTckDecisionRequest request, string? bearerToken) + private LlmTestServerDecisionResult DecideCore(LlmTestServerDecisionRequest request, string? bearerToken) { if (IsActiveRequestSuperseded()) { @@ -28,11 +28,11 @@ private LlmTckDecisionResult DecideCore(LlmTckDecisionRequest request, string? b } if (!HasRequiredToken(_configuration.RequiredBearerToken, bearerToken)) { - return DecisionFailure(request, 401, "invalid_api_key", "The bearer token did not match.", kind: LlmTckEventKind.AuthFailed); + return DecisionFailure(request, 401, "invalid_api_key", "The bearer token did not match.", kind: LlmTestServerEventKind.AuthFailed); } - if (!IsConfiguredModel(request.ModelId, LlmTckModelKind.Decision)) + if (!IsConfiguredModel(request.ModelId, LlmTestServerModelKind.Decision)) { - return DecisionFailure(request, 404, "model_not_found", "The decision model is not configured.", kind: LlmTckEventKind.ModelNotFound); + return DecisionFailure(request, 404, "model_not_found", "The decision model is not configured.", kind: LlmTestServerEventKind.ModelNotFound); } var fault = TryCreateFault(request.ModelId, [request.Input], JsonSerializer.Serialize(request)); if (fault is not null) @@ -43,7 +43,7 @@ private LlmTckDecisionResult DecideCore(LlmTckDecisionRequest request, string? b candidate.ModelId == request.ModelId && (candidate.ExpectedInput is null || candidate.ExpectedInput == request.Input)); if (scenario is null) { - return DecisionFailure(request, 404, "llm_tck_scenario_not_found", "No decision scenario matched.", kind: LlmTckEventKind.Unmatched); + return DecisionFailure(request, 404, "llm_test_server_scenario_not_found", "No decision scenario matched.", kind: LlmTestServerEventKind.Unmatched); } if (scenario.Error is { } error) { @@ -51,34 +51,34 @@ private LlmTckDecisionResult DecideCore(LlmTckDecisionRequest request, string? b } if (!DecisionAnswersMatch(request, scenario)) { - return DecisionFailure(request, 409, "llm_tck_fixture_mismatch", "Configured decision answers do not match the requested questions.", scenario.Id); + return DecisionFailure(request, 409, "llm_test_server_fixture_mismatch", "Configured decision answers do not match the requested questions.", scenario.Id); } var answers = scenario.Answers.ToDictionary(entry => entry.Key, entry => entry.Value with { Probabilities = new(entry.Value.Probabilities) }, StringComparer.Ordinal); - AddEvent(LlmTckEventKind.Matched, scenario.Id, request.ModelId, "Decision scenario matched.", + AddEvent(LlmTestServerEventKind.Matched, scenario.Id, request.ModelId, "Decision scenario matched.", JsonSerializer.Serialize(request), JsonSerializer.Serialize(answers), scenario.Usage); return new() { IsSuccess = true, ModelId = scenario.ResolvedModelId ?? request.ModelId, ScenarioId = scenario.Id, Answers = answers, Usage = scenario.Usage, Metadata = scenario.Metadata }; } - private static bool DecisionAnswersMatch(LlmTckDecisionRequest request, LlmTckDecisionScenario scenario) + private static bool DecisionAnswersMatch(LlmTestServerDecisionRequest request, LlmTestServerDecisionScenario scenario) { return request.Questions.Count == scenario.Answers.Count && request.Questions.Select(question => question.Name).Distinct(StringComparer.Ordinal).Count() == request.Questions.Count && request.Questions.All(question => scenario.Answers.TryGetValue(question.Name, out var answer) - && answer.Kind == question.Kind && (answer.Refused || (DecisionAnswerFitsQuestion(question, answer) && (scenario.AllowMalformedResponse || LlmTckDecisionFixtureValidation.IsConformant(request.Provider, question, answer))))); + && answer.Kind == question.Kind && (answer.Refused || (DecisionAnswerFitsQuestion(question, answer) && (scenario.AllowMalformedResponse || LlmTestServerDecisionFixtureValidation.IsConformant(request.Provider, question, answer))))); } - private static bool DecisionAnswerFitsQuestion(LlmTckDecisionQuestion question, LlmTckDecisionAnswer answer) + private static bool DecisionAnswerFitsQuestion(LlmTestServerDecisionQuestion question, LlmTestServerDecisionAnswer answer) { - if (question.Kind == LlmTckDecisionKind.Choice) + if (question.Kind == LlmTestServerDecisionKind.Choice) { return question.Options.Contains(answer.Choice!, StringComparer.Ordinal) && answer.Probabilities.Keys.All(key => question.Options.Contains(key, StringComparer.Ordinal)); } - if (question.Kind == LlmTckDecisionKind.Score) + if (question.Kind == LlmTestServerDecisionKind.Score) { return answer.Score <= question.Options.Count - 1 && answer.Probabilities.Keys.All(key => int.TryParse(key, NumberStyles.None, CultureInfo.InvariantCulture, out var index) && index >= 0 && index < question.Options.Count); } return true; } - private LlmTckDecisionResult DecisionFailure(LlmTckDecisionRequest request, int status, string code, - string message, string? scenario = null, LlmTckEventKind kind = LlmTckEventKind.ErrorReturned) + private LlmTestServerDecisionResult DecisionFailure(LlmTestServerDecisionRequest request, int status, string code, + string message, string? scenario = null, LlmTestServerEventKind kind = LlmTestServerEventKind.ErrorReturned) { AddEvent(kind, scenario, request.ModelId, message, JsonSerializer.Serialize(request), message); return new() { ModelId = request.ModelId, ScenarioId = scenario, StatusCode = status, ErrorCode = code, ErrorMessage = message }; diff --git a/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.EmbeddingDimensions.cs b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.EmbeddingDimensions.cs new file mode 100644 index 0000000..4c69c7f --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.EmbeddingDimensions.cs @@ -0,0 +1,23 @@ +namespace ManagedCode.LlmTestServer.Runtime; + +public sealed partial class LlmTestServerRuntime +{ + private LlmTestServerEmbeddingResult? ValidateEmbeddingDimensions(string modelId, int inputCount, IReadOnlyList? dimensions) + { + if (dimensions is null) { return null; } + var invalid = dimensions.Count != inputCount || dimensions.Any(dimension => dimension is < 1); + var incompatible = dimensions.Any(dimension => dimension > _configuration.DefaultEmbeddingVector.Count); + if (!invalid && !incompatible) { return null; } + var message = invalid ? "Embedding dimensions must contain one positive or omitted value per input." + : "The configured embedding vector has fewer coordinates than the requested dimensions."; + AddEvent(LlmTestServerEventKind.ErrorReturned, null, modelId, message); + return new() + { + IsSuccess = false, + StatusCode = invalid ? 400 : 409, + ModelId = modelId, + ErrorCode = invalid ? "invalid_request_error" : "llm_test_server_embedding_fixture_mismatch", + ErrorMessage = message, + }; + } +} diff --git a/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.NativeFixtures.cs b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.NativeFixtures.cs new file mode 100644 index 0000000..389887b --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.NativeFixtures.cs @@ -0,0 +1,103 @@ +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Scenarios; + +namespace ManagedCode.LlmTestServer.Runtime; + +public sealed partial class LlmTestServerRuntime +{ + private static string? ValidateNativeFixture(LlmTestServerChatRequest request, LlmTestServerScenarioResponse response) + { + ArgumentNullException.ThrowIfNull(request.NativeTools); + ArgumentNullException.ThrowIfNull(request.NativeIncludes); + var declarations = LlmTestServerResponseItems.SnapshotDeclarations(request.NativeTools, validate: true); + foreach (var message in request.Messages) + { + ArgumentNullException.ThrowIfNull(message); + var historyItems = LlmTestServerResponseItems.Snapshot(message.OutputItems); + LlmTestServerResponseItems.ValidateCompletedServerTools(historyItems); + } + var calls = response.OutputItems.Where(LlmTestServerResponseItems.IsNativeCall).ToList(); + var citations = response.OutputItems.OfType().SelectMany(message => message.Citations).ToList(); + var nativeHistory = request.Messages.Any(message => message.OutputItems.Any(LlmTestServerResponseItems.IsNative) + || message.OutputItems.OfType().Any(item => item.Citations.Count > 0)); + if (calls.Count == 0 && citations.Count == 0 && !nativeHistory && declarations.Count == 0 && request.RequiredNativeTool is null) { return null; } + if (request.NativeToolSurface == LlmTestServerNativeToolSurface.AnthropicMessages) + { return ValidateAnthropicNativeFixture(request, response, declarations, calls, citations); } + if (request.NativeToolSurface != LlmTestServerNativeToolSurface.OpenAiResponses) { return "Native fixtures require the qualified OpenAI Responses surface."; } + if (declarations.Select(tool => tool.Contract).Distinct().Count() != declarations.Count + || request.NativeIncludes.Any(include => !Enum.IsDefined(include))) { return "Native declarations and includes must use supported contracts."; } + if (response.OutputItems.Concat(request.Messages.SelectMany(message => message.OutputItems)).Any(item => + LlmTestServerResponseItems.Contract(item) == LlmTestServerNativeToolContract.AnthropicMessagesWebSearch20250305) + || request.Messages.SelectMany(message => message.OutputItems).OfType() + .SelectMany(message => message.Citations).Any(citation => citation is LlmTestServerAnthropicWebSearchCitation)) + { return "Anthropic native history and outputs require the Anthropic Messages surface."; } + foreach (var declaration in declarations) + { + var model = declaration.Contract switch + { + LlmTestServerNativeToolContract.OpenAiResponsesWebSearch => LlmTestServerKnownModelIds.Gpt41Mini, + LlmTestServerNativeToolContract.OpenAiResponsesFileSearch => LlmTestServerKnownModelIds.Gpt4OMini, + LlmTestServerNativeToolContract.OpenAiResponsesCodeInterpreter => LlmTestServerKnownModelIds.Gpt41, + _ => null, + }; + if (!string.Equals(request.ModelId, model, StringComparison.Ordinal)) { return "Native tool declaration is unsupported for this fixture model."; } + } + if (calls.Any(call => !declarations.Any(tool => tool.Contract == LlmTestServerResponseItems.Contract(call)))) + { return "The fixture calls a native tool that was not declared."; } + if (request.RequiredToolName is not null && calls.Count > 0) { return "A named function choice cannot select a native tool."; } + if (request.RequiredNativeTool is { } selected + && (calls.Count == 0 || response.ToolCalls.Count > 0 || calls.Any(call => LlmTestServerResponseItems.Contract(call) != selected))) + { return "The selected native tool must match every fixture call."; } + var webOptions = declarations.FirstOrDefault(tool => tool.Contract == LlmTestServerNativeToolContract.OpenAiResponsesWebSearch)?.WebSearchOptions; + if (calls.OfType().SelectMany(call => call.Sources).Any(source => !LlmTestServerResponseItems.AllowsUrl(webOptions, source.Url))) + { return "Fixture sources must respect the declared allowed domains."; } + foreach (var file in calls.OfType()) + { + var options = declarations.First(tool => tool.Contract == file.Contract).FileSearchOptions!; + if (file.VectorStoreIds.Count == 0 || file.VectorStoreIds.Any(store => !options.VectorStoreIds.Contains(store, StringComparer.Ordinal))) + { return "File search fixture stores must be declared by the request."; } + if (options.MaxNumResults is { } maximum && file.Results.Count > maximum + || file.Results.Any(result => options.ScoreThreshold is { } threshold && (result.Score is null || result.Score < threshold) + || !LlmTestServerResponseItems.MatchesFilter(options.Filters, result.Attributes))) + { return "File search results must satisfy the requested count, score and attribute filter."; } + } + foreach (var code in calls.OfType()) + { + var options = declarations.First(tool => tool.Contract == code.Contract).CodeInterpreterOptions!; + if (options.AutomaticContainer + ? !code.InputFileIds.Order(StringComparer.Ordinal).SequenceEqual(options.FileIds.Distinct(StringComparer.Ordinal).Order(StringComparer.Ordinal), StringComparer.Ordinal) + : !string.Equals(code.ContainerId, options.ContainerId, StringComparison.Ordinal)) + { return "Code fixture container or input file references do not match the request."; } + } + return ValidateNativeCitations(citations, calls, declarations, webOptions); + } + + private static string? ValidateNativeCitations(IReadOnlyList citations, + IReadOnlyList calls, IReadOnlyList declarations, LlmTestServerWebSearchOptions? webOptions) + { + foreach (var citation in citations) + { + switch (citation) + { + case LlmTestServerUrlCitation url: + if (!declarations.Any(tool => tool.Contract == LlmTestServerNativeToolContract.OpenAiResponsesWebSearch) + || !LlmTestServerResponseItems.AllowsUrl(webOptions, url.Url)) { return "URL citations require the declared web search domain contract."; } + break; + case LlmTestServerFileCitation file: + if (!declarations.Any(tool => tool.Contract == LlmTestServerNativeToolContract.OpenAiResponsesFileSearch) + || !calls.OfType().SelectMany(call => call.Results).Any(result => result.FileId == file.FileId + && (result.Filename is null || result.Filename == file.Filename))) { return "File citations must reference a configured search result."; } + break; + case LlmTestServerContainerFileCitation container: + if (!declarations.Any(tool => tool.Contract == LlmTestServerNativeToolContract.OpenAiResponsesCodeInterpreter) + || !calls.OfType().Any(call => call.ContainerId == container.ContainerId + && (call.InputFileIds.Contains(container.FileId, StringComparer.Ordinal) || call.GeneratedFileIds.Contains(container.FileId, StringComparer.Ordinal)))) + { return "Container citations must reference configured input or generated files in that container."; } + break; + default: + return "The selected citation kind is unsupported on the OpenAI Responses surface."; + } + } + return null; + } +} diff --git a/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.NativeTokenAccounting.cs b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.NativeTokenAccounting.cs new file mode 100644 index 0000000..550e9e0 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.NativeTokenAccounting.cs @@ -0,0 +1,12 @@ +using System.Text.Json; +using ManagedCode.LlmTestServer.Scenarios; + +namespace ManagedCode.LlmTestServer.Runtime; + +public sealed partial class LlmTestServerRuntime +{ + private static string NativeDeclarationTokenText(LlmTestServerChatRequest request) + { + return request.NativeTools.Count == 0 ? string.Empty : JsonSerializer.Serialize(request.NativeTools); + } +} diff --git a/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.Tools.cs b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.Tools.cs new file mode 100644 index 0000000..ba459a9 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.Tools.cs @@ -0,0 +1,164 @@ +using System.Text.Json; +using Json.Schema; +using ManagedCode.LlmTestServer.Scenarios; + +namespace ManagedCode.LlmTestServer.Runtime; + +public sealed partial class LlmTestServerRuntime +{ + private static string? ValidateFixture(LlmTestServerChatRequest request, LlmTestServerScenarioResponse response) + { + if (response.StreamDelivery is not null && (!request.Stream + || request.StreamDeliverySurface is not (LlmTestServerStreamDeliverySurface.OpenAiChatCompletions + or LlmTestServerStreamDeliverySurface.OpenAiResponses or LlmTestServerStreamDeliverySurface.AnthropicMessages + or LlmTestServerStreamDeliverySurface.GeminiGenerateContent) + || request.Tools is not { Count: 0 } || request.NativeTools is not { Count: 0 } + || request.NativeIncludes is not { Count: 0 } || request.RequiredNativeTool is not null + || request.RequireJson || request.ResponseSchemaJson is not null + || request.Messages is null || request.Messages.Any(message => message is null || message.Role == "tool" + || message.ToolCalls is not { Count: 0 } || message.OutputItems is not { Count: 0 }))) + { return "This stream delivery fixture requires a qualified OpenAI, Anthropic or Gemini text SSE route without tools or structured output."; } + if (response.Error is not null) + { + return null; + } + + try + { + response = LlmTestServerResponseItems.Project(response); + _ = LlmTestServerOutputDisposition.Snapshot(response); + foreach (var message in request.Messages) + { + if (message.OutputDisposition is not { } historyDisposition) { continue; } + LlmTestServerOutputDisposition.Validate(historyDisposition); + if (!string.Equals(message.Role, "assistant", StringComparison.OrdinalIgnoreCase)) { return "Output disposition history belongs to assistant messages."; } + } + if (response.OutputDisposition is { } disposition) + { + var supported = request.OutputDispositionSurface switch + { + LlmTestServerOutputDispositionSurface.OpenAiChatCompletions or LlmTestServerOutputDispositionSurface.OpenAiResponses + or LlmTestServerOutputDispositionSurface.AzureOpenAiDeploymentChatCompletions + or LlmTestServerOutputDispositionSurface.AzureOpenAiV1ChatCompletions or LlmTestServerOutputDispositionSurface.AzureOpenAiV1Responses + or LlmTestServerOutputDispositionSurface.MicrosoftFoundryV1ChatCompletions or LlmTestServerOutputDispositionSurface.MicrosoftFoundryV1Responses + => disposition.Kind is LlmTestServerOutputDispositionKind.Filtered or LlmTestServerOutputDispositionKind.Refused, + LlmTestServerOutputDispositionSurface.AnthropicMessages => disposition.Kind == LlmTestServerOutputDispositionKind.Refused, + LlmTestServerOutputDispositionSurface.GeminiGenerateContent => disposition.Kind == LlmTestServerOutputDispositionKind.Filtered, + LlmTestServerOutputDispositionSurface.DeepSeekChatCompletions + or LlmTestServerOutputDispositionSurface.MicrosoftFoundryChatCompletions + or LlmTestServerOutputDispositionSurface.MicrosoftFoundryModelsChatCompletions + => disposition.Kind == LlmTestServerOutputDispositionKind.Filtered, + _ => false, + }; + if (!supported || request.Tools is not { Count: 0 } || request.NativeTools is not { Count: 0 } + || request.NativeIncludes is not { Count: 0 } || request.RequiredNativeTool is not null + || request.RequireJson || request.ResponseSchemaJson is not null + || request.Messages.Any(message => message.Role == "tool" || message.ToolCalls is not { Count: 0 } || message.OutputItems is not { Count: 0 })) + { return "This output disposition requires a qualified provider text route and supported policy kind without tools, native items or structured output."; } + if (request.OutputDispositionSurface == LlmTestServerOutputDispositionSurface.AnthropicMessages + && disposition.Category is not (null or "cyber" or "bio" or "frontier_llm" or "reasoning_extraction" or "general_harms")) + { return "The configured category is not a supported Anthropic refusal category."; } + } + var nativeError = ValidateNativeFixture(request, response); + if (nativeError is not null) { return nativeError; } + if (request.RequireToolContinuationTokens && response.ToolCalls.Count > 0 + && !ValidToolContinuationToken(response.ToolCalls[0].ContinuationToken)) + { return "The selected fixture requires valid continuation metadata on its first function call."; } + if (response.ToolCalls.Any(call => call.ContinuationToken is not null && !ValidToolContinuationToken(call.ContinuationToken))) + { return "Fixture tool continuation metadata must be bounded, nonempty base64."; } + var callCount = response.ToolCalls.Count + response.OutputItems.Count(LlmTestServerResponseItems.IsNativeCall); + if (request.Tools is null || request.Tools.Any(tool => tool is null || string.IsNullOrWhiteSpace(tool.Name))) + { + return "Tool declarations require names."; + } + + if (request.Tools.Select(tool => tool.Name).Distinct(StringComparer.Ordinal).Count() != request.Tools.Count) + { + return "Tool names must be unique."; + } + + foreach (var tool in request.Tools) + { + _ = LlmTestServerJsonSchema.Matches("{}", tool.ParametersJson); + } + + if (request.ResponseSchemaJson is { } declaredSchema) + { + _ = LlmTestServerJsonSchema.Matches("{}", declaredSchema); + } + + if (!request.AllowParallelToolCalls && callCount > 1) + { + return "The selected fixture contains parallel tool calls, which the request disabled."; + } + + if (request.ToolChoice == LlmTestServerToolChoice.Required && callCount == 0) + { + return "The selected fixture must contain a tool call."; + } + + if (request.ToolChoice == LlmTestServerToolChoice.None && callCount > 0) + { + return "The selected fixture contains a tool call but tool choice is none."; + } + + foreach (var call in response.ToolCalls) + { + var definition = request.Tools.FirstOrDefault(tool => tool.Name == call.Name); + if (definition is null || request.RequiredToolName is { } required && required != call.Name) + { + return "The fixture calls a tool that was not selected or declared in the request."; + } + + if (!LlmTestServerJsonSchema.Matches(call.ArgumentsJson, definition.ParametersJson)) + { + return "Fixture tool arguments do not match the declared parameter schema."; + } + } + if (response.ToolCalls.Select(call => call.Id).Distinct(StringComparer.Ordinal).Count() != response.ToolCalls.Count) + { + return "Fixture tool call IDs must be unique."; + } + + if (response.ToolCalls.Count == 0 && (request.RequireJson || request.ResponseSchemaJson is not null)) + { + LlmTestServerJsonSchema.ValidateJson(response.Content); + if (request.ResponseSchemaJson is { } schema && !LlmTestServerJsonSchema.Matches(response.Content, schema)) + { + return "The selected fixture does not match the requested JSON schema."; + } + + if (response.StreamChunks.Count > 0 && string.Concat(response.StreamChunks) != response.Content) + { + return "Structured-output stream chunks must concatenate to the validated JSON fixture."; + } + } + return null; + } + catch (Exception exception) when (exception is JsonException or JsonSchemaException or ArgumentException) + { + return $"Invalid JSON or schema in request/fixture: {exception.Message}"; + } + } + // A repository retention bound, not a provider-advertised signature limit. + private static bool ValidToolContinuationToken(string? value) + { + if (string.IsNullOrEmpty(value) || value.Length > 16384 || value.Any(char.IsWhiteSpace)) { return false; } + return Convert.TryFromBase64String(value, new byte[value.Length], out var written) && written > 0; + } + + private static string? ValidateToolContinuationHistory(LlmTestServerChatRequest request, LlmTestServerChatResult priorResult) + { + if (!request.RequireToolContinuationTokens) { return null; } + var start = request.Messages.FindLastIndex(message => message.Role == "user"); + var currentCalls = request.Messages.Skip(start + 1).SelectMany(message => message.ToolCalls).ToList(); + foreach (var prior in priorResult.ToolCalls.Where(call => call.ContinuationToken is not null)) + { + var echoed = currentCalls.LastOrDefault(call => call.Id == prior.Id); + if (echoed is not null && !string.Equals(echoed.ContinuationToken, prior.ContinuationToken, StringComparison.Ordinal)) + { return "Tool continuation metadata differs from the last response retained in this conversation."; } + } + return null; + } + +} diff --git a/src/ManagedCode.LlmTck/Runtime/LlmTckRuntime.Videos.cs b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.Videos.cs similarity index 81% rename from src/ManagedCode.LlmTck/Runtime/LlmTckRuntime.Videos.cs rename to src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.Videos.cs index a7c32a3..152ae7a 100644 --- a/src/ManagedCode.LlmTck/Runtime/LlmTckRuntime.Videos.cs +++ b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.Videos.cs @@ -1,12 +1,12 @@ -namespace ManagedCode.LlmTck.Runtime; +namespace ManagedCode.LlmTestServer.Runtime; -public sealed partial class LlmTckRuntime +public sealed partial class LlmTestServerRuntime { - private readonly Dictionary<(string Provider, string Id), LlmTckVideoResult> _videos = []; + private readonly Dictionary<(string Provider, string Id), LlmTestServerVideoResult> _videos = []; private long _videoBytes; private readonly Dictionary _videoSequence = new(StringComparer.Ordinal); - public LlmTckVideoResult StoreVideo(string provider, LlmTckVideoResult video) + public LlmTestServerVideoResult StoreVideo(string provider, LlmTestServerVideoResult video) { ArgumentException.ThrowIfNullOrWhiteSpace(provider); ArgumentNullException.ThrowIfNull(video); @@ -25,8 +25,8 @@ public LlmTckVideoResult StoreVideo(string provider, LlmTckVideoResult video) if (_videos.Count >= _configuration.MaxVideoJobs || video.Bytes.LongLength > _configuration.MaxVideoBytes - _videoBytes) { const string message = "Video store capacity exceeded. Delete jobs or increase the configured video capacity."; - AddEvent(LlmTckEventKind.ErrorReturned, null, video.ModelId, message); - return video with { IsSuccess = false, StatusCode = 409, ErrorCode = "llm_tck_video_capacity_exceeded", ErrorMessage = message, Bytes = [] }; + AddEvent(LlmTestServerEventKind.ErrorReturned, null, video.ModelId, message); + return video with { IsSuccess = false, StatusCode = 409, ErrorCode = "llm_test_server_video_capacity_exceeded", ErrorMessage = message, Bytes = [] }; } var sequence = _videoSequence.GetValueOrDefault(provider); @@ -48,7 +48,7 @@ public LlmTckVideoResult StoreVideo(string provider, LlmTckVideoResult video) } } - public IReadOnlyList ListVideos(string provider) + public IReadOnlyList ListVideos(string provider) { lock (_gate) { @@ -57,7 +57,7 @@ public IReadOnlyList ListVideos(string provider) } } - public LlmTckVideoResult? FindVideo(string provider, string id, bool generation = false) + public LlmTestServerVideoResult? FindVideo(string provider, string id, bool generation = false) { lock (_gate) { diff --git a/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.WireMetadata.cs b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.WireMetadata.cs new file mode 100644 index 0000000..2db3b22 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.WireMetadata.cs @@ -0,0 +1,82 @@ +using ManagedCode.LlmTestServer.Determinism; +using ManagedCode.LlmTestServer.Scenarios; + +namespace ManagedCode.LlmTestServer.Runtime; + +public interface ILlmTestServerWireMetadataSource +{ + LlmTestServerResponseMetadata GetResponseMetadata(); +} + +public sealed partial class LlmTestServerRuntime : ILlmTestServerWireMetadataSource +{ + private readonly Dictionary _wireSequences = new(StringComparer.Ordinal); + + private LlmTestServerResponseMetadata ReserveResponseMetadata(string provider) + { + var sequence = checked(_wireSequences.GetValueOrDefault(provider) + 1); + _wireSequences[provider] = sequence; + return LlmTestServerResponseMetadata.Create(_configuration.Determinism, provider, sequence); + } + + public LlmTestServerResponseMetadata GetResponseMetadata() + { + lock (_gate) { return _activeRequest.Value?.WireMetadata ?? ReserveResponseMetadata(string.Empty); } + } + + public async Task CompleteChatAsync(LlmTestServerChatRequest request, string? bearerToken = null, CancellationToken cancellationToken = default) + { + using var operation = _activeRequest.Value is null ? BeginRequest("runtime-" + Guid.NewGuid().ToString("N")) : null; + var metadata = GetResponseMetadata(); + var result = await CompleteChatCoreAsync(request, bearerToken, cancellationToken).ConfigureAwait(false); + return result with { WireMetadata = result.WireMetadata ?? metadata }; + } + + public async Task CreateEmbeddingAsync(string modelId, IReadOnlyList inputs, string? bearerToken = null, IReadOnlyList? outputDimensions = null, CancellationToken cancellationToken = default) + { + using var operation = _activeRequest.Value is null ? BeginRequest("runtime-" + Guid.NewGuid().ToString("N")) : null; + var metadata = GetResponseMetadata(); + var result = await CreateEmbeddingCoreAsync(modelId, inputs, bearerToken, outputDimensions, cancellationToken).ConfigureAwait(false); + return result with { WireMetadata = result.WireMetadata ?? metadata }; + } + + public async Task GenerateImageAsync(string modelId, string prompt, string? bearerToken = null, CancellationToken cancellationToken = default) + { + using var operation = _activeRequest.Value is null ? BeginRequest("runtime-" + Guid.NewGuid().ToString("N")) : null; + var metadata = GetResponseMetadata(); + var result = await GenerateImageCoreAsync(modelId, prompt, bearerToken, cancellationToken).ConfigureAwait(false); + return result with { WireMetadata = result.WireMetadata ?? metadata }; + } + + public async Task GenerateAudioAsync(string modelId, string input, string? bearerToken = null, CancellationToken cancellationToken = default) + { + using var operation = _activeRequest.Value is null ? BeginRequest("runtime-" + Guid.NewGuid().ToString("N")) : null; + var metadata = GetResponseMetadata(); + var result = await GenerateAudioCoreAsync(modelId, input, bearerToken, cancellationToken).ConfigureAwait(false); + return result with { WireMetadata = result.WireMetadata ?? metadata }; + } + + public async Task TranscribeAudioAsync(string modelId, string fileName, string? prompt = null, string? bearerToken = null, CancellationToken cancellationToken = default) + { + using var operation = _activeRequest.Value is null ? BeginRequest("runtime-" + Guid.NewGuid().ToString("N")) : null; + var metadata = GetResponseMetadata(); + var result = await TranscribeAudioCoreAsync(modelId, fileName, prompt, bearerToken, cancellationToken).ConfigureAwait(false); + return result with { WireMetadata = result.WireMetadata ?? metadata }; + } + + public async Task TranslateAudioAsync(string modelId, string fileName, string? prompt = null, string? bearerToken = null, CancellationToken cancellationToken = default) + { + using var operation = _activeRequest.Value is null ? BeginRequest("runtime-" + Guid.NewGuid().ToString("N")) : null; + var metadata = GetResponseMetadata(); + var result = await TranslateAudioCoreAsync(modelId, fileName, prompt, bearerToken, cancellationToken).ConfigureAwait(false); + return result with { WireMetadata = result.WireMetadata ?? metadata }; + } + + public async Task GenerateVideoAsync(string modelId, string prompt, string? bearerToken = null, CancellationToken cancellationToken = default) + { + using var operation = _activeRequest.Value is null ? BeginRequest("runtime-" + Guid.NewGuid().ToString("N")) : null; + var metadata = GetResponseMetadata(); + var result = await GenerateVideoCoreAsync(modelId, prompt, bearerToken, cancellationToken).ConfigureAwait(false); + return result with { WireMetadata = result.WireMetadata ?? metadata }; + } +} diff --git a/src/ManagedCode.LlmTck/Runtime/LlmTckRuntime.cs b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.cs similarity index 60% rename from src/ManagedCode.LlmTck/Runtime/LlmTckRuntime.cs rename to src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.cs index 3a86c7d..6484407 100644 --- a/src/ManagedCode.LlmTck/Runtime/LlmTckRuntime.cs +++ b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntime.cs @@ -1,20 +1,20 @@ using System.Security.Cryptography; using System.Text; -using ManagedCode.LlmTck.Configuration; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.Scenarios; +using ManagedCode.LlmTestServer.Configuration; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Scenarios; -namespace ManagedCode.LlmTck.Runtime; +namespace ManagedCode.LlmTestServer.Runtime; -public sealed partial class LlmTckRuntime - : ILlmTckRuntime, - ILlmTckRuntimeRequestScope, - ILlmTckRuntimeEventLookup, - ILlmTckVideoStore +public sealed partial class LlmTestServerRuntime + : ILlmTestServerRuntime, + ILlmTestServerRuntimeRequestScope, + ILlmTestServerRuntimeEventLookup, + ILlmTestServerVideoStore { private const string _tooManyRequestsCode = "too_many_requests"; private const string _contentFilterCode = "content_filter"; - private const string _requestSupersededCode = "llm_tck_request_superseded"; + private const string _requestSupersededCode = "llm_test_server_request_superseded"; private const string _requestSupersededMessage = "The runtime was reset or reconfigured while the request was in flight."; private const int _openAiPromptCacheMinimumTokens = 1024; @@ -29,18 +29,19 @@ public sealed partial class LlmTckRuntime private const int _retainedMessageCost = 256; private const int _retainedToolCost = 256; private const int _retainedChunkCost = 128; + private const int _retainedOutputDispositionCost = 32; private readonly object _gate = new(); private readonly AsyncLocal _activeRequest = new(); - private readonly Dictionary> _scenarioReservations = new(ReferenceEqualityComparer.Instance); + private readonly Dictionary> _scenarioReservations = new(ReferenceEqualityComparer.Instance); private readonly Dictionary _promptCacheEntries = new(StringComparer.Ordinal); private readonly Queue _promptCacheOrder = []; - private readonly Queue _events = []; - private readonly Dictionary _eventsByRequestId = new( + private readonly Queue _events = []; + private readonly Dictionary _eventsByRequestId = new( StringComparer.Ordinal ); - private readonly int[] _eventKindCounts = new int[Enum.GetValues().Length]; - private LlmTckConfiguration _configuration = LlmTckConfiguration.CreateDefault(); + private readonly int[] _eventKindCounts = new int[Enum.GetValues().Length]; + private LlmTestServerConfiguration _configuration = LlmTestServerConfiguration.CreateDefault(); private int _requestCount; private int _totalEvents; private int _inputTokens; @@ -51,19 +52,20 @@ public sealed partial class LlmTckRuntime private int _totalTokens; private int _retainedEventCharacters; private long _generation; + private bool _retired; - public LlmTckRuntime() + public LlmTestServerRuntime() { } - public LlmTckRuntime(LlmTckConfiguration configuration) + public LlmTestServerRuntime(LlmTestServerConfiguration configuration) { ArgumentNullException.ThrowIfNull(configuration); - _configuration = LlmTckConfigurationBuilder.Snapshot(configuration); + _configuration = LlmTestServerConfigurationBuilder.Snapshot(configuration); } public Task ConfigureAsync( - LlmTckConfiguration configuration, + LlmTestServerConfiguration configuration, CancellationToken cancellationToken = default ) { @@ -72,8 +74,10 @@ public Task ConfigureAsync( lock (_gate) { - _configuration = LlmTckConfigurationBuilder.Snapshot(configuration); + _configuration = LlmTestServerConfigurationBuilder.Snapshot(configuration); _scenarioReservations.Clear(); + ClearConversations(); + _wireSequences.Clear(); _chatResponses.Clear(); _chatResponseBytes = 0; _videos.Clear(); @@ -89,6 +93,15 @@ public Task ConfigureAsync( return Task.CompletedTask; } + public Task RetireAsync() + { + lock (_gate) + { + _retired = true; + return ResetAsync(); + } + } + public Task ResetAsync(CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); @@ -96,6 +109,8 @@ public Task ResetAsync(CancellationToken cancellationToken = default) lock (_gate) { _scenarioReservations.Clear(); + ClearConversations(); + _wireSequences.Clear(); _chatResponses.Clear(); _chatResponseBytes = 0; _videos.Clear(); @@ -119,7 +134,7 @@ public bool IsBearerTokenAccepted(string? bearerToken) } } - public IReadOnlyList GetModels() + public IReadOnlyList GetModels() { lock (_gate) { @@ -127,19 +142,19 @@ public IReadOnlyList GetModels() } } - public IDisposable BeginRequest(string requestId) + public IDisposable BeginRequest(string requestId, string? conversationId = null, string? providerNamespace = null) { ArgumentException.ThrowIfNullOrWhiteSpace(requestId); var previousRequest = _activeRequest.Value; lock (_gate) { - _activeRequest.Value = new RuntimeRequestContext(requestId, _generation); + _activeRequest.Value = new RuntimeRequestContext(requestId, _generation, conversationId, providerNamespace, ReserveResponseMetadata(providerNamespace ?? string.Empty)); } return new RuntimeRequestScope(_activeRequest, previousRequest); } - public bool TryGetEvent(string requestId, out LlmTckRuntimeEvent? runtimeEvent) + public bool TryGetEvent(string requestId, out LlmTestServerRuntimeEvent? runtimeEvent) { ArgumentException.ThrowIfNullOrWhiteSpace(requestId); lock (_gate) @@ -150,8 +165,8 @@ public bool TryGetEvent(string requestId, out LlmTckRuntimeEvent? runtimeEvent) } } - public async Task CompleteChatAsync( - LlmTckChatRequest request, + private async Task CompleteChatCoreAsync( + LlmTestServerChatRequest request, string? bearerToken = null, CancellationToken cancellationToken = default ) @@ -159,15 +174,16 @@ public async Task CompleteChatAsync( ArgumentNullException.ThrowIfNull(request); cancellationToken.ThrowIfCancellationRequested(); - LlmTckScenarioResponse response; - LlmTckScenario? scenario; - int responsePosition; + LlmTestServerScenarioResponse response; + LlmTestServerScenario? scenario; + var responsePosition = -1; + ConversationLease? conversationLease = null; long generation; lock (_gate) { generation = _activeRequest.Value?.Generation ?? _generation; - if (generation != _generation) + if (_retired || generation != _generation) { return CreateSupersededChatResult(request, scenarioId: null); } @@ -176,7 +192,7 @@ public async Task CompleteChatAsync( { var usage = CreateChatUsage(request); AddEvent( - LlmTckEventKind.AuthFailed, + LlmTestServerEventKind.AuthFailed, null, request.ModelId, "Global bearer token requirement failed.", @@ -184,32 +200,32 @@ public async Task CompleteChatAsync( usage: usage, chatRequest: request ); - return LlmTckChatResult.Failure( + return LlmTestServerChatResult.Failure( request.ModelId, 401, "invalid_api_key", - "The supplied bearer token did not match the configured LLM TCK token.", + "The supplied bearer token did not match the configured LLM Test Server token.", usage: usage ); } - if (!IsConfiguredModel(request.ModelId, LlmTckModelKind.Chat)) + if (!IsConfiguredModel(request.ModelId, LlmTestServerModelKind.Chat)) { var usage = CreateChatUsage(request); AddModelNotFoundEvent( request.ModelId, - LlmTckModelKind.Chat, + LlmTestServerModelKind.Chat, FormatChatRequest(request), usage, chatRequest: request ); - return UnknownModel(request.ModelId, LlmTckModelKind.Chat, usage); + return UnknownModel(request.ModelId, LlmTestServerModelKind.Chat, usage); } var historyError = RestoreChatResponse(ref request, bearerToken); if (historyError is not null) { - AddEvent(LlmTckEventKind.ErrorReturned, null, request.ModelId, historyError.ErrorMessage!, + AddEvent(LlmTestServerEventKind.ErrorReturned, null, request.ModelId, historyError.ErrorMessage!, FormatChatRequest(request), historyError.ErrorMessage, chatRequest: request); return historyError; } @@ -223,7 +239,7 @@ public async Task CompleteChatAsync( ); if (fault is not null) { - return LlmTckChatResult.Failure( + return LlmTestServerChatResult.Failure( request.ModelId, fault.StatusCode, fault.Code, @@ -232,14 +248,13 @@ public async Task CompleteChatAsync( ); } - scenario = GetChatScenarios(_configuration) - .FirstOrDefault(candidate => ScenarioMatches(candidate, request)); + scenario = SelectChatScenario(request, bearerToken); if (scenario is null) { var usage = CreateChatUsage(request); AddEvent( - LlmTckEventKind.Unmatched, + LlmTestServerEventKind.Unmatched, null, request.ModelId, "No configured scenario matched the request.", @@ -247,11 +262,11 @@ public async Task CompleteChatAsync( usage: usage, chatRequest: request ); - return LlmTckChatResult.Failure( + return LlmTestServerChatResult.Failure( request.ModelId, 404, - "llm_tck_unmatched_request", - "No configured LLM TCK scenario matched the request.", + "llm_test_server_unmatched_request", + "No configured LLM Test Server scenario matched the request.", usage: usage ); } @@ -260,7 +275,7 @@ public async Task CompleteChatAsync( { var usage = CreateChatUsage(request); AddEvent( - LlmTckEventKind.AuthFailed, + LlmTestServerEventKind.AuthFailed, scenario.Id, request.ModelId, "Scenario bearer token requirement failed.", @@ -268,7 +283,7 @@ public async Task CompleteChatAsync( usage: usage, chatRequest: request ); - return LlmTckChatResult.Failure( + return LlmTestServerChatResult.Failure( request.ModelId, 401, "invalid_api_key", @@ -278,17 +293,6 @@ public async Task CompleteChatAsync( ); } - if (!_scenarioReservations.TryGetValue(scenario, out var reservations)) - { - reservations = []; - _scenarioReservations.Add(scenario, reservations); - } - - responsePosition = 0; - while (reservations.Contains(responsePosition)) - { - responsePosition++; - } if (request.PopulateCacheOnly) { var cacheUsage = CreateChatUsage(request, updatePromptCache: true); @@ -298,48 +302,70 @@ public async Task CompleteChatAsync( ReasoningTokens = 0, TotalTokens = cacheUsage.InputTokens, }; - AddEvent(LlmTckEventKind.Matched, scenario.Id, request.ModelId, + AddEvent(LlmTestServerEventKind.Matched, scenario.Id, request.ModelId, "Prompt cache populated without generating a response.", FormatChatRequest(request), string.Empty, usage, chatRequest: request); - return LlmTckChatResult.Success(request.ModelId, scenario.Id, string.Empty, [], usage); + return LlmTestServerChatResult.Success(request.ModelId, scenario.Id, string.Empty, [], usage); } - if (responsePosition >= scenario.Responses.Count) + if (scenario.Conversation is not null) { - var usage = CreateChatUsage(request); - AddEvent( - LlmTckEventKind.ScenarioExhausted, - scenario.Id, - request.ModelId, - "Scenario response queue is exhausted.", - FormatChatRequest(request), - usage: usage, - chatRequest: request - ); - return LlmTckChatResult.Failure( - request.ModelId, - 409, - "llm_tck_scenario_exhausted", - "The matched scenario has no responses left.", - scenario.Id, - usage - ); + var immediate = PrepareConversation(scenario, request, bearerToken, out conversationLease); + if (immediate is not null) { return immediate; } + response = conversationLease!.Step.Response; } - - response = scenario.Responses[responsePosition]; - var fixtureError = ValidateFixture(request, response); - if (fixtureError is not null) + else { - var usage = CreateChatUsage(request); - AddEvent(LlmTckEventKind.ErrorReturned, scenario.Id, request.ModelId, - "The selected fixture does not satisfy the request.", FormatChatRequest(request), - fixtureError, usage, chatRequest: request); - return LlmTckChatResult.Failure(request.ModelId, 409, "llm_tck_fixture_mismatch", fixtureError, scenario.Id, usage); + if (!_scenarioReservations.TryGetValue(scenario, out var reservations)) + { + reservations = []; + _scenarioReservations.Add(scenario, reservations); + } + + responsePosition = 0; + while (reservations.Contains(responsePosition)) + { + responsePosition++; + } + if (responsePosition >= scenario.Responses.Count) + { + var usage = CreateChatUsage(request); + AddEvent( + LlmTestServerEventKind.ScenarioExhausted, + scenario.Id, + request.ModelId, + "Scenario response queue is exhausted.", + FormatChatRequest(request), + usage: usage, + chatRequest: request + ); + return LlmTestServerChatResult.Failure( + request.ModelId, + 409, + "llm_test_server_scenario_exhausted", + "The matched scenario has no responses left.", + scenario.Id, + usage + ); + } + + response = scenario.Responses[responsePosition]; + var fixtureError = ValidateFixture(request, response); + if (fixtureError is not null) + { + var usage = CreateChatUsage(request); + AddEvent(LlmTestServerEventKind.ErrorReturned, scenario.Id, request.ModelId, + "The selected fixture does not satisfy the request.", FormatChatRequest(request), + fixtureError, usage, chatRequest: request); + return LlmTestServerChatResult.Failure(request.ModelId, 409, "llm_test_server_fixture_mismatch", fixtureError, scenario.Id, usage); + } + + reservations.Add(responsePosition); } - reservations.Add(responsePosition); } + response = LlmTestServerResponseItems.Project(response); try { if (response.DelayMilliseconds > 0) @@ -349,7 +375,7 @@ public async Task CompleteChatAsync( } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { - RollBackReservedResponse(scenario, responsePosition, generation); + RollBackResponse(scenario, responsePosition, generation, conversationLease); throw; } @@ -357,40 +383,46 @@ public async Task CompleteChatAsync( { if (cancellationToken.IsCancellationRequested) { - RollBackReservedResponse(scenario, responsePosition, generation); + RollBackResponse(scenario, responsePosition, generation, conversationLease); cancellationToken.ThrowIfCancellationRequested(); } - if (generation != _generation) + if (_retired || generation != _generation) { return CreateSupersededChatResult(request, scenario.Id); } var responseText = response.Error?.Message ?? response.Content; - var usage = CreateChatUsage( - request, - response.Error is null ? response.Content + string.Concat(response.ToolCalls.Select(call => call.Name + call.ArgumentsJson)) : null, - updatePromptCache: response.Error is null - ); + var responseOutput = response.Error is null + ? response.Content + string.Concat(response.ToolCalls.Select(call => call.Name + call.ArgumentsJson)) + LlmTestServerResponseItems.NativeTokenText(response.OutputItems) + : null; IReadOnlyList successfulStreamChunks = response.StreamChunks.Count > 0 ? response.StreamChunks : [response.Content]; - LlmTckChatResult? completed = null; + LlmTestServerChatResult? completed = null; if (response.Error is null) { - completed = RetainChatResponse(request, LlmTckChatResult.Success(request.ModelId, scenario.Id, - response.Content, successfulStreamChunks, usage) with - { ToolCalls = [.. response.ToolCalls] }, bearerToken); + completed = RetainChatResponse(request, LlmTestServerChatResult.Success(request.ModelId, scenario.Id, + response.Content, successfulStreamChunks) with + { OutputDisposition = response.OutputDisposition, StreamDelivery = response.StreamDelivery, ToolCalls = [.. response.ToolCalls], OutputItems = LlmTestServerResponseItems.Snapshot(response.OutputItems), WireMetadata = GetResponseMetadata() }, bearerToken); if (!completed.IsSuccess) { - RollBackReservedResponse(scenario, responsePosition, generation); - AddEvent(LlmTckEventKind.ErrorReturned, scenario.Id, request.ModelId, completed.ErrorMessage!, - FormatChatRequest(request), completed.ErrorMessage, usage, chatRequest: request); + RollBackResponse(scenario, responsePosition, generation, conversationLease); + var rejectedUsage = CreateChatUsage(request, responseOutput); + AddEvent(LlmTestServerEventKind.ErrorReturned, scenario.Id, request.ModelId, completed.ErrorMessage!, + FormatChatRequest(request), completed.ErrorMessage, rejectedUsage, chatRequest: request); return completed; } } + var usage = CreateChatUsage(request, responseOutput, updatePromptCache: response.Error is null); + if (completed is not null) { completed = completed with { Usage = usage }; } + var finalResult = response.Error is { } error + ? LlmTestServerChatResult.Failure(request.ModelId, error.StatusCode, error.Code, error.Message, scenario.Id, usage) + : completed!; + finalResult = finalResult with { WireMetadata = finalResult.WireMetadata ?? GetResponseMetadata() }; + if (conversationLease is not null) { CommitConversation(conversationLease, finalResult); } AddEvent( - response.Error is null ? LlmTckEventKind.Matched : LlmTckEventKind.ErrorReturned, + response.Error is null ? LlmTestServerEventKind.Matched : LlmTestServerEventKind.ErrorReturned, scenario.Id, request.ModelId, response.Error?.Message ?? "Scenario matched.", @@ -398,12 +430,18 @@ public async Task CompleteChatAsync( responseText, usage, chatRequest: request, - streamChunks: response.Error is null && request.Stream ? successfulStreamChunks : null + streamChunks: response.Error is null && request.Stream ? successfulStreamChunks : null, + stepId: conversationLease?.Step.Id, + logicalTurnIndex: conversationLease?.Turn, + responseToolCalls: response.Error is null ? response.ToolCalls : null, + retryAfterSeconds: response.Error?.RetryAfterSeconds, + wireMetadata: finalResult.WireMetadata, + responseOutputItems: finalResult.OutputItems, streamDelivery: finalResult.StreamDelivery, outputDisposition: finalResult.OutputDisposition ); if (response.Error is not null) { - return LlmTckChatResult.Failure( + return LlmTestServerChatResult.Failure( request.ModelId, response.Error.StatusCode, response.Error.Code, @@ -417,10 +455,11 @@ public async Task CompleteChatAsync( } } - public Task CreateEmbeddingAsync( + private Task CreateEmbeddingCoreAsync( string modelId, IReadOnlyList inputs, string? bearerToken = null, + IReadOnlyList? outputDimensions = null, CancellationToken cancellationToken = default ) { @@ -438,32 +477,35 @@ public Task CreateEmbeddingAsync( if (!HasRequiredToken(_configuration.RequiredBearerToken, bearerToken)) { AddEvent( - LlmTckEventKind.AuthFailed, + LlmTestServerEventKind.AuthFailed, null, modelId, "Embedding bearer token requirement failed." ); return Task.FromResult( - new LlmTckEmbeddingResult + new LlmTestServerEmbeddingResult { IsSuccess = false, StatusCode = 401, ModelId = modelId, ErrorCode = "invalid_api_key", - ErrorMessage = "The supplied bearer token did not match the configured LLM TCK token.", + ErrorMessage = "The supplied bearer token did not match the configured LLM Test Server token.", } ); } - if (!IsConfiguredModel(modelId, LlmTckModelKind.Embedding)) + if (!IsConfiguredModel(modelId, LlmTestServerModelKind.Embedding)) { AddModelNotFoundEvent( modelId, - LlmTckModelKind.Embedding + LlmTestServerModelKind.Embedding ); return Task.FromResult(UnknownEmbeddingModel(modelId)); } + var dimensionError = ValidateEmbeddingDimensions(modelId, inputs.Count, outputDimensions); + if (dimensionError is not null) { return Task.FromResult(dimensionError); } + var fault = TryCreateFault( modelId, inputs, @@ -472,7 +514,7 @@ public Task CreateEmbeddingAsync( if (fault is not null) { return Task.FromResult( - new LlmTckEmbeddingResult + new LlmTestServerEmbeddingResult { IsSuccess = false, StatusCode = fault.StatusCode, @@ -484,24 +526,25 @@ public Task CreateEmbeddingAsync( } AddEvent( - LlmTckEventKind.Matched, + LlmTestServerEventKind.Matched, null, modelId, $"Generated deterministic embeddings for {inputs.Count} input value(s)." ); return Task.FromResult( - new LlmTckEmbeddingResult + new LlmTestServerEmbeddingResult { IsSuccess = true, ModelId = modelId, - Vectors = inputs.Select(_ => _configuration.DefaultEmbeddingVector.ToList()).ToList(), + Vectors = inputs.Select((_, index) => _configuration.DefaultEmbeddingVector + .Take(outputDimensions?[index] ?? _configuration.DefaultEmbeddingVector.Count).ToList()).ToList(), } ); } } - public Task GenerateImageAsync( + private Task GenerateImageCoreAsync( string modelId, string prompt, string? bearerToken = null, @@ -522,26 +565,26 @@ public Task GenerateImageAsync( if (!HasRequiredToken(_configuration.RequiredBearerToken, bearerToken)) { AddEvent( - LlmTckEventKind.AuthFailed, + LlmTestServerEventKind.AuthFailed, null, modelId, "Image auth failed." ); return Task.FromResult( - new LlmTckImageResult + new LlmTestServerImageResult { IsSuccess = false, StatusCode = 401, ModelId = modelId, ErrorCode = "invalid_api_key", - ErrorMessage = "The supplied bearer token did not match the configured LLM TCK token.", + ErrorMessage = "The supplied bearer token did not match the configured LLM Test Server token.", } ); } - if (!IsConfiguredModel(modelId, LlmTckModelKind.Image)) + if (!IsConfiguredModel(modelId, LlmTestServerModelKind.Image)) { - AddModelNotFoundEvent(modelId, LlmTckModelKind.Image); + AddModelNotFoundEvent(modelId, LlmTestServerModelKind.Image); return Task.FromResult(UnknownImageModel(modelId)); } @@ -553,7 +596,7 @@ public Task GenerateImageAsync( if (fault is not null) { return Task.FromResult( - new LlmTckImageResult + new LlmTestServerImageResult { IsSuccess = false, StatusCode = fault.StatusCode, @@ -565,13 +608,13 @@ public Task GenerateImageAsync( } AddEvent( - LlmTckEventKind.Matched, + LlmTestServerEventKind.Matched, null, modelId, $"Generated image for '{prompt}'." ); return Task.FromResult( - new LlmTckImageResult + new LlmTestServerImageResult { IsSuccess = true, ModelId = modelId, @@ -581,7 +624,7 @@ public Task GenerateImageAsync( } } - public Task GenerateAudioAsync( + private Task GenerateAudioCoreAsync( string modelId, string input, string? bearerToken = null, @@ -602,26 +645,26 @@ public Task GenerateAudioAsync( if (!HasRequiredToken(_configuration.RequiredBearerToken, bearerToken)) { AddEvent( - LlmTckEventKind.AuthFailed, + LlmTestServerEventKind.AuthFailed, null, modelId, "Audio auth failed." ); return Task.FromResult( - new LlmTckAudioResult + new LlmTestServerAudioResult { IsSuccess = false, StatusCode = 401, ModelId = modelId, ErrorCode = "invalid_api_key", - ErrorMessage = "The supplied bearer token did not match the configured LLM TCK token.", + ErrorMessage = "The supplied bearer token did not match the configured LLM Test Server token.", } ); } - if (!IsConfiguredModel(modelId, LlmTckModelKind.Audio)) + if (!IsConfiguredModel(modelId, LlmTestServerModelKind.Audio)) { - AddModelNotFoundEvent(modelId, LlmTckModelKind.Audio); + AddModelNotFoundEvent(modelId, LlmTestServerModelKind.Audio); return Task.FromResult(UnknownAudioModel(modelId)); } @@ -633,7 +676,7 @@ public Task GenerateAudioAsync( if (fault is not null) { return Task.FromResult( - new LlmTckAudioResult + new LlmTestServerAudioResult { IsSuccess = false, StatusCode = fault.StatusCode, @@ -645,13 +688,13 @@ public Task GenerateAudioAsync( } AddEvent( - LlmTckEventKind.Matched, + LlmTestServerEventKind.Matched, null, modelId, $"Generated audio for '{input}'." ); return Task.FromResult( - new LlmTckAudioResult + new LlmTestServerAudioResult { IsSuccess = true, ModelId = modelId, @@ -662,7 +705,7 @@ public Task GenerateAudioAsync( } } - public Task TranscribeAudioAsync( + private Task TranscribeAudioCoreAsync( string modelId, string fileName, string? prompt = null, @@ -684,26 +727,26 @@ public Task TranscribeAudioAsync( if (!HasRequiredToken(_configuration.RequiredBearerToken, bearerToken)) { AddEvent( - LlmTckEventKind.AuthFailed, + LlmTestServerEventKind.AuthFailed, null, modelId, "Audio transcription auth failed." ); return Task.FromResult( - new LlmTckTranscriptionResult + new LlmTestServerTranscriptionResult { IsSuccess = false, StatusCode = 401, ModelId = modelId, ErrorCode = "invalid_api_key", - ErrorMessage = "The supplied bearer token did not match the configured LLM TCK token.", + ErrorMessage = "The supplied bearer token did not match the configured LLM Test Server token.", } ); } - if (!IsConfiguredModel(modelId, LlmTckModelKind.Audio)) + if (!IsConfiguredModel(modelId, LlmTestServerModelKind.Audio)) { - AddModelNotFoundEvent(modelId, LlmTckModelKind.Audio); + AddModelNotFoundEvent(modelId, LlmTestServerModelKind.Audio); return Task.FromResult(UnknownTranscriptionModel(modelId)); } @@ -715,7 +758,7 @@ public Task TranscribeAudioAsync( if (fault is not null) { return Task.FromResult( - new LlmTckTranscriptionResult + new LlmTestServerTranscriptionResult { IsSuccess = false, StatusCode = fault.StatusCode, @@ -727,7 +770,7 @@ public Task TranscribeAudioAsync( } AddEvent( - LlmTckEventKind.Matched, + LlmTestServerEventKind.Matched, null, modelId, string.IsNullOrWhiteSpace(prompt) @@ -735,7 +778,7 @@ public Task TranscribeAudioAsync( : $"Transcribed audio file '{fileName}' with prompt." ); return Task.FromResult( - new LlmTckTranscriptionResult + new LlmTestServerTranscriptionResult { IsSuccess = true, ModelId = modelId, @@ -746,7 +789,7 @@ public Task TranscribeAudioAsync( } } - public Task TranslateAudioAsync( + private Task TranslateAudioCoreAsync( string modelId, string fileName, string? prompt = null, @@ -768,26 +811,26 @@ public Task TranslateAudioAsync( if (!HasRequiredToken(_configuration.RequiredBearerToken, bearerToken)) { AddEvent( - LlmTckEventKind.AuthFailed, + LlmTestServerEventKind.AuthFailed, null, modelId, "Audio translation auth failed." ); return Task.FromResult( - new LlmTckTranscriptionResult + new LlmTestServerTranscriptionResult { IsSuccess = false, StatusCode = 401, ModelId = modelId, ErrorCode = "invalid_api_key", - ErrorMessage = "The supplied bearer token did not match the configured LLM TCK token.", + ErrorMessage = "The supplied bearer token did not match the configured LLM Test Server token.", } ); } - if (!IsConfiguredModel(modelId, LlmTckModelKind.Audio)) + if (!IsConfiguredModel(modelId, LlmTestServerModelKind.Audio)) { - AddModelNotFoundEvent(modelId, LlmTckModelKind.Audio); + AddModelNotFoundEvent(modelId, LlmTestServerModelKind.Audio); return Task.FromResult(UnknownTranscriptionModel(modelId)); } @@ -799,7 +842,7 @@ public Task TranslateAudioAsync( if (fault is not null) { return Task.FromResult( - new LlmTckTranscriptionResult + new LlmTestServerTranscriptionResult { IsSuccess = false, StatusCode = fault.StatusCode, @@ -811,7 +854,7 @@ public Task TranslateAudioAsync( } AddEvent( - LlmTckEventKind.Matched, + LlmTestServerEventKind.Matched, null, modelId, string.IsNullOrWhiteSpace(prompt) @@ -819,7 +862,7 @@ public Task TranslateAudioAsync( : $"Translated audio file '{fileName}' with prompt." ); return Task.FromResult( - new LlmTckTranscriptionResult + new LlmTestServerTranscriptionResult { IsSuccess = true, ModelId = modelId, @@ -829,7 +872,7 @@ public Task TranslateAudioAsync( } } - public Task GenerateVideoAsync( + private Task GenerateVideoCoreAsync( string modelId, string prompt, string? bearerToken = null, @@ -851,30 +894,30 @@ public Task GenerateVideoAsync( if (!HasRequiredToken(_configuration.RequiredBearerToken, bearerToken)) { AddEvent( - LlmTckEventKind.AuthFailed, + LlmTestServerEventKind.AuthFailed, null, modelId, "Video auth failed.", usage: usage ); return Task.FromResult( - new LlmTckVideoResult + new LlmTestServerVideoResult { IsSuccess = false, StatusCode = 401, ModelId = modelId, ErrorCode = "invalid_api_key", - ErrorMessage = "The supplied bearer token did not match the configured LLM TCK token.", + ErrorMessage = "The supplied bearer token did not match the configured LLM Test Server token.", Usage = usage, } ); } - if (!IsConfiguredModel(modelId, LlmTckModelKind.Video)) + if (!IsConfiguredModel(modelId, LlmTestServerModelKind.Video)) { AddModelNotFoundEvent( modelId, - LlmTckModelKind.Video, + LlmTestServerModelKind.Video, usage: usage ); return Task.FromResult(UnknownVideoModel(modelId, usage)); @@ -889,7 +932,7 @@ public Task GenerateVideoAsync( if (fault is not null) { return Task.FromResult( - new LlmTckVideoResult + new LlmTestServerVideoResult { IsSuccess = false, StatusCode = fault.StatusCode, @@ -902,14 +945,14 @@ public Task GenerateVideoAsync( } AddEvent( - LlmTckEventKind.Matched, + LlmTestServerEventKind.Matched, null, modelId, $"Generated video for '{prompt}'.", usage: usage ); return Task.FromResult( - new LlmTckVideoResult + new LlmTestServerVideoResult { IsSuccess = true, ModelId = modelId, @@ -927,19 +970,19 @@ public Task GenerateVideoAsync( } } - public LlmTckAssertionSummary GetAssertionSummary() + public LlmTestServerAssertionSummary GetAssertionSummary() { lock (_gate) { - return new LlmTckAssertionSummary + return new LlmTestServerAssertionSummary { TotalEvents = _totalEvents, - Matched = _eventKindCounts[(int)LlmTckEventKind.Matched], - Unmatched = _eventKindCounts[(int)LlmTckEventKind.Unmatched], - ModelNotFound = _eventKindCounts[(int)LlmTckEventKind.ModelNotFound], - AuthFailed = _eventKindCounts[(int)LlmTckEventKind.AuthFailed], - ScenarioExhausted = _eventKindCounts[(int)LlmTckEventKind.ScenarioExhausted], - ErrorsReturned = _eventKindCounts[(int)LlmTckEventKind.ErrorReturned], + Matched = _eventKindCounts[(int)LlmTestServerEventKind.Matched], + Unmatched = _eventKindCounts[(int)LlmTestServerEventKind.Unmatched], + ModelNotFound = _eventKindCounts[(int)LlmTestServerEventKind.ModelNotFound], + AuthFailed = _eventKindCounts[(int)LlmTestServerEventKind.AuthFailed], + ScenarioExhausted = _eventKindCounts[(int)LlmTestServerEventKind.ScenarioExhausted], + ErrorsReturned = _eventKindCounts[(int)LlmTestServerEventKind.ErrorReturned], InputTokens = _inputTokens, CachedInputTokens = _cachedInputTokens, CacheCreationInputTokens = _cacheCreationInputTokens, @@ -951,11 +994,13 @@ public LlmTckAssertionSummary GetAssertionSummary() } } - private static LlmTckRuntimeEvent SnapshotEvent(LlmTckRuntimeEvent item) + private static LlmTestServerRuntimeEvent SnapshotEvent(LlmTestServerRuntimeEvent item) { return item with { - Messages = item.Messages.Select(message => message with { ToolCalls = [.. message.ToolCalls] }).ToArray(), + Messages = item.Messages.Select(message => message with { ToolCalls = [.. message.ToolCalls], OutputItems = LlmTestServerResponseItems.Snapshot(message.OutputItems) }).ToArray(), + ResponseOutputItems = LlmTestServerResponseItems.Snapshot(item.ResponseOutputItems), + NativeTools = LlmTestServerResponseItems.SnapshotDeclarations(item.NativeTools), StreamChunks = item.StreamChunks.ToArray(), }; } @@ -965,14 +1010,14 @@ private static bool HasRequiredToken(string? required, string? supplied) return string.IsNullOrEmpty(required) || string.Equals(required, supplied, StringComparison.Ordinal); } - private static IEnumerable GetChatScenarios(LlmTckConfiguration configuration) + private static IEnumerable GetChatScenarios(LlmTestServerConfiguration configuration) { return configuration.ChatScenarios.Concat( configuration.Datasets.SelectMany(dataset => dataset.ChatScenarios) ); } - private bool IsConfiguredModel(string modelId, LlmTckModelKind kind) + private bool IsConfiguredModel(string modelId, LlmTestServerModelKind kind) { return _configuration.Models.Any(model => model.Kind == kind && string.Equals(model.Id, modelId, StringComparison.OrdinalIgnoreCase) @@ -984,18 +1029,18 @@ private int GetReasoningTokens(string modelId) return _configuration .Models .FirstOrDefault(model => - model.Kind == LlmTckModelKind.Chat + model.Kind == LlmTestServerModelKind.Chat && string.Equals(model.Id, modelId, StringComparison.OrdinalIgnoreCase) ) ?.ReasoningTokens ?? 0; } - private LlmTckRuntimeFault? TryCreateFault( + private LlmTestServerRuntimeFault? TryCreateFault( string modelId, IEnumerable requestContent, string? request = null, - LlmTckTokenUsage? usage = null, - LlmTckChatRequest? chatRequest = null + LlmTestServerTokenUsage? usage = null, + LlmTestServerChatRequest? chatRequest = null ) { var faultSimulation = _configuration.FaultSimulation; @@ -1006,9 +1051,9 @@ faultSimulation.MaxRequestsBeforeRateLimit is int allowedRequests && _requestCount >= allowedRequests ) { - var message = $"{_tooManyRequestsCode}: configured LLM TCK request limit was exceeded."; + var message = $"{_tooManyRequestsCode}: configured LLM Test Server request limit was exceeded."; AddEvent( - LlmTckEventKind.ErrorReturned, + LlmTestServerEventKind.ErrorReturned, null, modelId, message, @@ -1017,7 +1062,7 @@ faultSimulation.MaxRequestsBeforeRateLimit is int allowedRequests usage, chatRequest: chatRequest ); - return new LlmTckRuntimeFault(429, _tooManyRequestsCode, message, usage); + return new LlmTestServerRuntimeFault(429, _tooManyRequestsCode, message, usage); } _requestCount++; @@ -1032,9 +1077,9 @@ faultSimulation.MaxRequestsBeforeRateLimit is int allowedRequests } var filterMessage = - $"{_contentFilterCode}: request matched configured LLM TCK content filter term '{filteredTerm}'."; + $"{_contentFilterCode}: request matched configured LLM Test Server content filter term '{filteredTerm}'."; AddEvent( - LlmTckEventKind.ErrorReturned, + LlmTestServerEventKind.ErrorReturned, null, modelId, filterMessage, @@ -1043,7 +1088,7 @@ faultSimulation.MaxRequestsBeforeRateLimit is int allowedRequests usage, chatRequest: chatRequest ); - return new LlmTckRuntimeFault(400, _contentFilterCode, filterMessage, usage); + return new LlmTestServerRuntimeFault(400, _contentFilterCode, filterMessage, usage); } private static string? FindContentFilterTerm( @@ -1075,72 +1120,72 @@ IReadOnlyList contentFilterTerms return null; } - private static LlmTckChatResult UnknownModel( + private static LlmTestServerChatResult UnknownModel( string modelId, - LlmTckModelKind kind, - LlmTckTokenUsage? usage = null + LlmTestServerModelKind kind, + LlmTestServerTokenUsage? usage = null ) { - return LlmTckChatResult.Failure( + return LlmTestServerChatResult.Failure( modelId, 404, - "llm_tck_unknown_model", + "llm_test_server_unknown_model", CreateUnknownModelMessage(modelId, kind), usage: usage ); } - private static LlmTckEmbeddingResult UnknownEmbeddingModel(string modelId) + private static LlmTestServerEmbeddingResult UnknownEmbeddingModel(string modelId) { return new() { IsSuccess = false, StatusCode = 404, ModelId = modelId, - ErrorCode = "llm_tck_unknown_model", - ErrorMessage = CreateUnknownModelMessage(modelId, LlmTckModelKind.Embedding), + ErrorCode = "llm_test_server_unknown_model", + ErrorMessage = CreateUnknownModelMessage(modelId, LlmTestServerModelKind.Embedding), }; } - private static LlmTckImageResult UnknownImageModel(string modelId) + private static LlmTestServerImageResult UnknownImageModel(string modelId) { return new() { IsSuccess = false, StatusCode = 404, ModelId = modelId, - ErrorCode = "llm_tck_unknown_model", - ErrorMessage = CreateUnknownModelMessage(modelId, LlmTckModelKind.Image), + ErrorCode = "llm_test_server_unknown_model", + ErrorMessage = CreateUnknownModelMessage(modelId, LlmTestServerModelKind.Image), }; } - private static LlmTckAudioResult UnknownAudioModel(string modelId) + private static LlmTestServerAudioResult UnknownAudioModel(string modelId) { return new() { IsSuccess = false, StatusCode = 404, ModelId = modelId, - ErrorCode = "llm_tck_unknown_model", - ErrorMessage = CreateUnknownModelMessage(modelId, LlmTckModelKind.Audio), + ErrorCode = "llm_test_server_unknown_model", + ErrorMessage = CreateUnknownModelMessage(modelId, LlmTestServerModelKind.Audio), }; } - private static LlmTckTranscriptionResult UnknownTranscriptionModel(string modelId) + private static LlmTestServerTranscriptionResult UnknownTranscriptionModel(string modelId) { return new() { IsSuccess = false, StatusCode = 404, ModelId = modelId, - ErrorCode = "llm_tck_unknown_model", - ErrorMessage = CreateUnknownModelMessage(modelId, LlmTckModelKind.Audio), + ErrorCode = "llm_test_server_unknown_model", + ErrorMessage = CreateUnknownModelMessage(modelId, LlmTestServerModelKind.Audio), }; } - private static LlmTckVideoResult UnknownVideoModel( + private static LlmTestServerVideoResult UnknownVideoModel( string modelId, - LlmTckTokenUsage? usage = null + LlmTestServerTokenUsage? usage = null ) { return new() @@ -1148,27 +1193,27 @@ private static LlmTckVideoResult UnknownVideoModel( IsSuccess = false, StatusCode = 404, ModelId = modelId, - ErrorCode = "llm_tck_unknown_model", - ErrorMessage = CreateUnknownModelMessage(modelId, LlmTckModelKind.Video), - Usage = usage ?? new LlmTckTokenUsage(), + ErrorCode = "llm_test_server_unknown_model", + ErrorMessage = CreateUnknownModelMessage(modelId, LlmTestServerModelKind.Video), + Usage = usage ?? new LlmTestServerTokenUsage(), }; } - private static string CreateUnknownModelMessage(string modelId, LlmTckModelKind kind) + private static string CreateUnknownModelMessage(string modelId, LlmTestServerModelKind kind) { return $"Model '{modelId}' is not configured for {kind.ToString().ToLowerInvariant()} requests."; } private void AddModelNotFoundEvent( string modelId, - LlmTckModelKind kind, + LlmTestServerModelKind kind, string? request = null, - LlmTckTokenUsage? usage = null, - LlmTckChatRequest? chatRequest = null + LlmTestServerTokenUsage? usage = null, + LlmTestServerChatRequest? chatRequest = null ) { AddEvent( - LlmTckEventKind.ModelNotFound, + LlmTestServerEventKind.ModelNotFound, null, modelId, CreateUnknownModelMessage(modelId, kind), @@ -1179,14 +1224,14 @@ private void AddModelNotFoundEvent( } private void RollBackReservedResponse( - LlmTckScenario scenario, + LlmTestServerScenario scenario, int reservedPosition, long generation ) { lock (_gate) { - if (generation != _generation) + if (_retired || generation != _generation) { return; } @@ -1198,12 +1243,12 @@ long generation } } - private LlmTckChatResult CreateSupersededChatResult( - LlmTckChatRequest request, + private LlmTestServerChatResult CreateSupersededChatResult( + LlmTestServerChatRequest request, string? scenarioId ) { - return LlmTckChatResult.Failure( + return LlmTestServerChatResult.Failure( request.ModelId, 409, _requestSupersededCode, @@ -1215,12 +1260,12 @@ private LlmTckChatResult CreateSupersededChatResult( private bool IsActiveRequestSuperseded() { - return _activeRequest.Value is { } request && request.Generation != _generation; + return _retired || _activeRequest.Value is { } request && request.Generation != _generation; } - private static LlmTckEmbeddingResult CreateSupersededEmbeddingResult(string modelId) + private static LlmTestServerEmbeddingResult CreateSupersededEmbeddingResult(string modelId) { - return new LlmTckEmbeddingResult + return new LlmTestServerEmbeddingResult { IsSuccess = false, StatusCode = 409, @@ -1230,9 +1275,9 @@ private static LlmTckEmbeddingResult CreateSupersededEmbeddingResult(string mode }; } - private static LlmTckImageResult CreateSupersededImageResult(string modelId) + private static LlmTestServerImageResult CreateSupersededImageResult(string modelId) { - return new LlmTckImageResult + return new LlmTestServerImageResult { IsSuccess = false, StatusCode = 409, @@ -1242,9 +1287,9 @@ private static LlmTckImageResult CreateSupersededImageResult(string modelId) }; } - private static LlmTckAudioResult CreateSupersededAudioResult(string modelId) + private static LlmTestServerAudioResult CreateSupersededAudioResult(string modelId) { - return new LlmTckAudioResult + return new LlmTestServerAudioResult { IsSuccess = false, StatusCode = 409, @@ -1254,11 +1299,11 @@ private static LlmTckAudioResult CreateSupersededAudioResult(string modelId) }; } - private static LlmTckTranscriptionResult CreateSupersededTranscriptionResult( + private static LlmTestServerTranscriptionResult CreateSupersededTranscriptionResult( string modelId ) { - return new LlmTckTranscriptionResult + return new LlmTestServerTranscriptionResult { IsSuccess = false, StatusCode = 409, @@ -1268,9 +1313,9 @@ string modelId }; } - private static LlmTckVideoResult CreateSupersededVideoResult(string modelId) + private static LlmTestServerVideoResult CreateSupersededVideoResult(string modelId) { - return new LlmTckVideoResult + return new LlmTestServerVideoResult { IsSuccess = false, StatusCode = 409, @@ -1280,7 +1325,7 @@ private static LlmTckVideoResult CreateSupersededVideoResult(string modelId) }; } - private static bool ScenarioMatches(LlmTckScenario scenario, LlmTckChatRequest request) + private static bool ScenarioMatches(LlmTestServerScenario scenario, LlmTestServerChatRequest request) { if (!string.Equals(scenario.ModelId, request.ModelId, StringComparison.OrdinalIgnoreCase)) { @@ -1289,14 +1334,14 @@ private static bool ScenarioMatches(LlmTckScenario scenario, LlmTckChatRequest r return scenario.Match.Messages.Count == 0 || scenario.Match.Mode switch { - LlmTckMatchMode.Exact => ExactMessagesMatch(scenario.Match.Messages, request.Messages), + LlmTestServerMatchMode.Exact => ExactMessagesMatch(scenario.Match.Messages, request.Messages), _ => ContainsMessagesMatch(scenario.Match.Messages, request.Messages), }; } private static bool ExactMessagesMatch( - IReadOnlyList expected, - IReadOnlyList actual + IReadOnlyList expected, + IReadOnlyList actual ) { if (expected.Count != actual.Count) @@ -1306,7 +1351,7 @@ IReadOnlyList actual for (var i = 0; i < expected.Count; i++) { - if (!MessageMatches(expected[i], actual[i], LlmTckMatchMode.Exact)) + if (!MessageMatches(expected[i], actual[i], LlmTestServerMatchMode.Exact)) { return false; } @@ -1316,42 +1361,55 @@ IReadOnlyList actual } private static bool ContainsMessagesMatch( - IReadOnlyList expected, - IReadOnlyList actual + IReadOnlyList expected, + IReadOnlyList actual ) { return expected.All(item => - actual.Any(candidate => MessageMatches(item, candidate, LlmTckMatchMode.Contains)) + actual.Any(candidate => MessageMatches(item, candidate, LlmTestServerMatchMode.Contains)) ); } private static bool MessageMatches( - LlmTckMessage expected, - LlmTckMessage actual, - LlmTckMatchMode mode + LlmTestServerMessage expected, + LlmTestServerMessage actual, + LlmTestServerMatchMode mode ) { var roleMatches = string.IsNullOrWhiteSpace(expected.Role) || string.Equals(expected.Role, actual.Role, StringComparison.OrdinalIgnoreCase); - var contentMatches = mode == LlmTckMatchMode.Exact + var contentMatches = mode == LlmTestServerMatchMode.Exact ? string.Equals(expected.Content, actual.Content, StringComparison.Ordinal) : actual.Content.Contains(expected.Content, StringComparison.OrdinalIgnoreCase); + var expectedDisposition = LlmTestServerOutputDisposition.ForHistory(expected.OutputDisposition); + var actualDisposition = LlmTestServerOutputDisposition.ForHistory(actual.OutputDisposition); return roleMatches && contentMatches - && ((mode != LlmTckMatchMode.Exact && expected.ToolCallId is null) || expected.ToolCallId == actual.ToolCallId) - && ((mode != LlmTckMatchMode.Exact && expected.ToolCalls.Count == 0) || expected.ToolCalls.SequenceEqual(actual.ToolCalls)); + && ((mode != LlmTestServerMatchMode.Exact && expectedDisposition is null) || expectedDisposition == actualDisposition) + && ((mode != LlmTestServerMatchMode.Exact && expected.ToolCallId is null) || expected.ToolCallId == actual.ToolCallId) + && ((mode != LlmTestServerMatchMode.Exact && expected.ToolCalls.Count == 0) || expected.ToolCalls.SequenceEqual(actual.ToolCalls)) + && (expected.OutputItems.Count == 0 || System.Text.Json.JsonSerializer.Serialize(expected.OutputItems) == System.Text.Json.JsonSerializer.Serialize(actual.OutputItems)); } private void AddEvent( - LlmTckEventKind kind, + LlmTestServerEventKind kind, string? scenarioId, string modelId, string message, string? request = null, string? response = null, - LlmTckTokenUsage? usage = null, - LlmTckChatRequest? chatRequest = null, - IReadOnlyList? streamChunks = null + LlmTestServerTokenUsage? usage = null, + LlmTestServerChatRequest? chatRequest = null, + IReadOnlyList? streamChunks = null, + string? stepId = null, + int? logicalTurnIndex = null, + bool isReplay = false, + IReadOnlyList? responseToolCalls = null, + int? retryAfterSeconds = null, + ManagedCode.LlmTestServer.Determinism.LlmTestServerResponseMetadata? wireMetadata = null, + IReadOnlyList? responseOutputItems = null, + LlmTestServerStreamDelivery? streamDelivery = null, + LlmTestServerOutputDisposition? outputDisposition = null ) { var activeRequest = _activeRequest.Value; @@ -1360,21 +1418,32 @@ private void AddEvent( return; } - var runtimeEvent = BoundEventPayload(new LlmTckRuntimeEvent + var runtimeEvent = BoundEventPayload(new LlmTestServerRuntimeEvent { Timestamp = DateTimeOffset.UtcNow, Kind = kind, RequestId = activeRequest?.RequestId ?? chatRequest?.RequestId, ScenarioId = scenarioId, + ConversationId = activeRequest?.ConversationId ?? (stepId is null ? null : "default"), + StepId = stepId, + TurnIndex = logicalTurnIndex, + IsReplay = isReplay, + ResponseToolCalls = responseToolCalls ?? [], + ResponseOutputItems = responseOutputItems ?? [], + StreamDelivery = streamDelivery, + OutputDisposition = outputDisposition, + RetryAfterSeconds = retryAfterSeconds, + WireMetadata = wireMetadata ?? activeRequest?.WireMetadata, ModelId = modelId, Message = message, Request = request, Messages = chatRequest?.Messages ?? [], Tools = chatRequest?.Tools ?? [], + NativeTools = chatRequest?.NativeTools ?? [], Response = response, IsStreaming = chatRequest?.Stream ?? false, StreamChunks = streamChunks ?? [], - PromptCachePolicy = chatRequest?.PromptCachePolicy ?? LlmTckPromptCachePolicy.None, + PromptCachePolicy = chatRequest?.PromptCachePolicy ?? LlmTestServerPromptCachePolicy.None, PromptCacheKey = chatRequest?.PromptCacheKey, Usage = usage, }); @@ -1424,11 +1493,13 @@ private void RemoveOldestEvent() } } - private static LlmTckRuntimeEvent BoundEventPayload(LlmTckRuntimeEvent runtimeEvent) + private static LlmTestServerRuntimeEvent BoundEventPayload(LlmTestServerRuntimeEvent runtimeEvent) { var truncated = false; var requestId = TruncateMetadata(runtimeEvent.RequestId, ref truncated); var scenarioId = TruncateMetadata(runtimeEvent.ScenarioId, ref truncated); + var conversationId = TruncateMetadata(runtimeEvent.ConversationId, ref truncated); + var stepId = TruncateMetadata(runtimeEvent.StepId, ref truncated); var modelId = TruncateMetadata(runtimeEvent.ModelId, ref truncated) ?? string.Empty; var message = TruncateMetadata(runtimeEvent.Message, ref truncated) ?? string.Empty; var promptCacheKey = TruncateMetadata(runtimeEvent.PromptCacheKey, ref truncated); @@ -1437,34 +1508,48 @@ private static LlmTckRuntimeEvent BoundEventPayload(LlmTckRuntimeEvent runtimeEv _maxRetainedEventCharacters - CountCharacters(requestId) - CountCharacters(scenarioId) + - CountCharacters(conversationId) + - CountCharacters(stepId) - modelId.Length - message.Length - CountCharacters(promptCacheKey) + - CountCharacters(runtimeEvent.WireMetadata?.Identity) + - CountCharacters(runtimeEvent.WireMetadata?.ClockMode) ); var response = RetainText(runtimeEvent.Response, ref remaining, ref truncated); + var disposition = RetainOutputDisposition(runtimeEvent.OutputDisposition, ref remaining, ref truncated); var streamChunks = RetainChunks(runtimeEvent.StreamChunks, ref remaining, ref truncated); var messages = RetainMessages(runtimeEvent.Messages, ref remaining, ref truncated); var tools = RetainTools(runtimeEvent.Tools, ref remaining, ref truncated); + var nativeTools = RetainNativeTools(runtimeEvent.NativeTools, ref remaining, ref truncated); + var responseCalls = RetainResponseCalls(runtimeEvent.ResponseToolCalls, ref remaining, ref truncated); + var responseItems = RetainOutputItems(runtimeEvent.ResponseOutputItems, ref remaining, ref truncated); var request = RetainText(runtimeEvent.Request, ref remaining, ref truncated); return runtimeEvent with { RequestId = requestId, ScenarioId = scenarioId, + ConversationId = conversationId, + StepId = stepId, ModelId = modelId, Message = message, PromptCacheKey = promptCacheKey, Messages = messages, Tools = tools, + NativeTools = nativeTools, + ResponseToolCalls = responseCalls, + ResponseOutputItems = responseItems, Response = response, + OutputDisposition = disposition, StreamChunks = streamChunks, Request = request, PayloadTruncated = truncated, }; } - private static IReadOnlyList RetainMessages( - IReadOnlyList messages, + private static IReadOnlyList RetainMessages( + IReadOnlyList messages, ref int remaining, ref bool truncated ) @@ -1475,7 +1560,7 @@ ref bool truncated } var maximumItemCount = remaining / _retainedMessageCost; - var retained = new List(Math.Min(messages.Count, maximumItemCount)); + var retained = new List(Math.Min(messages.Count, maximumItemCount)); foreach (var source in messages) { if (remaining < _retainedMessageCost) @@ -1488,33 +1573,83 @@ ref bool truncated var role = RetainRequiredText(source.Role, ref remaining, ref truncated); var content = RetainRequiredText(source.Content, ref remaining, ref truncated); var callId = source.ToolCallId is null ? null : RetainRequiredText(source.ToolCallId, ref remaining, ref truncated); - var calls = new List(); + var calls = new List(); foreach (var call in source.ToolCalls) { if (remaining < _retainedMessageCost) { truncated = true; break; } remaining -= _retainedMessageCost; - calls.Add(new() { Id = RetainRequiredText(call.Id, ref remaining, ref truncated), Name = RetainRequiredText(call.Name, ref remaining, ref truncated), ArgumentsJson = RetainRequiredText(call.ArgumentsJson, ref remaining, ref truncated) }); + calls.Add(new() { Id = RetainRequiredText(call.Id, ref remaining, ref truncated), Name = RetainRequiredText(call.Name, ref remaining, ref truncated), ArgumentsJson = RetainRequiredText(call.ArgumentsJson, ref remaining, ref truncated), ContinuationToken = RetainText(call.ContinuationToken, ref remaining, ref truncated) }); } - retained.Add(new LlmTckMessage { Role = role, Content = content, ToolCallId = callId, ToolCalls = calls }); + var outputItems = RetainOutputItems(source.OutputItems, ref remaining, ref truncated); + var disposition = RetainOutputDisposition(source.OutputDisposition, ref remaining, ref truncated); + retained.Add(new LlmTestServerMessage { Role = role, Content = content, ToolCallId = callId, ToolCalls = calls, OutputItems = outputItems, OutputDisposition = disposition }); } return retained.Count == 0 ? [] : Array.AsReadOnly(retained.ToArray()); } - private static IReadOnlyList RetainTools( - IReadOnlyList tools, ref int remaining, ref bool truncated) + private static List RetainNativeTools(IReadOnlyList tools, ref int remaining, ref bool truncated) + { + var retained = new List(); + foreach (var tool in tools) + { + var cost = NativeToolCharacters(tool); + if (cost > remaining) { truncated = true; break; } + remaining -= cost; + retained.AddRange(LlmTestServerResponseItems.SnapshotDeclarations([tool])); + } + return retained; + } + + private static int NativeToolCharacters(LlmTestServerNativeToolDeclaration tool) + { + return _retainedToolCost + System.Text.Json.JsonSerializer.Serialize(tool).Length; + } + + private static List RetainOutputItems(IReadOnlyList items, ref int remaining, ref bool truncated) + { + var retained = new List(); + foreach (var item in items) + { + var cost = OutputItemCharacters(item); + if (cost > remaining) { truncated = true; break; } + remaining -= cost; + retained.AddRange(LlmTestServerResponseItems.Snapshot([item])); + } + return retained; + } + + private static int OutputItemCharacters(LlmTestServerResponseItem item) + { + return _retainedMessageCost + System.Text.Json.JsonSerializer.Serialize(item).Length; + } + + private static IReadOnlyList RetainResponseCalls(IReadOnlyList calls, ref int remaining, ref bool truncated) + { + var retained = new List(); + foreach (var call in calls) + { + if (remaining < _retainedMessageCost) { truncated = true; break; } + remaining -= _retainedMessageCost; + retained.Add(new() { Id = RetainRequiredText(call.Id, ref remaining, ref truncated), Name = RetainRequiredText(call.Name, ref remaining, ref truncated), ArgumentsJson = RetainRequiredText(call.ArgumentsJson, ref remaining, ref truncated), ContinuationToken = RetainText(call.ContinuationToken, ref remaining, ref truncated) }); + } + return retained.ToArray(); + } + + private static IReadOnlyList RetainTools( + IReadOnlyList tools, ref int remaining, ref bool truncated) { if (tools.Count == 0) { return []; } - var retained = new List(Math.Min(tools.Count, remaining / _retainedToolCost)); + var retained = new List(Math.Min(tools.Count, remaining / _retainedToolCost)); foreach (var tool in tools) { if (remaining < _retainedToolCost) { truncated = true; break; } remaining -= _retainedToolCost; - retained.Add(new LlmTckToolDefinition + retained.Add(new LlmTestServerToolDefinition { Name = RetainRequiredText(tool.Name, ref remaining, ref truncated), Description = RetainText(tool.Description, ref remaining, ref truncated), @@ -1591,21 +1726,48 @@ ref bool truncated return retained; } - private static int CountRetainedCharacters(LlmTckRuntimeEvent runtimeEvent) + private static LlmTestServerOutputDisposition? RetainOutputDisposition(LlmTestServerOutputDisposition? disposition, ref int remaining, ref bool truncated) + { + if (disposition is null) { return null; } + if (remaining < _retainedOutputDispositionCost) { truncated = true; return null; } + remaining -= _retainedOutputDispositionCost; + return disposition with + { + Category = RetainText(disposition.Category, ref remaining, ref truncated), + Explanation = RetainText(disposition.Explanation, ref remaining, ref truncated), + }; + } + + private static int OutputDispositionCharacters(LlmTestServerOutputDisposition? disposition) + { + return disposition is null ? 0 : _retainedOutputDispositionCost + CountCharacters(disposition.Category) + CountCharacters(disposition.Explanation); + } + + private static int CountRetainedCharacters(LlmTestServerRuntimeEvent runtimeEvent) { var total = CountCharacters(runtimeEvent.RequestId) + CountCharacters(runtimeEvent.ScenarioId) + + CountCharacters(runtimeEvent.ConversationId) + + CountCharacters(runtimeEvent.StepId) + runtimeEvent.ModelId.Length + runtimeEvent.Message.Length + CountCharacters(runtimeEvent.Request) + CountCharacters(runtimeEvent.Response) - + CountCharacters(runtimeEvent.PromptCacheKey); + + CountCharacters(runtimeEvent.PromptCacheKey) + + CountCharacters(runtimeEvent.WireMetadata?.Identity) + + CountCharacters(runtimeEvent.WireMetadata?.ClockMode) + + OutputDispositionCharacters(runtimeEvent.OutputDisposition); foreach (var item in runtimeEvent.Messages) { total += _retainedMessageCost + item.Role.Length + item.Content.Length + CountCharacters(item.ToolCallId) - + item.ToolCalls.Sum(call => _retainedMessageCost + call.Id.Length + call.Name.Length + call.ArgumentsJson.Length); + + OutputDispositionCharacters(item.OutputDisposition) + + item.ToolCalls.Sum(call => _retainedMessageCost + call.Id.Length + call.Name.Length + call.ArgumentsJson.Length + (call.ContinuationToken?.Length ?? 0)) + + item.OutputItems.Sum(OutputItemCharacters); } + total += runtimeEvent.ResponseOutputItems.Sum(OutputItemCharacters) + runtimeEvent.NativeTools.Sum(NativeToolCharacters); + foreach (var call in runtimeEvent.ResponseToolCalls) + { total += _retainedMessageCost + call.Id.Length + call.Name.Length + call.ArgumentsJson.Length + (call.ContinuationToken?.Length ?? 0); } foreach (var item in runtimeEvent.Tools) { total += _retainedToolCost + item.Name.Length + CountCharacters(item.Description) + item.ParametersJson.Length; @@ -1623,7 +1785,7 @@ private static int CountCharacters(string? value) return value?.Length ?? 0; } - private void RecordEventTotals(LlmTckRuntimeEvent runtimeEvent) + private void RecordEventTotals(LlmTestServerRuntimeEvent runtimeEvent) { _totalEvents++; _eventKindCounts[(int)runtimeEvent.Kind]++; @@ -1640,29 +1802,37 @@ private void RecordEventTotals(LlmTckRuntimeEvent runtimeEvent) _totalTokens += usage.TotalTokens; } - private static string MessageTokenText(LlmTckMessage message) + private static string MessageTokenText(LlmTestServerMessage message) { - return message.ToolCalls.Count == 0 && message.ToolCallId is null ? message.Content + var text = message.ToolCalls.Count == 0 && message.ToolCallId is null ? message.Content : message.Content + " " + message.ToolCallId + " " + string.Join(" ", message.ToolCalls.Select(call => call.Id + " " + call.Name + " " + call.ArgumentsJson)); + return text + LlmTestServerResponseItems.NativeTokenText(message.OutputItems); } - private LlmTckTokenUsage CreateChatUsage( - LlmTckChatRequest request, + private LlmTestServerTokenUsage CreateChatUsage( + LlmTestServerChatRequest request, string? response = null, bool updatePromptCache = false ) { - var inputTokens = request.Messages.Sum(message => - LlmTckTokenCounter.CountTextTokens(MessageTokenText(message)) - ) + request.Tools.Sum(tool => LlmTckTokenCounter.CountTextTokens(tool.Name + " " + tool.Description + " " + tool.ParametersJson)); + var composition = CreateInputComposition(request); + if (request.NativeTools.Count > 0) + { + composition = composition with + { + ToolDefinitionTokens = composition.ToolDefinitionTokens + + LlmTestServerTokenCounter.CountTextTokens(NativeDeclarationTokenText(request)), + }; + } + var inputTokens = composition.TotalTokens; var (cachedInputTokens, cacheCreationInputTokens) = CalculatePromptCacheUsage( request, updatePromptCache ); - var visibleOutputTokens = LlmTckTokenCounter.CountTextTokens(response); + var visibleOutputTokens = LlmTestServerTokenCounter.CountTextTokens(response); var reasoningTokens = response is null ? 0 : GetReasoningTokens(request.ModelId); var outputTokens = visibleOutputTokens + reasoningTokens; - return new LlmTckTokenUsage + return new LlmTestServerTokenUsage { InputTokens = inputTokens, CachedInputTokens = cachedInputTokens, @@ -1670,15 +1840,69 @@ private LlmTckTokenUsage CreateChatUsage( OutputTokens = outputTokens, ReasoningTokens = reasoningTokens, TotalTokens = inputTokens + outputTokens, + InputComposition = composition, + }; + } + + private static LlmTestServerInputTokenComposition CreateInputComposition(LlmTestServerChatRequest request) + { + int system = 0, user = 0, assistant = 0, toolCalls = 0, toolResults = 0, other = 0; + foreach (var message in request.Messages) + { + var tokens = LlmTestServerTokenCounter.CountTextTokens(MessageTokenText(message)); + if (message.ToolCallId is not null || IsRole(message, "tool") || IsRole(message, "function")) + { + toolResults += tokens; + continue; + } + + // Tool-call arguments are attributed separately; the remainder of the message stays with its role. + var roleTokens = message.ToolCalls.Count == 0 + ? tokens + : Math.Min(tokens, LlmTestServerTokenCounter.CountTextTokens(message.Content)); + toolCalls += tokens - roleTokens; + if (IsRole(message, "system") || IsRole(message, "developer")) + { + system += roleTokens; + } + else if (IsRole(message, "user")) + { + user += roleTokens; + } + else if (IsRole(message, "assistant") || IsRole(message, "model") || IsRole(message, "chatbot")) + { + assistant += roleTokens; + } + else + { + other += roleTokens; + } + } + + return new LlmTestServerInputTokenComposition + { + SystemTokens = system, + UserTokens = user, + AssistantTokens = assistant, + ToolCallTokens = toolCalls, + ToolResultTokens = toolResults, + ToolDefinitionTokens = request.Tools.Sum(tool => + LlmTestServerTokenCounter.CountTextTokens(tool.Name + " " + tool.Description + " " + tool.ParametersJson)), + OtherTokens = other, }; + + static bool IsRole(LlmTestServerMessage message, string role) + { + return string.Equals(message.Role, role, StringComparison.OrdinalIgnoreCase); + } } private (int CachedInputTokens, int CacheCreationInputTokens) CalculatePromptCacheUsage( - LlmTckChatRequest request, + LlmTestServerChatRequest request, bool updatePromptCache ) { - if (!updatePromptCache || _configuration.MaxPromptCacheEntries == 0 || request.PromptCachePolicy == LlmTckPromptCachePolicy.None) + if (!updatePromptCache || _configuration.MaxPromptCacheEntries == 0 || request.PromptCachePolicy == LlmTestServerPromptCachePolicy.None) { return (0, 0); } @@ -1700,6 +1924,10 @@ bool updatePromptCache } maxCacheableTokens = Math.Max(maxCacheableTokens, candidate.Tokens); + } + + foreach (var candidate in candidates) + { if (!_promptCacheEntries.ContainsKey(candidate.Key)) { while (_promptCacheEntries.Count >= _configuration.MaxPromptCacheEntries) @@ -1711,24 +1939,28 @@ bool updatePromptCache _promptCacheEntries[candidate.Key] = candidate.Tokens; } - return (cachedInputTokens, request.PromptCachePolicy == LlmTckPromptCachePolicy.Ollama + return (cachedInputTokens, request.PromptCachePolicy == LlmTestServerPromptCachePolicy.Ollama ? 0 : Math.Max(0, maxCacheableTokens - cachedInputTokens)); } - private static List CreatePromptCacheCandidates( - LlmTckChatRequest request + private List CreatePromptCacheCandidates( + LlmTestServerChatRequest request ) { var candidates = new List(); var cachePrefix = new StringBuilder(); var toolText = string.Join("\n", request.Tools.Select(tool => tool.Name + " " + tool.Description + " " + tool.ParametersJson)); + var nativeText = NativeDeclarationTokenText(request); + if (nativeText.Length > 0) { toolText += "\n" + nativeText; } cachePrefix.Append(toolText); - var cumulativeTokens = LlmTckTokenCounter.CountTextTokens(toolText); + var cumulativeTokens = request.Tools.Sum(tool => LlmTestServerTokenCounter.CountTextTokens( + tool.Name + " " + tool.Description + " " + tool.ParametersJson)) + + LlmTestServerTokenCounter.CountTextTokens(nativeText); foreach (var message in request.Messages) { - cumulativeTokens += LlmTckTokenCounter.CountTextTokens(MessageTokenText(message)); + cumulativeTokens += LlmTestServerTokenCounter.CountTextTokens(MessageTokenText(message)); cachePrefix .Append("role:") .Append(message.Role) @@ -1737,6 +1969,20 @@ LlmTckChatRequest request .Append(MessageTokenText(message)) .Append('\u001e'); + var historyDisposition = LlmTestServerOutputDisposition.ForHistory(message.OutputDisposition); + if (historyDisposition is not null) + { + cachePrefix.Append("output-disposition:") + .Append(System.Text.Json.JsonSerializer.Serialize(historyDisposition)).Append('\u001e'); + } + if (message.ToolCalls.Any(call => call.ContinuationToken is not null)) + { + cachePrefix.Append("continuations:") + .Append(System.Text.Json.JsonSerializer.Serialize(message.ToolCalls + .Where(call => call.ContinuationToken is not null).Select(call => new { call.Id, call.ContinuationToken }))) + .Append('\u001e'); + } + var cacheableTokens = RoundPromptCacheTokens( request.PromptCachePolicy, cumulativeTokens @@ -1758,7 +2004,7 @@ LlmTckChatRequest request } private static int RoundPromptCacheTokens( - LlmTckPromptCachePolicy policy, + LlmTestServerPromptCachePolicy policy, int tokenCount ) { @@ -1772,36 +2018,37 @@ int tokenCount return minimumTokens + ((tokenCount - minimumTokens) / incrementTokens * incrementTokens); } - private static int GetPromptCacheMinimumTokens(LlmTckPromptCachePolicy policy) + private static int GetPromptCacheMinimumTokens(LlmTestServerPromptCachePolicy policy) { return policy switch { - LlmTckPromptCachePolicy.Mistral => _mistralPromptCacheMinimumTokens, - LlmTckPromptCachePolicy.Gemini => _geminiPromptCacheMinimumTokens, - LlmTckPromptCachePolicy.Ollama => 1, + LlmTestServerPromptCachePolicy.Mistral => _mistralPromptCacheMinimumTokens, + LlmTestServerPromptCachePolicy.Gemini => _geminiPromptCacheMinimumTokens, + LlmTestServerPromptCachePolicy.Ollama => 1, _ => _openAiPromptCacheMinimumTokens, }; } - private static int GetPromptCacheIncrementTokens(LlmTckPromptCachePolicy policy) + private static int GetPromptCacheIncrementTokens(LlmTestServerPromptCachePolicy policy) { return policy switch { - LlmTckPromptCachePolicy.Mistral => _mistralPromptCacheIncrementTokens, - LlmTckPromptCachePolicy.Gemini => _geminiPromptCacheIncrementTokens, - LlmTckPromptCachePolicy.Ollama => 1, + LlmTestServerPromptCachePolicy.Mistral => _mistralPromptCacheIncrementTokens, + LlmTestServerPromptCachePolicy.Gemini => _geminiPromptCacheIncrementTokens, + LlmTestServerPromptCachePolicy.Ollama => 1, _ => _openAiPromptCacheIncrementTokens, }; } - private static string CreatePromptCacheEntryKey( - LlmTckChatRequest request, + private string CreatePromptCacheEntryKey( + LlmTestServerChatRequest request, string prefix, int cacheableTokens ) { var keyMaterial = string.Join( '\u001d', + _activeRequest.Value?.ProviderNamespace ?? string.Empty, request.PromptCachePolicy.ToString(), request.ModelId, request.PromptCacheKey ?? string.Empty, @@ -1813,19 +2060,19 @@ int cacheableTokens private readonly record struct PromptCacheCandidate(string Key, int Tokens); - private readonly record struct RuntimeRequestContext(string RequestId, long Generation); + private readonly record struct RuntimeRequestContext(string RequestId, long Generation, string? ConversationId, string? ProviderNamespace, ManagedCode.LlmTestServer.Determinism.LlmTestServerResponseMetadata WireMetadata); - private static LlmTckTokenUsage CreateVideoUsage(string prompt) + private static LlmTestServerTokenUsage CreateVideoUsage(string prompt) { - var inputTokens = LlmTckTokenCounter.CountTextTokens(prompt); - return new LlmTckTokenUsage + var inputTokens = LlmTestServerTokenCounter.CountTextTokens(prompt); + return new LlmTestServerTokenUsage { InputTokens = inputTokens, TotalTokens = inputTokens, }; } - private static string FormatChatRequest(LlmTckChatRequest request) + private static string FormatChatRequest(LlmTestServerChatRequest request) { if (request.Messages.Count == 0) { @@ -1859,11 +2106,11 @@ private static string Truncate(string value, int maxLength) return trimmed.Length <= maxLength ? trimmed : trimmed[..maxLength] + "…"; } - private sealed record LlmTckRuntimeFault( + private sealed record LlmTestServerRuntimeFault( int StatusCode, string Code, string Message, - LlmTckTokenUsage? Usage = null + LlmTestServerTokenUsage? Usage = null ); private sealed class RuntimeRequestScope( diff --git a/src/ManagedCode.LlmTck/Runtime/LlmTckRuntimeEvent.cs b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntimeEvent.cs similarity index 65% rename from src/ManagedCode.LlmTck/Runtime/LlmTckRuntimeEvent.cs rename to src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntimeEvent.cs index 81d0257..2cd7b1b 100644 --- a/src/ManagedCode.LlmTck/Runtime/LlmTckRuntimeEvent.cs +++ b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerRuntimeEvent.cs @@ -1,18 +1,36 @@ -using ManagedCode.LlmTck.Scenarios; +using ManagedCode.LlmTestServer.Scenarios; -namespace ManagedCode.LlmTck.Runtime; +namespace ManagedCode.LlmTestServer.Runtime; -public sealed record LlmTckRuntimeEvent +public sealed record LlmTestServerRuntimeEvent { + public ManagedCode.LlmTestServer.Determinism.LlmTestServerResponseMetadata? WireMetadata { get; init; } + + public LlmTestServerOutputDisposition? OutputDisposition { get; init; } + + public LlmTestServerStreamDelivery? StreamDelivery { get; init; } + + public IReadOnlyList NativeTools { get; init; } = []; + + public IReadOnlyList ResponseOutputItems { get; init; } = []; + + public IReadOnlyList ResponseToolCalls { get; init; } = []; + public int? RetryAfterSeconds { get; init; } + public DateTimeOffset Timestamp { get; init; } - public LlmTckEventKind Kind { get; init; } + public LlmTestServerEventKind Kind { get; init; } /// /// Optional caller-provided identifier used to correlate this event with its provider request. /// public string? RequestId { get; init; } + public string? ConversationId { get; init; } + public string? StepId { get; init; } + public int? TurnIndex { get; init; } + public bool IsReplay { get; init; } + public string? ScenarioId { get; init; } public string ModelId { get; init; } = string.Empty; @@ -29,10 +47,10 @@ public sealed record LlmTckRuntimeEvent /// Independent, ordered copy of the chat conversation that produced this event. See /// for the exceptional journal-budget case. /// - public IReadOnlyList Messages { get; init; } = []; + public IReadOnlyList Messages { get; init; } = []; /// Independent tool definitions offered in the originating chat request, within the journal payload budget. - public IReadOnlyList Tools { get; init; } = []; + public IReadOnlyList Tools { get; init; } = []; /// /// The response the model returned for this event (matched content or the error text), @@ -60,7 +78,7 @@ public sealed record LlmTckRuntimeEvent /// /// Provider-neutral prompt-cache behavior selected for the originating chat request. /// - public LlmTckPromptCachePolicy PromptCachePolicy { get; init; } + public LlmTestServerPromptCachePolicy PromptCachePolicy { get; init; } /// /// Optional caller-provided prompt-cache key used by the originating chat request. @@ -70,5 +88,5 @@ public sealed record LlmTckRuntimeEvent /// /// Deterministic token usage observed for this event, when the runtime can derive it. /// - public LlmTckTokenUsage? Usage { get; init; } + public LlmTestServerTokenUsage? Usage { get; init; } } diff --git a/src/ManagedCode.LlmTck/Runtime/LlmTckTokenCounter.cs b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerTokenCounter.cs similarity index 80% rename from src/ManagedCode.LlmTck/Runtime/LlmTckTokenCounter.cs rename to src/ManagedCode.LlmTestServer/Runtime/LlmTestServerTokenCounter.cs index 8caced5..84e20de 100644 --- a/src/ManagedCode.LlmTck/Runtime/LlmTckTokenCounter.cs +++ b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerTokenCounter.cs @@ -1,8 +1,8 @@ using Microsoft.ML.Tokenizers; -namespace ManagedCode.LlmTck.Runtime; +namespace ManagedCode.LlmTestServer.Runtime; -public static class LlmTckTokenCounter +public static class LlmTestServerTokenCounter { public const string DefaultTokenizerModel = "gpt-5"; diff --git a/src/ManagedCode.LlmTck/Runtime/LlmTckTokenUsage.cs b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerTokenUsage.cs similarity index 62% rename from src/ManagedCode.LlmTck/Runtime/LlmTckTokenUsage.cs rename to src/ManagedCode.LlmTestServer/Runtime/LlmTestServerTokenUsage.cs index 4668c51..b26fe27 100644 --- a/src/ManagedCode.LlmTck/Runtime/LlmTckTokenUsage.cs +++ b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerTokenUsage.cs @@ -1,6 +1,6 @@ -namespace ManagedCode.LlmTck.Runtime; +namespace ManagedCode.LlmTestServer.Runtime; -public sealed record LlmTckTokenUsage +public sealed record LlmTestServerTokenUsage { public int InputTokens { get; init; } @@ -13,4 +13,6 @@ public sealed record LlmTckTokenUsage public int ReasoningTokens { get; init; } public int TotalTokens { get; init; } + + public LlmTestServerInputTokenComposition? InputComposition { get; init; } } diff --git a/src/ManagedCode.LlmTck/Runtime/LlmTckTranscriptionUsage.cs b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerTranscriptionUsage.cs similarity index 77% rename from src/ManagedCode.LlmTck/Runtime/LlmTckTranscriptionUsage.cs rename to src/ManagedCode.LlmTestServer/Runtime/LlmTestServerTranscriptionUsage.cs index c8c9f31..4b12ecd 100644 --- a/src/ManagedCode.LlmTck/Runtime/LlmTckTranscriptionUsage.cs +++ b/src/ManagedCode.LlmTestServer/Runtime/LlmTestServerTranscriptionUsage.cs @@ -1,7 +1,7 @@ -namespace ManagedCode.LlmTck.Runtime; +namespace ManagedCode.LlmTestServer.Runtime; /// Explicit transcription fixture usage: duration or complete audio/text/output token counts. -public sealed record LlmTckTranscriptionUsage +public sealed record LlmTestServerTranscriptionUsage { public double? DurationSeconds { get; init; } = 1; @@ -11,16 +11,16 @@ public sealed record LlmTckTranscriptionUsage public long? OutputTokens { get; init; } - public static LlmTckTranscriptionUsage ForDuration(double seconds) + public static LlmTestServerTranscriptionUsage ForDuration(double seconds) { - var usage = new LlmTckTranscriptionUsage { DurationSeconds = seconds }; + var usage = new LlmTestServerTranscriptionUsage { DurationSeconds = seconds }; usage.Validate(); return usage; } - public static LlmTckTranscriptionUsage ForTokens(long audio, long text, long output) + public static LlmTestServerTranscriptionUsage ForTokens(long audio, long text, long output) { - var usage = new LlmTckTranscriptionUsage + var usage = new LlmTestServerTranscriptionUsage { DurationSeconds = null, InputAudioTokens = audio, diff --git a/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerAnthropicResponseItems.cs b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerAnthropicResponseItems.cs new file mode 100644 index 0000000..ee5f70d --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerAnthropicResponseItems.cs @@ -0,0 +1,73 @@ +using System.Text.Json.Serialization; + +namespace ManagedCode.LlmTestServer.Scenarios; + +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record LlmTestServerResponseServerToolUseItem : LlmTestServerResponseItem +{ + public string Id { get; init; } = string.Empty; + public LlmTestServerNativeToolContract Contract { get; init; } = LlmTestServerNativeToolContract.AnthropicMessagesWebSearch20250305; + public LlmTestServerServerToolInput Input { get; init; } = new LlmTestServerWebSearchInput(); +} + +[JsonPolymorphic(TypeDiscriminatorPropertyName = "kind")] +[JsonDerivedType(typeof(LlmTestServerWebSearchInput), "web_search")] +public abstract record LlmTestServerServerToolInput; + +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record LlmTestServerWebSearchInput : LlmTestServerServerToolInput +{ + public string Query { get; init; } = string.Empty; +} + +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record LlmTestServerResponseServerToolResultItem : LlmTestServerResponseItem +{ + public string ToolUseId { get; init; } = string.Empty; + public LlmTestServerNativeToolContract Contract { get; init; } = LlmTestServerNativeToolContract.AnthropicMessagesWebSearch20250305; + public LlmTestServerServerToolResult Result { get; init; } = new LlmTestServerWebSearchResults(); +} + +[JsonPolymorphic(TypeDiscriminatorPropertyName = "kind")] +[JsonDerivedType(typeof(LlmTestServerWebSearchResults), "web_search_results")] +[JsonDerivedType(typeof(LlmTestServerWebSearchErrorResult), "web_search_error")] +public abstract record LlmTestServerServerToolResult; + +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record LlmTestServerWebSearchResults : LlmTestServerServerToolResult +{ + public List Results { get; init; } = []; +} + +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record LlmTestServerAnthropicWebSearchResult +{ + public string Url { get; init; } = string.Empty; + public string Title { get; init; } = string.Empty; + public string EncryptedContent { get; init; } = string.Empty; + public string? PageAge { get; init; } +} + +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record LlmTestServerWebSearchErrorResult : LlmTestServerServerToolResult +{ + public string ErrorCode { get; init; } = string.Empty; +} + +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record LlmTestServerAnthropicWebSearchCitation : LlmTestServerResponseCitation +{ + public string Url { get; init; } = string.Empty; + public string EncryptedIndex { get; init; } = string.Empty; + public string CitedText { get; init; } = string.Empty; + public string? Title { get; init; } +} + +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record LlmTestServerAnthropicWebSearchOptions +{ + public int? MaxUses { get; init; } + public List? AllowedDomains { get; init; } + public List? BlockedDomains { get; init; } + public LlmTestServerWebSearchLocation? UserLocation { get; init; } +} diff --git a/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerChatRequest.cs b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerChatRequest.cs new file mode 100644 index 0000000..adf1641 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerChatRequest.cs @@ -0,0 +1,61 @@ +using ManagedCode.LlmTestServer.Models; + +namespace ManagedCode.LlmTestServer.Scenarios; + +public sealed record LlmTestServerChatRequest +{ + /// + /// Optional caller-provided identifier copied to runtime trace events for stable correlation. + /// + public string? RequestId { get; init; } + + public string ModelId { get; init; } = LlmTestServerKnownModelIds.Gpt41Mini; + + public List Messages { get; init; } = []; + + /// Exact provider/transport namespace for optional stored-response continuation. + public string? HistoryNamespace { get; init; } + public string? PreviousResponseId { get; init; } + public bool StoreResponse { get; init; } + + /// The qualified native route contract; null on surfaces without native support. + public LlmTestServerNativeToolSurface? NativeToolSurface { get; init; } + public List NativeTools { get; init; } = []; + public LlmTestServerNativeToolContract? RequiredNativeTool { get; init; } + public List NativeIncludes { get; init; } = []; + + public LlmTestServerStreamDeliverySurface? StreamDeliverySurface { get; init; } + + public LlmTestServerOutputDispositionSurface? OutputDispositionSurface { get; init; } + + public bool Stream { get; init; } + public List Tools { get; init; } = []; + public LlmTestServerToolChoice ToolChoice { get; init; } + /// Whether a fixture may contain more than one tool call in a response. + public bool AllowParallelToolCalls { get; init; } = true; + public string? RequiredToolName { get; init; } + /// Whether the qualified provider requires continuation metadata on the first function call. + public bool RequireToolContinuationTokens { get; init; } + public bool RequireJson { get; init; } + public string? ResponseSchemaJson { get; init; } + + /// Warms the prompt cache without consuming a configured response. + public bool PopulateCacheOnly { get; init; } + + public LlmTestServerPromptCachePolicy PromptCachePolicy { get; init; } + + public string? PromptCacheKey { get; init; } +} + +public enum LlmTestServerPromptCachePolicy +{ + None, + OpenAiCompatible, + Mistral, + Anthropic, + Gemini, + Bedrock, + DeepSeek, + OpenRouter, + Ollama, +} diff --git a/src/ManagedCode.LlmTck/Scenarios/LlmTckJsonSchema.cs b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerJsonSchema.cs similarity index 94% rename from src/ManagedCode.LlmTck/Scenarios/LlmTckJsonSchema.cs rename to src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerJsonSchema.cs index 4b3f62c..aca3cc8 100644 --- a/src/ManagedCode.LlmTck/Scenarios/LlmTckJsonSchema.cs +++ b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerJsonSchema.cs @@ -1,9 +1,9 @@ using System.Text.Json; using Json.Schema; -namespace ManagedCode.LlmTck.Scenarios; +namespace ManagedCode.LlmTestServer.Scenarios; -public static class LlmTckJsonSchema +public static class LlmTestServerJsonSchema { public static void ValidateJson(string json) { diff --git a/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerMatchMode.cs b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerMatchMode.cs new file mode 100644 index 0000000..eece0e0 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerMatchMode.cs @@ -0,0 +1,7 @@ +namespace ManagedCode.LlmTestServer.Scenarios; + +public enum LlmTestServerMatchMode +{ + Exact, + Contains, +} diff --git a/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerMessage.cs b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerMessage.cs new file mode 100644 index 0000000..7dbeb41 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerMessage.cs @@ -0,0 +1,16 @@ +namespace ManagedCode.LlmTestServer.Scenarios; + +public sealed record LlmTestServerMessage +{ + public List OutputItems { get; init; } = []; + + public LlmTestServerOutputDisposition? OutputDisposition { get; init; } + + public string? ToolCallId { get; init; } + + public string Role { get; init; } = "user"; + + public List ToolCalls { get; init; } = []; + + public string Content { get; init; } = string.Empty; +} diff --git a/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerOutputDisposition.cs b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerOutputDisposition.cs new file mode 100644 index 0000000..98f0f6b --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerOutputDisposition.cs @@ -0,0 +1,60 @@ +using System.Text.Json.Serialization; + +namespace ManagedCode.LlmTestServer.Scenarios; + +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum LlmTestServerOutputDispositionKind +{ + Filtered = 1, + Refused = 2, +} + +public enum LlmTestServerOutputDispositionSurface +{ + OpenAiChatCompletions = 1, + OpenAiResponses = 2, + AnthropicMessages = 3, + GeminiGenerateContent = 4, + DeepSeekChatCompletions = 5, + AzureOpenAiDeploymentChatCompletions = 6, + MicrosoftFoundryChatCompletions = 7, + MicrosoftFoundryModelsChatCompletions = 8, + AzureOpenAiV1ChatCompletions = 9, + AzureOpenAiV1Responses = 10, + MicrosoftFoundryV1ChatCompletions = 11, + MicrosoftFoundryV1Responses = 12, +} + +/// A provider policy outcome; fixture Content supplies generated text or refusal text for its native mapping. +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record LlmTestServerOutputDisposition +{ + public LlmTestServerOutputDispositionKind Kind { get; init; } + public string? Category { get; init; } + public string? Explanation { get; init; } + + internal static void Validate(LlmTestServerOutputDisposition disposition) + { + if (!Enum.IsDefined(disposition.Kind) || disposition.Category?.Length > 256 || disposition.Explanation?.Length > 4096) + { throw new ArgumentException("Invalid or oversized output disposition metadata.", nameof(disposition)); } + } + + /// Only native input semantics belong in history; outcome diagnostics remain on the result. + internal static LlmTestServerOutputDisposition? ForHistory(LlmTestServerOutputDisposition? disposition) + { + if (disposition is null) { return null; } + Validate(disposition); + return disposition.Kind == LlmTestServerOutputDispositionKind.Refused + ? new() { Kind = LlmTestServerOutputDispositionKind.Refused } : null; + } + + internal static LlmTestServerOutputDisposition? Snapshot(LlmTestServerScenarioResponse response) + { + if (response.OutputDisposition is not { } disposition) { return null; } + Validate(disposition); + if (response.Error is not null || response.StreamDelivery is not null || response.ToolCalls.Count > 0 || response.OutputItems.Count > 0 + || response.StreamChunks.Count > 0 && string.Concat(response.StreamChunks) != response.Content) + { throw new ArgumentException("Output disposition requires a plain text response with matching chunks and no error, tools, native items or stream delivery plan.", nameof(response)); } + return disposition with { }; + } +} diff --git a/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerRequestMatch.cs b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerRequestMatch.cs new file mode 100644 index 0000000..07ee27f --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerRequestMatch.cs @@ -0,0 +1,8 @@ +namespace ManagedCode.LlmTestServer.Scenarios; + +public sealed record LlmTestServerRequestMatch +{ + public LlmTestServerMatchMode Mode { get; init; } = LlmTestServerMatchMode.Contains; + + public List Messages { get; init; } = []; +} diff --git a/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerResponseItem.cs b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerResponseItem.cs new file mode 100644 index 0000000..2f7a517 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerResponseItem.cs @@ -0,0 +1,182 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace ManagedCode.LlmTestServer.Scenarios; + +[JsonPolymorphic(TypeDiscriminatorPropertyName = "kind")] +[JsonDerivedType(typeof(LlmTestServerResponseMessageItem), "message")] +[JsonDerivedType(typeof(LlmTestServerResponseFunctionCallItem), "function_call")] +[JsonDerivedType(typeof(LlmTestServerResponseWebSearchItem), "web_search")] +[JsonDerivedType(typeof(LlmTestServerResponseFileSearchItem), "file_search")] +[JsonDerivedType(typeof(LlmTestServerResponseCodeInterpreterItem), "code_interpreter")] +[JsonDerivedType(typeof(LlmTestServerResponseServerToolUseItem), "server_tool_use")] +[JsonDerivedType(typeof(LlmTestServerResponseServerToolResultItem), "server_tool_result")] +public abstract record LlmTestServerResponseItem; + +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record LlmTestServerResponseMessageItem : LlmTestServerResponseItem +{ + public string Id { get; init; } = string.Empty; + public string Content { get; init; } = string.Empty; + public List StreamChunks { get; init; } = []; + public List Citations { get; init; } = []; +} + +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record LlmTestServerResponseFunctionCallItem : LlmTestServerResponseItem +{ + public string Id { get; init; } = string.Empty; + public LlmTestServerToolCall Call { get; init; } = new(); +} + +/// A completed, explicitly scripted search. It never performs network access. +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record LlmTestServerResponseWebSearchItem : LlmTestServerResponseItem +{ + public string Id { get; init; } = string.Empty; + public LlmTestServerNativeToolContract Contract { get; init; } = LlmTestServerNativeToolContract.OpenAiResponsesWebSearch; + public List Queries { get; init; } = []; + public List Sources { get; init; } = []; +} + +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record LlmTestServerWebSearchSource +{ + public string Url { get; init; } = string.Empty; +} + +/// Typed provider citations; indexed citation variants use their explicit bounded fixture ranges. +[JsonPolymorphic(TypeDiscriminatorPropertyName = "kind")] +[JsonDerivedType(typeof(LlmTestServerUrlCitation), "url")] +[JsonDerivedType(typeof(LlmTestServerFileCitation), "file")] +[JsonDerivedType(typeof(LlmTestServerContainerFileCitation), "container_file")] +[JsonDerivedType(typeof(LlmTestServerAnthropicWebSearchCitation), "anthropic_web_search")] +public abstract record LlmTestServerResponseCitation; + +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record LlmTestServerUrlCitation : LlmTestServerResponseCitation +{ + public int StartIndex { get; init; } + public int EndIndex { get; init; } + public string Url { get; init; } = string.Empty; + public string Title { get; init; } = string.Empty; +} + +public enum LlmTestServerNativeToolContract { OpenAiResponsesWebSearch, OpenAiResponsesFileSearch, OpenAiResponsesCodeInterpreter, AnthropicMessagesWebSearch20250305 } +public enum LlmTestServerNativeToolSurface { OpenAiResponses, AnthropicMessages } +public enum LlmTestServerNativeToolInclude { WebSearchSources, FileSearchResults, CodeInterpreterOutputs } + +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record LlmTestServerNativeToolDeclaration +{ + public LlmTestServerNativeToolContract Contract { get; init; } + public LlmTestServerWebSearchOptions? WebSearchOptions { get; init; } + public LlmTestServerAnthropicWebSearchOptions? AnthropicWebSearchOptions { get; init; } + public LlmTestServerFileSearchOptions? FileSearchOptions { get; init; } + public LlmTestServerCodeInterpreterOptions? CodeInterpreterOptions { get; init; } +} + +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record LlmTestServerWebSearchOptions +{ + public string? SearchContextSize { get; init; } + public LlmTestServerWebSearchLocation? UserLocation { get; init; } + public List AllowedDomains { get; init; } = []; +} + +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record LlmTestServerWebSearchLocation +{ + public string Type { get; init; } = "approximate"; + public string? Country { get; init; } + public string? Region { get; init; } + public string? City { get; init; } + public string? Timezone { get; init; } +} + + +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record LlmTestServerFileCitation : LlmTestServerResponseCitation +{ + public string FileId { get; init; } = string.Empty; + public string Filename { get; init; } = string.Empty; + public int Index { get; init; } +} + +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record LlmTestServerContainerFileCitation : LlmTestServerResponseCitation +{ + public string ContainerId { get; init; } = string.Empty; + public string FileId { get; init; } = string.Empty; + public string Filename { get; init; } = string.Empty; + public int StartIndex { get; init; } + public int EndIndex { get; init; } +} + +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record LlmTestServerResponseFileSearchItem : LlmTestServerResponseItem +{ + public string Id { get; init; } = string.Empty; + public LlmTestServerNativeToolContract Contract { get; init; } = LlmTestServerNativeToolContract.OpenAiResponsesFileSearch; + public List VectorStoreIds { get; init; } = []; + public List Queries { get; init; } = []; + public List Results { get; init; } = []; +} + +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record LlmTestServerFileSearchResult +{ + public string? FileId { get; init; } + public string? Filename { get; init; } + public string? Text { get; init; } + public double? Score { get; init; } + public Dictionary? Attributes { get; init; } +} + +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record LlmTestServerResponseCodeInterpreterItem : LlmTestServerResponseItem +{ + public string Id { get; init; } = string.Empty; + public LlmTestServerNativeToolContract Contract { get; init; } = LlmTestServerNativeToolContract.OpenAiResponsesCodeInterpreter; + public string ContainerId { get; init; } = string.Empty; + public List InputFileIds { get; init; } = []; + public List GeneratedFileIds { get; init; } = []; + public string? Code { get; init; } + public List CodeChunks { get; init; } = []; + public List? Outputs { get; init; } +} + +[JsonPolymorphic(TypeDiscriminatorPropertyName = "kind")] +[JsonDerivedType(typeof(LlmTestServerCodeInterpreterLogsOutput), "logs")] +[JsonDerivedType(typeof(LlmTestServerCodeInterpreterImageOutput), "image")] +public abstract record LlmTestServerCodeInterpreterOutput; + +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record LlmTestServerCodeInterpreterLogsOutput : LlmTestServerCodeInterpreterOutput +{ + public string Logs { get; init; } = string.Empty; +} + +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record LlmTestServerCodeInterpreterImageOutput : LlmTestServerCodeInterpreterOutput +{ + public string Url { get; init; } = string.Empty; +} + +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record LlmTestServerFileSearchOptions +{ + public List VectorStoreIds { get; init; } = []; + public int? MaxNumResults { get; init; } + public string? Ranker { get; init; } + public double? ScoreThreshold { get; init; } + public JsonElement? Filters { get; init; } +} + +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record LlmTestServerCodeInterpreterOptions +{ + public string? ContainerId { get; init; } + public bool AutomaticContainer { get; init; } + public List FileIds { get; init; } = []; +} diff --git a/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerResponseItems.Anthropic.cs b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerResponseItems.Anthropic.cs new file mode 100644 index 0000000..8edcf51 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerResponseItems.Anthropic.cs @@ -0,0 +1,160 @@ +using System.Text.Json; + +namespace ManagedCode.LlmTestServer.Scenarios; + +internal static partial class LlmTestServerResponseItems +{ + private static LlmTestServerResponseServerToolUseItem SnapshotServerToolUse(LlmTestServerResponseServerToolUseItem call) + { + Identifier(call.Id); + if (call.Contract != LlmTestServerNativeToolContract.AnthropicMessagesWebSearch20250305 || call.Input is not LlmTestServerWebSearchInput input) + { throw new ArgumentException("Unknown server tool call contract or input."); } + ArgumentException.ThrowIfNullOrWhiteSpace(input.Query); + return call with { Input = input with { } }; + } + + private static LlmTestServerResponseServerToolResultItem SnapshotServerToolResult(LlmTestServerResponseServerToolResultItem item) + { + Identifier(item.ToolUseId); + if (item.Contract != LlmTestServerNativeToolContract.AnthropicMessagesWebSearch20250305) + { throw new ArgumentException("Unknown server tool result contract."); } + LlmTestServerServerToolResult result; + switch (item.Result) + { + case LlmTestServerWebSearchResults web: + ArgumentNullException.ThrowIfNull(web.Results); + if (web.Results.Count > 128) { throw new ArgumentException("Too many server search results."); } + result = web with + { + Results = web.Results.Select(source => + { + ArgumentNullException.ThrowIfNull(source); + ValidateUrl(source.Url); + ArgumentNullException.ThrowIfNull(source.Title); + ArgumentException.ThrowIfNullOrWhiteSpace(source.EncryptedContent); + return source with { }; + }).ToList(), + }; + break; + case LlmTestServerWebSearchErrorResult error: + if (error.ErrorCode is not ("invalid_tool_input" or "unavailable" or "max_uses_exceeded" or "too_many_requests" or "query_too_long" or "request_too_large")) + { throw new ArgumentException("Unknown server web search error code."); } + result = error with { }; + break; + default: + throw new ArgumentException("Unknown server tool result."); + } + return item with { Result = result }; + } + + private static LlmTestServerAnthropicWebSearchCitation SnapshotAnthropicCitation(LlmTestServerAnthropicWebSearchCitation citation) + { + ValidateUrl(citation.Url); + ArgumentException.ThrowIfNullOrWhiteSpace(citation.EncryptedIndex); + ArgumentNullException.ThrowIfNull(citation.CitedText); + return citation with { }; + } + + internal static void ValidateCompletedServerTools(IReadOnlyList items) + { + var pending = new Dictionary(StringComparer.Ordinal); + var callIds = new HashSet(StringComparer.Ordinal); + var functionIds = items.OfType().Select(item => item.Call.Id).ToHashSet(StringComparer.Ordinal); + foreach (var item in items) + { + if (item is LlmTestServerResponseServerToolUseItem call) + { + if (!callIds.Add(call.Id) || functionIds.Contains(call.Id)) { throw new ArgumentException("Server and function tool call IDs must be distinct."); } + pending.Add(call.Id, call.Contract); + } + else if (item is LlmTestServerResponseServerToolResultItem result) + { + if (!pending.Remove(result.ToolUseId, out var contract) || contract != result.Contract) + { throw new ArgumentException("Each server tool result must follow its matching call exactly once."); } + } + } + if (pending.Count > 0) { throw new ArgumentException("This fixture contract requires completed server tool call/result pairs."); } + } + + private static LlmTestServerAnthropicWebSearchOptions? SnapshotAnthropicOptions(LlmTestServerAnthropicWebSearchOptions? options) + { + if (options is null) { return null; } + if (options.MaxUses is < 0 or > 128 || options.AllowedDomains is not null && options.BlockedDomains is not null + || options.UserLocation is { Type: not "approximate" }) + { throw new ArgumentException("Invalid Anthropic web search options."); } + foreach (var patterns in new[] { options.AllowedDomains, options.BlockedDomains }) + { + if (patterns is null) { continue; } + if (patterns.Count > 128) { throw new ArgumentException("Too many web domain filters."); } + foreach (var pattern in patterns) { ValidateDomainPattern(pattern); } + } + return CloneAnthropicOptions(options); + } + + private static LlmTestServerAnthropicWebSearchOptions? CloneAnthropicOptions(LlmTestServerAnthropicWebSearchOptions? options) + { + return options is null ? null : options with + { + AllowedDomains = options.AllowedDomains is null ? null : [.. options.AllowedDomains], + BlockedDomains = options.BlockedDomains is null ? null : [.. options.BlockedDomains], + UserLocation = options.UserLocation is null ? null : options.UserLocation with { }, + }; + } + + private static void ValidateDomainPattern(string pattern) + { + ArgumentException.ThrowIfNullOrWhiteSpace(pattern); + var slash = pattern.IndexOf('/'); + var host = slash < 0 ? pattern : pattern[..slash]; + if (pattern.Contains("://", StringComparison.Ordinal) || pattern.IndexOfAny(['\\', '?', '#']) >= 0 + || pattern.Any(char.IsWhiteSpace) || host.Length > 253 || Uri.CheckHostName(host) != UriHostNameType.Dns + || host.Split('.').Any(label => label.Length is < 1 or > 63 || label[0] == '-' || label[^1] == '-' + || label.Any(character => !char.IsAsciiLetterOrDigit(character) && character != '-'))) + { throw new ArgumentException("Domain filters require an ASCII DNS host and optional path; wildcards are allowed only in the path."); } + } + + internal static bool AllowsAnthropicUrl(LlmTestServerAnthropicWebSearchOptions? options, string url) + { + var uri = new Uri(url); + return (options?.AllowedDomains is null || options.AllowedDomains.Any(pattern => MatchesDomainPattern(uri, pattern))) + && (options?.BlockedDomains is null || !options.BlockedDomains.Any(pattern => MatchesDomainPattern(uri, pattern))); + } + + private static bool MatchesDomainPattern(Uri uri, string pattern) + { + var slash = pattern.IndexOf('/'); + var host = slash < 0 ? pattern : pattern[..slash]; + if (!uri.IdnHost.Equals(host, StringComparison.OrdinalIgnoreCase) + && !uri.IdnHost.EndsWith("." + host, StringComparison.OrdinalIgnoreCase)) { return false; } + if (slash < 0) { return true; } + var path = new Uri("https://" + pattern).AbsolutePath; + return GlobPrefix(uri.AbsolutePath, path); + } + + private static bool GlobPrefix(string text, string pattern) + { + var position = 0; + var index = 0; + var star = -1; + var restart = 0; + while (position < text.Length) + { + if (index == pattern.Length) { return true; } + if (pattern[index] == '*') { star = index++; restart = position; } + else if (pattern[index] == text[position]) { index++; position++; } + else if (star >= 0) { index = star + 1; position = ++restart; } + else { return false; } + } + while (index < pattern.Length && pattern[index] == '*') { index++; } + return index == pattern.Length; + } + + private static string ServerToolInputTokenText(LlmTestServerServerToolInput input) + { + return input switch + { + LlmTestServerWebSearchInput web => JsonSerializer.Serialize(new { query = web.Query }), + _ => string.Empty, + }; + } +} diff --git a/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerResponseItems.FileCode.cs b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerResponseItems.FileCode.cs new file mode 100644 index 0000000..2d8c5f2 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerResponseItems.FileCode.cs @@ -0,0 +1,241 @@ +using System.Text.Json; + +namespace ManagedCode.LlmTestServer.Scenarios; + +internal static partial class LlmTestServerResponseItems +{ + internal static bool IsNative(LlmTestServerResponseItem item) + { + return item is LlmTestServerResponseWebSearchItem or LlmTestServerResponseFileSearchItem or LlmTestServerResponseCodeInterpreterItem + or LlmTestServerResponseServerToolUseItem or LlmTestServerResponseServerToolResultItem; + } + + internal static bool IsNativeCall(LlmTestServerResponseItem item) + { + return IsNative(item) && item is not LlmTestServerResponseServerToolResultItem; + } + + internal static LlmTestServerNativeToolContract? Contract(LlmTestServerResponseItem item) + { + return item switch + { + LlmTestServerResponseWebSearchItem web => web.Contract, + LlmTestServerResponseFileSearchItem file => file.Contract, + LlmTestServerResponseCodeInterpreterItem code => code.Contract, + LlmTestServerResponseServerToolUseItem call => call.Contract, + LlmTestServerResponseServerToolResultItem result => result.Contract, + _ => null, + }; + } + + private static LlmTestServerResponseCitation SnapshotCitation(LlmTestServerResponseCitation citation, string content) + { + ArgumentNullException.ThrowIfNull(citation); + if (citation is LlmTestServerAnthropicWebSearchCitation anthropic) { return SnapshotAnthropicCitation(anthropic); } + if (content.Any(character => character > 127)) { throw new ArgumentException("Citation text must be ASCII.", nameof(citation)); } + switch (citation) + { + case LlmTestServerUrlCitation url: + ArgumentNullException.ThrowIfNull(url.Title); + ValidateUrl(url.Url); + Range(url.StartIndex, url.EndIndex, content.Length, false); + return url with { }; + case LlmTestServerFileCitation file: + Identifier(file.FileId); + ArgumentException.ThrowIfNullOrWhiteSpace(file.Filename); + if (file.Index < 0 || file.Index > content.Length) { throw new ArgumentException("File citation index exceeds output text.", nameof(citation)); } + return file with { }; + case LlmTestServerContainerFileCitation container: + Identifier(container.ContainerId); + Identifier(container.FileId); + ArgumentException.ThrowIfNullOrWhiteSpace(container.Filename); + Range(container.StartIndex, container.EndIndex, content.Length, true); + return container with { }; + default: + throw new ArgumentException("Unknown citation kind.", nameof(citation)); + } + } + + private static void Range(int start, int end, int length, bool allowEmpty) + { + if (start < 0 || end < start || !allowEmpty && end == start || end > length) + { throw new ArgumentException("Citation range exceeds output text."); } + } + + private static LlmTestServerResponseFileSearchItem SnapshotFileSearch(LlmTestServerResponseFileSearchItem file, bool fixture) + { + Identifier(file.Id); + if (file.Contract != LlmTestServerNativeToolContract.OpenAiResponsesFileSearch) { throw new ArgumentException("Invalid file search contract."); } + Strings(file.VectorStoreIds, fixture ? 1 : 0, true); + Strings(file.Queries, 1, false); + ArgumentNullException.ThrowIfNull(file.Results); + if (file.Results.Count > 128) { throw new ArgumentException("Too many file search results."); } + var results = file.Results.Select(result => + { + ArgumentNullException.ThrowIfNull(result); + if (result.FileId is not null) { Identifier(result.FileId); } + if (result.Filename is not null) { ArgumentException.ThrowIfNullOrWhiteSpace(result.Filename); } + Score(result.Score); + if (result.Attributes is { } attributes) + { + if (attributes.Count > 16) { throw new ArgumentException("At most 16 file attributes are supported."); } + foreach (var pair in attributes) + { + if (string.IsNullOrWhiteSpace(pair.Key) || pair.Key.Length > 64) { throw new ArgumentException("Invalid file attribute key."); } + Scalar(pair.Value); + if (pair.Value.ValueKind == JsonValueKind.String && pair.Value.GetString()!.Length > 512) + { throw new ArgumentException("File attribute text exceeds 512 characters."); } + } + } + return result with { Attributes = result.Attributes?.ToDictionary(pair => pair.Key, pair => pair.Value.Clone(), StringComparer.Ordinal) }; + }).ToList(); + return file with { VectorStoreIds = [.. file.VectorStoreIds], Queries = [.. file.Queries], Results = results }; + } + + private static LlmTestServerResponseCodeInterpreterItem SnapshotCode(LlmTestServerResponseCodeInterpreterItem code) + { + Identifier(code.Id); + Identifier(code.ContainerId); + if (code.Contract != LlmTestServerNativeToolContract.OpenAiResponsesCodeInterpreter) { throw new ArgumentException("Invalid code interpreter contract."); } + Strings(code.InputFileIds, 0, true); + Strings(code.GeneratedFileIds, 0, true); + ArgumentNullException.ThrowIfNull(code.CodeChunks); + if (code.CodeChunks.Count > 128) { throw new ArgumentException("Too many code chunks."); } + foreach (var chunk in code.CodeChunks) { ArgumentNullException.ThrowIfNull(chunk); } + if (code.CodeChunks.Count > 0 && (code.Code is null || string.Concat(code.CodeChunks) != code.Code)) + { throw new ArgumentException("Code chunks must concatenate to the configured code."); } + if (code.Outputs?.Count > 128) { throw new ArgumentException("Too many code outputs."); } + return code with + { + InputFileIds = [.. code.InputFileIds], + GeneratedFileIds = [.. code.GeneratedFileIds], + CodeChunks = [.. code.CodeChunks], + Outputs = code.Outputs?.Select(SnapshotCodeOutput).ToList(), + }; + } + + private static LlmTestServerCodeInterpreterOutput SnapshotCodeOutput(LlmTestServerCodeInterpreterOutput output) + { + ArgumentNullException.ThrowIfNull(output); + switch (output) + { + case LlmTestServerCodeInterpreterLogsOutput logs: + ArgumentNullException.ThrowIfNull(logs.Logs); + return logs with { }; + case LlmTestServerCodeInterpreterImageOutput image: + ValidateUrl(image.Url); + return image with { }; + default: + throw new ArgumentException("Unknown code output kind.", nameof(output)); + } + } + + private static LlmTestServerNativeToolDeclaration CloneDeclaration(LlmTestServerNativeToolDeclaration declaration) + { + var web = declaration.WebSearchOptions; + var file = declaration.FileSearchOptions; + var code = declaration.CodeInterpreterOptions; + return declaration with + { + WebSearchOptions = web is null ? null : web with + { + AllowedDomains = web.AllowedDomains is null ? [] : [.. web.AllowedDomains], + UserLocation = web.UserLocation is null ? null : web.UserLocation with { }, + }, + FileSearchOptions = file is null ? null : file with + { + VectorStoreIds = file.VectorStoreIds is null ? [] : [.. file.VectorStoreIds], + Filters = file.Filters is { ValueKind: not JsonValueKind.Undefined } filter ? filter.Clone() : file.Filters, + }, + CodeInterpreterOptions = code is null ? null : code with { FileIds = code.FileIds is null ? [] : [.. code.FileIds] }, + AnthropicWebSearchOptions = CloneAnthropicOptions(declaration.AnthropicWebSearchOptions), + }; + } + + private static LlmTestServerNativeToolDeclaration SnapshotDeclaration(LlmTestServerNativeToolDeclaration declaration) + { + ArgumentNullException.ThrowIfNull(declaration); + if (!Enum.IsDefined(declaration.Contract)) { throw new ArgumentException("Unknown native tool contract."); } + if (declaration.WebSearchOptions is not null && declaration.Contract != LlmTestServerNativeToolContract.OpenAiResponsesWebSearch + || declaration.FileSearchOptions is not null && declaration.Contract != LlmTestServerNativeToolContract.OpenAiResponsesFileSearch + || declaration.CodeInterpreterOptions is not null && declaration.Contract != LlmTestServerNativeToolContract.OpenAiResponsesCodeInterpreter + || declaration.AnthropicWebSearchOptions is not null && declaration.Contract != LlmTestServerNativeToolContract.AnthropicMessagesWebSearch20250305) + { throw new ArgumentException("Native declaration options must match their contract."); } + var web = declaration.WebSearchOptions; + if (web is not null) + { + ArgumentNullException.ThrowIfNull(web.AllowedDomains); + if (web.AllowedDomains.Count > 100 || web.SearchContextSize is not (null or "low" or "medium" or "high") || web.UserLocation is { Type: not "approximate" }) + { throw new ArgumentException("Invalid web search options."); } + foreach (var domain in web.AllowedDomains) + { + ArgumentException.ThrowIfNullOrWhiteSpace(domain); + if (Uri.CheckHostName(domain) != UriHostNameType.Dns || domain.Contains(':') || domain.Contains('/')) { throw new ArgumentException("A plain DNS domain is required."); } + } + web = web with { AllowedDomains = [.. web.AllowedDomains], UserLocation = web.UserLocation is null ? null : web.UserLocation with { } }; + } + var file = declaration.FileSearchOptions; + if (file is not null) + { + Strings(file.VectorStoreIds, 0, true, unique: false); + if (file.MaxNumResults is < 1 or > 50 || file.Ranker is not (null or "auto" or "default-2024-11-15")) { throw new ArgumentException("Invalid file search options."); } + Score(file.ScoreThreshold); + if (file.Filters is { ValueKind: not JsonValueKind.Null } filter) { ValidateFilter(filter); } + file = file with { VectorStoreIds = [.. file.VectorStoreIds], Filters = file.Filters?.Clone() }; + } + if (declaration.Contract == LlmTestServerNativeToolContract.OpenAiResponsesFileSearch && file is null) { throw new ArgumentException("File search options are required."); } + var code = declaration.CodeInterpreterOptions; + if (code is not null) + { + Strings(code.FileIds, 0, true, unique: false); + if (code.AutomaticContainer) + { + if (code.ContainerId is not null) { throw new ArgumentException("Automatic containers cannot also specify a container ID."); } + } + else + { + Identifier(code.ContainerId!); + if (code.FileIds.Count > 0) { throw new ArgumentException("Explicit containers cannot specify auto file IDs."); } + } + code = code with { FileIds = [.. code.FileIds] }; + } + if (declaration.Contract == LlmTestServerNativeToolContract.OpenAiResponsesCodeInterpreter && code is null) { throw new ArgumentException("Code interpreter options are required."); } + return declaration with + { + WebSearchOptions = web, + FileSearchOptions = file, + CodeInterpreterOptions = code, + AnthropicWebSearchOptions = SnapshotAnthropicOptions(declaration.AnthropicWebSearchOptions) + }; + } + + private static void Identifier(string value) + { + ArgumentException.ThrowIfNullOrWhiteSpace(value); + if (value.Length > 128) { throw new ArgumentException("Fixture identifiers are bounded to 128 characters.", nameof(value)); } + } + + private static void Strings(List values, int minimum, bool identifiers, bool unique = true) + { + ArgumentNullException.ThrowIfNull(values); + if (values.Count < minimum || values.Count > 128) { throw new ArgumentException("Fixture collections must stay within their bounded count."); } + foreach (var value in values) + { + ArgumentException.ThrowIfNullOrWhiteSpace(value); + if (identifiers) { Identifier(value); } + } + if (identifiers && unique && values.Distinct(StringComparer.Ordinal).Count() != values.Count) { throw new ArgumentException("Resource identifiers must be distinct."); } + } + + private static void Score(double? score) + { + if (score is { } value && (!double.IsFinite(value) || value is < 0 or > 1)) { throw new ArgumentException("Scores must be finite values between zero and one."); } + } + + private static void Scalar(JsonElement value) + { + if (value.ValueKind is JsonValueKind.String or JsonValueKind.True or JsonValueKind.False) { return; } + if (value.ValueKind == JsonValueKind.Number && value.TryGetDouble(out var number) && double.IsFinite(number)) { return; } + throw new ArgumentException("A finite JSON scalar is required."); + } +} diff --git a/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerResponseItems.Filters.cs b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerResponseItems.Filters.cs new file mode 100644 index 0000000..b58e404 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerResponseItems.Filters.cs @@ -0,0 +1,145 @@ +using System.Globalization; +using System.Numerics; +using System.Text.Json; + +namespace ManagedCode.LlmTestServer.Scenarios; + +internal static partial class LlmTestServerResponseItems +{ + private static void ValidateFilter(JsonElement filter) + { + var nodes = 0; + ValidateFilterNode(filter, 1, ref nodes); + } + + private static void ValidateFilterNode(JsonElement node, int depth, ref int nodes) + { + if (++nodes > 128 || depth > 8 || node.ValueKind != JsonValueKind.Object) { throw new ArgumentException("Filter depth or node count exceeds its bounded contract."); } + if (!node.TryGetProperty("type", out var type) || type.ValueKind != JsonValueKind.String) { throw new ArgumentException("A filter type is required."); } + var operation = type.GetString(); + var compound = operation is "and" or "or"; + var allowed = compound ? new[] { "type", "filters" } : ["type", "key", "value"]; + var fields = new HashSet(StringComparer.Ordinal); + foreach (var property in node.EnumerateObject()) + { + if (!allowed.Contains(property.Name, StringComparer.Ordinal) || !fields.Add(property.Name)) { throw new ArgumentException("Unknown or duplicate filter field."); } + } + if (fields.Count != allowed.Length) { throw new ArgumentException("Required filter fields are missing."); } + if (compound) + { + var children = node.GetProperty("filters"); + if (children.ValueKind != JsonValueKind.Array || children.GetArrayLength() is < 1 or > 128) { throw new ArgumentException("A nonempty bounded filter group is required."); } + foreach (var child in children.EnumerateArray()) { ValidateFilterNode(child, depth + 1, ref nodes); } + return; + } + var key = node.GetProperty("key"); + if (key.ValueKind != JsonValueKind.String || string.IsNullOrWhiteSpace(key.GetString()) || key.GetString()!.Length > 64) + { throw new ArgumentException("A bounded filter attribute key is required."); } + var value = node.GetProperty("value"); + if (operation is "in" or "nin") + { + if (value.ValueKind != JsonValueKind.Array || value.GetArrayLength() is < 1 or > 128) { throw new ArgumentException("Membership filters require a nonempty bounded array."); } + var kind = value[0].ValueKind; + if (kind is not (JsonValueKind.String or JsonValueKind.Number)) { throw new ArgumentException("Membership filters require strings or numbers."); } + foreach (var entry in value.EnumerateArray()) + { + Scalar(entry); + if (entry.ValueKind != kind) { throw new ArgumentException("Membership filter values must have a common kind."); } + } + return; + } + if (operation is not ("eq" or "ne" or "gt" or "gte" or "lt" or "lte")) { throw new ArgumentException("Unsupported filter operation."); } + Scalar(value); + if (operation is not ("eq" or "ne") && value.ValueKind is not (JsonValueKind.String or JsonValueKind.Number)) + { throw new ArgumentException("Ordering filters require numbers or strings."); } + } + + internal static bool MatchesFilter(JsonElement? filter, IReadOnlyDictionary? attributes) + { + if (filter is null || filter.Value.ValueKind == JsonValueKind.Null) { return true; } + return MatchFilterNode(filter.Value, attributes); + } + + private static bool MatchFilterNode(JsonElement filter, IReadOnlyDictionary? attributes) + { + var operation = filter.GetProperty("type").GetString(); + if (operation is "and" or "or") + { + var children = filter.GetProperty("filters").EnumerateArray(); + return operation == "and" ? children.All(child => MatchFilterNode(child, attributes)) : children.Any(child => MatchFilterNode(child, attributes)); + } + if (attributes is null || !attributes.TryGetValue(filter.GetProperty("key").GetString()!, out var actual)) { return false; } + var expected = filter.GetProperty("value"); + if (operation is "in" or "nin") + { + if (actual.ValueKind != expected[0].ValueKind) { return false; } + var contains = expected.EnumerateArray().Any(entry => Compare(actual, entry) == 0); + return operation == "in" ? contains : !contains; + } + var comparison = Compare(actual, expected); + if (comparison is null) { return false; } + return operation switch + { + "eq" => comparison == 0, + "ne" => comparison != 0, + "gt" => comparison > 0, + "gte" => comparison >= 0, + "lt" => comparison < 0, + "lte" => comparison <= 0, + _ => false, + }; + } + + private static int? Compare(JsonElement actual, JsonElement expected) + { + if (actual.ValueKind == JsonValueKind.Number && expected.ValueKind == JsonValueKind.Number) + { + return CompareNumbers(actual, expected); + } + if (actual.ValueKind == JsonValueKind.String && expected.ValueKind == JsonValueKind.String) + { return string.Compare(actual.GetString(), expected.GetString(), StringComparison.Ordinal); } + if (actual.ValueKind is JsonValueKind.True or JsonValueKind.False && expected.ValueKind is JsonValueKind.True or JsonValueKind.False) + { return actual.GetBoolean().CompareTo(expected.GetBoolean()); } + return null; + } + + private static int CompareNumbers(JsonElement actual, JsonElement expected) + { + var left = NormalizeNumber(actual); + var right = NormalizeNumber(expected); + var sign = left.Sign.CompareTo(right.Sign); + if (sign != 0) { return sign; } + if (left.Sign == 0) { return 0; } + var magnitude = left.Order.CompareTo(right.Order); + if (magnitude == 0) + { + var length = Math.Max(left.Digits.Length, right.Digits.Length); + for (var index = 0; index < length; index++) + { + var leftDigit = index < left.Digits.Length ? left.Digits[index] : '0'; + var rightDigit = index < right.Digits.Length ? right.Digits[index] : '0'; + magnitude = leftDigit.CompareTo(rightDigit); + if (magnitude != 0) { break; } + } + } + return left.Sign * magnitude; + } + + private static (int Sign, string Digits, BigInteger Order) NormalizeNumber(JsonElement value) + { + var text = value.GetRawText(); + var start = text[0] == '-' ? 1 : 0; + var exponentIndex = text.IndexOf('e'); + if (exponentIndex < 0) { exponentIndex = text.IndexOf('E'); } + var end = exponentIndex < 0 ? text.Length : exponentIndex; + var point = text.IndexOf('.', start, end - start); + var fractionDigits = point < 0 ? 0 : end - point - 1; + var digits = text[start..end].Replace(".", string.Empty, StringComparison.Ordinal).TrimStart('0'); + if (digits.Length == 0) { return (0, string.Empty, BigInteger.Zero); } + // Compare decimal order without expanding powers, including very small finite JSON numbers. + var exponent = exponentIndex < 0 ? BigInteger.Zero + : BigInteger.Parse(text.AsSpan(exponentIndex + 1), NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture); + return (start == 0 ? 1 : -1, digits, exponent + digits.Length - fractionDigits); + } + +} diff --git a/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerResponseItems.cs b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerResponseItems.cs new file mode 100644 index 0000000..531a41c --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerResponseItems.cs @@ -0,0 +1,134 @@ +using System.Text.Json; + +namespace ManagedCode.LlmTestServer.Scenarios; + +internal static partial class LlmTestServerResponseItems +{ + internal static List Snapshot(IEnumerable items, bool validateChunks = false) + { + ArgumentNullException.ThrowIfNull(items); + var sourceItems = items.ToList(); + var nativePlan = sourceItems.Any(IsNative); + var result = new List(); + var ids = new HashSet(StringComparer.Ordinal); + foreach (var item in sourceItems) + { + if (result.Count >= 128) { throw new ArgumentException("At most 128 response output items are supported.", nameof(items)); } + ArgumentNullException.ThrowIfNull(item); + string id; + switch (item) + { + case LlmTestServerResponseMessageItem message: + ArgumentNullException.ThrowIfNull(message.Id); + ArgumentNullException.ThrowIfNull(message.Content); + ArgumentNullException.ThrowIfNull(message.StreamChunks); + ArgumentNullException.ThrowIfNull(message.Citations); + foreach (var chunk in message.StreamChunks) { ArgumentNullException.ThrowIfNull(chunk); } + if (validateChunks && message.StreamChunks.Count > 0 && string.Concat(message.StreamChunks) != message.Content) + { throw new ArgumentException("Message chunks must concatenate to the output text.", nameof(items)); } + if (message.Citations.Count > 128) { throw new ArgumentException("Too many URL citations.", nameof(items)); } + var citations = message.Citations.Select(citation => SnapshotCitation(citation, message.Content)).ToList(); + result.Add(message with { StreamChunks = [.. message.StreamChunks], Citations = citations }); + id = message.Id; + break; + case LlmTestServerResponseFunctionCallItem function: + ArgumentNullException.ThrowIfNull(function.Id); + if (nativePlan) { ArgumentException.ThrowIfNullOrWhiteSpace(function.Id); } + ArgumentNullException.ThrowIfNull(function.Call); + ArgumentException.ThrowIfNullOrWhiteSpace(function.Call.Id); + ArgumentException.ThrowIfNullOrWhiteSpace(function.Call.Name); + LlmTestServerJsonSchema.ValidateJson(function.Call.ArgumentsJson); + result.Add(function with { Call = function.Call with { } }); + id = function.Id.Length > 0 ? function.Id : function.Call.Id; + break; + case LlmTestServerResponseWebSearchItem web: + ArgumentException.ThrowIfNullOrWhiteSpace(web.Id); + ArgumentNullException.ThrowIfNull(web.Queries); + ArgumentNullException.ThrowIfNull(web.Sources); + if (web.Contract != LlmTestServerNativeToolContract.OpenAiResponsesWebSearch || web.Queries.Count is < 1 or > 32 || web.Sources.Count > 128) + { throw new ArgumentException("Invalid native search fixture.", nameof(items)); } + foreach (var query in web.Queries) { ArgumentException.ThrowIfNullOrWhiteSpace(query); } + foreach (var source in web.Sources) { ArgumentNullException.ThrowIfNull(source); ValidateUrl(source.Url); } + result.Add(web with { Queries = [.. web.Queries], Sources = [.. web.Sources.Select(source => source with { })] }); + id = web.Id; + break; + case LlmTestServerResponseFileSearchItem file: + result.Add(SnapshotFileSearch(file, validateChunks)); + id = file.Id; + break; + case LlmTestServerResponseCodeInterpreterItem code: + result.Add(SnapshotCode(code)); + id = code.Id; + break; + case LlmTestServerResponseServerToolUseItem serverCall: + result.Add(SnapshotServerToolUse(serverCall)); + id = serverCall.Id; + break; + case LlmTestServerResponseServerToolResultItem serverResult: + result.Add(SnapshotServerToolResult(serverResult)); + id = string.Empty; + break; + default: + throw new ArgumentException("Unknown response output item.", nameof(items)); + } + var legacyFunctionId = item is LlmTestServerResponseFunctionCallItem { Id.Length: 0 }; + if (!legacyFunctionId && id.Length > 128 || id.Length > 0 && (string.IsNullOrWhiteSpace(id) || !ids.Add(id))) { throw new ArgumentException("Output item IDs must be distinct and bounded.", nameof(items)); } + } + if (validateChunks) { ValidateCompletedServerTools(result); } + return result; + } + + internal static LlmTestServerScenarioResponse Project(LlmTestServerScenarioResponse response) + { + if (response.OutputItems.Count == 0) { return response; } + var items = Snapshot(response.OutputItems); + return response with + { + OutputItems = items, + Content = string.Concat(items.OfType().Select(message => message.Content)), + StreamChunks = items.OfType().SelectMany(message => message.StreamChunks.Count > 0 ? message.StreamChunks : [message.Content]).ToList(), + ToolCalls = items.OfType().Select(function => function.Call with { }).ToList(), + }; + } + + internal static List SnapshotDeclarations(IEnumerable declarations, bool validate = false) + { + ArgumentNullException.ThrowIfNull(declarations); + return validate + ? declarations.Select(SnapshotDeclaration).ToList() + : declarations.Where(declaration => declaration is not null).Select(CloneDeclaration).ToList(); + } + + internal static bool AllowsUrl(LlmTestServerWebSearchOptions? options, string url) + { + if (options is null || options.AllowedDomains.Count == 0) { return true; } + var host = new Uri(url).IdnHost; + return options.AllowedDomains.Any(domain => + { + var allowed = new Uri("https://" + domain).IdnHost; + return host.Equals(allowed, StringComparison.OrdinalIgnoreCase) || host.EndsWith("." + allowed, StringComparison.OrdinalIgnoreCase); + }); + } + + internal static string NativeTokenText(IEnumerable items) + { + return string.Concat(items.Where(IsNative).Select(item => item switch + { + LlmTestServerResponseServerToolUseItem call => ServerToolInputTokenText(call.Input), + LlmTestServerResponseServerToolResultItem => string.Empty, + _ => JsonSerializer.Serialize(item), + })); + } + + internal static long Bytes(IEnumerable items) + { + return items.Sum(item => 128L + JsonSerializer.SerializeToUtf8Bytes(item).LongLength); + } + + private static void ValidateUrl(string value) + { + if (string.IsNullOrWhiteSpace(value) || value.Any(char.IsWhiteSpace) + || !Uri.TryCreate(value, UriKind.Absolute, out var uri) || uri.Scheme is not ("http" or "https")) + { throw new ArgumentException("Native fixture URLs must be absolute HTTP or HTTPS URLs.", nameof(value)); } + } +} diff --git a/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerScenario.cs b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerScenario.cs new file mode 100644 index 0000000..ff178d3 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerScenario.cs @@ -0,0 +1,18 @@ +using ManagedCode.LlmTestServer.Models; + +namespace ManagedCode.LlmTestServer.Scenarios; + +public sealed record LlmTestServerScenario +{ + public string Id { get; init; } = Guid.NewGuid().ToString("N"); + + public string ModelId { get; init; } = LlmTestServerKnownModelIds.Gpt41Mini; + + public string? RequiredBearerToken { get; init; } + + public LlmTestServerRequestMatch Match { get; init; } = new(); + + public ManagedCode.LlmTestServer.Conversations.LlmTestServerConversation? Conversation { get; init; } + + public List Responses { get; init; } = []; +} diff --git a/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerScenarioBuilder.cs b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerScenarioBuilder.cs new file mode 100644 index 0000000..6dba5f2 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerScenarioBuilder.cs @@ -0,0 +1,174 @@ +namespace ManagedCode.LlmTestServer.Scenarios; + +public sealed class LlmTestServerScenarioBuilder(string id) +{ + private LlmTestServerScenario _scenario = new() { Id = id }; + + public LlmTestServerScenarioBuilder ForModel(string modelId) + { + ArgumentException.ThrowIfNullOrWhiteSpace(modelId); + _scenario = _scenario with { ModelId = modelId }; + return this; + } + + public LlmTestServerScenarioBuilder RequireBearerToken(string token) + { + ArgumentException.ThrowIfNullOrWhiteSpace(token); + _scenario = _scenario with { RequiredBearerToken = token }; + return this; + } + + public LlmTestServerScenarioBuilder WhenUserContains(string content) + { + ArgumentException.ThrowIfNullOrWhiteSpace(content); + _scenario.Match.Messages.Add(new LlmTestServerMessage { Role = "user", Content = content }); + return this; + } + + public LlmTestServerScenarioBuilder WithExactMatch(params LlmTestServerMessage[] messages) + { + ArgumentNullException.ThrowIfNull(messages); + _scenario.Match.Messages.Clear(); + _scenario.Match.Messages.AddRange(messages); + _scenario = _scenario with + { + Match = _scenario.Match with { Mode = LlmTestServerMatchMode.Exact }, + }; + return this; + } + + public LlmTestServerScenarioBuilder RespondsWithItems(params LlmTestServerResponseItem[] items) + { + ArgumentNullException.ThrowIfNull(items); + if (items.Length == 0) { throw new ArgumentException("At least one output item is required.", nameof(items)); } + _scenario.Responses.Add(new() { OutputItems = LlmTestServerResponseItems.Snapshot(items, validateChunks: true) }); + return this; + } + + public LlmTestServerScenarioBuilder Responds(string content, params string[] streamChunks) + { + ArgumentNullException.ThrowIfNull(content); + _scenario.Responses.Add( + new LlmTestServerScenarioResponse + { + Content = content, + StreamChunks = streamChunks.Length > 0 ? [.. streamChunks] : [content], + } + ); + return this; + } + + public LlmTestServerScenarioBuilder CallsTool(string name, string argumentsJson, string callId = "call_fixture") + { + return CallsTools(new LlmTestServerToolCall { Id = callId, Name = name, ArgumentsJson = argumentsJson }); + } + + public LlmTestServerScenarioBuilder CallsTools(params LlmTestServerToolCall[] calls) + { + ArgumentNullException.ThrowIfNull(calls); + if (calls.Length == 0) + { + throw new ArgumentException("At least one tool call is required.", nameof(calls)); + } + + foreach (var call in calls) + { + ArgumentNullException.ThrowIfNull(call); + ArgumentException.ThrowIfNullOrWhiteSpace(call.Name); + ArgumentException.ThrowIfNullOrWhiteSpace(call.Id); + LlmTestServerJsonSchema.ValidateJson(call.ArgumentsJson); + } + _scenario.Responses.Add(new() { ToolCalls = [.. calls] }); + return this; + } + + public LlmTestServerScenarioBuilder RespondsJson(string json) + { + LlmTestServerJsonSchema.ValidateJson(json); + return Responds(json); + } + + public LlmTestServerScenarioBuilder RetryAfter(int seconds) + { + if (seconds is < 0 or > 86400) { throw new ArgumentOutOfRangeException(nameof(seconds)); } + if (_scenario.Responses.LastOrDefault()?.Error is not { } error) { throw new InvalidOperationException("RetryAfter requires a preceding scripted error."); } + var response = _scenario.Responses[^1]; + _scenario.Responses[^1] = response with { Error = error with { RetryAfterSeconds = seconds } }; + return this; + } + + public LlmTestServerScenarioBuilder Fails(int statusCode, string code, string message) + { + ArgumentException.ThrowIfNullOrWhiteSpace(code); + ArgumentException.ThrowIfNullOrWhiteSpace(message); + _scenario.Responses.Add( + new LlmTestServerScenarioResponse + { + Error = new LlmTestServerScenarioError + { + StatusCode = statusCode, + Code = code, + Message = message, + }, + } + ); + return this; + } + + public LlmTestServerScenarioBuilder DelaysBy(int milliseconds) + { + if (milliseconds < 0) + { + throw new ArgumentOutOfRangeException(nameof(milliseconds), "Delay cannot be negative."); + } + + if (_scenario.Responses.Count == 0) + { + _scenario.Responses.Add(new LlmTestServerScenarioResponse()); + } + + var index = _scenario.Responses.Count - 1; + _scenario.Responses[index] = _scenario.Responses[index] with + { + DelayMilliseconds = milliseconds, + }; + return this; + } + + public LlmTestServerScenarioBuilder WithStreamDelivery(LlmTestServerStreamDelivery delivery) + { + ArgumentNullException.ThrowIfNull(delivery); + if (_scenario.Responses.Count == 0) { throw new InvalidOperationException("Stream delivery requires a preceding response."); } + _scenario.Responses[^1] = _scenario.Responses[^1] with { StreamDelivery = delivery with { } }; + return this; + } + + public LlmTestServerScenarioBuilder WithOutputDisposition(LlmTestServerOutputDisposition disposition) + { + ArgumentNullException.ThrowIfNull(disposition); + if (_scenario.Responses.Count == 0) { throw new InvalidOperationException("Output disposition requires a preceding response."); } + _scenario.Responses[^1] = _scenario.Responses[^1] with { OutputDisposition = disposition with { } }; + return this; + } + + public LlmTestServerScenario Build() + { + if (_scenario.Responses.Count == 0) + { + throw new InvalidOperationException("A scenario requires at least one response."); + } + + return _scenario with + { + Match = _scenario.Match with { Messages = _scenario.Match.Messages.Select(message => message with { ToolCalls = [.. message.ToolCalls] }).ToList() }, + Responses = + [ + .. _scenario.Responses.Select(response => response with + { + StreamChunks = [.. response.StreamChunks], + ToolCalls = [.. response.ToolCalls], + }), + ], + }; + } +} diff --git a/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerScenarioDataset.cs b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerScenarioDataset.cs new file mode 100644 index 0000000..f6d8123 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerScenarioDataset.cs @@ -0,0 +1,8 @@ +namespace ManagedCode.LlmTestServer.Scenarios; + +public sealed record LlmTestServerScenarioDataset +{ + public string Id { get; init; } = Guid.NewGuid().ToString("N"); + + public List ChatScenarios { get; init; } = []; +} diff --git a/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerScenarioDatasetBuilder.cs b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerScenarioDatasetBuilder.cs new file mode 100644 index 0000000..a6bc819 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerScenarioDatasetBuilder.cs @@ -0,0 +1,50 @@ +using ManagedCode.LlmTestServer.Configuration; + +namespace ManagedCode.LlmTestServer.Scenarios; + +public sealed class LlmTestServerScenarioDatasetBuilder(string id) +{ + private readonly LlmTestServerScenarioDataset _dataset = new() { Id = id }; + + public LlmTestServerScenarioDatasetBuilder AddChatScenario( + string scenarioId, + Action configure + ) + { + ArgumentException.ThrowIfNullOrWhiteSpace(scenarioId); + ArgumentNullException.ThrowIfNull(configure); + + var builder = new LlmTestServerScenarioBuilder(scenarioId); + configure(builder); + var scenario = builder.Build(); + + _dataset.ChatScenarios.RemoveAll(existing => + string.Equals(existing.Id, scenarioId, StringComparison.OrdinalIgnoreCase) + ); + _dataset.ChatScenarios.Add(scenario); + + return this; + } + + public LlmTestServerScenarioDatasetBuilder AddConversation(string scenarioId, Action configure) + { + ArgumentNullException.ThrowIfNull(configure); + var builder = new ManagedCode.LlmTestServer.Conversations.LlmTestServerConversationBuilder(scenarioId); + configure(builder); + var scenario = builder.Build(); + _dataset.ChatScenarios.RemoveAll(existing => string.Equals(existing.Id, scenarioId, StringComparison.OrdinalIgnoreCase)); + _dataset.ChatScenarios.Add(scenario); + return this; + } + + public LlmTestServerScenarioDataset Build() + { + return _dataset with + { + ChatScenarios = + [ + .. _dataset.ChatScenarios.Select(LlmTestServerConfigurationBuilder.SnapshotScenario), + ], + }; + } +} diff --git a/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerScenarioError.cs b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerScenarioError.cs new file mode 100644 index 0000000..8eea5ae --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerScenarioError.cs @@ -0,0 +1,12 @@ +namespace ManagedCode.LlmTestServer.Scenarios; + +public sealed record LlmTestServerScenarioError +{ + public int? RetryAfterSeconds { get; init; } + + public int StatusCode { get; init; } = 500; + + public string Code { get; init; } = "llm_test_server_error"; + + public string Message { get; init; } = "Scripted LLM Test Server error."; +} diff --git a/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerScenarioResponse.cs b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerScenarioResponse.cs new file mode 100644 index 0000000..490dbd6 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerScenarioResponse.cs @@ -0,0 +1,20 @@ +namespace ManagedCode.LlmTestServer.Scenarios; + +public sealed record LlmTestServerScenarioResponse +{ + public List OutputItems { get; init; } = []; + + public List ToolCalls { get; init; } = []; + + public string Content { get; init; } = string.Empty; + + public List StreamChunks { get; init; } = []; + + public LlmTestServerScenarioError? Error { get; init; } + + public LlmTestServerStreamDelivery? StreamDelivery { get; init; } + + public LlmTestServerOutputDisposition? OutputDisposition { get; init; } + + public int DelayMilliseconds { get; init; } +} diff --git a/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerStreamDelivery.cs b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerStreamDelivery.cs new file mode 100644 index 0000000..55a6bb3 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerStreamDelivery.cs @@ -0,0 +1,42 @@ +using System.Text.Json.Serialization; + +namespace ManagedCode.LlmTestServer.Scenarios; + +public enum LlmTestServerStreamDeliveryMode +{ + EarlyCleanEnd = 1, + DelayThenContinue = 2, + AbortConnection = 3, +} + +public enum LlmTestServerStreamDeliverySurface +{ + OpenAiChatCompletions = 1, + OpenAiResponses = 2, + AnthropicMessages = 3, + GeminiGenerateContent = 4, +} + +/// Bounded delivery behavior applied after an actual text content frame is flushed. +[JsonUnmappedMemberHandling(JsonUnmappedMemberHandling.Disallow)] +public sealed record LlmTestServerStreamDelivery +{ + public LlmTestServerStreamDeliveryMode Mode { get; init; } + public int AfterContentFrames { get; init; } = 1; + public int DelayMilliseconds { get; init; } + + internal static LlmTestServerStreamDelivery? Snapshot(LlmTestServerScenarioResponse response) + { + if (response.StreamDelivery is not { } plan) { return null; } + if (!Enum.IsDefined(plan.Mode) || plan.AfterContentFrames is < 1 or > 128 + || plan.AfterContentFrames > (response.StreamChunks.Count > 0 + ? response.StreamChunks.Count(chunk => !string.IsNullOrEmpty(chunk)) : response.Content.Length > 0 ? 1 : 0) + || (plan.Mode == LlmTestServerStreamDeliveryMode.DelayThenContinue + ? plan.DelayMilliseconds is < 1 or > 60000 : plan.DelayMilliseconds != 0)) + { throw new ArgumentException("Invalid bounded stream delivery plan."); } + if (response.Error is not null || response.ToolCalls.Count > 0 || response.OutputItems.Count > 0 + || response.StreamChunks.Count > 0 && string.Concat(response.StreamChunks) != response.Content) + { throw new ArgumentException("Stream delivery requires a text-only response with matching content chunks."); } + return plan with { }; + } +} diff --git a/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerTools.cs b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerTools.cs new file mode 100644 index 0000000..81e5d7c --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Scenarios/LlmTestServerTools.cs @@ -0,0 +1,20 @@ +namespace ManagedCode.LlmTestServer.Scenarios; + +public sealed record LlmTestServerToolDefinition +{ + public string Name { get; init; } = string.Empty; + public string? Description { get; init; } + public string ParametersJson { get; init; } = "{\"type\":\"object\"}"; +} + +public sealed record LlmTestServerToolCall +{ + public string Id { get; init; } = string.Empty; + public string Name { get; init; } = string.Empty; + public string ArgumentsJson { get; init; } = "{}"; + + /// Optional opaque base64 continuation metadata, excluded from text token accounting. + public string? ContinuationToken { get; init; } +} + +public enum LlmTestServerToolChoice { Auto, None, Required } diff --git a/src/ManagedCode.LlmTestServer/Sessions/LlmTestServerSession.cs b/src/ManagedCode.LlmTestServer/Sessions/LlmTestServerSession.cs new file mode 100644 index 0000000..dc690e2 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Sessions/LlmTestServerSession.cs @@ -0,0 +1,21 @@ +using ManagedCode.LlmTestServer.Configuration; + +namespace ManagedCode.LlmTestServer.Sessions; + +public static class LlmTestServerSessionHeaders +{ + public const string Id = "X-Llm-Test-Server-Session"; + public const string AccessKey = "X-Llm-Test-Server-Session-Key"; +} + +public sealed class LlmTestServerSession +{ + public required string Id { get; init; } + public required string AccessKey { get; init; } + public long ReservedStateBytes { get; init; } +} + +public sealed class LlmTestServerSessionCreateRequest +{ + public LlmTestServerConfiguration? Configuration { get; init; } +} diff --git a/src/ManagedCode.LlmTestServer/Tools/WellKnown/LlmTestServerWellKnownTool.cs b/src/ManagedCode.LlmTestServer/Tools/WellKnown/LlmTestServerWellKnownTool.cs new file mode 100644 index 0000000..7d644ca --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Tools/WellKnown/LlmTestServerWellKnownTool.cs @@ -0,0 +1,129 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using ManagedCode.LlmTestServer.Scenarios; + +namespace ManagedCode.LlmTestServer.Tools.WellKnown; + +public sealed record LlmTestServerWellKnownToolDescriptor +{ + public string Name { get; init; } = string.Empty; + public string Description { get; init; } = string.Empty; + public string ParametersJson { get; init; } = "{}"; + public string ResultSchemaJson { get; init; } = "{}"; + public string ErrorSchemaJson { get; init; } = "{}"; +} + +/// Scripted function-tool data consumed by existing scenario and conversation APIs. +public sealed record LlmTestServerWellKnownToolFixture +{ + public required LlmTestServerToolDefinition Definition { get; init; } + public required LlmTestServerToolCall Call { get; init; } + public required LlmTestServerMessage Result { get; init; } + public bool IsError { get; init; } + public int DelayMilliseconds { get; init; } +} + +/// Defines typed fixture contracts. It does not execute the represented operation. +public sealed class LlmTestServerWellKnownTool + where TArguments : class + where TResult : class +{ + private static readonly JsonSerializerOptions _json = new(JsonSerializerDefaults.Web) + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + }; + + internal LlmTestServerWellKnownTool(LlmTestServerWellKnownToolDescriptor descriptor) + { + Descriptor = descriptor; + } + + public LlmTestServerWellKnownToolDescriptor Descriptor { get; } + + public LlmTestServerToolDefinition Definition => new() + { + Name = Descriptor.Name, + Description = Descriptor.Description, + ParametersJson = Descriptor.ParametersJson, + }; + + public LlmTestServerToolCall CreateCall(TArguments arguments, string? callId = null) + { + ArgumentNullException.ThrowIfNull(arguments); + return new() + { + Id = ValidateCallId(callId ?? "call_" + Descriptor.Name), + Name = Descriptor.Name, + ArgumentsJson = Serialize(arguments, Descriptor.ParametersJson), + }; + } + + /// Serializes a successful result as its typed JSON root, without an envelope. + public LlmTestServerMessage CreateResult(TResult result, string? callId = null) + { + ArgumentNullException.ThrowIfNull(result); + return new() + { + Role = "tool", + ToolCallId = ValidateCallId(callId ?? "call_" + Descriptor.Name), + Content = Serialize(result, Descriptor.ResultSchemaJson), + }; + } + + /// Creates an explicit JSON null result, distinct from an omitted tool result. + public LlmTestServerMessage CreateNullResult(string? callId = null) + { + return new() + { + Role = "tool", + ToolCallId = ValidateCallId(callId ?? "call_" + Descriptor.Name), + Content = "null", + }; + } + + /// Serializes the explicitly selected {"error":{"code","message","retryable"}} fixture envelope. + public LlmTestServerMessage CreateError(LlmTestServerToolFixtureError error, string? callId = null) + { + ArgumentNullException.ThrowIfNull(error); + return new() + { + Role = "tool", + ToolCallId = ValidateCallId(callId ?? "call_" + Descriptor.Name), + Content = Serialize(new LlmTestServerToolFixtureErrorResult { Error = error }, Descriptor.ErrorSchemaJson), + }; + } + + public LlmTestServerWellKnownToolFixture Success(TArguments arguments, TResult result, string? callId = null, int delayMilliseconds = 0) + { + ArgumentOutOfRangeException.ThrowIfNegative(delayMilliseconds); + return new() { Definition = Definition, Call = CreateCall(arguments, callId), Result = CreateResult(result, callId), DelayMilliseconds = delayMilliseconds }; + } + + /// Selects a literal JSON null outcome without weakening the typed success schema. + public LlmTestServerWellKnownToolFixture NullResult(TArguments arguments, string? callId = null, int delayMilliseconds = 0) + { + ArgumentOutOfRangeException.ThrowIfNegative(delayMilliseconds); + return new() { Definition = Definition, Call = CreateCall(arguments, callId), Result = CreateNullResult(callId), DelayMilliseconds = delayMilliseconds }; + } + + public LlmTestServerWellKnownToolFixture Failure(TArguments arguments, LlmTestServerToolFixtureError error, string? callId = null, int delayMilliseconds = 0) + { + ArgumentOutOfRangeException.ThrowIfNegative(delayMilliseconds); + return new() { Definition = Definition, Call = CreateCall(arguments, callId), Result = CreateError(error, callId), IsError = true, DelayMilliseconds = delayMilliseconds }; + } + + private static string Serialize(T value, string schema) + { + var json = JsonSerializer.Serialize(value, _json); + if (!LlmTestServerJsonSchema.Matches(json, schema)) + { throw new ArgumentException("The typed fixture does not satisfy its well-known tool schema.", nameof(value)); } + return json; + } + + private static string ValidateCallId(string callId) + { + ArgumentException.ThrowIfNullOrWhiteSpace(callId); + if (callId.Length > 128) { throw new ArgumentException("Tool call IDs are limited to 128 characters.", nameof(callId)); } + return callId; + } +} diff --git a/src/ManagedCode.LlmTestServer/Tools/WellKnown/LlmTestServerWellKnownToolContracts.cs b/src/ManagedCode.LlmTestServer/Tools/WellKnown/LlmTestServerWellKnownToolContracts.cs new file mode 100644 index 0000000..07f82ed --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Tools/WellKnown/LlmTestServerWellKnownToolContracts.cs @@ -0,0 +1,179 @@ +namespace ManagedCode.LlmTestServer.Tools.WellKnown; + +public sealed record LlmTestServerCalculatorArguments +{ + public required string Expression { get; init; } +} + +public sealed record LlmTestServerCalculatorResult +{ + public decimal Value { get; init; } +} + +public sealed record LlmTestServerCurrentTimeArguments +{ + public string TimeZone { get; init; } = "UTC"; +} + +public sealed record LlmTestServerCurrentTimeResult +{ + public required DateTimeOffset CurrentTime { get; init; } + public string TimeZone { get; init; } = "UTC"; +} + +public sealed record LlmTestServerWeatherArguments +{ + public required string Location { get; init; } + public string Unit { get; init; } = "celsius"; +} + +public sealed record LlmTestServerWeatherResult +{ + public required string Location { get; init; } + public decimal Temperature { get; init; } + public string Unit { get; init; } = "celsius"; + public required string Condition { get; init; } +} + +public sealed record LlmTestServerWebSearchArguments +{ + public required string Query { get; init; } + public int Limit { get; init; } = 5; +} + +public sealed record LlmTestServerWebSearchHit +{ + public required string Title { get; init; } + public required string Url { get; init; } + public required string Snippet { get; init; } +} + +public sealed record LlmTestServerWebSearchResult +{ + public List Results { get; init; } = []; +} + +public sealed record LlmTestServerFetchUrlArguments +{ + public required string Url { get; init; } +} + +public sealed record LlmTestServerFetchUrlResult +{ + public required string Url { get; init; } + public int StatusCode { get; init; } = 200; + public string ContentType { get; init; } = "text/plain"; + public string Content { get; init; } = string.Empty; +} + +public sealed record LlmTestServerFileSearchArguments +{ + public required string Query { get; init; } + public int Limit { get; init; } = 5; +} + +public sealed record LlmTestServerFileSearchHit +{ + public required string FileId { get; init; } + public required string Path { get; init; } + public required string Snippet { get; init; } +} + +public sealed record LlmTestServerFileSearchResult +{ + public List Results { get; init; } = []; +} + +public sealed record LlmTestServerReadFileArguments +{ + public required string Path { get; init; } + public int Offset { get; init; } + public int? Length { get; init; } +} + +public sealed record LlmTestServerReadFileResult +{ + public required string Path { get; init; } + public string Content { get; init; } = string.Empty; + public bool Truncated { get; init; } +} + +public sealed record LlmTestServerWriteFileArguments +{ + public required string Path { get; init; } + public string Content { get; init; } = string.Empty; + public bool Append { get; init; } +} + +public sealed record LlmTestServerWriteFileResult +{ + public required string Path { get; init; } + public int BytesWritten { get; init; } +} + +public sealed record LlmTestServerListFilesArguments +{ + public required string Path { get; init; } + public bool Recursive { get; init; } +} + +public sealed record LlmTestServerFileEntry +{ + public required string Path { get; init; } + public bool IsDirectory { get; init; } + public long? Bytes { get; init; } +} + +public sealed record LlmTestServerListFilesResult +{ + public List Entries { get; init; } = []; +} + +public sealed record LlmTestServerRunCodeArguments +{ + public required string Language { get; init; } + public required string Code { get; init; } +} + +public sealed record LlmTestServerRunCodeResult +{ + public string Stdout { get; init; } = string.Empty; + public string Stderr { get; init; } = string.Empty; + public int ExitCode { get; init; } +} + +public sealed record LlmTestServerShellArguments +{ + public required string Command { get; init; } + public string? WorkingDirectory { get; init; } +} + +public sealed record LlmTestServerShellResult +{ + public string Stdout { get; init; } = string.Empty; + public string Stderr { get; init; } = string.Empty; + public int ExitCode { get; init; } +} + +public sealed record LlmTestServerApplyPatchArguments +{ + public required string Patch { get; init; } +} + +public sealed record LlmTestServerApplyPatchResult +{ + public List UpdatedPaths { get; init; } = []; +} + +public sealed record LlmTestServerToolFixtureError +{ + public required string Code { get; init; } + public required string Message { get; init; } + public bool Retryable { get; init; } +} + +/// An explicitly selected tool error payload; successful results retain their own typed root shape. +public sealed record LlmTestServerToolFixtureErrorResult +{ + public required LlmTestServerToolFixtureError Error { get; init; } +} diff --git a/src/ManagedCode.LlmTestServer/Tools/WellKnown/LlmTestServerWellKnownToolNames.cs b/src/ManagedCode.LlmTestServer/Tools/WellKnown/LlmTestServerWellKnownToolNames.cs new file mode 100644 index 0000000..9d2b090 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Tools/WellKnown/LlmTestServerWellKnownToolNames.cs @@ -0,0 +1,17 @@ +namespace ManagedCode.LlmTestServer.Tools.WellKnown; + +public static class LlmTestServerWellKnownToolNames +{ + public const string Calculator = "calculator"; + public const string CurrentTime = "current_time"; + public const string Weather = "weather"; + public const string WebSearch = "web_search"; + public const string FetchUrl = "fetch_url"; + public const string FileSearch = "file_search"; + public const string ReadFile = "read_file"; + public const string WriteFile = "write_file"; + public const string ListFiles = "list_files"; + public const string RunCode = "run_code"; + public const string Shell = "shell"; + public const string ApplyPatch = "apply_patch"; +} diff --git a/src/ManagedCode.LlmTestServer/Tools/WellKnown/LlmTestServerWellKnownTools.cs b/src/ManagedCode.LlmTestServer/Tools/WellKnown/LlmTestServerWellKnownTools.cs new file mode 100644 index 0000000..b297abe --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Tools/WellKnown/LlmTestServerWellKnownTools.cs @@ -0,0 +1,102 @@ +namespace ManagedCode.LlmTestServer.Tools.WellKnown; + +/// Provider-neutral function-tool presets. Every result is explicitly scripted by the caller. +public static class LlmTestServerWellKnownTools +{ + public const string ErrorSchemaJson = """{"type":"object","properties":{"error":{"type":"object","properties":{"code":{"type":"string","minLength":1},"message":{"type":"string","minLength":1},"retryable":{"type":"boolean"}},"required":["code","message","retryable"],"additionalProperties":false}},"required":["error"],"additionalProperties":false}"""; + + public static LlmTestServerWellKnownTool Calculator { get; } = Create( + LlmTestServerWellKnownToolNames.Calculator, + "Scripted expression and numeric result; no expression is evaluated.", + """{"type":"object","properties":{"expression":{"type":"string","minLength":1}},"required":["expression"],"additionalProperties":false}""", + """{"type":"object","properties":{"value":{"type":"number"}},"required":["value"],"additionalProperties":false}"""); + + public static LlmTestServerWellKnownTool CurrentTime { get; } = Create( + LlmTestServerWellKnownToolNames.CurrentTime, + "Scripted timestamp supplied by the fixture; no system clock is consulted.", + """{"type":"object","properties":{"timeZone":{"type":"string","minLength":1}},"required":["timeZone"],"additionalProperties":false}""", + """{"type":"object","properties":{"currentTime":{"type":"string","format":"date-time"},"timeZone":{"type":"string","minLength":1}},"required":["currentTime","timeZone"],"additionalProperties":false}"""); + + public static LlmTestServerWellKnownTool Weather { get; } = Create( + LlmTestServerWellKnownToolNames.Weather, + "Scripted weather data for a location; no weather service is called.", + """{"type":"object","properties":{"location":{"type":"string","minLength":1},"unit":{"type":"string","enum":["celsius","fahrenheit"]}},"required":["location","unit"],"additionalProperties":false}""", + """{"type":"object","properties":{"location":{"type":"string","minLength":1},"temperature":{"type":"number"},"unit":{"type":"string","enum":["celsius","fahrenheit"]},"condition":{"type":"string","minLength":1}},"required":["location","temperature","unit","condition"],"additionalProperties":false}"""); + + public static LlmTestServerWellKnownTool WebSearch { get; } = Create( + LlmTestServerWellKnownToolNames.WebSearch, + "Scripted search hits from test data; no network search is performed.", + """{"type":"object","properties":{"query":{"type":"string","minLength":1},"limit":{"type":"integer","minimum":1,"maximum":100}},"required":["query","limit"],"additionalProperties":false}""", + """{"type":"object","properties":{"results":{"type":"array","items":{"type":"object","properties":{"title":{"type":"string","minLength":1},"url":{"type":"string","minLength":1},"snippet":{"type":"string","minLength":0}},"required":["title","url","snippet"],"additionalProperties":false}}},"required":["results"],"additionalProperties":false}"""); + + public static LlmTestServerWellKnownTool FetchUrl { get; } = Create( + LlmTestServerWellKnownToolNames.FetchUrl, + "Scripted URL content; no URL is fetched.", + """{"type":"object","properties":{"url":{"type":"string","minLength":1}},"required":["url"],"additionalProperties":false}""", + """{"type":"object","properties":{"url":{"type":"string","minLength":1},"statusCode":{"type":"integer","minimum":100,"maximum":599},"contentType":{"type":"string","minLength":1},"content":{"type":"string","minLength":0}},"required":["url","statusCode","contentType","content"],"additionalProperties":false}"""); + + public static LlmTestServerWellKnownTool FileSearch { get; } = Create( + LlmTestServerWellKnownToolNames.FileSearch, + "Scripted file search hits from test data; no file system is searched.", + """{"type":"object","properties":{"query":{"type":"string","minLength":1},"limit":{"type":"integer","minimum":1,"maximum":100}},"required":["query","limit"],"additionalProperties":false}""", + """{"type":"object","properties":{"results":{"type":"array","items":{"type":"object","properties":{"fileId":{"type":"string","minLength":1},"path":{"type":"string","minLength":1},"snippet":{"type":"string","minLength":0}},"required":["fileId","path","snippet"],"additionalProperties":false}}},"required":["results"],"additionalProperties":false}"""); + + public static LlmTestServerWellKnownTool ReadFile { get; } = Create( + LlmTestServerWellKnownToolNames.ReadFile, + "Scripted file content; no file is opened.", + """{"type":"object","properties":{"path":{"type":"string","minLength":1},"offset":{"type":"integer","minimum":0},"length":{"type":"integer","minimum":0}},"required":["path","offset"],"additionalProperties":false}""", + """{"type":"object","properties":{"path":{"type":"string","minLength":1},"content":{"type":"string","minLength":0},"truncated":{"type":"boolean"}},"required":["path","content","truncated"],"additionalProperties":false}"""); + + public static LlmTestServerWellKnownTool WriteFile { get; } = Create( + LlmTestServerWellKnownToolNames.WriteFile, + "Scripted write receipt; no file is created or changed.", + """{"type":"object","properties":{"path":{"type":"string","minLength":1},"content":{"type":"string","minLength":0},"append":{"type":"boolean"}},"required":["path","content","append"],"additionalProperties":false}""", + """{"type":"object","properties":{"path":{"type":"string","minLength":1},"bytesWritten":{"type":"integer","minimum":0}},"required":["path","bytesWritten"],"additionalProperties":false}"""); + + public static LlmTestServerWellKnownTool ListFiles { get; } = Create( + LlmTestServerWellKnownToolNames.ListFiles, + "Scripted directory entries; no directory is enumerated.", + """{"type":"object","properties":{"path":{"type":"string","minLength":1},"recursive":{"type":"boolean"}},"required":["path","recursive"],"additionalProperties":false}""", + """{"type":"object","properties":{"entries":{"type":"array","items":{"type":"object","properties":{"path":{"type":"string","minLength":1},"isDirectory":{"type":"boolean"},"bytes":{"type":"integer","minimum":0}},"required":["path","isDirectory"],"additionalProperties":false}}},"required":["entries"],"additionalProperties":false}"""); + + public static LlmTestServerWellKnownTool RunCode { get; } = Create( + LlmTestServerWellKnownToolNames.RunCode, + "Scripted process output; code is never evaluated or executed.", + """{"type":"object","properties":{"language":{"type":"string","minLength":1},"code":{"type":"string","minLength":1}},"required":["language","code"],"additionalProperties":false}""", + """{"type":"object","properties":{"stdout":{"type":"string","minLength":0},"stderr":{"type":"string","minLength":0},"exitCode":{"type":"integer"}},"required":["stdout","stderr","exitCode"],"additionalProperties":false}"""); + + public static LlmTestServerWellKnownTool Shell { get; } = Create( + LlmTestServerWellKnownToolNames.Shell, + "Scripted command output; no shell or process is started.", + """{"type":"object","properties":{"command":{"type":"string","minLength":1},"workingDirectory":{"type":"string","minLength":1}},"required":["command"],"additionalProperties":false}""", + """{"type":"object","properties":{"stdout":{"type":"string","minLength":0},"stderr":{"type":"string","minLength":0},"exitCode":{"type":"integer"}},"required":["stdout","stderr","exitCode"],"additionalProperties":false}"""); + + public static LlmTestServerWellKnownTool ApplyPatch { get; } = Create( + LlmTestServerWellKnownToolNames.ApplyPatch, + "Scripted patch receipt; no patch is applied.", + """{"type":"object","properties":{"patch":{"type":"string","minLength":1}},"required":["patch"],"additionalProperties":false}""", + """{"type":"object","properties":{"updatedPaths":{"type":"array","items":{"type":"string","minLength":1}}},"required":["updatedPaths"],"additionalProperties":false}"""); + + public static IReadOnlyList All { get; } = Array.AsReadOnly(new[] + { + Calculator.Descriptor, + CurrentTime.Descriptor, + Weather.Descriptor, + WebSearch.Descriptor, + FetchUrl.Descriptor, + FileSearch.Descriptor, + ReadFile.Descriptor, + WriteFile.Descriptor, + ListFiles.Descriptor, + RunCode.Descriptor, + Shell.Descriptor, + ApplyPatch.Descriptor, + }); + + private static LlmTestServerWellKnownTool Create(string name, string description, string parametersJson, string resultSchemaJson) + where TArguments : class + where TResult : class + { + return new(new() { Name = name, Description = description, ParametersJson = parametersJson, ResultSchemaJson = resultSchemaJson, ErrorSchemaJson = ErrorSchemaJson }); + } +} diff --git a/src/ManagedCode.LlmTestServer/Verification/LlmTestServerVerification.cs b/src/ManagedCode.LlmTestServer/Verification/LlmTestServerVerification.cs new file mode 100644 index 0000000..40b6382 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Verification/LlmTestServerVerification.cs @@ -0,0 +1,80 @@ +using ManagedCode.LlmTestServer.Conversations; +using ManagedCode.LlmTestServer.Diagnostics; + +namespace ManagedCode.LlmTestServer.Verification; + +public sealed record LlmTestServerVerificationRequest +{ + public LlmTestServerRequestQuery Scope { get; init; } = new(); + public List Requests { get; init; } = []; + public bool InOrder { get; init; } + public bool NoUnexpectedCalls { get; init; } + public List Conversations { get; init; } = []; + public List Retries { get; init; } = []; +} + +public sealed record LlmTestServerRequestExpectation +{ + public string Name { get; init; } = "request"; + public LlmTestServerRequestQuery Query { get; init; } = new(); + public int MinimumCount { get; init; } = 1; + public int? MaximumCount { get; init; } = 1; + public bool RequireStreamingTerminalMarker { get; init; } + public List Tools { get; init; } = []; +} + +public enum LlmTestServerToolEvidenceKind +{ + ResponseCall, + RequestCall, + Result, +} + +public sealed record LlmTestServerToolExpectation +{ + public LlmTestServerToolEvidenceKind Kind { get; init; } + public string? Name { get; init; } + public string? CallId { get; init; } + public string? ArgumentsJson { get; init; } + public string? ResultJson { get; init; } + public List ResultFields { get; init; } = []; + public int MinimumCount { get; init; } = 1; + public int? MaximumCount { get; init; } = 1; +} + +public sealed record LlmTestServerConversationExpectation +{ + public string ScenarioId { get; init; } = string.Empty; + public string? ConversationId { get; init; } + public string? ProviderNamespace { get; init; } + public List VisitedSteps { get; init; } = []; + public bool RequireTerminal { get; init; } +} + +public sealed record LlmTestServerRetryExpectation +{ + public string GroupId { get; init; } = string.Empty; + public LlmTestServerRequestQuery Query { get; init; } = new(); + public int MinimumAttempts { get; init; } = 1; + public int MaximumAttempts { get; init; } = 3; + public List StatusSequence { get; init; } = []; + public double? MinimumRetryAfterSeconds { get; init; } + public bool RespectRetryAfter { get; init; } +} + +public sealed record LlmTestServerVerificationResult +{ + public bool Passed { get; init; } + public bool EvidenceComplete { get; init; } + public List Failures { get; init; } = []; + public List MatchedRequestIds { get; init; } = []; +} + +public sealed record LlmTestServerVerificationFailure +{ + public string Code { get; init; } = string.Empty; + public string Expectation { get; init; } = string.Empty; + public string Expected { get; init; } = string.Empty; + public string Actual { get; init; } = string.Empty; + public List RequestIds { get; init; } = []; +} diff --git a/src/ManagedCode.LlmTestServer/Verification/LlmTestServerVerificationEvaluator.cs b/src/ManagedCode.LlmTestServer/Verification/LlmTestServerVerificationEvaluator.cs new file mode 100644 index 0000000..9a02975 --- /dev/null +++ b/src/ManagedCode.LlmTestServer/Verification/LlmTestServerVerificationEvaluator.cs @@ -0,0 +1,253 @@ +using System.Globalization; +using System.Text.Json; +using ManagedCode.LlmTestServer.Conversations; +using ManagedCode.LlmTestServer.Diagnostics; + +namespace ManagedCode.LlmTestServer.Verification; + +public static class LlmTestServerVerificationEvaluator +{ + public static LlmTestServerVerificationResult Verify( + LlmTestServerVerificationRequest verification, + LlmTestServerRequestSnapshot snapshot, + IReadOnlyList conversations) + { + ArgumentNullException.ThrowIfNull(verification); + ArgumentNullException.ThrowIfNull(snapshot); + ArgumentNullException.ThrowIfNull(conversations); + Validate(verification); + var scope = LlmTestServerRequestQueries.Apply(snapshot, verification.Scope); + var failures = new List(); + var matched = new HashSet(StringComparer.Ordinal); + var complete = snapshot.EvictedCount == 0 && !snapshot.RetentionTruncated + && scope.TotalMatched == scope.Requests.Count + && scope.Requests.All(trace => trace.IsCompleted && !trace.RetentionTruncated && !trace.RuntimePayloadTruncated); + void Fail(string code, string expectation, string expected, string actual, IEnumerable? ids = null) + { + failures.Add(new() { Code = code, Expectation = expectation, Expected = expected, Actual = actual, RequestIds = ids?.ToList() ?? [] }); + } + + if (!complete) + { + Fail("evidence.incomplete", "journal", "A complete retained journal with completed requests", + $"evicted={snapshot.EvictedCount}; truncated={snapshot.RetentionTruncated}; selected={scope.Requests.Count}/{scope.TotalMatched}"); + } + long? previousLastSequence = null; + foreach (var expectation in verification.Requests) + { + var selected = LlmTestServerRequestQueries.Apply(scope, expectation.Query); + var requests = selected.Requests.OrderBy(trace => trace.Sequence).ToArray(); + var ids = requests.Select(trace => trace.RequestId).ToArray(); + matched.UnionWith(ids); + if (selected.TotalMatched != requests.Length) + { complete = false; Fail("evidence.page", expectation.Name, "All matching requests retained in the verification page", selected.TotalMatched.ToString(CultureInfo.InvariantCulture), ids); } + if (!InRange(requests.Length, expectation.MinimumCount, expectation.MaximumCount)) + { Fail("request.count", expectation.Name, Range(expectation.MinimumCount, expectation.MaximumCount), requests.Length.ToString(CultureInfo.InvariantCulture), ids); } + if (verification.InOrder && requests.Length > 0) + { + if (previousLastSequence is { } previous && requests[0].Sequence <= previous) + { Fail("request.order", expectation.Name, $"Sequence greater than {previous}", requests[0].Sequence.ToString(CultureInfo.InvariantCulture), ids); } + previousLastSequence = requests[^1].Sequence; + } + foreach (var trace in requests) + { + if (expectation.RequireStreamingTerminalMarker && (!trace.IsStreaming || trace.TerminalMarkerObserved != true)) + { Fail("stream.terminal", expectation.Name, "A streaming response with its terminal marker observed", $"streaming={trace.IsStreaming}; terminal={trace.TerminalMarkerObserved}", [trace.RequestId]); } + foreach (var tool in expectation.Tools) + { + var count = Tools(trace, tool.Kind).Count(evidence => ToolMatches(evidence, tool)); + if (!InRange(count, tool.MinimumCount, tool.MaximumCount)) + { Fail("tool.count", expectation.Name, $"{tool.Kind} {tool.Name ?? "*"}/{tool.CallId ?? "*"}: {Range(tool.MinimumCount, tool.MaximumCount)}", count.ToString(CultureInfo.InvariantCulture), [trace.RequestId]); } + } + } + } + foreach (var expectation in verification.Conversations) + { + var states = conversations.Where(state => state.ScenarioId == expectation.ScenarioId + && (expectation.ConversationId is null || state.ConversationId == expectation.ConversationId) + && (expectation.ProviderNamespace is null || state.ProviderNamespace == expectation.ProviderNamespace)).ToArray(); + if (states.Length == 0) + { Fail("conversation.missing", expectation.ScenarioId, "At least one matching conversation", "0"); continue; } + foreach (var state in states) + { + var missing = expectation.VisitedSteps.Except(state.VisitedSteps, StringComparer.Ordinal).ToArray(); + if (missing.Length > 0) + { Fail("conversation.steps", expectation.ScenarioId, string.Join(",", expectation.VisitedSteps), $"Missing: {string.Join(",", missing)}"); } + if (expectation.RequireTerminal && !state.IsTerminal) + { Fail("conversation.terminal", expectation.ScenarioId, "Terminal", state.IsBusy ? "Busy" : "Active"); } + } + } + foreach (var retry in verification.Retries) + { + var selected = LlmTestServerRequestQueries.Apply(scope, retry.Query); + var attempts = selected.Requests.Where(trace => trace.RetryGroupId == retry.GroupId).OrderBy(trace => trace.Sequence).ToArray(); + var ids = attempts.Select(trace => trace.RequestId).ToArray(); + matched.UnionWith(ids); + if (selected.TotalMatched != selected.Requests.Count) + { complete = false; Fail("evidence.page", retry.GroupId, "All matching retry requests retained", selected.TotalMatched.ToString(CultureInfo.InvariantCulture), ids); } + if (!InRange(attempts.Length, retry.MinimumAttempts, retry.MaximumAttempts)) + { Fail("retry.attempts", retry.GroupId, Range(retry.MinimumAttempts, retry.MaximumAttempts), attempts.Length.ToString(CultureInfo.InvariantCulture), ids); } + if (retry.StatusSequence.Count > 0 && !attempts.Select(trace => trace.StatusCode).SequenceEqual(retry.StatusSequence.Select(status => (int?)status))) + { Fail("retry.statuses", retry.GroupId, string.Join(",", retry.StatusSequence), string.Join(",", attempts.Select(trace => trace.StatusCode)), ids); } + for (var index = 0; index < attempts.Length; index++) + { + var attempt = attempts[index]; + if (attempt.StatusCode is not (429 or 503)) { continue; } + var delay = RetryAfterSeconds(attempt); + if (retry.MinimumRetryAfterSeconds is { } minimum && (delay is null || delay < minimum)) + { Fail("retry.after", retry.GroupId, $"Retry-After >= {minimum.ToString(CultureInfo.InvariantCulture)}s", attempt.RetryAfter ?? "missing", [attempt.RequestId]); } + if (retry.RespectRetryAfter && index + 1 < attempts.Length) + { + var next = attempts[index + 1]; + if (!TryGetMonotonicBounds(attempt, out _, out var completed) + || !TryGetMonotonicBounds(next, out var started, out _)) + { + complete = false; + Fail("retry.timing", retry.GroupId, "Finite monotonic start and completion timestamps in chronological order", + $"previous={attempt.StartedMonotonicSeconds}/{attempt.CompletedMonotonicSeconds}; next={next.StartedMonotonicSeconds}/{next.CompletedMonotonicSeconds}", [attempt.RequestId, next.RequestId]); + continue; + } + var elapsed = started - completed; + if (delay is null || elapsed < delay) + { Fail("retry.delay", retry.GroupId, delay is null ? "A valid Retry-After header" : $"At least {delay.Value.ToString(CultureInfo.InvariantCulture)}s", elapsed.ToString(CultureInfo.InvariantCulture), [attempt.RequestId, next.RequestId]); } + } + } + } + // Retry expectations also account for their calls when asserting that nothing unexpected happened. + if (verification.NoUnexpectedCalls) + { + var unexpected = scope.Requests.Where(trace => !matched.Contains(trace.RequestId)).Select(trace => trace.RequestId).ToArray(); + if (unexpected.Length > 0) { Fail("request.unexpected", "journal", "No unexpected provider requests", unexpected.Length.ToString(CultureInfo.InvariantCulture), unexpected); } + } + return new() + { + Passed = complete && failures.Count == 0, + EvidenceComplete = complete, + Failures = failures, + MatchedRequestIds = scope.Requests.OrderBy(trace => trace.Sequence).Where(trace => matched.Contains(trace.RequestId)).Select(trace => trace.RequestId).ToList() + }; + } + + private static bool InRange(int count, int minimum, int? maximum) + { + return count >= minimum && (maximum is null || count <= maximum); + } + + private static string Range(int minimum, int? maximum) + { + return maximum == minimum + ? minimum.ToString(CultureInfo.InvariantCulture) : $"{minimum}..{maximum?.ToString(CultureInfo.InvariantCulture) ?? "unbounded"}"; + } + + private static IEnumerable Tools(LlmTestServerProviderHttpTrace trace, LlmTestServerToolEvidenceKind kind) + { + if (kind == LlmTestServerToolEvidenceKind.ResponseCall) + { return trace.ResponseToolCalls.Select(call => new ToolEvidence(call.Name, call.Id, call.ArgumentsJson, null)); } + if (kind == LlmTestServerToolEvidenceKind.RequestCall) + { return trace.Messages.SelectMany(message => message.ToolCalls).Select(call => new ToolEvidence(call.Name, call.Id, call.ArgumentsJson, null)); } + return trace.Messages.Select((message, index) => (message, index)).Where(item => item.message.Role == "tool").Select(item => + { + var call = trace.Messages.Take(item.index).SelectMany(message => message.ToolCalls).LastOrDefault(candidate => candidate.Id == item.message.ToolCallId); + return new ToolEvidence(call?.Name, item.message.ToolCallId, call?.ArgumentsJson, item.message.Content); + }); + } + + private static bool ToolMatches(ToolEvidence evidence, LlmTestServerToolExpectation expectation) + { + return (expectation.Name is null || evidence.Name == expectation.Name) + && (expectation.CallId is null || evidence.CallId == expectation.CallId) + && (expectation.ArgumentsJson is null || evidence.ArgumentsJson is not null && LlmTestServerJsonFieldSelector.Matches(evidence.ArgumentsJson, "$", expectation.ArgumentsJson)) + && (expectation.ResultJson is null || evidence.ResultJson is not null && LlmTestServerJsonFieldSelector.Matches(evidence.ResultJson, "$", expectation.ResultJson)) + && expectation.ResultFields.All(field => evidence.ResultJson is not null && LlmTestServerJsonFieldSelector.Matches(evidence.ResultJson, field.Path, field.EqualsJson)); + } + + private static bool TryGetMonotonicBounds(LlmTestServerProviderHttpTrace trace, out double started, out double completed) + { + started = trace.StartedMonotonicSeconds ?? double.NaN; + completed = trace.CompletedMonotonicSeconds ?? double.NaN; + return double.IsFinite(started) && double.IsFinite(completed) && started >= 0 && completed >= started; + } + + private static double? RetryAfterSeconds(LlmTestServerProviderHttpTrace trace) + { + if (trace.RetryAfter is null) { return null; } + if (double.TryParse(trace.RetryAfter, NumberStyles.None, CultureInfo.InvariantCulture, out var seconds) && double.IsFinite(seconds) && seconds >= 0) + { return seconds; } + if (DateTimeOffset.TryParse(trace.RetryAfter, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var date)) + { return Math.Max(0, (date - (trace.CompletedAt ?? trace.StartedAt)).TotalSeconds); } + return null; + } + + private static void Validate(LlmTestServerVerificationRequest verification) + { + ArgumentNullException.ThrowIfNull(verification.Scope); + ArgumentNullException.ThrowIfNull(verification.Requests); + ArgumentNullException.ThrowIfNull(verification.Conversations); + ArgumentNullException.ThrowIfNull(verification.Retries); + var expectations = verification.Requests.Count + verification.Conversations.Count + verification.Retries.Count; + if (expectations > 256 || expectations == 0 && !verification.NoUnexpectedCalls) + { throw new ArgumentException("Verification requires at most 256 expectations and a request, graph, retry, or no-unexpected-calls assertion.", nameof(verification)); } + foreach (var expectation in verification.Requests) + { + ArgumentNullException.ThrowIfNull(expectation); + ArgumentException.ThrowIfNullOrWhiteSpace(expectation.Name); + ArgumentNullException.ThrowIfNull(expectation.Query); + ValidateRange(expectation.MinimumCount, expectation.MaximumCount); + ArgumentNullException.ThrowIfNull(expectation.Tools); + if (expectation.Tools.Count > 64) { throw new ArgumentException("A request expectation supports at most 64 tool expectations.", nameof(verification)); } + foreach (var tool in expectation.Tools) + { + ArgumentNullException.ThrowIfNull(tool); + if (!Enum.IsDefined(tool.Kind)) { throw new ArgumentException("Unknown tool evidence kind.", nameof(verification)); } + ValidateRange(tool.MinimumCount, tool.MaximumCount); + ValidateJson(tool.ArgumentsJson); + ValidateJson(tool.ResultJson); + ArgumentNullException.ThrowIfNull(tool.ResultFields); + if (tool.ResultFields.Count > 64) { throw new ArgumentException("A tool expectation supports at most 64 result fields.", nameof(verification)); } + foreach (var field in tool.ResultFields) + { + ArgumentNullException.ThrowIfNull(field); + LlmTestServerJsonFieldSelector.Parse(field.Path); + ArgumentNullException.ThrowIfNull(field.EqualsJson); + ValidateJson(field.EqualsJson); + } + if (tool.Kind != LlmTestServerToolEvidenceKind.Result && (tool.ResultJson is not null || tool.ResultFields.Count != 0)) + { throw new ArgumentException("Result predicates require tool-result evidence.", nameof(verification)); } + } + } + foreach (var conversation in verification.Conversations) + { + ArgumentNullException.ThrowIfNull(conversation); + ArgumentException.ThrowIfNullOrWhiteSpace(conversation.ScenarioId); + ArgumentNullException.ThrowIfNull(conversation.VisitedSteps); + if (conversation.VisitedSteps.Count > 256) { throw new ArgumentException("A graph supports at most 256 required steps.", nameof(verification)); } + foreach (var step in conversation.VisitedSteps) { ArgumentException.ThrowIfNullOrWhiteSpace(step); } + } + foreach (var retry in verification.Retries) + { + ArgumentNullException.ThrowIfNull(retry); + ArgumentException.ThrowIfNullOrWhiteSpace(retry.GroupId); + ArgumentNullException.ThrowIfNull(retry.Query); + ValidateRange(retry.MinimumAttempts, retry.MaximumAttempts); + ArgumentNullException.ThrowIfNull(retry.StatusSequence); + if (retry.StatusSequence.Count > 500 || retry.StatusSequence.Any(status => status is < 100 or > 599) + || retry.MinimumRetryAfterSeconds is { } seconds && (!double.IsFinite(seconds) || seconds < 0)) + { throw new ArgumentException("Invalid retry verification limits or statuses.", nameof(verification)); } + } + } + + private static void ValidateRange(int minimum, int? maximum) + { + ArgumentOutOfRangeException.ThrowIfNegative(minimum); + if (maximum < minimum) { throw new ArgumentException("The maximum count must be at least the minimum count.", nameof(maximum)); } + } + + private static void ValidateJson(string? json) + { + if (json is null) { return; } + try { using var parsed = JsonDocument.Parse(json); } + catch (JsonException exception) { throw new ArgumentException("Verification JSON predicates must contain valid JSON.", nameof(json), exception); } + } + + private sealed record ToolEvidence(string? Name, string? CallId, string? ArgumentsJson, string? ResultJson); +} diff --git a/src/Shared/LlmTckJsonValidation.cs b/src/Shared/LlmTestServerJsonValidation.cs similarity index 90% rename from src/Shared/LlmTckJsonValidation.cs rename to src/Shared/LlmTestServerJsonValidation.cs index 91e8758..a530e73 100644 --- a/src/Shared/LlmTckJsonValidation.cs +++ b/src/Shared/LlmTestServerJsonValidation.cs @@ -1,12 +1,12 @@ using System.Text.Json; -using ManagedCode.LlmTck.Providers; +using ManagedCode.LlmTestServer.Providers; -namespace ManagedCode.LlmTck.ProviderValidation; +namespace ManagedCode.LlmTestServer.ProviderValidation; // Linked into provider assemblies: JSON primitives only, no provider-specific field names. -internal static class LlmTckJsonValidation +internal static class LlmTestServerJsonValidation { - public static LlmTckRequestValidationResult Check(JsonElement body, Action validate) + public static LlmTestServerRequestValidationResult Check(JsonElement body, Action validate) { try { diff --git a/tests/AGENTS.md b/tests/AGENTS.md index a47130d..b3a5689 100644 --- a/tests/AGENTS.md +++ b/tests/AGENTS.md @@ -2,4 +2,6 @@ Tests use TUnit and Microsoft.Testing.Platform. -Use manual fakes for `IChatClient` callback assertions. Keep OpenAI endpoint tests on `Microsoft.AspNetCore.TestHost`, and keep Aspire tests on `Aspire.Hosting.Testing` with the AppHost model built directly in test code or a test fixture. +Keep only integration tests. Start the server and call its APIs through HTTP or real provider SDKs, with supported parameters, streaming, failures, and full tool/conversation flows. Do not retain unit tests, fake clients, direct runtime/builder tests, source/document scans, or reflection-based test guards. + +Keep HTTP endpoint integration tests on `Microsoft.AspNetCore.TestHost`, and Aspire integration tests on `Aspire.Hosting.Testing` with the AppHost model built directly in test code or a test fixture. Inspect state through the server's public control APIs instead of resolving internal runtime services as a substitute for observable behavior. diff --git a/tests/ManagedCode.LlmTck.Tests/Aspire/AspireEndpointGuardTests.cs b/tests/ManagedCode.LlmTck.Tests/Aspire/AspireEndpointGuardTests.cs deleted file mode 100644 index 11ee08d..0000000 --- a/tests/ManagedCode.LlmTck.Tests/Aspire/AspireEndpointGuardTests.cs +++ /dev/null @@ -1,104 +0,0 @@ -namespace ManagedCode.LlmTck.Tests.AspireIntegration; - -public sealed class AspireEndpointGuardTests -{ - [Test] - public async Task AspireWiring_DoesNotUseManualProviderEndpointConfigurationAsync() - { - var repositoryRoot = FindRepositoryRoot(); - var violations = new List(); - string[] guardedFiles = - [ - "src/ManagedCode.LlmTck.Aspire/LlmTckAspireExtensions.cs", - "samples/ManagedCode.LlmTck.AppHost/AppHost.cs", - "samples/ManagedCode.LlmTck.Service/Program.cs", - "tests/ManagedCode.LlmTck.Tests/Aspire/AspireIntegrationTests.cs", - "docs/Architecture.md", - "docs/Testing/AcceptanceCriteriaTestInventory.md", - "README.md", - ]; - string[] forbiddenFragments = - [ - ".WithEndpoint(", - "127.0.0.1", - "LlmTck__Endpoint", - "LLM_TCK_ENDPOINT", - ".WithOpenAICompatibility(", - ".WithAzureOpenAICompatibility(", - ".WithFoundryCompatibility(", - ".WithAnthropicCompatibility(", - ".WithGeminiCompatibility(", - ".WithGroqCompatibility(", - ".WithMistralCompatibility(", - ".WithOllamaCompatibility(", - ".WithCohereCompatibility(", - ".WithBedrockCompatibility(", - ".WithOpenRouterCompatibility(", - ".WithDeepSeekCompatibility(", - ".WithPerplexityCompatibility(", - "LLM_TCK_OPENAI_COMPATIBILITY", - "LLM_TCK_AZURE_OPENAI_COMPATIBILITY", - "LLM_TCK_MICROSOFT_FOUNDRY_COMPATIBILITY", - "LLM_TCK_RESOURCE_NAME", - ]; - - foreach (var relativePath in guardedFiles) - { - var path = Path.Combine(repositoryRoot, relativePath); - var text = File.ReadAllText(path); - - foreach (var forbiddenFragment in forbiddenFragments) - { - var searchIndex = text.IndexOf(forbiddenFragment, StringComparison.Ordinal); - while (searchIndex >= 0) - { - violations.Add( - $"{relativePath}:{GetLineNumber(text, searchIndex)} contains {forbiddenFragment}; use the Aspire resource endpoint instead." - ); - searchIndex = text.IndexOf( - forbiddenFragment, - searchIndex + forbiddenFragment.Length, - StringComparison.Ordinal - ); - } - } - } - - var readme = File.ReadAllText(Path.Combine(repositoryRoot, "README.md")); - await Assert.That(readme).Contains(".WithReference(llmTck)"); - await Assert.That(readme).Contains("llmTck.GetHttpEndpoint()"); - await Assert.That(string.Join(Environment.NewLine, violations)).IsEqualTo(string.Empty); - } - - private static string FindRepositoryRoot() - { - var directory = new DirectoryInfo(AppContext.BaseDirectory); - - while (directory is not null) - { - if (File.Exists(Path.Combine(directory.FullName, "ManagedCode.LlmTck.slnx"))) - { - return directory.FullName; - } - - directory = directory.Parent; - } - - throw new DirectoryNotFoundException("Could not find ManagedCode.LlmTck.slnx."); - } - - private static int GetLineNumber(string sourceText, int characterIndex) - { - var lineNumber = 1; - - for (var i = 0; i < characterIndex; i++) - { - if (sourceText[i] == '\n') - { - lineNumber++; - } - } - - return lineNumber; - } -} diff --git a/tests/ManagedCode.LlmTck.Tests/Aspire/AspireLaunchProfileTests.cs b/tests/ManagedCode.LlmTck.Tests/Aspire/AspireLaunchProfileTests.cs deleted file mode 100644 index 5506be7..0000000 --- a/tests/ManagedCode.LlmTck.Tests/Aspire/AspireLaunchProfileTests.cs +++ /dev/null @@ -1,52 +0,0 @@ -using System.Text.Json; - -namespace ManagedCode.LlmTck.Tests.AspireIntegration; - -public sealed class AspireLaunchProfileTests -{ - [Test] - public async Task SampleAppHost_HttpLaunchProfileAllowsUnsecuredTransportAsync() - { - var launchSettingsPath = Path.Combine( - FindRepositoryRoot(), - "samples", - "ManagedCode.LlmTck.AppHost", - "Properties", - "launchSettings.json" - ); - await using var stream = File.OpenRead(launchSettingsPath); - using var document = await JsonDocument.ParseAsync(stream); - var httpProfile = document.RootElement - .GetProperty("profiles") - .GetProperty("http"); - var applicationUrl = httpProfile.GetProperty("applicationUrl").GetString(); - var environmentVariables = httpProfile.GetProperty("environmentVariables"); - - if (applicationUrl?.StartsWith("http://", StringComparison.OrdinalIgnoreCase) is true) - { - await Assert.That( - environmentVariables - .GetProperty("ASPIRE_ALLOW_UNSECURED_TRANSPORT") - .GetString() - ) - .IsEqualTo("true"); - } - } - - private static string FindRepositoryRoot() - { - var current = new DirectoryInfo(AppContext.BaseDirectory); - while (current is not null) - { - var solutionPath = Path.Combine(current.FullName, "ManagedCode.LlmTck.slnx"); - if (File.Exists(solutionPath)) - { - return current.FullName; - } - - current = current.Parent; - } - - throw new InvalidOperationException("Could not locate the ManagedCode.LlmTck repository root."); - } -} diff --git a/tests/ManagedCode.LlmTck.Tests/Client/LlmTckClientConfigurationTests.cs b/tests/ManagedCode.LlmTck.Tests/Client/LlmTckClientConfigurationTests.cs deleted file mode 100644 index fa72eb1..0000000 --- a/tests/ManagedCode.LlmTck.Tests/Client/LlmTckClientConfigurationTests.cs +++ /dev/null @@ -1,146 +0,0 @@ -using ManagedCode.LlmTck.Client; -using ManagedCode.LlmTck.Configuration; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.Tests.TestSupport; -using Microsoft.Extensions.AI; - -namespace ManagedCode.LlmTck.Tests.Client; - -public sealed class LlmTckClientConfigurationTests -{ - [Test] - public async Task ClientBuilder_ExposesOfficialModelConvenienceMethodsAsync() - { - var configuration = new LlmTckClientConfigurationBuilder() - .UseDefaultOpenAiModels() - .UseReasoningChatModel("gpt-5-nano", 11) - .Build(); - - await Assert.That(configuration.Models.Any(model => model.Id == LlmTckKnownModelIds.Gpt41Mini)) - .IsTrue(); - await Assert.That(configuration.Models.Any(model => model.Id == LlmTckKnownModelIds.TextEmbedding3Small)) - .IsTrue(); - await Assert.That(configuration.Models.Any(model => model.Id == LlmTckKnownModelIds.GptImage1)) - .IsTrue(); - await Assert.That(configuration.Models.Any(model => model.Id == LlmTckKnownModelIds.Gpt4OMiniTts)) - .IsTrue(); - await Assert.That(configuration.Models.Single(model => model.Id == "gpt-5-nano").ReasoningTokens) - .IsEqualTo(11); - } - - [Test] - public async Task ClientBuilder_ExposesKnownOpenAiModelConvenienceMethodsAsync() - { - var configuration = new LlmTckClientConfigurationBuilder() - .UseKnownOpenAiModels(reasoningTokens: 19) - .Build(); - - await Assert.That(HasModel(configuration, LlmTckKnownModelIds.Gpt55, LlmTckModelKind.Chat)) - .IsTrue(); - await Assert.That(HasModel(configuration, LlmTckKnownModelIds.Gpt54Nano, LlmTckModelKind.Chat)) - .IsTrue(); - await Assert.That(HasModel(configuration, LlmTckKnownModelIds.TextEmbedding3Large, LlmTckModelKind.Embedding)) - .IsTrue(); - await Assert.That(HasModel(configuration, LlmTckKnownModelIds.GptImage2, LlmTckModelKind.Image)) - .IsTrue(); - await Assert.That(HasModel(configuration, LlmTckKnownModelIds.Tts1Hd, LlmTckModelKind.Audio)) - .IsTrue(); - await Assert.That(HasModel(configuration, LlmTckKnownModelIds.Sora2Pro, LlmTckModelKind.Video)) - .IsTrue(); - await Assert.That(configuration.Models.Single(model => model.Id == LlmTckKnownModelIds.Gpt55).ReasoningTokens) - .IsEqualTo(19); - } - - [Test] - public async Task ConfigureAsync_WithFluentClientApi_LoadsDatasetAndFixturesAsync() - { - using var host = await LlmTckTestHost.StartAsync(); - using var httpClient = host.GetTestClient(); - var control = LlmTckClient.Create(httpClient, "test-key"); - - await control.ConfigureAsync( - tck => tck - .RequireBearerToken("test-key") - .UseChatModel("dataset-chat") - .UseEmbeddingModel("dataset-embedding") - .UseImageModel("dataset-image") - .UseAudioModel("dataset-audio") - .UseVideoModel("dataset-video") - .UseEmbeddingVector(0.9f, 0.8f) - .UseImageDataUri("data:image/png;base64,Zm9v") - .UseAudio([1, 2, 3, 4], "audio/test") - .UseVideo([0, 0, 0, 24, 102, 116, 121, 112], "video/test") - .UseTranscriptionText("dataset transcript") - .UseTranslationText("dataset translation") - .UseDataset( - "support-flow", - dataset => dataset - .AddChatScenario( - "dataset-blue", - scenario => scenario - .ForModel("dataset-chat") - .WhenUserContains("support") - .Responds("dataset answer", "dataset ", "answer") - .Responds("dataset answer", "dataset ", "answer") - ) - ), - resetFirst: true - ); - - using var chatClient = control.CreateChatClient("dataset-chat"); - using var embeddingGenerator = control.CreateEmbeddingGenerator("dataset-embedding"); - using var imageGenerator = control.CreateImageGenerator("dataset-image"); - - var chat = await chatClient.GetResponseAsync( - [new ChatMessage(ChatRole.User, "support request")] - ); - var streamChunks = new List(); - await foreach ( - var update in chatClient.GetStreamingResponseAsync( - [new ChatMessage(ChatRole.User, "support request")] - ) - ) - { - streamChunks.Add(update.Text); - } - - var embeddings = await embeddingGenerator.GenerateAsync(["alpha"]); - var image = await imageGenerator.GenerateAsync( - new ImageGenerationRequest { Prompt = "fixture prompt" } - ); - var audio = await control.GenerateAudioAsync("dataset-audio", "say this"); - var videoId = await control.GenerateVideoAsync("dataset-video", "make a fixture video"); - var transcription = await control.TranscribeAudioAsync( - "dataset-audio", - [82, 73, 70, 70], - "fixture.wav" - ); - var translation = await control.TranslateAudioAsync( - "dataset-audio", - [82, 73, 70, 70], - "fixture.wav" - ); - var assertions = await control.GetAssertionsAsync(); - - await Assert.That(chat.Text).IsEqualTo("dataset answer"); - await Assert.That(string.Concat(streamChunks)).IsEqualTo("dataset answer"); - await Assert.That(embeddings).Count().IsEqualTo(1); - await Assert.That(embeddings[0].Vector.ToArray()).IsEquivalentTo([0.9f, 0.8f]); - await Assert.That(image.Contents).Count().IsEqualTo(1); - await Assert.That(audio.Bytes).IsEquivalentTo((byte[])[1, 2, 3, 4]); - await Assert.That(audio.MediaType).IsEqualTo("audio/test"); - await Assert.That(videoId).IsEqualTo("video_llm_tck"); - await Assert.That(transcription).IsEqualTo("dataset transcript"); - await Assert.That(translation).IsEqualTo("dataset translation"); - await Assert.That(assertions.Matched).IsGreaterThanOrEqualTo(8); - } - - private static bool HasModel( - LlmTckConfiguration configuration, - string id, - LlmTckModelKind kind - ) - { - return configuration.Models.Any(model => model.Id == id && model.Kind == kind); - } -} diff --git a/tests/ManagedCode.LlmTck.Tests/Configuration/LlmTckConfigurationBuilderTests.cs b/tests/ManagedCode.LlmTck.Tests/Configuration/LlmTckConfigurationBuilderTests.cs deleted file mode 100644 index a58e07e..0000000 --- a/tests/ManagedCode.LlmTck.Tests/Configuration/LlmTckConfigurationBuilderTests.cs +++ /dev/null @@ -1,284 +0,0 @@ -using ManagedCode.LlmTck.Configuration; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.Scenarios; - -namespace ManagedCode.LlmTck.Tests.Configuration; - -public sealed class LlmTckConfigurationBuilderTests -{ - [Test] - public async Task CreateDefault_UsesOfficialOpenAiFixtureModelIdsAsync() - { - var configuration = LlmTckConfiguration.CreateDefault(); - - await Assert.That(configuration.Models.Select(model => (model.Id, model.Kind))) - .IsEquivalentTo( - [ - (LlmTckKnownModelIds.Gpt41Mini, LlmTckModelKind.Chat), - (LlmTckKnownModelIds.TextEmbedding3Small, LlmTckModelKind.Embedding), - (LlmTckKnownModelIds.GptImage1, LlmTckModelKind.Image), - (LlmTckKnownModelIds.Gpt4OMiniTts, LlmTckModelKind.Audio), - (LlmTckKnownModelIds.Sora2, LlmTckModelKind.Video), - ] - ); - } - - [Test] - public async Task ConvenienceMethods_AddOfficialFixtureModelsAsync() - { - var configuration = new LlmTckConfigurationBuilder() - .AddGpt41Mini() - .AddTextEmbedding3Small() - .AddGptImage1() - .AddGpt4OMiniTts() - .AddSora2() - .AddReasoningChatModel("gpt-5-nano", 13) - .Build(); - - await Assert.That(configuration.Models.Single(model => model.Id == LlmTckKnownModelIds.Gpt41Mini).Kind) - .IsEqualTo(LlmTckModelKind.Chat); - await Assert.That(configuration.Models.Single(model => model.Id == LlmTckKnownModelIds.TextEmbedding3Small).Kind) - .IsEqualTo(LlmTckModelKind.Embedding); - await Assert.That(configuration.Models.Single(model => model.Id == LlmTckKnownModelIds.GptImage1).Kind) - .IsEqualTo(LlmTckModelKind.Image); - await Assert.That(configuration.Models.Single(model => model.Id == LlmTckKnownModelIds.Gpt4OMiniTts).Kind) - .IsEqualTo(LlmTckModelKind.Audio); - await Assert.That(configuration.Models.Single(model => model.Id == LlmTckKnownModelIds.Sora2).Kind) - .IsEqualTo(LlmTckModelKind.Video); - await Assert.That(configuration.Models.Single(model => model.Id == "gpt-5-nano").ReasoningTokens) - .IsEqualTo(13); - } - - [Test] - public async Task KnownOpenAiModelConvenienceMethods_AddSupportedAliasesAsync() - { - var configuration = new LlmTckConfigurationBuilder() - .AddKnownOpenAiModels(reasoningTokens: 17) - .Build(); - - await Assert.That(HasModel(configuration, LlmTckKnownModelIds.Gpt55, LlmTckModelKind.Chat)) - .IsTrue(); - await Assert.That(HasModel(configuration, LlmTckKnownModelIds.Gpt54Mini, LlmTckModelKind.Chat)) - .IsTrue(); - await Assert.That(HasModel(configuration, LlmTckKnownModelIds.Gpt5Nano, LlmTckModelKind.Chat)) - .IsTrue(); - await Assert.That(HasModel(configuration, LlmTckKnownModelIds.Gpt41, LlmTckModelKind.Chat)) - .IsTrue(); - await Assert.That(HasModel(configuration, LlmTckKnownModelIds.Gpt4OMini, LlmTckModelKind.Chat)) - .IsTrue(); - await Assert.That(HasModel(configuration, LlmTckKnownModelIds.TextEmbedding3Large, LlmTckModelKind.Embedding)) - .IsTrue(); - await Assert.That(HasModel(configuration, LlmTckKnownModelIds.TextEmbeddingAda002, LlmTckModelKind.Embedding)) - .IsTrue(); - await Assert.That(HasModel(configuration, LlmTckKnownModelIds.GptImage2, LlmTckModelKind.Image)) - .IsTrue(); - await Assert.That(HasModel(configuration, LlmTckKnownModelIds.GptImage15, LlmTckModelKind.Image)) - .IsTrue(); - await Assert.That(HasModel(configuration, LlmTckKnownModelIds.GptImage1Mini, LlmTckModelKind.Image)) - .IsTrue(); - await Assert.That(HasModel(configuration, LlmTckKnownModelIds.Tts1, LlmTckModelKind.Audio)) - .IsTrue(); - await Assert.That(HasModel(configuration, LlmTckKnownModelIds.Tts1Hd, LlmTckModelKind.Audio)) - .IsTrue(); - await Assert.That(HasModel(configuration, LlmTckKnownModelIds.Sora2Pro, LlmTckModelKind.Video)) - .IsTrue(); - await Assert.That(configuration.Models.Single(model => model.Id == LlmTckKnownModelIds.Gpt55).ReasoningTokens) - .IsEqualTo(17); - await Assert.That(configuration.Models.Single(model => model.Id == LlmTckKnownModelIds.Gpt41).ReasoningTokens) - .IsEqualTo(0); - } - - [Test] - public async Task Build_ReplacesDuplicateModelsAndScenariosAndSnapshotsNestedStateAsync() - { - var builder = new LlmTckConfigurationBuilder() - .AddModel("docs-model", LlmTckModelKind.Chat) - .AddModel("docs-model", LlmTckModelKind.Embedding) - .WithDefaultEmbeddingVector(0.5f, 0.75f) - .WithDefaultImageDataUri("data:image/png;base64,Zm9v") - .WithDefaultAudio([1, 2, 3], "audio/test") - .WithDefaultVideo([4, 5, 6], "video/test") - .WithDefaultTranscriptionText("fixture transcript") - .WithDefaultTranslationText("fixture translation") - .SimulateRateLimitAfter(2) - .SimulateContentFilter("blocked", " BLOCKED ", "forbidden") - .AddDataset( - "docs-dataset", - dataset => dataset - .AddChatScenario( - "dataset-scenario", - scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) - .WhenUserContains("dataset") - .Responds("dataset response") - ) - ) - .AddChatScenario( - "duplicate-scenario", - scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) - .WhenUserContains("old") - .Responds("old") - ) - .AddChatScenario( - "duplicate-scenario", - scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) - .WhenUserContains("new") - .Fails(418, "teapot", "Short and stout.") - .DelaysBy(25) - ); - - var configuration = builder.Build(); - - configuration.Models.Clear(); - configuration.ChatScenarios[0].Match.Messages.Clear(); - configuration.Datasets[0].ChatScenarios[0].Match.Messages.Clear(); - configuration.ChatScenarios[0].Responses[0].StreamChunks.Add("mutated"); - configuration.DefaultEmbeddingVector.Clear(); - configuration.DefaultAudioBytes[0] = 99; - configuration.DefaultVideoBytes[0] = 99; - configuration.FaultSimulation.ContentFilterTerms.Clear(); - - var rebuilt = builder.Build(); - - await Assert.That(rebuilt.Models.Single(model => model.Id == "docs-model").Kind) - .IsEqualTo(LlmTckModelKind.Embedding); - await Assert.That(rebuilt.ChatScenarios).Count().IsEqualTo(1); - await Assert.That(rebuilt.ChatScenarios[0].Match.Messages[0].Content).IsEqualTo("new"); - await Assert.That(rebuilt.ChatScenarios[0].Responses[0].Error?.Code).IsEqualTo("teapot"); - await Assert.That(rebuilt.ChatScenarios[0].Responses[0].DelayMilliseconds).IsEqualTo(25); - await Assert.That(rebuilt.Datasets).Count().IsEqualTo(1); - await Assert.That(rebuilt.Datasets[0].ChatScenarios[0].Match.Messages[0].Content) - .IsEqualTo("dataset"); - await Assert.That(rebuilt.DefaultEmbeddingVector).IsEquivalentTo([0.5f, 0.75f]); - await Assert.That(rebuilt.DefaultImageDataUri).IsEqualTo("data:image/png;base64,Zm9v"); - await Assert.That(rebuilt.DefaultAudioBytes).IsEquivalentTo((byte[])[1, 2, 3]); - await Assert.That(rebuilt.DefaultAudioMediaType).IsEqualTo("audio/test"); - await Assert.That(rebuilt.DefaultVideoBytes).IsEquivalentTo((byte[])[4, 5, 6]); - await Assert.That(rebuilt.DefaultVideoMediaType).IsEqualTo("video/test"); - await Assert.That(rebuilt.DefaultTranscriptionText).IsEqualTo("fixture transcript"); - await Assert.That(rebuilt.DefaultTranslationText).IsEqualTo("fixture translation"); - await Assert.That(rebuilt.FaultSimulation.MaxRequestsBeforeRateLimit).IsEqualTo(2); - await Assert.That(rebuilt.FaultSimulation.ContentFilterTerms) - .IsEquivalentTo(["blocked", "forbidden"]); - } - - [Test] - public async Task Builders_RejectInvalidInputsAsync() - { - await ShouldThrowAsync( - () => new LlmTckConfigurationBuilder().AddModel("", LlmTckModelKind.Chat) - ); - await ShouldThrowAsync( - () => new LlmTckConfigurationBuilder().AddModel("gpt-5-nano", LlmTckModelKind.Chat, -1) - ); - await ShouldThrowAsync( - () => new LlmTckConfigurationBuilder().AddReasoningChatModel("gpt-5-nano", 0) - ); - await ShouldThrowAsync( - () => new LlmTckConfigurationBuilder().RequireBearerToken(" ") - ); - await ShouldThrowAsync( - () => new LlmTckConfigurationBuilder().AddChatScenario("id", null!) - ); - await ShouldThrowAsync( - () => new LlmTckConfigurationBuilder().WithDefaultEmbeddingVector() - ); - await ShouldThrowAsync( - () => new LlmTckConfigurationBuilder().WithDefaultImageDataUri("https://example.com") - ); - await ShouldThrowAsync( - () => new LlmTckConfigurationBuilder().WithDefaultAudio([], "audio/wav") - ); - await ShouldThrowAsync( - () => new LlmTckConfigurationBuilder().WithDefaultAudio([1], "") - ); - await ShouldThrowAsync( - () => new LlmTckConfigurationBuilder().WithDefaultVideo([], "video/mp4") - ); - await ShouldThrowAsync( - () => new LlmTckConfigurationBuilder().WithDefaultVideo([1], "") - ); - await ShouldThrowAsync( - () => new LlmTckConfigurationBuilder().WithDefaultTranscriptionText("") - ); - await ShouldThrowAsync( - () => new LlmTckConfigurationBuilder().WithDefaultTranslationText("") - ); - await ShouldThrowAsync( - () => new LlmTckConfigurationBuilder().SimulateRateLimitAfter(-1) - ); - await ShouldThrowAsync( - () => new LlmTckConfigurationBuilder().SimulateContentFilter() - ); - await ShouldThrowAsync( - () => new LlmTckConfigurationBuilder().SimulateContentFilter(" ") - ); - - await ShouldThrowAsync(() => new LlmTckScenarioBuilder("id").ForModel("")); - await ShouldThrowAsync( - () => new LlmTckScenarioBuilder("id").RequireBearerToken("") - ); - await ShouldThrowAsync( - () => new LlmTckScenarioBuilder("id").WhenUserContains("") - ); - await ShouldThrowAsync( - () => new LlmTckScenarioBuilder("id").WithExactMatch(null!) - ); - await ShouldThrowAsync( - () => new LlmTckScenarioBuilder("id").Responds(null!) - ); - await ShouldThrowAsync( - () => new LlmTckScenarioBuilder("id").Fails(400, "", "message") - ); - await ShouldThrowAsync( - () => new LlmTckScenarioBuilder("id").Fails(400, "code", "") - ); - await ShouldThrowAsync( - () => new LlmTckScenarioBuilder("id").DelaysBy(-1) - ); - await ShouldThrowAsync( - () => new LlmTckScenarioBuilder("id").Build() - ); - await ShouldThrowAsync( - () => new LlmTckScenarioDatasetBuilder("id").AddChatScenario("scenario", null!) - ); - } - - [Test] - public async Task DelaysBy_CreatesAResponseWhenNoResponseExistsYetAsync() - { - var scenario = new LlmTckScenarioBuilder("delayed-empty-response") - .DelaysBy(10) - .Build(); - - await Assert.That(scenario.Responses).Count().IsEqualTo(1); - await Assert.That(scenario.Responses[0].DelayMilliseconds).IsEqualTo(10); - } - - private static async Task ShouldThrowAsync(Action action) - where TException : Exception - { - Exception? exception = null; - try - { - action(); - } - catch (Exception ex) - { - exception = ex; - } - - await Assert.That(exception?.GetType()).IsEqualTo(typeof(TException)); - } - - private static bool HasModel( - LlmTckConfiguration configuration, - string id, - LlmTckModelKind kind - ) - { - return configuration.Models.Any(model => model.Id == id && model.Kind == kind); - } -} diff --git a/tests/ManagedCode.LlmTck.Tests/Configuration/TranscriptionUsageConfigurationTests.cs b/tests/ManagedCode.LlmTck.Tests/Configuration/TranscriptionUsageConfigurationTests.cs deleted file mode 100644 index e67503e..0000000 --- a/tests/ManagedCode.LlmTck.Tests/Configuration/TranscriptionUsageConfigurationTests.cs +++ /dev/null @@ -1,47 +0,0 @@ -using ManagedCode.LlmTck.Configuration; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.Runtime; - -namespace ManagedCode.LlmTck.Tests.Configuration; - -public sealed class TranscriptionUsageConfigurationTests -{ - [Test] - [Arguments(0)] - [Arguments(-1)] - [Arguments(double.NaN)] - [Arguments(double.PositiveInfinity)] - public async Task Duration_RejectsNonpositiveOrNonfiniteUsageAsync(double seconds) - { - await Assert.That(() => LlmTckTranscriptionUsage.ForDuration(seconds)).Throws(); - } - - [Test] - [Arguments(-1, 0, 1)] - [Arguments(1, -1, 1)] - [Arguments(1, 0, -1)] - [Arguments(0, 0, 0)] - [Arguments(long.MaxValue, 1, 1)] - public async Task Tokens_RejectsNegativeEmptyOrOverflowUsageAsync(long audio, long text, long output) - { - await Assert.That(() => LlmTckTranscriptionUsage.ForTokens(audio, text, output)).Throws(); - } - - [Test] - public async Task Configuration_RejectsInvalidUsageAndAllowsAbsentUsageAsync() - { - await Assert.That(() => new LlmTckConfigurationBuilder() - .WithDefaultTranscriptionUsage(new() { InputAudioTokens = 12 })).Throws(); - await Assert.That(() => new LlmTckConfigurationBuilder() - .WithDefaultTranscriptionUsage(new() { DurationSeconds = null, InputAudioTokens = 12, OutputTokens = 1 })) - .Throws(); - var withoutUsage = new LlmTckConfigurationBuilder() - .WithoutDefaultTranscriptionUsage() - .Build(); - await Assert.That(withoutUsage.DefaultTranscriptionUsage).IsNull(); - var runtime = new LlmTckRuntime(withoutUsage); - var result = await runtime.TranscribeAudioAsync(LlmTckKnownModelIds.Gpt4OMiniTts, "fixture.wav"); - await Assert.That(result.IsSuccess).IsTrue(); - await Assert.That(result.Usage).IsNull(); - } -} diff --git a/tests/ManagedCode.LlmTck.Tests/Decisions/DecisionRuntimeTests.cs b/tests/ManagedCode.LlmTck.Tests/Decisions/DecisionRuntimeTests.cs deleted file mode 100644 index 6205327..0000000 --- a/tests/ManagedCode.LlmTck.Tests/Decisions/DecisionRuntimeTests.cs +++ /dev/null @@ -1,105 +0,0 @@ -using System.Net; -using System.Net.Http.Json; -using System.Text.Json; -using ManagedCode.LlmTck.Configuration; -using ManagedCode.LlmTck.Decisions; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Tests.TestSupport; - -namespace ManagedCode.LlmTck.Tests.Decisions; - -public sealed class DecisionRuntimeTests -{ - private const string _model = "jev-latest"; - - [Test] - [Arguments("unknown", 404)] - [Arguments("unmatched", 404)] - [Arguments("error", 503)] - [Arguments("mismatch", 409)] - [Arguments("score-key", 409)] - [Arguments("score-high", 409)] - [Arguments("wrong-input", 404)] - public async Task DecisionFailures_AreObservableAsync(string kind, int status) - { - var builder = new LlmTckConfigurationBuilder(); - if (kind != "unknown") { builder.AddModel(_model, LlmTckModelKind.Decision); } - var scenario = DecisionEndpointTests.Scenario(_model); - if (kind == "error") { scenario = scenario with { Error = new() { StatusCode = status, Code = "configured", Message = "configured outage" } }; } - if (kind == "mismatch") { scenario.Answers.Remove("binary"); } - if (kind == "score-key") { scenario.Answers["score"] = scenario.Answers["score"] with { Probabilities = new() { ["invalid"] = 0.5 } }; } - if (kind == "score-high") { scenario.Answers["score"] = scenario.Answers["score"] with { Score = 100 }; } - if (kind == "wrong-input") { scenario = scenario with { ExpectedInput = "other" }; } - if (kind != "unmatched") { builder.AddDecisionScenario(scenario); } - var runtime = new LlmTckRuntime(builder.Build()); - var result = await runtime.DecideAsync(Request()); - await Assert.That(result.StatusCode).IsEqualTo(status); - await Assert.That(result.IsSuccess).IsFalse(); - await Assert.That(runtime.GetAssertionSummary().Events.Count).IsEqualTo(1); - } - - [Test] - public async Task DecisionFixtures_CopyInputsAndReturnedAnswersAsync() - { - var scenario = DecisionEndpointTests.Scenario(_model); - var builder = new LlmTckConfigurationBuilder().AddModel(_model, LlmTckModelKind.Decision).AddDecisionScenario(scenario); - scenario.Answers["choice"].Probabilities["opaque-id"] = 0; - var runtime = new LlmTckRuntime(builder.Build()); - var first = await runtime.DecideAsync(Request()); first.Answers["choice"].Probabilities["opaque-id"] = 0; - var second = await runtime.DecideAsync(Request()); - await Assert.That(second.Answers["choice"].Probabilities["opaque-id"]).IsEqualTo(0.95); - await runtime.ResetAsync(); - await Assert.That((await runtime.DecideAsync(Request())).IsSuccess).IsTrue(); - using var active = ((ILlmTckRuntimeRequestScope)runtime).BeginRequest("obsolete"); - await runtime.ResetAsync(); - await Assert.That((await runtime.DecideAsync(Request())).ErrorCode).IsEqualTo("llm_tck_request_superseded"); - } - - [Test] - [Arguments(-0.1)] - [Arguments(1.1)] - [Arguments(double.NaN)] - [Arguments(double.PositiveInfinity)] - public async Task DecisionFixtureValidation_RejectsInvalidProbabilityAsync(double value) - { - var scenario = DecisionEndpointTests.Scenario(_model); scenario.Answers["binary"] = new() { Probability = value }; - await Assert.That(() => new LlmTckConfigurationBuilder().AddDecisionScenario(scenario)).Throws(); - } - - [Test] - [Arguments("score")] - [Arguments("choice")] - [Arguments("kind")] - public async Task DecisionFixtureValidation_RejectsMissingValuesAsync(string kind) - { - var scenario = DecisionEndpointTests.Scenario(_model); - scenario.Answers["binary"] = new() { Kind = kind switch { "score" => LlmTckDecisionKind.Score, "choice" => LlmTckDecisionKind.Choice, _ => (LlmTckDecisionKind)100 } }; - await Assert.That(() => new LlmTckConfigurationBuilder().AddDecisionScenario(scenario)).Throws(); - } - - [Test] - public async Task OpenAiDecisions_EmitNativeRefusalAnswersAsync() - { - var scenario = DecisionEndpointTests.Scenario("gpt-6-luna"); scenario.Answers["0"] = new() { Refused = true }; - using var host = await LlmTckTestHost.StartAsync(builder => builder.AddModel("gpt-6-luna", LlmTckModelKind.Decision).AddDecisionScenario(scenario)); - using var client = host.GetTestClient(); - using var response = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor("gpt-6-luna"), DecisionEndpointTests.Body("gpt-6-luna")); - await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.OK); - var payload = await response.Content.ReadFromJsonAsync(); - await Assert.That(payload.GetProperty("answers")[0].GetProperty("type").GetString()).IsEqualTo("refusal"); - await Assert.That(payload.GetProperty("answers")[0].TryGetProperty("probability", out _)).IsFalse(); - } - - private static LlmTckDecisionRequest Request() - { - return new() - { - ModelId = _model, - Input = "evidence", - Questions = - [new() { Name = "binary" }, new() { Name = "choice", Kind = LlmTckDecisionKind.Choice, Options = ["opaque-id", "other-id"] }, - new() { Name = "score", Kind = LlmTckDecisionKind.Score, Options = ["low", "middle", "high"] }], - }; - } -} diff --git a/tests/ManagedCode.LlmTck.Tests/Hosting/BoundedTeeStreamTests.cs b/tests/ManagedCode.LlmTck.Tests/Hosting/BoundedTeeStreamTests.cs deleted file mode 100644 index bcdab79..0000000 --- a/tests/ManagedCode.LlmTck.Tests/Hosting/BoundedTeeStreamTests.cs +++ /dev/null @@ -1,56 +0,0 @@ -using ManagedCode.LlmTck.Hosting; - -namespace ManagedCode.LlmTck.Tests.Hosting; - -public sealed class BoundedTeeStreamTests -{ - [Test] - public async Task WriteTee_ForwardsEveryWriteAndBoundsTheCapturedPreviewAsync() - { - using var inner = new MemoryStream(); - using var tee = new LlmTckBoundedWriteTeeStream(inner, maxCaptureBytes: 4); - - tee.WriteByte(1); - tee.Write((byte[])[2, 3], 0, 2); - tee.Write((ReadOnlySpan)[4]); - await tee.WriteAsync(new byte[] { 5 }.AsMemory(), CancellationToken.None); - await tee.WriteAsync(new byte[] { 6 }.AsMemory(), CancellationToken.None); - tee.Flush(); - await tee.FlushAsync(CancellationToken.None); - - await Assert.That(inner.ToArray()).IsEquivalentTo((byte[])[1, 2, 3, 4, 5, 6]); - await Assert.That(tee.GetCapturedBytes()).IsEquivalentTo((byte[])[1, 2, 3, 4]); - await Assert.That(tee.BytesObserved).IsEqualTo(6); - await Assert.That(tee.SuccessfulFlushCount).IsEqualTo(2); - await Assert.That(tee.IsTruncated).IsTrue(); - await Assert.That(tee.CaptureFailed).IsFalse(); - await Assert.That(tee.CanRead).IsFalse(); - await Assert.That(tee.CanSeek).IsFalse(); - await Assert.That(tee.CanWrite).IsTrue(); - } - - [Test] - public async Task ReadTee_ForwardsSequentialReadsAndBoundsTheCapturedPreviewAsync() - { - using var inner = new MemoryStream([1, 2, 3, 4, 5, 6]); - using var tee = new LlmTckBoundedReadTeeStream(inner, maxCaptureBytes: 4); - var pair = new byte[2]; - var single = new byte[1]; - - await Assert.That(tee.ReadByte()).IsEqualTo(1); - await Assert.That(tee.Read(pair, 0, pair.Length)).IsEqualTo(2); - await Assert.That(tee.Read(single.AsSpan())).IsEqualTo(1); - await Assert.That(await tee.ReadAsync(single.AsMemory(), CancellationToken.None)).IsEqualTo(1); - await Assert.That(await tee.ReadAsync(single.AsMemory(), CancellationToken.None)).IsEqualTo(1); - - await Assert.That(tee.GetCapturedBytes()).IsEquivalentTo((byte[])[1, 2, 3, 4]); - await Assert.That(tee.BytesObserved).IsEqualTo(6); - await Assert.That(tee.IsTruncated).IsTrue(); - await Assert.That(tee.CaptureFailed).IsFalse(); - await Assert.That(tee.CanRead).IsTrue(); - await Assert.That(tee.CanSeek).IsTrue(); - await Assert.That(tee.CanWrite).IsTrue(); - await Assert.That(tee.Length).IsEqualTo(6); - await Assert.That(tee.Position).IsEqualTo(6); - } -} diff --git a/tests/ManagedCode.LlmTck.Tests/ManagedCode.LlmTck.Tests.csproj b/tests/ManagedCode.LlmTck.Tests/ManagedCode.LlmTck.Tests.csproj deleted file mode 100644 index 67095ca..0000000 --- a/tests/ManagedCode.LlmTck.Tests/ManagedCode.LlmTck.Tests.csproj +++ /dev/null @@ -1,62 +0,0 @@ - - - - true - true - DnxPinned - Exe - false - false - true - false - true - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/tests/ManagedCode.LlmTck.Tests/MicrosoftExtensionsAI/MicrosoftExtensionsAiClientTests.cs b/tests/ManagedCode.LlmTck.Tests/MicrosoftExtensionsAI/MicrosoftExtensionsAiClientTests.cs deleted file mode 100644 index 05f3549..0000000 --- a/tests/ManagedCode.LlmTck.Tests/MicrosoftExtensionsAI/MicrosoftExtensionsAiClientTests.cs +++ /dev/null @@ -1,131 +0,0 @@ -using ManagedCode.LlmTck.Client; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.Tests.TestSupport; -using Microsoft.Extensions.AI; - -namespace ManagedCode.LlmTck.Tests.MicrosoftExtensionsAI; - -public sealed class MicrosoftExtensionsAiClientTests -{ - [Test] - public async Task TestChatClient_UsesDotnetExtensionsCallbackPatternAsync() - { - ChatMessage? capturedMessage = null; - ChatOptions? capturedOptions = null; - CancellationToken capturedToken = default; - - using IChatClient innerClient = new TestChatClient - { - GetResponseAsyncCallback = (messages, options, cancellationToken) => - { - capturedMessage = messages.Single(); - capturedOptions = options; - capturedToken = cancellationToken; - return Task.FromResult( - new ChatResponse(new ChatMessage(ChatRole.Assistant, "blue whale")) - ); - }, - }; - using var cts = new CancellationTokenSource(); - - var response = await innerClient.GetResponseAsync( - [new ChatMessage(ChatRole.User, "What is the largest animal?")], - new ChatOptions { ModelId = LlmTckKnownModelIds.Gpt41Mini }, - cts.Token - ); - - await Assert.That(capturedMessage?.Text).IsEqualTo("What is the largest animal?"); - await Assert.That(capturedOptions?.ModelId).IsEqualTo(LlmTckKnownModelIds.Gpt41Mini); - await Assert.That(capturedToken.CanBeCanceled).IsTrue(); - await Assert.That(response.Text).IsEqualTo("blue whale"); - } - - [Test] - public async Task TestChatClient_StreamsThroughCallbackPatternAsync() - { - ChatMessage? capturedMessage = null; - ChatOptions? capturedOptions = null; - CancellationToken capturedToken = default; - - using IChatClient innerClient = new TestChatClient - { - GetStreamingResponseAsyncCallback = (messages, options, cancellationToken) => - { - capturedMessage = messages.Single(); - capturedOptions = options; - capturedToken = cancellationToken; - return GetUpdatesAsync(); - }, - }; - using var cts = new CancellationTokenSource(); - var updates = new List(); - - await foreach ( - var update in innerClient.GetStreamingResponseAsync( - [new ChatMessage(ChatRole.User, "stream")], - new ChatOptions { ModelId = LlmTckKnownModelIds.Gpt41Mini }, - cts.Token - ) - ) - { - updates.Add(update.Text); - } - - await Assert.That(capturedMessage?.Text).IsEqualTo("stream"); - await Assert.That(capturedOptions?.ModelId).IsEqualTo(LlmTckKnownModelIds.Gpt41Mini); - await Assert.That(capturedToken.CanBeCanceled).IsTrue(); - await Assert.That(string.Concat(updates)).IsEqualTo("blue whale"); - - static async IAsyncEnumerable GetUpdatesAsync() - { - await Task.Yield(); - yield return new(ChatRole.Assistant, "blue "); - yield return new(ChatRole.Assistant, "whale"); - } - } - - [Test] - public async Task MicrosoftExtensionsAiClients_InvokeLlmTckThroughOpenAiCompatibilityEndpointsAsync() - { - using var host = await LlmTckTestHost.StartAsync(options => options - .WithDefaultEmbeddingVector(0.125f, 0.25f, 0.5f) - .AddChatScenario( - "extensions-ai-blue-whale", - scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) - .WhenUserContains("largest animal") - .Responds("blue whale", "blue ", "whale") - .Responds("blue whale", "blue ", "whale") - )); - using var httpClient = host.GetTestClient(); - using IChatClient chatClient = new LlmTckChatClient(httpClient); - using IEmbeddingGenerator> embeddingGenerator = - new LlmTckEmbeddingGenerator(httpClient); - using IImageGenerator imageGenerator = new LlmTckImageGenerator(httpClient); - - var chatResponse = await chatClient.GetResponseAsync( - [new ChatMessage(ChatRole.User, "What is the largest animal?")] - ); - var streamUpdates = new List(); - await foreach ( - var update in chatClient.GetStreamingResponseAsync( - [new ChatMessage(ChatRole.User, "What is the largest animal?")] - ) - ) - { - streamUpdates.Add(update.Text); - } - - var embeddings = await embeddingGenerator.GenerateAsync(["alpha", "beta"]); - var image = await imageGenerator.GenerateAsync( - new ImageGenerationRequest { Prompt = "blue compatibility marker" } - ); - - await Assert.That(chatResponse.Text).IsEqualTo("blue whale"); - await Assert.That(string.Concat(streamUpdates)).IsEqualTo("blue whale"); - await Assert.That(embeddings).Count().IsEqualTo(2); - await Assert.That(embeddings[0].Vector.Length).IsEqualTo(3); - await Assert.That(image.Contents).Count().IsEqualTo(1); - await Assert.That(image.Contents[0]).IsTypeOf(); - } -} diff --git a/tests/ManagedCode.LlmTck.Tests/Providers/CompatibilityTagConstantTests.cs b/tests/ManagedCode.LlmTck.Tests/Providers/CompatibilityTagConstantTests.cs deleted file mode 100644 index f233af6..0000000 --- a/tests/ManagedCode.LlmTck.Tests/Providers/CompatibilityTagConstantTests.cs +++ /dev/null @@ -1,117 +0,0 @@ -using System.Reflection; -using ManagedCode.LlmTck.Providers; - -namespace ManagedCode.LlmTck.Tests.Providers; - -public sealed class CompatibilityTagConstantTests -{ - [Test] - public async Task CompatibilityTagValues_AreNotTypedAsInlineStringLiteralsAsync() - { - var repositoryRoot = FindRepositoryRoot(); - var tagValues = typeof(LlmTckCompatibilityTags) - .GetFields(BindingFlags.Public | BindingFlags.Static | BindingFlags.FlattenHierarchy) - .Where(field => field is { IsLiteral: true, IsInitOnly: false }) - .Select(field => new - { - field.Name, - Value = field.GetRawConstantValue() as string, - }) - .Where(tag => !string.IsNullOrWhiteSpace(tag.Value)) - .ToArray(); - var violations = new List(); - - foreach (var filePath in EnumerateCSharpSourceFiles(repositoryRoot)) - { - var sourceText = File.ReadAllText(filePath); - var relativePath = Path.GetRelativePath(repositoryRoot, filePath); - - foreach (var tag in tagValues) - { - var literal = $"\"{tag.Value}\""; - var searchIndex = sourceText.IndexOf(literal, StringComparison.Ordinal); - - while (searchIndex >= 0) - { - violations.Add( - $"{relativePath}:{GetLineNumber(sourceText, searchIndex)} uses {literal}; use {nameof(LlmTckCompatibilityTags)}.{tag.Name}." - ); - searchIndex = sourceText.IndexOf( - literal, - searchIndex + literal.Length, - StringComparison.Ordinal - ); - } - } - } - - await Assert.That(string.Join(Environment.NewLine, violations)).IsEqualTo(string.Empty); - } - - private static string FindRepositoryRoot() - { - var directory = new DirectoryInfo(AppContext.BaseDirectory); - - while (directory is not null) - { - if (File.Exists(Path.Combine(directory.FullName, "ManagedCode.LlmTck.slnx"))) - { - return directory.FullName; - } - - directory = directory.Parent; - } - - throw new DirectoryNotFoundException("Could not find ManagedCode.LlmTck.slnx."); - } - - private static IEnumerable EnumerateCSharpSourceFiles(string repositoryRoot) - { - string[] sourceDirectories = - [ - Path.Combine(repositoryRoot, "src"), - Path.Combine(repositoryRoot, "tests"), - Path.Combine(repositoryRoot, "samples"), - ]; - - return sourceDirectories - .Where(Directory.Exists) - .SelectMany(directory => Directory.EnumerateFiles(directory, "*.cs", SearchOption.AllDirectories)) - .Where(path => !IsBuildOutput(path)) - .Where(path => - !path.EndsWith( - Path.Combine("Providers", "LlmTckCompatibilityTags.cs"), - StringComparison.Ordinal - ) - ) - .Where(path => - !path.EndsWith( - Path.Combine("Providers", "CompatibilityTagConstantTests.cs"), - StringComparison.Ordinal - ) - ); - } - - private static bool IsBuildOutput(string path) - { - var normalizedPath = path.Replace(Path.DirectorySeparatorChar, '/'); - - return normalizedPath.Contains("/bin/", StringComparison.Ordinal) - || normalizedPath.Contains("/obj/", StringComparison.Ordinal); - } - - private static int GetLineNumber(string sourceText, int characterIndex) - { - var lineNumber = 1; - - for (var i = 0; i < characterIndex; i++) - { - if (sourceText[i] == '\n') - { - lineNumber++; - } - } - - return lineNumber; - } -} diff --git a/tests/ManagedCode.LlmTck.Tests/Providers/ProviderApiContractTests.cs b/tests/ManagedCode.LlmTck.Tests/Providers/ProviderApiContractTests.cs deleted file mode 100644 index e5f47b6..0000000 --- a/tests/ManagedCode.LlmTck.Tests/Providers/ProviderApiContractTests.cs +++ /dev/null @@ -1,996 +0,0 @@ -using ManagedCode.LlmTck.Anthropic; -using ManagedCode.LlmTck.AzureOpenAI; -using ManagedCode.LlmTck.Bedrock; -using ManagedCode.LlmTck.Cloudflare; -using ManagedCode.LlmTck.Cohere; -using ManagedCode.LlmTck.Control; -using ManagedCode.LlmTck.DeepSeek; -using ManagedCode.LlmTck.Foundry; -using ManagedCode.LlmTck.Gemini; -using ManagedCode.LlmTck.Groq; -using ManagedCode.LlmTck.Hosting; -using ManagedCode.LlmTck.Mistral; -using ManagedCode.LlmTck.Ollama; -using ManagedCode.LlmTck.OpenAI; -using ManagedCode.LlmTck.OpenRouter; -using ManagedCode.LlmTck.Perplexity; -using ManagedCode.LlmTck.Providers; -using ManagedCode.LlmTck.SystemOne; -using ManagedCode.LlmTck.Tests.Anthropic; -using ManagedCode.LlmTck.Tests.Azure; -using ManagedCode.LlmTck.Tests.Bedrock; -using ManagedCode.LlmTck.Tests.Cohere; -using ManagedCode.LlmTck.Tests.Decisions; -using ManagedCode.LlmTck.Tests.Gemini; -using ManagedCode.LlmTck.Tests.Ollama; -using ManagedCode.LlmTck.Tests.OpenAI; -using ManagedCode.LlmTck.Tests.TestSupport; -using Microsoft.AspNetCore.Routing; -using Microsoft.Extensions.DependencyInjection; - -namespace ManagedCode.LlmTck.Tests.Providers; - -public sealed class ProviderApiContractTests -{ - private const int _maxDocumentationReviewAgeDays = 180; - private static readonly string[] _controlRoutePaths = - [ - LlmTckControlRoutes.Admin, - LlmTckControlRoutes.Models, - LlmTckControlRoutes.Assertions, - LlmTckControlRoutes.Configure, - LlmTckControlRoutes.Reset, - ]; - - private static readonly Uri[] _officialDocumentationHosts = - [ - new("https://api.typesafe.ai"), - new("https://docs.typesafe.ai"), - new("https://developers.cloudflare.com"), - new("https://developers.openai.com"), - new("https://learn.microsoft.com"), - new("https://docs.anthropic.com"), - new("https://platform.claude.com"), - new("https://ai.google.dev"), - new("https://console.groq.com"), - new("https://docs.mistral.ai"), - new("https://docs.ollama.com"), - new("https://github.com/ollama/ollama"), - new("https://docs.cohere.com"), - new("https://docs.aws.amazon.com"), - new("https://openrouter.ai"), - new("https://api-docs.deepseek.com"), - new("https://docs.perplexity.ai"), - ]; - - [Test] - public async Task ProviderCapabilities_DeclareTestedToolsAndStructuredOutputAsync() - { - foreach (var profile in GetProfiles().Where(profile => profile.Capabilities.Contains(LlmTckProviderCapability.Chat))) - { - await Assert.That(profile.Capabilities.Contains(LlmTckProviderCapability.StructuredOutput)).IsTrue(); - await Assert.That(profile.Capabilities.Contains(LlmTckProviderCapability.Tools)).IsEqualTo(profile.Id != LlmTckCompatibilityTags.Perplexity); - } - } - - [Test] - public async Task ProviderApiContracts_AreDocBackedAndCoverClaimedCapabilitiesAsync() - { - foreach (var profile in GetProfiles()) - { - await Assert.That(IsOfficialDocumentationUrl(profile.ApiContract.DocumentationUrl)).IsTrue(); - var hasDocumentationDate = DateOnly.TryParse( - profile.ApiContract.DocumentationRetrievedOn, - out var documentationRetrievedOn - ); - - await Assert.That(hasDocumentationDate).IsTrue(); - await Assert.That(IsFreshDocumentationReview(documentationRetrievedOn)).IsTrue(); - await Assert.That(profile.ApiContract.DocumentationVersion).IsNotNull(); - await Assert.That(profile.ApiContract.DocumentationVersion).IsNotEmpty(); - await Assert.That(profile.ApiContract.Operations).IsNotEmpty(); - - var documentedCapabilities = profile.ApiContract.Operations - .SelectMany(operation => operation.Capabilities) - .Distinct() - .ToArray(); - var duplicateOperations = profile.ApiContract.Operations - .GroupBy(operation => new { operation.Method, operation.Path }) - .Where(group => group.Count() > 1) - .Select(group => $"{group.Key.Method} {group.Key.Path}") - .ToArray(); - var unimplementedOperations = profile.ApiContract.Operations - .Where(operation => !operation.ImplementedByHosting) - .Select(operation => operation.Id) - .ToArray(); - - await Assert.That(profile.Capabilities.Except(documentedCapabilities)).IsEmpty(); - await Assert.That(duplicateOperations).IsEmpty(); - await Assert.That(unimplementedOperations).IsEmpty(); - await Assert.That( - profile.ApiContract.Operations.Any(operation => - string.Equals( - operation.Path, - profile.DefaultEndpointPath, - StringComparison.Ordinal - ) - ) || profile.ApiContract.Operations.Any(operation => - operation.Path.StartsWith(profile.DefaultEndpointPath, StringComparison.Ordinal) - ) - ) - .IsTrue(); - - foreach (var operation in profile.ApiContract.Operations) - { - await Assert.That(operation.Id).IsNotEmpty(); - await Assert.That(operation.Path).StartsWith("/"); - await Assert.That(IsSupportedHttpMethod(operation.Method)).IsTrue(); - await Assert.That(IsOfficialDocumentationUrl(operation.DocumentationUrl)).IsTrue(); - - if (operation.SupportsStreaming) - { - await Assert.That( - operation.Capabilities.Any(capability => - capability - is LlmTckProviderCapability.StreamingChat - or LlmTckProviderCapability.StreamingImages - or LlmTckProviderCapability.StreamingAudio - ) - ) - .IsTrue(); - } - } - } - } - - [Test] - public async Task HostingProviderRoutes_AreCoveredByImplementedDocumentedOperationsAsync() - { - using var host = await LlmTckTestHost.StartAsync(); - var providerNamespaces = GetProfiles() - .Select(profile => GetExpectedProviderNamespace(profile.Id)) - .ToArray(); - var routes = host.Services - .GetRequiredService() - .Endpoints.OfType() - .Where(endpoint => - endpoint.Metadata.GetMetadata() is null - ) - .SelectMany(endpoint => - { - var methods = endpoint.Metadata.GetMetadata()?.HttpMethods - ?? ["GET"]; - - return methods.Select(method => new ProviderRoute( - method, - endpoint.RoutePattern.RawText ?? string.Empty - )); - }) - .Where(route => !_controlRoutePaths.Contains(route.Path, StringComparer.Ordinal)) - .Where(route => IsProviderRoute(route.Path, providerNamespaces)) - .OrderBy(route => route.Method, StringComparer.Ordinal) - .ThenBy(route => route.Path, StringComparer.Ordinal) - .ToArray(); - - var implementedOperations = GetProfiles() - .SelectMany(profile => profile.ApiContract.Operations) - .Where(operation => operation.ImplementedByHosting) - .Select(operation => new ProviderRoute(operation.Method, operation.Path)) - .Distinct() - .OrderBy(route => route.Method, StringComparer.Ordinal) - .ThenBy(route => route.Path, StringComparer.Ordinal) - .ToArray(); - - await Assert.That(routes).IsEquivalentTo(implementedOperations); - } - - [Test] - public async Task HostingProviderFallbackRoutes_AreExplicitlyMarkedDiagnosticsAsync() - { - using var host = await LlmTckTestHost.StartAsync(); - var providerNamespaces = GetProfiles() - .Select(profile => GetExpectedProviderNamespace(profile.Id)) - .ToArray(); - var providerEndpoints = host.Services - .GetRequiredService() - .Endpoints.OfType() - .Where(endpoint => - IsProviderRoute(endpoint.RoutePattern.RawText ?? string.Empty, providerNamespaces) - ) - .ToArray(); - var fallbackEndpoints = providerEndpoints - .Where(endpoint => endpoint.RoutePattern.Parameters.Any(parameter => parameter.IsCatchAll)) - .ToArray(); - var markedEndpoints = providerEndpoints - .Where(endpoint => - endpoint.Metadata.GetMetadata() is not null - ) - .ToArray(); - - await Assert.That(fallbackEndpoints.Length).IsEqualTo(providerNamespaces.Length); - await Assert.That(markedEndpoints.Length).IsEqualTo(providerNamespaces.Length); - foreach (var endpoint in fallbackEndpoints) - { - await Assert.That(endpoint.Metadata.GetMetadata()) - .IsNotNull(); - } - - foreach (var endpoint in markedEndpoints) - { - await Assert.That( - endpoint.RoutePattern.Parameters.Any(parameter => parameter.IsCatchAll) - ) - .IsTrue(); - } - } - - [Test] - public async Task ProviderApiContracts_UseExplicitProviderNamespacesAsync() - { - foreach (var profile in GetProfiles()) - { - var expectedNamespace = GetExpectedProviderNamespace(profile.Id); - - await Assert.That(profile.DefaultEndpointPath).StartsWith(expectedNamespace); - - foreach (var operation in profile.ApiContract.Operations) - { - await Assert.That(operation.Path).StartsWith(expectedNamespace); - } - } - } - - [Test] - public async Task ProviderBehaviorEvidence_CoversEveryImplementedOperationAsync() - { - var implementedOperations = GetImplementedOperations(); - var implementedOperationKeys = implementedOperations - .Select(item => item.Key) - .Distinct() - .ToArray(); - var streamingOperationKeys = implementedOperations - .Where(item => item.Operation.SupportsStreaming) - .Select(item => item.Key) - .Distinct() - .ToArray(); - var evidence = GetBehaviorEvidence(); - var evidenceKeys = evidence - .Select(item => new OperationKey(item.ProviderId, item.OperationId)) - .Distinct() - .ToArray(); - var streamingEvidenceKeys = evidence - .Where(item => item.CoversStreaming) - .Select(item => new OperationKey(item.ProviderId, item.OperationId)) - .Distinct() - .ToArray(); - var duplicateEvidence = evidence - .GroupBy(item => new OperationKey(item.ProviderId, item.OperationId)) - .Where(group => group.Count() > 1) - .Select(group => group.Key.ToString()) - .ToArray(); - var missingEvidence = implementedOperationKeys - .Except(evidenceKeys) - .Select(key => key.ToString()) - .ToArray(); - var staleEvidence = evidenceKeys - .Except(implementedOperationKeys) - .Select(key => key.ToString()) - .ToArray(); - var missingStreamingEvidence = streamingOperationKeys - .Except(streamingEvidenceKeys) - .Select(key => key.ToString()) - .ToArray(); - var missingTestMethods = evidence - .SelectMany(item => item.TestMethods.Select(method => new { item.TestType, Method = method })) - .Where(item => item.TestType.GetMethod(item.Method) is null) - .Select(item => $"{item.TestType.Name}.{item.Method}") - .ToArray(); - var inventory = await File.ReadAllTextAsync( - Path.Combine( - FindRepositoryRoot(), - "docs", - "Testing", - "AcceptanceCriteriaTestInventory.md" - ) - ); - var undocumentedTestMethods = evidence - .SelectMany(item => item.TestMethods) - .Distinct() - .Where(method => !inventory.Contains(method, StringComparison.Ordinal)) - .ToArray(); - - await Assert.That(duplicateEvidence).IsEmpty(); - await Assert.That(missingEvidence).IsEmpty(); - await Assert.That(staleEvidence).IsEmpty(); - await Assert.That(missingStreamingEvidence).IsEmpty(); - await Assert.That(missingTestMethods).IsEmpty(); - await Assert.That(undocumentedTestMethods).IsEmpty(); - } - - private static bool IsFreshDocumentationReview(DateOnly retrievedOn) - { - var today = DateOnly.FromDateTime(DateTimeOffset.UtcNow.UtcDateTime); - var oldestAccepted = today.AddDays(-_maxDocumentationReviewAgeDays); - return retrievedOn >= oldestAccepted && retrievedOn <= today; - } - - private static bool IsOfficialDocumentationUrl(string value) - { - return Uri.TryCreate(value, UriKind.Absolute, out var uri) - && uri.Scheme == Uri.UriSchemeHttps - && _officialDocumentationHosts.Any(host => - string.Equals(uri.Host, host.Host, StringComparison.OrdinalIgnoreCase) - ); - } - - private static bool IsProviderRoute(string path, IReadOnlyCollection providerNamespaces) - { - return providerNamespaces.Any(providerNamespace => - path.StartsWith(providerNamespace, StringComparison.Ordinal) - ); - } - - private static bool IsSupportedHttpMethod(string method) - { - return method is "DELETE" or "GET" or "HEAD" or "POST"; - } - - private static string GetExpectedProviderNamespace(string providerId) - { - return providerId switch - { - LlmTckCompatibilityTags.SystemOne => LlmTckProviderRouteNamespaces.SystemOne, - LlmTckCompatibilityTags.Cloudflare => LlmTckProviderRouteNamespaces.Cloudflare, - LlmTckCompatibilityTags.OpenAI => LlmTckProviderRouteNamespaces.OpenAI, - LlmTckCompatibilityTags.AzureOpenAI => LlmTckProviderRouteNamespaces.AzureOpenAI, - LlmTckCompatibilityTags.MicrosoftFoundry => LlmTckProviderRouteNamespaces.MicrosoftFoundry, - LlmTckCompatibilityTags.Anthropic => LlmTckProviderRouteNamespaces.Anthropic, - LlmTckCompatibilityTags.Gemini => LlmTckProviderRouteNamespaces.Gemini, - LlmTckCompatibilityTags.Groq => LlmTckProviderRouteNamespaces.Groq, - LlmTckCompatibilityTags.Mistral => LlmTckProviderRouteNamespaces.Mistral, - LlmTckCompatibilityTags.Ollama => LlmTckProviderRouteNamespaces.Ollama, - LlmTckCompatibilityTags.Cohere => LlmTckProviderRouteNamespaces.Cohere, - LlmTckCompatibilityTags.Bedrock => LlmTckProviderRouteNamespaces.Bedrock, - LlmTckCompatibilityTags.OpenRouter => LlmTckProviderRouteNamespaces.OpenRouter, - LlmTckCompatibilityTags.DeepSeek => LlmTckProviderRouteNamespaces.DeepSeek, - LlmTckCompatibilityTags.Perplexity => LlmTckProviderRouteNamespaces.Perplexity, - _ => throw new InvalidOperationException($"Unknown provider '{providerId}'."), - }; - } - - private static (OperationKey Key, LlmTckProviderOperation Operation)[] GetImplementedOperations() - { - return GetProfiles() - .SelectMany(profile => profile - .ApiContract - .Operations - .Where(operation => operation.ImplementedByHosting) - .Select(operation => ( - new OperationKey(profile.Id, operation.Id), - Operation: operation - ))) - .ToArray(); - } - - private static ProviderOperationEvidence[] GetBehaviorEvidence() - { - return - [ - Evidence(LlmTckCompatibilityTags.SystemOne, LlmTckProviderOperationIds.SystemOne.DecisionsCreate, typeof(Jev114EndpointTests), [nameof(Jev114EndpointTests.Jev114_JsonImagesAndBoxPreserveNativeAnswersAsync), nameof(Jev114EndpointTests.Jev114_RejectsInvalidImageAndBoxRequestsAsync)]), - Evidence(LlmTckCompatibilityTags.Cloudflare, LlmTckProviderOperationIds.Cloudflare.ClefDecisionsCreate, typeof(DecisionEndpointTests), [nameof(DecisionEndpointTests.NativeDecisionEndpoints_PreserveAnswersAndUsageAsync)]), - Evidence(LlmTckCompatibilityTags.Cloudflare, LlmTckProviderOperationIds.Cloudflare.ClefFlashDecisionsCreate, typeof(DecisionEndpointTests), [nameof(DecisionEndpointTests.NativeDecisionEndpoints_PreserveAnswersAndUsageAsync)]), - Evidence(LlmTckCompatibilityTags.OpenAI, LlmTckProviderOperationIds.OpenAI.DecisionsCreate, typeof(DecisionEndpointTests), [nameof(DecisionEndpointTests.NativeDecisionEndpoints_PreserveAnswersAndUsageAsync)]), - Evidence( - LlmTckCompatibilityTags.AzureOpenAI, - LlmTckProviderOperationIds.AzureOpenAI.V1ChatCompletionsCreate, - typeof(AzureV1SdkCompatibilityTests), - [nameof(AzureV1SdkCompatibilityTests.V1ChatAndEmbeddings_UseOfficialSdkWithoutApiVersionAsync)], - coversStreaming: true - ), - Evidence( - LlmTckCompatibilityTags.AzureOpenAI, - LlmTckProviderOperationIds.AzureOpenAI.V1AudioTranscriptionsCreate, - typeof(V1AudioTranscriptionCompatibilityTests), - [nameof(V1AudioTranscriptionCompatibilityTests.OfficialSdk_PreservesTranscriptionUsageAsync), - nameof(V1AudioTranscriptionCompatibilityTests.Streaming_PreservesFinalUsageAsync)], - coversStreaming: true - ), - Evidence( - LlmTckCompatibilityTags.AzureOpenAI, - LlmTckProviderOperationIds.AzureOpenAI.V1EmbeddingsCreate, - typeof(AzureV1SdkCompatibilityTests), - [nameof(AzureV1SdkCompatibilityTests.V1ChatAndEmbeddings_UseOfficialSdkWithoutApiVersionAsync)], - coversStreaming: false - ), - Evidence( - LlmTckCompatibilityTags.AzureOpenAI, - LlmTckProviderOperationIds.AzureOpenAI.V1ImagesCreate, - typeof(AzureV1SdkCompatibilityTests), - [nameof(AzureV1SdkCompatibilityTests.AzureOpenAiV1Images_UseOfficialOpenAiSdkAndReturnImageBytesAsync)] - ), - Evidence( - LlmTckCompatibilityTags.AzureOpenAI, - LlmTckProviderOperationIds.AzureOpenAI.V1ResponsesCreate, - typeof(AzureV1SdkCompatibilityTests), - [nameof(AzureV1SdkCompatibilityTests.V1Responses_UseOfficialSdkAndStreamAsync)], - coversStreaming: true - ), - Evidence( - LlmTckCompatibilityTags.MicrosoftFoundry, - LlmTckProviderOperationIds.MicrosoftFoundry.V1ChatCompletionsCreate, - typeof(AzureV1SdkCompatibilityTests), - [nameof(AzureV1SdkCompatibilityTests.V1ChatAndEmbeddings_UseOfficialSdkWithoutApiVersionAsync)], - coversStreaming: true - ), - Evidence( - LlmTckCompatibilityTags.MicrosoftFoundry, - LlmTckProviderOperationIds.MicrosoftFoundry.V1AudioTranscriptionsCreate, - typeof(V1AudioTranscriptionCompatibilityTests), - [nameof(V1AudioTranscriptionCompatibilityTests.OfficialSdk_PreservesTranscriptionUsageAsync), - nameof(V1AudioTranscriptionCompatibilityTests.Streaming_PreservesFinalUsageAsync)], - coversStreaming: true - ), - Evidence( - LlmTckCompatibilityTags.MicrosoftFoundry, - LlmTckProviderOperationIds.MicrosoftFoundry.V1EmbeddingsCreate, - typeof(AzureV1SdkCompatibilityTests), - [nameof(AzureV1SdkCompatibilityTests.V1ChatAndEmbeddings_UseOfficialSdkWithoutApiVersionAsync)], - coversStreaming: false - ), - Evidence( - LlmTckCompatibilityTags.MicrosoftFoundry, - LlmTckProviderOperationIds.MicrosoftFoundry.V1ResponsesCreate, - typeof(AzureV1SdkCompatibilityTests), - [nameof(AzureV1SdkCompatibilityTests.V1Responses_UseOfficialSdkAndStreamAsync)], - coversStreaming: true - ), - Evidence( - LlmTckCompatibilityTags.OpenAI, - LlmTckProviderOperationIds.OpenAI.ModelsList, - typeof(OpenAiEndpointTests), - [nameof(OpenAiEndpointTests.OpenAiEndpoints_ExposeModelsAndAllModalitiesAsync)] - ), - Evidence( - LlmTckCompatibilityTags.OpenAI, - LlmTckProviderOperationIds.OpenAI.ChatCompletionsCreate, - typeof(OpenAiEndpointTests), - [nameof(OpenAiEndpointTests.StreamingChatCompletion_ReturnsServerSentChunksAsync)], - coversStreaming: true - ), - Evidence( - LlmTckCompatibilityTags.OpenAI, - LlmTckProviderOperationIds.OpenAI.ResponsesCreate, - typeof(OpenAiCompatibleProviderRouteTests), - [ - nameof(OpenAiCompatibleProviderRouteTests.OpenAiCompatibleResponsesRoutes_ReturnResponseShapeAsync), - nameof(OpenAiCompatibleProviderRouteTests.OpenAiCompatibleResponsesRoutes_StreamResponseServerSentEventsAsync), - ], - coversStreaming: true - ), - Evidence( - LlmTckCompatibilityTags.OpenAI, - LlmTckProviderOperationIds.OpenAI.EmbeddingsCreate, - typeof(OpenAiEndpointTests), - [nameof(OpenAiEndpointTests.OpenAiEndpoints_ExposeModelsAndAllModalitiesAsync)] - ), - Evidence( - LlmTckCompatibilityTags.OpenAI, - LlmTckProviderOperationIds.OpenAI.ImagesCreate, - typeof(OpenAiEndpointTests), - [nameof(OpenAiEndpointTests.ImageRoutes_ReturnDocumentedEditVariationAndStreamingShapesAsync)], - coversStreaming: true - ), - Evidence( - LlmTckCompatibilityTags.OpenAI, - LlmTckProviderOperationIds.OpenAI.ImagesEditsCreate, - typeof(OpenAiEndpointTests), - [nameof(OpenAiEndpointTests.ImageRoutes_ReturnDocumentedEditVariationAndStreamingShapesAsync)], - coversStreaming: true - ), - Evidence( - LlmTckCompatibilityTags.OpenAI, - LlmTckProviderOperationIds.OpenAI.ImagesVariationsCreate, - typeof(OpenAiEndpointTests), - [nameof(OpenAiEndpointTests.ImageRoutes_ReturnDocumentedEditVariationAndStreamingShapesAsync)] - ), - Evidence( - LlmTckCompatibilityTags.OpenAI, - LlmTckProviderOperationIds.OpenAI.AudioSpeechCreate, - typeof(OpenAiEndpointTests), - [nameof(OpenAiEndpointTests.AudioSpeech_ReturnsTruthfulWavFixtureAsync)] - ), - Evidence( - LlmTckCompatibilityTags.OpenAI, - LlmTckProviderOperationIds.OpenAI.AudioTranscriptionsCreate, - typeof(OpenAiEndpointTests), - [nameof(OpenAiEndpointTests.AudioTranscription_ReturnsJsonTextAndStreamingShapesAsync)], - coversStreaming: true - ), - Evidence( - LlmTckCompatibilityTags.OpenAI, - LlmTckProviderOperationIds.OpenAI.AudioTranslationsCreate, - typeof(OpenAiEndpointTests), - [nameof(OpenAiEndpointTests.AudioTranslation_ReturnsJsonTextAndRejectsStreamingAsync)] - ), - Evidence( - LlmTckCompatibilityTags.OpenAI, - LlmTckProviderOperationIds.OpenAI.VideosCreate, - typeof(OpenAiEndpointTests), - [nameof(OpenAiEndpointTests.VideoRoutes_ReturnDocumentedOpenAiShapesAndValidateEnumsAsync)] - ), - Evidence( - LlmTckCompatibilityTags.OpenAI, - LlmTckProviderOperationIds.OpenAI.VideosList, - typeof(OpenAiEndpointTests), - [nameof(OpenAiEndpointTests.VideoRoutes_ReturnDocumentedOpenAiShapesAndValidateEnumsAsync)] - ), - Evidence( - LlmTckCompatibilityTags.OpenAI, - LlmTckProviderOperationIds.OpenAI.VideosRetrieve, - typeof(OpenAiEndpointTests), - [nameof(OpenAiEndpointTests.VideoRoutes_ReturnDocumentedOpenAiShapesAndValidateEnumsAsync)] - ), - Evidence( - LlmTckCompatibilityTags.OpenAI, - LlmTckProviderOperationIds.OpenAI.VideosDelete, - typeof(OpenAiEndpointTests), - [nameof(OpenAiEndpointTests.VideoRoutes_ReturnDocumentedOpenAiShapesAndValidateEnumsAsync)] - ), - Evidence( - LlmTckCompatibilityTags.OpenAI, - LlmTckProviderOperationIds.OpenAI.VideosContentRetrieve, - typeof(OpenAiEndpointTests), - [nameof(OpenAiEndpointTests.VideoRoutes_ReturnDocumentedOpenAiShapesAndValidateEnumsAsync)] - ), - Evidence( - LlmTckCompatibilityTags.OpenAI, - LlmTckProviderOperationIds.OpenAI.VideosEditsCreate, - typeof(OpenAiEndpointTests), - [nameof(OpenAiEndpointTests.VideoRoutes_ReturnDocumentedOpenAiShapesAndValidateEnumsAsync)] - ), - Evidence( - LlmTckCompatibilityTags.OpenAI, - LlmTckProviderOperationIds.OpenAI.VideosExtensionsCreate, - typeof(OpenAiEndpointTests), - [nameof(OpenAiEndpointTests.VideoRoutes_ReturnDocumentedOpenAiShapesAndValidateEnumsAsync)] - ), - Evidence( - LlmTckCompatibilityTags.OpenAI, - LlmTckProviderOperationIds.OpenAI.VideosRemix, - typeof(OpenAiEndpointTests), - [nameof(OpenAiEndpointTests.VideoRoutes_ReturnDocumentedOpenAiShapesAndValidateEnumsAsync)] - ), - Evidence( - LlmTckCompatibilityTags.OpenAI, - LlmTckProviderOperationIds.OpenAI.VideosCharactersCreate, - typeof(OpenAiEndpointTests), - [nameof(OpenAiEndpointTests.VideoRoutes_ReturnDocumentedOpenAiShapesAndValidateEnumsAsync)] - ), - Evidence( - LlmTckCompatibilityTags.OpenAI, - LlmTckProviderOperationIds.OpenAI.VideosCharactersRetrieve, - typeof(OpenAiEndpointTests), - [nameof(OpenAiEndpointTests.VideoRoutes_ReturnDocumentedOpenAiShapesAndValidateEnumsAsync)] - ), - Evidence( - LlmTckCompatibilityTags.AzureOpenAI, - LlmTckProviderOperationIds.AzureOpenAI.ChatCompletionsCreate, - typeof(AzureSdkCompatibilityTests), - [nameof(AzureSdkCompatibilityTests.AzureOpenAiClient_CanUseDeploymentChatAndEmbeddingsAsync)], - coversStreaming: true - ), - Evidence( - LlmTckCompatibilityTags.AzureOpenAI, - LlmTckProviderOperationIds.AzureOpenAI.EmbeddingsCreate, - typeof(AzureSdkCompatibilityTests), - [nameof(AzureSdkCompatibilityTests.AzureOpenAiClient_CanUseDeploymentChatAndEmbeddingsAsync)] - ), - Evidence( - LlmTckCompatibilityTags.AzureOpenAI, - LlmTckProviderOperationIds.AzureOpenAI.ImagesCreate, - typeof(AzureSdkCompatibilityTests), - [nameof(AzureSdkCompatibilityTests.AzureOpenAiDeploymentRoutes_UseApiKeyAndDeploymentModelForModalitiesAsync)] - ), - Evidence( - LlmTckCompatibilityTags.AzureOpenAI, - LlmTckProviderOperationIds.AzureOpenAI.AudioSpeechCreate, - typeof(AzureSdkCompatibilityTests), - [nameof(AzureSdkCompatibilityTests.AzureOpenAiDeploymentRoutes_UseApiKeyAndDeploymentModelForModalitiesAsync)] - ), - Evidence( - LlmTckCompatibilityTags.AzureOpenAI, - LlmTckProviderOperationIds.AzureOpenAI.AudioTranscriptionsCreate, - typeof(OpenAiCompatibleProviderRouteTests), - [ - nameof(OpenAiCompatibleProviderRouteTests.AzureOpenAiAudioTranscriptionRoute_UsesDeploymentModelAndApiKeyAsync), - ] - ), - Evidence( - LlmTckCompatibilityTags.AzureOpenAI, - LlmTckProviderOperationIds.AzureOpenAI.AudioTranslationsCreate, - typeof(OpenAiCompatibleProviderRouteTests), - [ - nameof(OpenAiCompatibleProviderRouteTests.AzureOpenAiAudioTranslationRoute_UsesDeploymentModelAndApiKeyAsync), - ] - ), - Evidence( - LlmTckCompatibilityTags.AzureOpenAI, - LlmTckProviderOperationIds.AzureOpenAI.VideoGenerationJobsCreate, - typeof(OpenAiCompatibleProviderRouteTests), - [nameof(OpenAiCompatibleProviderRouteTests.AzureOpenAiVideoRoutes_FollowPreviewJobAndContentShapesAsync)] - ), - Evidence( - LlmTckCompatibilityTags.AzureOpenAI, - LlmTckProviderOperationIds.AzureOpenAI.VideoGenerationJobsList, - typeof(OpenAiCompatibleProviderRouteTests), - [nameof(OpenAiCompatibleProviderRouteTests.AzureOpenAiVideoRoutes_FollowPreviewJobAndContentShapesAsync)] - ), - Evidence( - LlmTckCompatibilityTags.AzureOpenAI, - LlmTckProviderOperationIds.AzureOpenAI.VideoGenerationJobsRetrieve, - typeof(OpenAiCompatibleProviderRouteTests), - [nameof(OpenAiCompatibleProviderRouteTests.AzureOpenAiVideoRoutes_FollowPreviewJobAndContentShapesAsync)] - ), - Evidence( - LlmTckCompatibilityTags.AzureOpenAI, - LlmTckProviderOperationIds.AzureOpenAI.VideoGenerationJobsDelete, - typeof(OpenAiCompatibleProviderRouteTests), - [nameof(OpenAiCompatibleProviderRouteTests.AzureOpenAiVideoRoutes_FollowPreviewJobAndContentShapesAsync)] - ), - Evidence( - LlmTckCompatibilityTags.AzureOpenAI, - LlmTckProviderOperationIds.AzureOpenAI.VideoGenerationsRetrieve, - typeof(OpenAiCompatibleProviderRouteTests), - [nameof(OpenAiCompatibleProviderRouteTests.AzureOpenAiVideoRoutes_FollowPreviewJobAndContentShapesAsync)] - ), - Evidence( - LlmTckCompatibilityTags.AzureOpenAI, - LlmTckProviderOperationIds.AzureOpenAI.VideoGenerationsThumbnailRetrieve, - typeof(OpenAiCompatibleProviderRouteTests), - [nameof(OpenAiCompatibleProviderRouteTests.AzureOpenAiVideoRoutes_FollowPreviewJobAndContentShapesAsync)] - ), - Evidence( - LlmTckCompatibilityTags.AzureOpenAI, - LlmTckProviderOperationIds.AzureOpenAI.VideoGenerationsContentRetrieve, - typeof(OpenAiCompatibleProviderRouteTests), - [nameof(OpenAiCompatibleProviderRouteTests.AzureOpenAiVideoRoutes_FollowPreviewJobAndContentShapesAsync)] - ), - Evidence( - LlmTckCompatibilityTags.AzureOpenAI, - LlmTckProviderOperationIds.AzureOpenAI.VideoGenerationsContentHead, - typeof(OpenAiCompatibleProviderRouteTests), - [nameof(OpenAiCompatibleProviderRouteTests.AzureOpenAiVideoRoutes_FollowPreviewJobAndContentShapesAsync)] - ), - Evidence( - LlmTckCompatibilityTags.MicrosoftFoundry, - LlmTckProviderOperationIds.MicrosoftFoundry.ChatCompletionsCreate, - typeof(OpenAiCompatibleProviderRouteTests), - [ - nameof(OpenAiCompatibleProviderRouteTests.OpenAiCompatibleChatRoutes_ReturnChatCompletionShapeAsync), - nameof(OpenAiCompatibleProviderRouteTests.OpenAiCompatibleChatRoutes_StreamServerSentChunksAsync), - ], - coversStreaming: true - ), - Evidence( - LlmTckCompatibilityTags.MicrosoftFoundry, - LlmTckProviderOperationIds.MicrosoftFoundry.EmbeddingsCreate, - typeof(AzureSdkCompatibilityTests), - [nameof(AzureSdkCompatibilityTests.AzureAiInferenceClients_CanUseFoundryChatAndEmbeddingsAsync)] - ), - Evidence( - LlmTckCompatibilityTags.MicrosoftFoundry, - LlmTckProviderOperationIds.MicrosoftFoundry.ModelsChatCompletionsCreate, - typeof(OpenAiCompatibleProviderRouteTests), - [ - nameof(OpenAiCompatibleProviderRouteTests.OpenAiCompatibleChatRoutes_ReturnChatCompletionShapeAsync), - nameof(OpenAiCompatibleProviderRouteTests.OpenAiCompatibleChatRoutes_StreamServerSentChunksAsync), - ], - coversStreaming: true - ), - Evidence( - LlmTckCompatibilityTags.MicrosoftFoundry, - LlmTckProviderOperationIds.MicrosoftFoundry.ModelsEmbeddingsCreate, - typeof(OpenAiCompatibleProviderRouteTests), - [nameof(OpenAiCompatibleProviderRouteTests.FoundryEmbeddingRoutes_ReturnEmbeddingShapeAsync)] - ), - Evidence( - LlmTckCompatibilityTags.Anthropic, - LlmTckProviderOperationIds.Anthropic.MessagesCreate, - typeof(AnthropicEndpointTests), - [ - nameof(AnthropicEndpointTests.MessagesEndpoint_ReturnsAnthropicMessageShapeAsync), - nameof(AnthropicEndpointTests.MessagesEndpoint_StreamsAnthropicServerSentEventsAsync), - ], - coversStreaming: true - ), - Evidence( - LlmTckCompatibilityTags.Gemini, - LlmTckProviderOperationIds.Gemini.ModelsGenerateContent, - typeof(GeminiEndpointTests), - [nameof(GeminiEndpointTests.GenerateContentEndpoint_ReturnsGeminiCandidateShapeAsync)] - ), - Evidence( - LlmTckCompatibilityTags.Gemini, - LlmTckProviderOperationIds.Gemini.ModelsStreamGenerateContent, - typeof(GeminiEndpointTests), - [nameof(GeminiEndpointTests.StreamGenerateContentEndpoint_ReturnsGeminiServerSentEventsAsync)], - coversStreaming: true - ), - Evidence( - LlmTckCompatibilityTags.Gemini, - LlmTckProviderOperationIds.Gemini.ModelsEmbedContent, - typeof(GeminiEndpointTests), - [nameof(GeminiEndpointTests.EmbedContentEndpoint_ReturnsGeminiEmbeddingShapeAsync)] - ), - Evidence( - LlmTckCompatibilityTags.Gemini, - LlmTckProviderOperationIds.Gemini.ModelsPredictLongRunningVideo, - typeof(GeminiEndpointTests), - [nameof(GeminiEndpointTests.PredictLongRunningVideoEndpoint_ReturnsGeminiOperationAndGeneratedFileAsync)] - ), - Evidence( - LlmTckCompatibilityTags.Gemini, - LlmTckProviderOperationIds.Gemini.ModelsOperationsGetVideo, - typeof(GeminiEndpointTests), - [nameof(GeminiEndpointTests.PredictLongRunningVideoEndpoint_ReturnsGeminiOperationAndGeneratedFileAsync)] - ), - Evidence( - LlmTckCompatibilityTags.Gemini, - LlmTckProviderOperationIds.Gemini.FilesGetGeneratedVideo, - typeof(GeminiEndpointTests), - [nameof(GeminiEndpointTests.PredictLongRunningVideoEndpoint_ReturnsGeminiOperationAndGeneratedFileAsync)] - ), - Evidence( - LlmTckCompatibilityTags.Groq, - LlmTckProviderOperationIds.Groq.ChatCompletionsCreate, - typeof(OpenAiCompatibleProviderRouteTests), - [ - nameof(OpenAiCompatibleProviderRouteTests.OpenAiCompatibleChatRoutes_ReturnChatCompletionShapeAsync), - nameof(OpenAiCompatibleProviderRouteTests.OpenAiCompatibleChatRoutes_StreamServerSentChunksAsync), - ], - coversStreaming: true - ), - Evidence( - LlmTckCompatibilityTags.Groq, - LlmTckProviderOperationIds.Groq.ResponsesCreate, - typeof(OpenAiCompatibleProviderRouteTests), - [ - nameof(OpenAiCompatibleProviderRouteTests.OpenAiCompatibleResponsesRoutes_ReturnResponseShapeAsync), - nameof(OpenAiCompatibleProviderRouteTests.OpenAiCompatibleResponsesRoutes_StreamResponseServerSentEventsAsync), - ], - coversStreaming: true - ), - Evidence( - LlmTckCompatibilityTags.Groq, - LlmTckProviderOperationIds.Groq.AudioSpeechCreate, - typeof(OpenAiCompatibleProviderRouteTests), - [nameof(OpenAiCompatibleProviderRouteTests.GroqAudioSpeechRoute_ReturnsAudioAndValidatesDocumentedRequestAsync)] - ), - Evidence( - LlmTckCompatibilityTags.Groq, - LlmTckProviderOperationIds.Groq.AudioTranscriptionsCreate, - typeof(OpenAiCompatibleProviderRouteTests), - [nameof(OpenAiCompatibleProviderRouteTests.GroqAudioTranscriptionRoute_ReturnsGroqTranscriptionShapeAsync)] - ), - Evidence( - LlmTckCompatibilityTags.Groq, - LlmTckProviderOperationIds.Groq.AudioTranslationsCreate, - typeof(OpenAiCompatibleProviderRouteTests), - [nameof(OpenAiCompatibleProviderRouteTests.GroqAudioTranslationRoute_ReturnsGroqTranslationShapeAsync)] - ), - Evidence( - LlmTckCompatibilityTags.Groq, - LlmTckProviderOperationIds.Groq.ModelsList, - typeof(OpenAiCompatibleProviderRouteTests), - [nameof(OpenAiCompatibleProviderRouteTests.OpenAiCompatibleModelRoutes_ReturnModelListShapeAsync)] - ), - Evidence( - LlmTckCompatibilityTags.Mistral, - LlmTckProviderOperationIds.Mistral.ChatComplete, - typeof(OpenAiCompatibleProviderRouteTests), - [ - nameof(OpenAiCompatibleProviderRouteTests.OpenAiCompatibleChatRoutes_ReturnChatCompletionShapeAsync), - nameof(OpenAiCompatibleProviderRouteTests.OpenAiCompatibleChatRoutes_StreamServerSentChunksAsync), - ], - coversStreaming: true - ), - Evidence( - LlmTckCompatibilityTags.Mistral, - LlmTckProviderOperationIds.Mistral.EmbeddingsCreate, - typeof(OpenAiCompatibleProviderRouteTests), - [nameof(OpenAiCompatibleProviderRouteTests.MistralEmbeddingRoute_ReturnsEmbeddingShapeAsync)] - ), - Evidence( - LlmTckCompatibilityTags.Ollama, - LlmTckProviderOperationIds.Ollama.ChatCreate, - typeof(OllamaEndpointTests), - [ - nameof(OllamaEndpointTests.ChatEndpoint_WithStreamFalse_ReturnsOllamaChatShapeAsync), - nameof(OllamaEndpointTests.ChatEndpoint_DefaultStreamsOllamaJsonLinesAsync), - ], - coversStreaming: true - ), - Evidence( - LlmTckCompatibilityTags.Ollama, - LlmTckProviderOperationIds.Ollama.EmbeddingsCreate, - typeof(OllamaEndpointTests), - [nameof(OllamaEndpointTests.EmbedEndpoint_ReturnsOllamaEmbeddingShapeAsync)] - ), - Evidence( - LlmTckCompatibilityTags.Cohere, - LlmTckProviderOperationIds.Cohere.ChatCreate, - typeof(CohereEndpointTests), - [ - nameof(CohereEndpointTests.ChatEndpoint_ReturnsCohereChatShapeAsync), - nameof(CohereEndpointTests.ChatEndpoint_StreamsCohereServerSentEventsAsync), - ], - coversStreaming: true - ), - Evidence( - LlmTckCompatibilityTags.Cohere, - LlmTckProviderOperationIds.Cohere.EmbedCreate, - typeof(CohereEndpointTests), - [ - nameof(CohereEndpointTests.EmbedEndpoint_ReturnsCohereEmbeddingShapeForTextsAsync), - nameof(CohereEndpointTests.EmbedEndpoint_ReturnsCohereEmbeddingShapeForInputsAsync), - ] - ), - Evidence( - LlmTckCompatibilityTags.Bedrock, - LlmTckProviderOperationIds.Bedrock.Converse, - typeof(BedrockEndpointTests), - [nameof(BedrockEndpointTests.ConverseEndpoint_ReturnsBedrockConverseShapeAsync)] - ), - Evidence( - LlmTckCompatibilityTags.Bedrock, - LlmTckProviderOperationIds.Bedrock.ConverseStream, - typeof(BedrockEndpointTests), - [nameof(BedrockEndpointTests.ConverseStreamEndpoint_ReturnsBedrockEventStreamShapeAsync)], - coversStreaming: true - ), - Evidence( - LlmTckCompatibilityTags.Bedrock, - LlmTckProviderOperationIds.Bedrock.InvokeModel, - typeof(BedrockEndpointTests), - [ - nameof(BedrockEndpointTests.InvokeEndpoint_ReturnsTitanTextShapeForChatModelAsync), - nameof(BedrockEndpointTests.InvokeEndpoint_ReturnsTitanEmbeddingShapeForEmbeddingModelAsync), - nameof(BedrockEndpointTests.InvokeEndpoint_ReturnsImageShapeForImageModelAsync), - ] - ), - Evidence( - LlmTckCompatibilityTags.Bedrock, - LlmTckProviderOperationIds.Bedrock.InvokeModelWithResponseStream, - typeof(BedrockEndpointTests), - [nameof(BedrockEndpointTests.InvokeModelWithResponseStreamEndpoint_ReturnsChunkBytesAsync)], - coversStreaming: true - ), - Evidence( - LlmTckCompatibilityTags.OpenRouter, - LlmTckProviderOperationIds.OpenRouter.ChatCompletionsCreate, - typeof(OpenAiCompatibleProviderRouteTests), - [ - nameof(OpenAiCompatibleProviderRouteTests.OpenAiCompatibleChatRoutes_ReturnChatCompletionShapeAsync), - nameof(OpenAiCompatibleProviderRouteTests.OpenAiCompatibleChatRoutes_StreamServerSentChunksAsync), - ], - coversStreaming: true - ), - Evidence( - LlmTckCompatibilityTags.OpenRouter, - LlmTckProviderOperationIds.OpenRouter.ResponsesCreate, - typeof(OpenAiCompatibleProviderRouteTests), - [ - nameof(OpenAiCompatibleProviderRouteTests.OpenAiCompatibleResponsesRoutes_ReturnResponseShapeAsync), - nameof(OpenAiCompatibleProviderRouteTests.OpenAiCompatibleResponsesRoutes_StreamResponseServerSentEventsAsync), - ], - coversStreaming: true - ), - Evidence( - LlmTckCompatibilityTags.OpenRouter, - LlmTckProviderOperationIds.OpenRouter.ModelsList, - typeof(OpenAiCompatibleProviderRouteTests), - [nameof(OpenAiCompatibleProviderRouteTests.OpenAiCompatibleModelRoutes_ReturnModelListShapeAsync)] - ), - Evidence( - LlmTckCompatibilityTags.DeepSeek, - LlmTckProviderOperationIds.DeepSeek.ChatCompletionsCreate, - typeof(OpenAiCompatibleProviderRouteTests), - [ - nameof(OpenAiCompatibleProviderRouteTests.OpenAiCompatibleChatRoutes_ReturnChatCompletionShapeAsync), - nameof(OpenAiCompatibleProviderRouteTests.OpenAiCompatibleChatRoutes_StreamServerSentChunksAsync), - ], - coversStreaming: true - ), - Evidence( - LlmTckCompatibilityTags.DeepSeek, - LlmTckProviderOperationIds.DeepSeek.ModelsList, - typeof(OpenAiCompatibleProviderRouteTests), - [nameof(OpenAiCompatibleProviderRouteTests.OpenAiCompatibleModelRoutes_ReturnModelListShapeAsync)] - ), - Evidence( - LlmTckCompatibilityTags.Perplexity, - LlmTckProviderOperationIds.Perplexity.SonarCreate, - typeof(OpenAiCompatibleProviderRouteTests), - [ - nameof(OpenAiCompatibleProviderRouteTests.OpenAiCompatibleChatRoutes_ReturnChatCompletionShapeAsync), - nameof(OpenAiCompatibleProviderRouteTests.OpenAiCompatibleChatRoutes_StreamServerSentChunksAsync), - ], - coversStreaming: true - ), - ]; - } - - private static ProviderOperationEvidence Evidence( - string providerId, - string operationId, - Type testType, - IReadOnlyList testMethods, - bool coversStreaming = false - ) - { - return new ProviderOperationEvidence( - providerId, - operationId, - testType, - testMethods, - coversStreaming - ); - } - - private static string FindRepositoryRoot() - { - var directory = new DirectoryInfo(AppContext.BaseDirectory); - - while (directory is not null) - { - if (File.Exists(Path.Combine(directory.FullName, "ManagedCode.LlmTck.slnx"))) - { - return directory.FullName; - } - - directory = directory.Parent; - } - - throw new DirectoryNotFoundException("Could not find ManagedCode.LlmTck.slnx."); - } - - internal static LlmTckProviderProfile[] GetProfiles() - { - return - [ - SystemOneCompatibility.Profile, - CloudflareCompatibility.Profile, - OpenAiCompatibility.Profile, - AzureOpenAiCompatibility.Profile, - MicrosoftFoundryCompatibility.Profile, - AnthropicCompatibility.Profile, - GeminiCompatibility.Profile, - GroqCompatibility.Profile, - MistralCompatibility.Profile, - OllamaCompatibility.Profile, - CohereCompatibility.Profile, - BedrockCompatibility.Profile, - OpenRouterCompatibility.Profile, - DeepSeekCompatibility.Profile, - PerplexityCompatibility.Profile, - ]; - } - - private sealed record OperationKey(string ProviderId, string OperationId) - { - public override string ToString() - { - return $"{ProviderId}:{OperationId}"; - } - } - - private sealed record ProviderOperationEvidence( - string ProviderId, - string OperationId, - Type TestType, - IReadOnlyList TestMethods, - bool CoversStreaming - ); - - private sealed record ProviderRoute(string Method, string Path); -} diff --git a/tests/ManagedCode.LlmTck.Tests/Providers/ProviderCapabilityEvidenceTests.cs b/tests/ManagedCode.LlmTck.Tests/Providers/ProviderCapabilityEvidenceTests.cs deleted file mode 100644 index 9144afe..0000000 --- a/tests/ManagedCode.LlmTck.Tests/Providers/ProviderCapabilityEvidenceTests.cs +++ /dev/null @@ -1,113 +0,0 @@ -using System.Net; -using System.Text; -using ManagedCode.LlmTck.Providers; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Tests.Regression; -using ManagedCode.LlmTck.Tests.TestSupport; -using Microsoft.Extensions.DependencyInjection; - -namespace ManagedCode.LlmTck.Tests.Providers; - -public sealed class ProviderCapabilityEvidenceTests -{ - [Test] - [Arguments(LlmTckCompatibilityTags.SystemOne)] - [Arguments(LlmTckCompatibilityTags.Cloudflare)] - [Arguments(LlmTckCompatibilityTags.OpenAI)] - [Arguments(LlmTckCompatibilityTags.AzureOpenAI)] - [Arguments(LlmTckCompatibilityTags.MicrosoftFoundry)] - [Arguments(LlmTckCompatibilityTags.Anthropic)] - [Arguments(LlmTckCompatibilityTags.Gemini)] - [Arguments(LlmTckCompatibilityTags.Groq)] - [Arguments(LlmTckCompatibilityTags.Mistral)] - [Arguments(LlmTckCompatibilityTags.Ollama)] - [Arguments(LlmTckCompatibilityTags.Cohere)] - [Arguments(LlmTckCompatibilityTags.Bedrock)] - [Arguments(LlmTckCompatibilityTags.OpenRouter)] - [Arguments(LlmTckCompatibilityTags.DeepSeek)] - [Arguments(LlmTckCompatibilityTags.Perplexity)] - public async Task ClaimedToolAndSchemaCases_ExecutePositiveAndNegativeHttpEvidenceAsync(string provider) - { - var profile = ProviderApiContractTests.GetProfiles().Single(item => item.Id == provider); - foreach (var operation in profile.ApiContract.Operations.Where(operation => operation.ImplementedByHosting)) - { - foreach (var capability in operation.Capabilities.Where(capability => capability is LlmTckProviderCapability.Tools or LlmTckProviderCapability.StructuredOutput)) - { - var streamingOnly = operation.Id is LlmTckProviderOperationIds.Gemini.ModelsStreamGenerateContent or LlmTckProviderOperationIds.Bedrock.ConverseStream; - var modes = streamingOnly ? new[] { true } : operation.SupportsStreaming ? [false, true] : [false]; - foreach (var stream in modes) - { - await VerifyCaseAsync(provider, operation, capability, stream); - } - } - } - } - - private static async Task VerifyCaseAsync(string provider, LlmTckProviderOperation operation, LlmTckProviderCapability capability, bool stream) - { - var caseId = $"{provider}/{operation.Id}/{capability}/{(stream ? "stream" : "json")}"; - var tools = capability == LlmTckProviderCapability.Tools; - const string answer = "{\"city\":\"Paris\"}"; - using var host = await LlmTckTestHost.StartAsync(b => b.AddChatScenario(caseId, s => - { - if (tools) { s.CallsTool("weather", answer); } - else { s.RespondsJson(answer); } - })); - using var http = host.GetTestClient(); - http.DefaultRequestHeaders.Add("anthropic-version", "2023-06-01"); - var format = provider switch - { - LlmTckCompatibilityTags.Anthropic => 2, - LlmTckCompatibilityTags.Gemini => 3, - LlmTckCompatibilityTags.Ollama => 4, - LlmTckCompatibilityTags.Cohere => 5, - LlmTckCompatibilityTags.Bedrock => 6, - _ => operation.Id is LlmTckProviderOperationIds.OpenAI.ResponsesCreate or LlmTckProviderOperationIds.AzureOpenAI.V1ResponsesCreate ? 1 : 0, - }; - var body = ToolFixtureEndpointTests.Request(format, tools); - body["stream"] = stream; - var path = operation.Path.Replace("{model}", "gpt-4.1-mini", StringComparison.Ordinal) - .Replace("{modelId}", "gpt-4.1-mini", StringComparison.Ordinal) - .Replace("{deployment}", "gpt-4.1-mini", StringComparison.Ordinal); - if (operation.ApiVersion is { } version && version != "v1") { path += "?api-version=" + version; } - if (provider == LlmTckCompatibilityTags.Gemini && stream) { path += "?alt=sse"; } - var invalid = body.ToJsonString().Replace("Paris", "London", StringComparison.Ordinal); - using var rejected = await http.PostAsync(path, new StringContent(invalid, Encoding.UTF8, "application/json")); - await Assert.That((caseId + "/reject", rejected.StatusCode)).IsEqualTo((caseId + "/reject", HttpStatusCode.Conflict)); - using var accepted = await http.PostAsync(path, new StringContent(body.ToJsonString(), Encoding.UTF8, "application/json")); - await Assert.That((caseId + "/accept", accepted.StatusCode)).IsEqualTo((caseId + "/accept", HttpStatusCode.OK)); - var content = await accepted.Content.ReadAsStringAsync(); - await Assert.That(content).Contains(tools ? "weather" : "Paris"); - if (stream) - { - var marker = format switch { 1 => "response.completed", 2 => "message_stop", 3 => "data:", 4 => "\"done\":true", 5 => "message-end", 6 => "messageStop", _ => "[DONE]" }; - await Assert.That(content).Contains(marker); - } - var summary = host.Services.GetRequiredService().GetAssertionSummary(); - await Assert.That(summary.ErrorsReturned).IsEqualTo(1); - await Assert.That(summary.Matched).IsEqualTo(1); - await Assert.That(summary.Events.All(entry => entry.ScenarioId == caseId)).IsTrue(); - } - - [Test] - public async Task EvidenceGuard_RequiresExecutableProviderAndEdgeCasesAsync() - { - var method = typeof(ProviderCapabilityEvidenceTests).GetMethod(nameof(ClaimedToolAndSchemaCases_ExecutePositiveAndNegativeHttpEvidenceAsync))!; - await Assert.That(method.IsDefined(typeof(TestAttribute), false)).IsTrue(); - var providers = ArgumentRows(method).Select(row => (string)row[0].Value!).Order().ToArray(); - await Assert.That(providers).IsEquivalentTo(ProviderApiContractTests.GetProfiles().Select(profile => profile.Id)); - var schema = typeof(FollowUpEndpointTests).GetMethod(nameof(FollowUpEndpointTests.GeminiSchema_ValidatesNullableNestedAlternativesAndStringLimitsAsync))!; - await Assert.That(schema.IsDefined(typeof(TestAttribute), false)).IsTrue(); - await Assert.That(ArgumentRows(schema).Select(row => (string)row[0].Value!)).IsEquivalentTo(new[] { "nullable", "string-limits", "nested-anyof" }); - var parallel = typeof(FollowUpEndpointTests).GetMethod(nameof(FollowUpEndpointTests.ParallelTools_RespectRequestLimitAndPreserveQueueAsync))!; - await Assert.That(parallel.IsDefined(typeof(TestAttribute), false)).IsTrue(); - await Assert.That(ArgumentRows(parallel).Select(row => ((int)row[1].Value!, (bool)row[2].Value!))) - .IsEquivalentTo(new[] { (0, false), (0, true), (1, false), (1, true), (2, false), (2, true) }); - } - - private static IEnumerable> ArgumentRows(System.Reflection.MethodInfo method) - { - return method.GetCustomAttributesData().Where(attribute => attribute.AttributeType == typeof(ArgumentsAttribute)) - .Select(attribute => (IList)attribute.ConstructorArguments[0].Value!); - } -} diff --git a/tests/ManagedCode.LlmTck.Tests/Providers/ProviderOperationIdConstantTests.cs b/tests/ManagedCode.LlmTck.Tests/Providers/ProviderOperationIdConstantTests.cs deleted file mode 100644 index 8409c68..0000000 --- a/tests/ManagedCode.LlmTck.Tests/Providers/ProviderOperationIdConstantTests.cs +++ /dev/null @@ -1,147 +0,0 @@ -using System.Reflection; -using ManagedCode.LlmTck.Providers; - -namespace ManagedCode.LlmTck.Tests.Providers; - -public sealed class ProviderOperationIdConstantTests -{ - [Test] - public async Task ProviderOperationIds_AreNotTypedAsInlineStringLiteralsAsync() - { - var repositoryRoot = FindRepositoryRoot(); - var operationIds = EnumerateOperationIdConstants() - .Where(operationId => !string.IsNullOrWhiteSpace(operationId.Value)) - .DistinctBy(operationId => operationId.Value) - .ToArray(); - var violations = new List(); - - foreach (var filePath in EnumerateCSharpSourceFiles(repositoryRoot)) - { - var sourceText = File.ReadAllText(filePath); - var relativePath = Path.GetRelativePath(repositoryRoot, filePath); - - foreach (var operationId in operationIds) - { - var literal = $"\"{operationId.Value}\""; - var searchIndex = sourceText.IndexOf(literal, StringComparison.Ordinal); - - while (searchIndex >= 0) - { - violations.Add( - $"{relativePath}:{GetLineNumber(sourceText, searchIndex)} uses {literal}; use {operationId.Name}." - ); - searchIndex = sourceText.IndexOf( - literal, - searchIndex + literal.Length, - StringComparison.Ordinal - ); - } - } - } - - await Assert.That(string.Join(Environment.NewLine, violations)).IsEqualTo(string.Empty); - } - - private static IEnumerable<(string Name, string? Value)> EnumerateOperationIdConstants() - { - return EnumerateConstantStringFields( - typeof(LlmTckProviderOperationIds), - nameof(LlmTckProviderOperationIds) - ); - } - - private static IEnumerable<(string Name, string? Value)> EnumerateConstantStringFields( - Type type, - string qualifiedName - ) - { - foreach (var field in type.GetFields(BindingFlags.Public | BindingFlags.Static)) - { - if (field is { IsLiteral: true, IsInitOnly: false } && field.FieldType == typeof(string)) - { - yield return ( - $"{qualifiedName}.{field.Name}", - field.GetRawConstantValue() as string - ); - } - } - - foreach (var nestedType in type.GetNestedTypes(BindingFlags.Public)) - { - foreach (var field in EnumerateConstantStringFields( - nestedType, - $"{qualifiedName}.{nestedType.Name}" - )) - { - yield return field; - } - } - } - - private static string FindRepositoryRoot() - { - var directory = new DirectoryInfo(AppContext.BaseDirectory); - - while (directory is not null) - { - if (File.Exists(Path.Combine(directory.FullName, "ManagedCode.LlmTck.slnx"))) - { - return directory.FullName; - } - - directory = directory.Parent; - } - - throw new DirectoryNotFoundException("Could not find ManagedCode.LlmTck.slnx."); - } - - private static IEnumerable EnumerateCSharpSourceFiles(string repositoryRoot) - { - string[] sourceDirectories = - [ - Path.Combine(repositoryRoot, "src"), - Path.Combine(repositoryRoot, "tests"), - Path.Combine(repositoryRoot, "samples"), - ]; - - return sourceDirectories - .Where(Directory.Exists) - .SelectMany(directory => Directory.EnumerateFiles(directory, "*.cs", SearchOption.AllDirectories)) - .Where(path => !IsBuildOutput(path)) - .Where(path => - !path.EndsWith( - Path.Combine("Providers", "LlmTckProviderOperationIds.cs"), - StringComparison.Ordinal - ) - ) - .Where(path => - !path.EndsWith( - Path.Combine("Providers", "ProviderOperationIdConstantTests.cs"), - StringComparison.Ordinal - ) - ); - } - - private static bool IsBuildOutput(string path) - { - var normalizedPath = path.Replace(Path.DirectorySeparatorChar, '/'); - - return normalizedPath.Contains("/bin/", StringComparison.Ordinal) - || normalizedPath.Contains("/obj/", StringComparison.Ordinal); - } - - private static int GetLineNumber(string sourceText, int characterIndex) - { - var lineNumber = 1; - - for (var i = 0; i < characterIndex; i++) - { - if (sourceText[i] == '\n') - { - lineNumber++; - } - } - - return lineNumber; - } -} diff --git a/tests/ManagedCode.LlmTck.Tests/Providers/ProviderPackageCatalogTests.cs b/tests/ManagedCode.LlmTck.Tests/Providers/ProviderPackageCatalogTests.cs deleted file mode 100644 index e281769..0000000 --- a/tests/ManagedCode.LlmTck.Tests/Providers/ProviderPackageCatalogTests.cs +++ /dev/null @@ -1,103 +0,0 @@ -using ManagedCode.LlmTck.Anthropic; -using ManagedCode.LlmTck.AzureOpenAI; -using ManagedCode.LlmTck.Bedrock; -using ManagedCode.LlmTck.Cloudflare; -using ManagedCode.LlmTck.Cohere; -using ManagedCode.LlmTck.DeepSeek; -using ManagedCode.LlmTck.Foundry; -using ManagedCode.LlmTck.Gemini; -using ManagedCode.LlmTck.Groq; -using ManagedCode.LlmTck.Mistral; -using ManagedCode.LlmTck.Ollama; -using ManagedCode.LlmTck.OpenAI; -using ManagedCode.LlmTck.OpenRouter; -using ManagedCode.LlmTck.Perplexity; -using ManagedCode.LlmTck.Providers; -using ManagedCode.LlmTck.SystemOne; - -namespace ManagedCode.LlmTck.Tests.Providers; - -public sealed class ProviderPackageCatalogTests -{ - [Test] - public async Task ProviderPackages_ExposeExpectedCompatibilityProfilesAsync() - { - LlmTckProviderProfile[] profiles = - [ - SystemOneCompatibility.Profile, - CloudflareCompatibility.Profile, - OpenAiCompatibility.Profile, - AzureOpenAiCompatibility.Profile, - MicrosoftFoundryCompatibility.Profile, - AnthropicCompatibility.Profile, - GeminiCompatibility.Profile, - GroqCompatibility.Profile, - MistralCompatibility.Profile, - OllamaCompatibility.Profile, - CohereCompatibility.Profile, - BedrockCompatibility.Profile, - OpenRouterCompatibility.Profile, - DeepSeekCompatibility.Profile, - PerplexityCompatibility.Profile, - ]; - - await Assert.That(profiles.Select(profile => profile.Id)) - .IsEquivalentTo( - [ - SystemOneCompatibility.ProviderId, - CloudflareCompatibility.ProviderId, - OpenAiCompatibility.ProviderId, - AzureOpenAiCompatibility.ProviderId, - MicrosoftFoundryCompatibility.ProviderId, - AnthropicCompatibility.ProviderId, - GeminiCompatibility.ProviderId, - GroqCompatibility.ProviderId, - MistralCompatibility.ProviderId, - OllamaCompatibility.ProviderId, - CohereCompatibility.ProviderId, - BedrockCompatibility.ProviderId, - OpenRouterCompatibility.ProviderId, - DeepSeekCompatibility.ProviderId, - PerplexityCompatibility.ProviderId, - ] - ); - await Assert.That(AzureOpenAiCompatibility.Profile.CompatibilityTags) - .IsEquivalentTo( - [ - AzureOpenAiCompatibility.ProviderId, - LlmTckCompatibilityTags.OpenAICompatible, - LlmTckCompatibilityTags.Foundry, - ] - ); - await Assert.That(profiles.All(profile => !string.IsNullOrWhiteSpace(profile.DisplayName))) - .IsTrue(); - await Assert.That( - profiles.All(profile => !string.IsNullOrWhiteSpace(profile.DefaultEndpointPath)) - ) - .IsTrue(); - await Assert.That( - profiles.All(profile => profile.Capabilities.Contains(LlmTckProviderCapability.Chat) || profile.Capabilities.Contains(LlmTckProviderCapability.Decisions)) - ) - .IsTrue(); - await Assert.That( - profiles - .Where(profile => - profile.Protocol - is LlmTckProtocolFamily.OpenAI - or LlmTckProtocolFamily.AzureOpenAI - or LlmTckProtocolFamily.MicrosoftFoundry - or LlmTckProtocolFamily.Gemini - or LlmTckProtocolFamily.Groq - or LlmTckProtocolFamily.Mistral - or LlmTckProtocolFamily.Ollama - or LlmTckProtocolFamily.OpenRouter - or LlmTckProtocolFamily.DeepSeek - or LlmTckProtocolFamily.Perplexity - ) - .All(profile => - profile.CompatibilityTags.Contains(LlmTckCompatibilityTags.OpenAICompatible) - ) - ) - .IsTrue(); - } -} diff --git a/tests/ManagedCode.LlmTck.Tests/Regression/FollowUpRuntimeTests.cs b/tests/ManagedCode.LlmTck.Tests/Regression/FollowUpRuntimeTests.cs deleted file mode 100644 index 8756c29..0000000 --- a/tests/ManagedCode.LlmTck.Tests/Regression/FollowUpRuntimeTests.cs +++ /dev/null @@ -1,117 +0,0 @@ -using ManagedCode.LlmTck.Configuration; -using ManagedCode.LlmTck.Providers; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Scenarios; - -namespace ManagedCode.LlmTck.Tests.Regression; - -public sealed class FollowUpRuntimeTests -{ - [Test] - public async Task VideoStore_DoesNotOverwriteCollidingCallerIdentifiersAsync() - { - var runtime = new LlmTckRuntime(); - var first = runtime.StoreVideo("provider", new() { IsSuccess = true, VideoId = "video_2", GenerationId = "generation_3", Bytes = [1] }); - var second = runtime.StoreVideo("provider", new() { IsSuccess = true, VideoId = "video", GenerationId = "generation", Bytes = [2] }); - await Assert.That(second.VideoId).IsEqualTo("video_4"); - await Assert.That(runtime.FindVideo("provider", first.VideoId)!.Bytes.Single()).IsEqualTo((byte)1); - await Assert.That(runtime.FindVideo("provider", second.GenerationId, generation: true)!.Bytes.Single()).IsEqualTo((byte)2); - } - - [Test] - public async Task SummaryMutations_CannotChangeRetainedMessagesOrChunksAsync() - { - var runtime = new LlmTckRuntime(new LlmTckConfigurationBuilder().AddChatScenario("snapshot", s => s.Responds("answer")).Build()); - var request = new LlmTckChatRequest { RequestId = "snapshot", Stream = true, Messages = [new() { Role = "assistant", ToolCalls = [new() { Id = "original", Name = "weather" }] }] }; - await runtime.CompleteChatAsync(request); - var snapshot = runtime.GetAssertionSummary(); - var entry = snapshot.Events.Single(); - entry.Messages.Single().ToolCalls.Clear(); - entry.Messages.Single().ToolCalls.Add(new() { Id = "injected", Name = "changed" }); - ((IList)entry.Messages)[0] = new() { Content = "replaced" }; - ((IList)entry.StreamChunks)[0] = "replaced"; - await Assert.That(runtime.TryGetEvent("snapshot", out var indexed)).IsTrue(); - indexed!.Messages.Single().ToolCalls.Clear(); - ((IList)indexed.StreamChunks)[0] = "index mutation"; - var fresh = runtime.GetAssertionSummary().Events.Single(); - await Assert.That(fresh.Messages.Single().ToolCalls.Single().Id).IsEqualTo("original"); - await Assert.That(fresh.StreamChunks.Single()).IsEqualTo("answer"); - await Assert.That(fresh.Usage).IsEqualTo(entry.Usage); - } - - [Test] - public async Task FixtureMismatch_IsCountedAndJournaledWithoutConsumingResponseAsync() - { - var runtime = new LlmTckRuntime(new LlmTckConfigurationBuilder().AddChatScenario("json", s => s.Responds("not json")).Build()); - var request = new LlmTckChatRequest { RequestId = "request-1", RequireJson = true, Messages = [new() { Content = "hello" }] }; - var rejected = await runtime.CompleteChatAsync(request); - var summary = runtime.GetAssertionSummary(); - await Assert.That(rejected.ErrorCode).IsEqualTo("llm_tck_fixture_mismatch"); - await Assert.That(summary.TotalEvents).IsEqualTo(1); - await Assert.That(summary.ErrorsReturned).IsEqualTo(1); - await Assert.That(summary.TotalTokens).IsEqualTo(rejected.Usage.TotalTokens); - await Assert.That(summary.Events.Single().RequestId).IsEqualTo("request-1"); - await Assert.That(summary.Events.Single().ScenarioId).IsEqualTo("json"); - await Assert.That(summary.Events.Single().Response).IsEqualTo(rejected.ErrorMessage); - await Assert.That((await runtime.CompleteChatAsync(request with { RequireJson = false })).Content).IsEqualTo("not json"); - } - - [Test] - [Arguments(false)] - [Arguments(true)] - public async Task ExactMatch_RejectsUnexpectedToolMetadataWhileContainsKeepsWildcardsAsync(bool toolResult) - { - var expected = new LlmTckMessage { Role = "assistant" }; - var actual = toolResult ? expected with { ToolCallId = "extra" } : expected with { ToolCalls = [new() { Name = "weather" }] }; - var runtime = new LlmTckRuntime(new LlmTckConfigurationBuilder().AddChatScenario("exact", s => s.WithExactMatch(expected).Responds("exact")).Build()); - await Assert.That((await runtime.CompleteChatAsync(new() { Messages = [actual] })).StatusCode).IsEqualTo(404); - await Assert.That((await runtime.CompleteChatAsync(new() { Messages = [expected] })).Content).IsEqualTo("exact"); - var containsConfiguration = new LlmTckConfigurationBuilder().AddChatScenario("contains", s => s.WithExactMatch(expected).Responds("contains")).Build(); - containsConfiguration.ChatScenarios[0] = containsConfiguration.ChatScenarios[0] with { Match = containsConfiguration.ChatScenarios[0].Match with { Mode = LlmTckMatchMode.Contains } }; - var contains = new LlmTckRuntime(containsConfiguration); - await Assert.That((await contains.CompleteChatAsync(new() { Messages = [actual] })).Content).IsEqualTo("contains"); - } - - [Test] - [Arguments(1, 100L)] - [Arguments(10, 4L)] - public async Task VideoCapacity_RejectsOverflowAndReclaimsDeletedBytesAsync(int jobs, long bytes) - { - var configuration = new LlmTckConfigurationBuilder().WithVideoCapacity(jobs, bytes).Build(); - var runtime = new LlmTckRuntime(configuration); - var video = new LlmTckVideoResult { IsSuccess = true, VideoId = "video", GenerationId = "generation", ModelId = "sora-2", Bytes = [1, 2, 3, 4] }; - var first = runtime.StoreVideo(LlmTckCompatibilityTags.OpenAI, video); - var overflow = runtime.StoreVideo(LlmTckCompatibilityTags.AzureOpenAI, video); - await Assert.That(first.IsSuccess).IsTrue(); - await Assert.That(overflow.StatusCode).IsEqualTo(409); - await Assert.That(overflow.ErrorCode).IsEqualTo("llm_tck_video_capacity_exceeded"); - await Assert.That(overflow.Bytes.Length).IsEqualTo(0); - await Assert.That(runtime.GetAssertionSummary().ErrorsReturned).IsEqualTo(1); - await Assert.That(runtime.ListVideos(LlmTckCompatibilityTags.OpenAI).Single().VideoId).IsEqualTo(first.VideoId); - await Assert.That(runtime.ListVideos(LlmTckCompatibilityTags.AzureOpenAI).Count).IsEqualTo(0); - await Assert.That(runtime.DeleteVideo(LlmTckCompatibilityTags.OpenAI, first.VideoId)).IsTrue(); - await Assert.That(runtime.DeleteVideo(LlmTckCompatibilityTags.OpenAI, first.VideoId)).IsFalse(); - await Assert.That(runtime.StoreVideo(LlmTckCompatibilityTags.AzureOpenAI, video).VideoId).IsEqualTo("video"); - await runtime.ResetAsync(); - await Assert.That(runtime.StoreVideo(LlmTckCompatibilityTags.OpenAI, video).IsSuccess).IsTrue(); - await runtime.ConfigureAsync(configuration); - await Assert.That(runtime.StoreVideo(LlmTckCompatibilityTags.OpenAI, video).VideoId).IsEqualTo("video"); - } - - [Test] - public async Task VideoCapacity_ValidatesConfigurationAndHandlesOversizedOrFailedResultsAsync() - { - await Assert.That(() => new LlmTckConfigurationBuilder().WithVideoCapacity(-1, 1)).Throws(); - await Assert.That(() => new LlmTckConfigurationBuilder().WithVideoCapacity(1, -1)).Throws(); - await Assert.That(() => new LlmTckRuntime(new() { MaxVideoJobs = -1 })).Throws(); - await Assert.That(() => new LlmTckRuntime(new() { MaxVideoBytes = -1 })).Throws(); - var runtime = new LlmTckRuntime(new LlmTckConfigurationBuilder().WithVideoCapacity(2, 1).Build()); - var video = new LlmTckVideoResult { IsSuccess = true, Bytes = [1, 2] }; - await Assert.That(runtime.StoreVideo("provider", video).IsSuccess).IsFalse(); - await Assert.That(runtime.ListVideos("provider").Count).IsEqualTo(0); - var failed = video with { IsSuccess = false, StatusCode = 429 }; - await Assert.That(runtime.StoreVideo("provider", failed)).IsEqualTo(failed); - var disabled = new LlmTckRuntime(new() { MaxVideoJobs = 0 }); - await Assert.That(disabled.StoreVideo("provider", video with { Bytes = [] }).IsSuccess).IsFalse(); - } -} diff --git a/tests/ManagedCode.LlmTck.Tests/Regression/FollowUpSchemaTests.cs b/tests/ManagedCode.LlmTck.Tests/Regression/FollowUpSchemaTests.cs deleted file mode 100644 index 75faa7e..0000000 --- a/tests/ManagedCode.LlmTck.Tests/Regression/FollowUpSchemaTests.cs +++ /dev/null @@ -1,61 +0,0 @@ -using System.Text.Json; -using ManagedCode.LlmTck.Gemini; -using ManagedCode.LlmTck.Scenarios; - -namespace ManagedCode.LlmTck.Tests.Regression; - -public sealed class FollowUpSchemaTests -{ - [Test] - [Arguments("minItems", "[]", "[1]", "ARRAY")] - [Arguments("maxItems", "[1,2]", "[1]", "ARRAY")] - [Arguments("minProperties", "{}", "{\"x\":1}", "OBJECT")] - [Arguments("maxProperties", "{\"x\":1,\"y\":2}", "{\"x\":1}", "OBJECT")] - [Arguments("minLength", "\"\"", "\"x\"", "STRING")] - [Arguments("maxLength", "\"xx\"", "\"x\"", "STRING")] - public async Task GeminiStringLimits_RejectOutOfRangeFixturesAsync(string keyword, string invalid, string valid, string type) - { - var schema = GeminiSchemaMapper.ToJsonSchema(JsonSerializer.Deserialize($$"""{"type":"{{type}}","{{keyword}}":"1"}"""))!; - await Assert.That(LlmTckJsonSchema.Matches(invalid, schema)).IsFalse(); - await Assert.That(LlmTckJsonSchema.Matches(valid, schema)).IsTrue(); - } - - [Test] - public async Task GeminiNullable_HandlesEnumsAlternativesAndLeavesExampleDataUntouchedAsync() - { - var source = JsonSerializer.Deserialize("""{"type":"STRING","nullable":true,"enum":["Paris"],"default":{"type":"OBJECT","minItems":"1"},"example":{"type":"STRING"}}"""); - var mapped = GeminiSchemaMapper.ToJsonSchema(source)!; - await Assert.That(LlmTckJsonSchema.Matches("null", mapped)).IsTrue(); - await Assert.That(LlmTckJsonSchema.Matches("\"Paris\"", mapped)).IsTrue(); - await Assert.That(LlmTckJsonSchema.Matches("\"London\"", mapped)).IsFalse(); - var document = JsonSerializer.Deserialize(mapped); - await Assert.That(document.GetProperty("anyOf")[0].GetProperty("default").GetProperty("type").GetString()).IsEqualTo("OBJECT"); - await Assert.That(document.GetProperty("anyOf")[0].GetProperty("default").GetProperty("minItems").GetString()).IsEqualTo("1"); - var alternatives = GeminiSchemaMapper.ToJsonSchema(JsonSerializer.Deserialize("""{"nullable":true,"anyOf":[{"type":"INTEGER","minimum":1,"maximum":2},{"type":"STRING","enum":["Paris"]}]}"""))!; - foreach (var valid in new[] { "null", "1", "\"Paris\"" }) { await Assert.That(LlmTckJsonSchema.Matches(valid, alternatives)).IsTrue(); } - foreach (var invalid in new[] { "3", "true", "\"London\"" }) { await Assert.That(LlmTckJsonSchema.Matches(invalid, alternatives)).IsFalse(); } - await Assert.That(GeminiSchemaMapper.ToJsonSchema(null)).IsNull(); - } - - [Test] - [Arguments("true")] - [Arguments("{\"type\":\"ARRAY\",\"minItems\":\"-1\"}")] - [Arguments("{\"type\":\"ARRAY\",\"minItems\":\"9223372036854775808\"}")] - public async Task GeminiInvalidSchema_ReturnsValidationFailureBeforeMappingAsync(string schema) - { - var request = JsonSerializer.Deserialize("{\"generationConfig\":{\"responseSchema\":" + schema + "}}"); - var validation = GeminiRequestValidation.Validate(request); - await Assert.That(validation.IsValid).IsFalse(); - await Assert.That(validation.Error).IsNotNull(); - } - - [Test] - public async Task GeminiJsonSchema_IsNotConvertedAsOpenApiSchemaAsync() - { - var request = JsonSerializer.Deserialize("""{"generationConfig":{"responseJsonSchema":{"type":["string","null"],"minLength":2}},"tools":[{"functionDeclarations":[{"name":"weather","parameters":{"type":"OBJECT","properties":{"city":{"type":"STRING","nullable":true}}}}]}]}""")!; - var mapped = GeminiWireMapper.ToRuntimeRequest("gpt-4.1-mini", request, false); - await Assert.That(mapped.ResponseSchemaJson).IsEqualTo("{\"type\":[\"string\",\"null\"],\"minLength\":2}"); - await Assert.That(LlmTckJsonSchema.Matches("{\"city\":null}", mapped.Tools.Single().ParametersJson)).IsTrue(); - await Assert.That(LlmTckJsonSchema.Matches("{\"city\":42}", mapped.Tools.Single().ParametersJson)).IsFalse(); - } -} diff --git a/tests/ManagedCode.LlmTck.Tests/Regression/ReviewRuntimeRegressionTests.cs b/tests/ManagedCode.LlmTck.Tests/Regression/ReviewRuntimeRegressionTests.cs deleted file mode 100644 index 49dbce6..0000000 --- a/tests/ManagedCode.LlmTck.Tests/Regression/ReviewRuntimeRegressionTests.cs +++ /dev/null @@ -1,96 +0,0 @@ -using ManagedCode.LlmTck.Configuration; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Scenarios; - -namespace ManagedCode.LlmTck.Tests.Regression; - -public sealed class ReviewRuntimeRegressionTests -{ - private static LlmTckChatRequest Request(string content = "whale") - { - return new() - { - ModelId = LlmTckKnownModelIds.Gpt41Mini, - Messages = [new() { Role = "user", Content = content }], - }; - } - - [Test] - public async Task PromptCache_EvictsOldestPrefixAtConfiguredCapacityAsync() - { - var runtime = new LlmTckRuntime(new LlmTckConfigurationBuilder().WithPromptCacheCapacity(1) - .AddChatScenario("queue", s => s.Responds("a").Responds("b").Responds("c").Responds("d")).Build()); - var first = Request("first whale") with { PromptCachePolicy = LlmTckPromptCachePolicy.Ollama }; - var second = Request("second whale") with { PromptCachePolicy = LlmTckPromptCachePolicy.Ollama }; - await runtime.CompleteChatAsync(first); - await Assert.That((await runtime.CompleteChatAsync(first)).Usage.CachedInputTokens).IsGreaterThan(0); - await runtime.CompleteChatAsync(second); - await Assert.That((await runtime.CompleteChatAsync(first)).Usage.CachedInputTokens).IsEqualTo(0); - } - - [Test] - public async Task PreCancelledRequest_PreservesFirstResponseAndJournalAsync() - { - var runtime = new LlmTckRuntime(new LlmTckConfigurationBuilder() - .AddChatScenario("queue", s => s.Responds("first").Responds("second")).Build()); - await Assert.That(async () => { await runtime.CompleteChatAsync(Request(), cancellationToken: new CancellationToken(true)); }) - .Throws(); - await Assert.That(runtime.GetAssertionSummary().TotalEvents).IsEqualTo(0); - await Assert.That((await runtime.CompleteChatAsync(Request())).Content).IsEqualTo("first"); - } - - [Test] - public async Task ConcurrentCancellation_ReturnsReservationWithoutReplayingCompletedResponseAsync() - { - var runtime = new LlmTckRuntime(new LlmTckConfigurationBuilder() - .AddChatScenario("queue", s => s.Responds("first").DelaysBy(100).Responds("second")).Build()); - using var cancellation = new CancellationTokenSource(); - var first = runtime.CompleteChatAsync(Request(), cancellationToken: cancellation.Token); - // CompleteChatAsync reserves synchronously before its first await; no timing sleep is needed. - await Assert.That((await runtime.CompleteChatAsync(Request())).Content).IsEqualTo("second"); - await cancellation.CancelAsync(); - await Assert.That(async () => await first).Throws(); - await Assert.That((await runtime.CompleteChatAsync(Request())).Content).IsEqualTo("first"); - await Assert.That((await runtime.CompleteChatAsync(Request())).StatusCode).IsEqualTo(409); - await Assert.That(runtime.GetAssertionSummary().Matched).IsEqualTo(2); - } - - [Test] - public async Task SameLocalScenarioId_RemainsIndependentAcrossDatasetsAndResetAsync() - { - var runtime = new LlmTckRuntime(new LlmTckConfigurationBuilder() - .AddDataset("a", d => d.AddChatScenario("greeting", s => s.WhenUserContains("first").Responds("A"))) - .AddDataset("b", d => d.AddChatScenario("greeting", s => s.WhenUserContains("second").Responds("B"))) - .Build()); - for (var iteration = 0; iteration < 2; iteration++) - { - await Assert.That((await runtime.CompleteChatAsync(Request("first"))).Content).IsEqualTo("A"); - await Assert.That((await runtime.CompleteChatAsync(Request("second"))).Content).IsEqualTo("B"); - await runtime.ResetAsync(); - } - } - - [Test] - public async Task RejectedAndScriptedErrorRequests_DoNotGenerateReasoningOrOutputUsageAsync() - { - var runtime = new LlmTckRuntime(new LlmTckConfigurationBuilder() - .RequireBearerToken("test-key").AddReasoningChatModel(LlmTckKnownModelIds.Gpt41Mini, 17) - .AddChatScenario("queue", s => s.WhenUserContains("whale").Fails(503, "unavailable", "failed request").Responds("blue whale")) - .Build()); - var unauthorized = await runtime.CompleteChatAsync(Request()); - var unmatched = await runtime.CompleteChatAsync(Request("other"), "test-key"); - var scripted = await runtime.CompleteChatAsync(Request(), "test-key"); - foreach (var response in new[] { unauthorized, unmatched, scripted }) - { - await Assert.That(response.IsSuccess).IsFalse(); - await Assert.That(response.Usage.OutputTokens).IsEqualTo(0); - await Assert.That(response.Usage.ReasoningTokens).IsEqualTo(0); - await Assert.That(response.Usage.TotalTokens).IsEqualTo(response.Usage.InputTokens); - } - await Assert.That(runtime.GetAssertionSummary().OutputTokens).IsEqualTo(0); - var successful = await runtime.CompleteChatAsync(Request(), "test-key"); - await Assert.That(successful.Usage.ReasoningTokens).IsEqualTo(17); - await Assert.That(successful.Usage.OutputTokens).IsGreaterThan(17); - } -} diff --git a/tests/ManagedCode.LlmTck.Tests/Regression/ToolFixtureStateTests.cs b/tests/ManagedCode.LlmTck.Tests/Regression/ToolFixtureStateTests.cs deleted file mode 100644 index 722381f..0000000 --- a/tests/ManagedCode.LlmTck.Tests/Regression/ToolFixtureStateTests.cs +++ /dev/null @@ -1,82 +0,0 @@ -using System.Text.Json; -using ManagedCode.LlmTck.Anthropic; -using ManagedCode.LlmTck.Bedrock; -using ManagedCode.LlmTck.Cohere; -using ManagedCode.LlmTck.Configuration; -using ManagedCode.LlmTck.Gemini; -using ManagedCode.LlmTck.Ollama; -using ManagedCode.LlmTck.OpenAI; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Scenarios; - -namespace ManagedCode.LlmTck.Tests.Regression; - -public sealed class ToolFixtureStateTests -{ - [Test] - public async Task ParallelCallsAndText_SurviveEveryMapperAndStreamingEventAsync() - { - var result = LlmTckChatResult.Success("gpt-4.1-mini", "tools", "checking", ["check", "ing"], new()) with - { ToolCalls = [new() { Id = "first", Name = "weather", ArgumentsJson = "{\"city\":\"Paris\"}" }, new() { Id = "second", Name = "weather", ArgumentsJson = "{\"city\":\"London\"}" }] }; - var outputs = new object[] - { - OpenAiWireMapper.ToChatResponse(result), OpenAiWireMapper.ToResponse(result, "resp_fixture", 0), - AnthropicWireMapper.ToMessageResponse(result), BedrockWireMapper.ToConverseResponse(result), - GeminiWireMapper.ToGenerateContentResponse(result, result.Content), OllamaWireMapper.ToChatResponse(result), CohereWireMapper.ToChatResponse(result), - }; - foreach (var output in outputs) - { - var json = JsonSerializer.Serialize(output); - await Assert.That(json).Contains("checking"); - await Assert.That(json).Contains("Paris"); - await Assert.That(json).Contains("London"); - } - var responseEvents = JsonSerializer.Serialize(OpenAiWireMapper.ToResponseStreamEvents(result, "resp_fixture", 0).ToArray()); - await Assert.That(responseEvents).Contains("response.output_text.delta"); - await Assert.That(responseEvents).Contains("response.function_call_arguments.delta"); - foreach (var events in new[] { AnthropicWireMapper.ToToolStreamEvents(result, 1), BedrockWireMapper.ToToolStreamEvents(result, 1), CohereWireMapper.ToToolStreamEvents(result) }) - { - var json = JsonSerializer.Serialize(events.ToArray()); - await Assert.That(json).Contains("Paris"); - await Assert.That(json).Contains("London"); - } - } - - [Test] - public async Task ToolHistory_IsSnapshottedAndParticipatesInExactMatchingAsync() - { - var call = new LlmTckToolCall { Id = "first", Name = "weather", ArgumentsJson = "{}" }; - var history = new LlmTckMessage { Role = "assistant", ToolCalls = [call] }; - var configuration = new LlmTckConfigurationBuilder().AddChatScenario("history", s => s.WithExactMatch(history).Responds("correct")).Build(); - var runtime = new LlmTckRuntime(configuration); - history.ToolCalls.Clear(); - configuration.ChatScenarios.Single().Match.Messages.Single().ToolCalls.Clear(); - var wrong = await runtime.CompleteChatAsync(new() { Messages = [history with { ToolCalls = [call with { Id = "other" }] }] }); - await Assert.That(wrong.StatusCode).IsEqualTo(404); - var right = await runtime.CompleteChatAsync(new() { Messages = [history with { ToolCalls = [call] }] }); - await Assert.That(right.Content).IsEqualTo("correct"); - await Assert.That(right.Usage.InputTokens).IsGreaterThan(0); - await Assert.That(runtime.GetAssertionSummary().Events.Last().Messages.Single().ToolCalls.Single().Id).IsEqualTo("first"); - } - - [Test] - public async Task ToolSchemas_ArePartOfPromptCacheIdentityAsync() - { - var runtime = new LlmTckRuntime(new LlmTckConfigurationBuilder().AddChatScenario("cache", s => s.Responds("one").Responds("two").Responds("three")).Build()); - var request = new LlmTckChatRequest { Messages = [new() { Role = "user", Content = "weather" }], Tools = [new() { Name = "weather" }], PromptCachePolicy = LlmTckPromptCachePolicy.Ollama }; - var first = await runtime.CompleteChatAsync(request); - await Assert.That(first.Usage.InputTokens).IsGreaterThan(1); - await Assert.That((await runtime.CompleteChatAsync(request)).Usage.CachedInputTokens).IsGreaterThan(0); - await Assert.That((await runtime.CompleteChatAsync(request with { Tools = [new() { Name = "other" }] })).Usage.CachedInputTokens).IsEqualTo(0); - } - - [Test] - public async Task Builder_ParallelCallsRemainIndependentOfCallerCollectionAsync() - { - var calls = new[] { new LlmTckToolCall { Id = "first", Name = "weather" }, new LlmTckToolCall { Id = "second", Name = "weather" } }; - var builder = new LlmTckScenarioBuilder("parallel").CallsTools(calls); - calls[0] = calls[0] with { Name = "changed" }; - await Assert.That(builder.Build().Responses.Single().ToolCalls.Select(call => call.Name)).IsEquivalentTo(new[] { "weather", "weather" }); - await Assert.That(() => new LlmTckScenarioBuilder("empty").CallsTools()).Throws(); - } -} diff --git a/tests/ManagedCode.LlmTck.Tests/Regression/ToolFixtureValidationTests.cs b/tests/ManagedCode.LlmTck.Tests/Regression/ToolFixtureValidationTests.cs deleted file mode 100644 index 39c3ebe..0000000 --- a/tests/ManagedCode.LlmTck.Tests/Regression/ToolFixtureValidationTests.cs +++ /dev/null @@ -1,90 +0,0 @@ -using System.Net; -using System.Text; -using ManagedCode.LlmTck.Configuration; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Scenarios; -using ManagedCode.LlmTck.Tests.TestSupport; - -namespace ManagedCode.LlmTck.Tests.Regression; - -public sealed class ToolFixtureValidationTests -{ - [Test] - [Arguments("/openai/v1/chat/completions", "\"tools\":null")] - [Arguments("/openai/v1/chat/completions", "\"tools\":[null]")] - [Arguments("/openai/v1/chat/completions", "\"tools\":[{\"type\":\"function\",\"function\":null}]")] - [Arguments("/openai/v1/chat/completions", "\"tools\":[{\"type\":\"web_search\"}]")] - [Arguments("/openai/v1/chat/completions", "\"tool_choice\":\"maybe\"")] - [Arguments("/openai/v1/chat/completions", "\"tool_choice\":{\"type\":\"function\",\"function\":{}}")] - [Arguments("/openai/v1/chat/completions", "\"response_format\":{\"type\":\"json_schema\"}")] - [Arguments("/openai/v1/chat/completions", "\"response_format\":{\"type\":\"xml\"}")] - [Arguments("/openai/v1/responses", "\"text\":{\"format\":{\"type\":\"json_schema\"}}")] - [Arguments("/anthropic/v1/messages", "\"tools\":[{\"name\":\"weather\"}]")] - [Arguments("/anthropic/v1/messages", "\"tool_choice\":{\"type\":\"tool\"}")] - [Arguments("/gemini/v1beta/models/gpt-4.1-mini:generateContent", "\"tools\":[{}]")] - [Arguments("/gemini/v1beta/models/gpt-4.1-mini:generateContent", "\"toolConfig\":{\"functionCallingConfig\":{\"mode\":\"WRONG\"}}")] - [Arguments("/ollama/api/chat", "\"format\":\"xml\"")] - [Arguments("/cohere/v2/chat", "\"tool_choice\":\"MAYBE\"")] - [Arguments("/bedrock/model/gpt-4.1-mini/converse", "\"toolConfig\":{\"tools\":[null]}")] - [Arguments("/bedrock/model/gpt-4.1-mini/converse", "\"outputConfig\":{\"textFormat\":{\"type\":\"json_schema\"}}")] - [Arguments("/openai/v1/chat/completions", "\"messages\":[{\"tool_calls\":[null]}]")] - [Arguments("/openai/v1/chat/completions", "\"messages\":[{\"tool_calls\":[{\"id\":\"call\",\"function\":{\"name\":\"weather\",\"arguments\":null}}]}]")] - [Arguments("/openai/v1/responses", "\"input\":[{\"type\":\"function_call\",\"name\":\"weather\"}]")] - [Arguments("/anthropic/v1/messages", "\"messages\":[{\"content\":[{\"type\":\"tool_use\"}]}]")] - [Arguments("/gemini/v1beta/models/gpt-4.1-mini:generateContent", "\"contents\":[{\"parts\":[{\"functionCall\":{}}]}]")] - [Arguments("/bedrock/model/gpt-4.1-mini/converse", "\"messages\":[{\"content\":[{\"toolResult\":{\"toolUseId\":\"call\",\"content\":[null]}}]}]")] - public async Task InvalidToolOptions_Return400BeforeRuntimeAsync(string path, string fields) - { - using var host = await LlmTckTestHost.StartAsync(); - using var http = host.GetTestClient(); - http.DefaultRequestHeaders.Add("anthropic-version", "2023-06-01"); - using var response = await http.PostAsync(path, new StringContent("{" + fields + "}", Encoding.UTF8, "application/json")); - await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); - } - - [Test] - [Arguments("{\"$ref\":\"https://example.com/schema\"}")] - [Arguments("{\"type\":42}")] - [Arguments("not json")] - public async Task InvalidSchema_ReturnsFixtureMismatchWithoutConsumingResponseAsync(string schema) - { - var runtime = new LlmTckRuntime(new LlmTckConfigurationBuilder().AddChatScenario("json", s => s.RespondsJson("{}")).Build()); - var request = new LlmTckChatRequest { Messages = [new() { Role = "user", Content = "json" }], RequireJson = true, ResponseSchemaJson = schema }; - await Assert.That((await runtime.CompleteChatAsync(request)).StatusCode).IsEqualTo(409); - await Assert.That((await runtime.CompleteChatAsync(request with { ResponseSchemaJson = "{\"type\":\"object\"}" })).Content).IsEqualTo("{}"); - } - - [Test] - public async Task RequiredAndNoneChoices_DoNotConsumeMismatchedToolFixturesAsync() - { - var runtime = new LlmTckRuntime(new LlmTckConfigurationBuilder().AddChatScenario("tools", s => s.CallsTool("weather", "{}").Responds("done")).Build()); - var request = new LlmTckChatRequest { Messages = [new() { Role = "user", Content = "weather" }], Tools = [new() { Name = "weather" }] }; - await Assert.That((await runtime.CompleteChatAsync(request with { ToolChoice = LlmTckToolChoice.None })).StatusCode).IsEqualTo(409); - await Assert.That((await runtime.CompleteChatAsync(request with { RequiredToolName = "other" })).StatusCode).IsEqualTo(409); - await Assert.That((await runtime.CompleteChatAsync(request with { Tools = [] })).StatusCode).IsEqualTo(409); - await Assert.That((await runtime.CompleteChatAsync(request with { ToolChoice = LlmTckToolChoice.Required })).ToolCalls.Single().Name).IsEqualTo("weather"); - await Assert.That((await runtime.CompleteChatAsync(request with { ToolChoice = LlmTckToolChoice.Required })).StatusCode).IsEqualTo(409); - await Assert.That((await runtime.CompleteChatAsync(request with { ToolChoice = LlmTckToolChoice.None })).Content).IsEqualTo("done"); - } - - [Test] - public async Task LocalSchemaReferences_AreEvaluatedAndInvalidArgumentsDoNotConsumeAsync() - { - var schema = """{"$defs":{"city":{"const":"Paris"}},"type":"object","properties":{"city":{"$ref":"#/$defs/city"}},"required":["city"]}"""; - await Assert.That(LlmTckJsonSchema.Matches("""{"city":"Paris"}""", schema)).IsTrue(); - await Assert.That(LlmTckJsonSchema.Matches("""{"city":"London"}""", schema)).IsFalse(); - var runtime = new LlmTckRuntime(new LlmTckConfigurationBuilder().AddChatScenario("tools", s => s.CallsTool("weather", """{"city":"London"}""")).Build()); - var request = new LlmTckChatRequest { Messages = [new() { Role = "user", Content = "weather" }], Tools = [new() { Name = "weather", ParametersJson = schema }] }; - await Assert.That((await runtime.CompleteChatAsync(request)).StatusCode).IsEqualTo(409); - await Assert.That((await runtime.CompleteChatAsync(request with { Tools = [new() { Name = "weather" }] })).ToolCalls.Single().ArgumentsJson).Contains("London"); - } - - [Test] - public async Task StructuredStream_ValidatesActualConcatenatedContentAsync() - { - var runtime = new LlmTckRuntime(new LlmTckConfigurationBuilder().AddChatScenario("json", s => s.Responds("{}", "not json")).Build()); - var request = new LlmTckChatRequest { Messages = [new() { Role = "user", Content = "json" }], RequireJson = true, Stream = true }; - await Assert.That((await runtime.CompleteChatAsync(request)).StatusCode).IsEqualTo(409); - await Assert.That((await runtime.CompleteChatAsync(request with { RequireJson = false })).Content).IsEqualTo("{}"); - } -} diff --git a/tests/ManagedCode.LlmTck.Tests/Runtime/LlmTckChatHistoryTests.cs b/tests/ManagedCode.LlmTck.Tests/Runtime/LlmTckChatHistoryTests.cs deleted file mode 100644 index a1f59b0..0000000 --- a/tests/ManagedCode.LlmTck.Tests/Runtime/LlmTckChatHistoryTests.cs +++ /dev/null @@ -1,154 +0,0 @@ -using ManagedCode.LlmTck.Client; -using ManagedCode.LlmTck.Configuration; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Scenarios; - -namespace ManagedCode.LlmTck.Tests.Runtime; - -public sealed class LlmTckChatHistoryTests -{ - private const string _prompt = "history-private-evidence"; - private const string _scope = "history-scope"; - private const string _token = "history-owner-token"; - - [Test] - public async Task Continuation_SnapshotsMessagesAndReplacesInstructionsAsync() - { - var runtime = CreateRuntime(); - var request = Request(); - request.Messages.Insert(0, new() { Role = "system", Content = "old-instruction" }); - request.Messages.Add(new() { Role = "assistant", ToolCalls = [new() { Id = "call_prior", Name = "read_document", ArgumentsJson = "{}" }] }); - var first = await runtime.CompleteChatAsync(request, _token); - request.Messages[1] = new() { Role = "user", Content = "mutated" }; - request.Messages.Last().ToolCalls.Clear(); - var next = Request() with - { - PreviousResponseId = first.ResponseId, - Messages = [new() { Role = "developer", Content = "new-instruction" }, new() { Role = "tool", ToolCallId = "call_prior", Content = "read result" }] - }; - var result = await runtime.CompleteChatAsync(next, _token); - await Assert.That(result.IsSuccess).IsTrue(); - await Assert.That(result.Content).IsEqualTo("second"); - var messages = runtime.GetAssertionSummary().Events.Last().Messages; - await Assert.That(messages.Any(message => message.Content == _prompt)).IsTrue(); - await Assert.That(messages.Any(message => message.Content is "mutated" or "old-instruction")).IsFalse(); - await Assert.That(messages[0].Content).IsEqualTo("new-instruction"); - await Assert.That(messages.SelectMany(message => message.ToolCalls).Single().Id).IsEqualTo("call_prior"); - } - - [Test] - [Arguments("missing")] - [Arguments("namespace")] - [Arguments("token")] - [Arguments("reset")] - [Arguments("configure")] - public async Task ForeignOrExpiredParent_FailsWithoutConsumingAResponseAsync(string change) - { - var runtime = CreateRuntime(); - var first = await runtime.CompleteChatAsync(Request(), _token); - if (change == "reset") { await runtime.ResetAsync(); } - if (change == "configure") { await runtime.ConfigureAsync(Configuration()); } - var request = Request() with { PreviousResponseId = first.ResponseId, Messages = [new() { Content = "new input only" }] }; - request = change switch - { - "missing" => request with { PreviousResponseId = "resp_missing" }, - "namespace" => request with { HistoryNamespace = "another-scope" }, - _ => request - }; - var rejected = await runtime.CompleteChatAsync(request, change == "token" ? "foreign-token" : _token); - await Assert.That(rejected.StatusCode).IsEqualTo(404); - await Assert.That(rejected.ErrorCode).IsEqualTo("response_not_found"); - await Assert.That(runtime.GetAssertionSummary().Events.Last().Messages.Any(message => message.Content == _prompt)).IsFalse(); - var accepted = await runtime.CompleteChatAsync(Request() with { StoreResponse = false }, _token); - await Assert.That(accepted.Content).IsEqualTo(change is "reset" or "configure" ? "first" : "second"); - } - - [Test] - public async Task Continuation_AllowsAnotherModelWithinTheSameCredentialAndProviderAsync() - { - var configuration = Configuration(); - configuration.ChatScenarios.Add(new LlmTckScenarioBuilder("changed-model") - .ForModel(LlmTckKnownModelIds.Gpt41).WhenUserContains(_prompt).Responds("changed model").Build()); - var runtime = new LlmTckRuntime(configuration); - var first = await runtime.CompleteChatAsync(Request(), _token); - var result = await runtime.CompleteChatAsync(Request() with - { - ModelId = LlmTckKnownModelIds.Gpt41, - PreviousResponseId = first.ResponseId, - Messages = [new() { Role = "user", Content = "continue with another model" }] - }, _token); - - await Assert.That(result.IsSuccess).IsTrue(); - await Assert.That(result.ModelId).IsEqualTo(LlmTckKnownModelIds.Gpt41); - await Assert.That(result.Content).IsEqualTo("changed model"); - var messages = runtime.GetAssertionSummary().Events.Last().Messages; - await Assert.That(messages.Any(message => message.Content == _prompt)).IsTrue(); - await Assert.That(messages.Any(message => message.Role == "assistant" && message.Content == "first")).IsTrue(); - } - - [Test] - public async Task StoreFalse_DoesNotRetainAResponseAsync() - { - var runtime = CreateRuntime(); - var result = await runtime.CompleteChatAsync(Request() with { StoreResponse = false }, _token); - await Assert.That(result.IsSuccess).IsTrue(); - await Assert.That(result.ResponseId).IsNull(); - } - - [Test] - [Arguments(0, 1000000L)] - [Arguments(100, 0L)] - [Arguments(100, 1L)] - public async Task CapacityRejection_DoesNotConsumeTheFixtureAsync(int count, long bytes) - { - var runtime = CreateRuntime(count, bytes); - var rejected = await runtime.CompleteChatAsync(Request(), _token); - await Assert.That(rejected.StatusCode).IsEqualTo(409); - await Assert.That(rejected.ErrorCode).IsEqualTo("llm_tck_response_capacity_exceeded"); - var accepted = await runtime.CompleteChatAsync(Request() with { StoreResponse = false }, _token); - await Assert.That(accepted.Content).IsEqualTo("first"); - } - - [Test] - public async Task CapacityAndNamespace_AreValidatedAndResetReleasesCapacityAsync() - { - await Assert.That(() => new LlmTckConfigurationBuilder().WithChatResponseCapacity(-1, 1)).Throws(); - await Assert.That(() => new LlmTckConfigurationBuilder().WithChatResponseCapacity(1, -1)).Throws(); - await Assert.That(() => new LlmTckRuntime(new() { MaxStoredChatResponses = -1 })).Throws(); - await Assert.That(() => new LlmTckRuntime(new() { MaxStoredChatResponseBytes = -1 })).Throws(); - var options = new LlmTckClientConfigurationBuilder().UseChatResponseCapacity(1, 1000000).Build(); - await Assert.That(options.MaxStoredChatResponses).IsEqualTo(1); - var runtime = CreateRuntime(1, 1000000); - await Assert.That((await runtime.CompleteChatAsync(Request() with { HistoryNamespace = null }, _token)).StatusCode).IsEqualTo(400); - await Assert.That((await runtime.CompleteChatAsync(Request(), _token)).IsSuccess).IsTrue(); - await Assert.That((await runtime.CompleteChatAsync(Request(), _token)).StatusCode).IsEqualTo(409); - await runtime.ResetAsync(); - await Assert.That((await runtime.CompleteChatAsync(Request(), _token)).IsSuccess).IsTrue(); - } - - private static LlmTckChatRequest Request() - { - return new() - { - ModelId = LlmTckKnownModelIds.Gpt41Mini, - HistoryNamespace = _scope, - StoreResponse = true, - Messages = [new() { Role = "user", Content = _prompt }] - }; - } - - private static LlmTckRuntime CreateRuntime(int count = 256, long bytes = 1000000) - { - return new(Configuration(count, bytes)); - } - - private static LlmTckConfiguration Configuration(int count = 256, long bytes = 1000000) - { - return new LlmTckConfigurationBuilder().WithChatResponseCapacity(count, bytes) - .AddModel(LlmTckKnownModelIds.Gpt41Mini, LlmTckModelKind.Chat) - .AddModel(LlmTckKnownModelIds.Gpt41, LlmTckModelKind.Chat) - .AddChatScenario(_prompt, scenario => scenario.ForModel(LlmTckKnownModelIds.Gpt41Mini) - .WhenUserContains(_prompt).Responds("first").Responds("second")).Build(); - } -} diff --git a/tests/ManagedCode.LlmTck.Tests/Runtime/LlmTckRuntimeTests.cs b/tests/ManagedCode.LlmTck.Tests/Runtime/LlmTckRuntimeTests.cs deleted file mode 100644 index 5fc83cd..0000000 --- a/tests/ManagedCode.LlmTck.Tests/Runtime/LlmTckRuntimeTests.cs +++ /dev/null @@ -1,942 +0,0 @@ -using System.Text; -using ManagedCode.LlmTck.Configuration; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Scenarios; - -namespace ManagedCode.LlmTck.Tests.Runtime; - -public sealed class LlmTckRuntimeTests -{ - [Test] - public async Task ConfiguredConstructor_SnapshotsInitialConfigurationAsync() - { - var configuration = new LlmTckConfigurationBuilder() - .AddModel(LlmTckKnownModelIds.Gpt41Mini, LlmTckModelKind.Chat) - .Build(); - var expectedModels = configuration.Models.Count; - var runtime = new LlmTckRuntime(configuration); - - configuration.Models.Clear(); - - await Assert.That(runtime.GetModels().Count).IsEqualTo(expectedModels); - await Assert.That(runtime.GetModels().Any(model => model.Id == LlmTckKnownModelIds.Gpt41Mini)) - .IsTrue(); - } - - private static readonly LlmTckChatRequest _largestAnimalRequest = new() - { - ModelId = LlmTckKnownModelIds.Gpt41Mini, - Messages = [new LlmTckMessage { Role = "user", Content = "What is the largest animal?" }], - }; - - [Test] - public async Task CompleteChatAsync_MatchesScenarioAndRecordsAssertionAsync() - { - const string userMessage = "What is the largest animal?"; - const string assistantMessage = "blue whale"; - var inputTokens = LlmTckTokenCounter.CountTextTokens(userMessage); - var outputTokens = LlmTckTokenCounter.CountTextTokens(assistantMessage); - - var runtime = new LlmTckRuntime(); - await runtime.ConfigureAsync( - new LlmTckConfigurationBuilder() - .AddChatScenario( - "blue-whale", - scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) - .WhenUserContains("largest animal") - .Responds(assistantMessage, "blue ", "whale") - ) - .Build() - ); - - var result = await runtime.CompleteChatAsync( - new LlmTckChatRequest - { - ModelId = LlmTckKnownModelIds.Gpt41Mini, - Messages = - [ - new LlmTckMessage - { - Role = "user", - Content = userMessage, - }, - ], - } - ); - - await Assert.That(result.IsSuccess).IsTrue(); - await Assert.That(result.Content).IsEqualTo("blue whale"); - await Assert.That(result.StreamChunks).IsEquivalentTo(["blue ", "whale"]); - await Assert.That(result.Usage.InputTokens).IsEqualTo(inputTokens); - await Assert.That(result.Usage.OutputTokens).IsEqualTo(outputTokens); - await Assert.That(result.Usage.TotalTokens).IsEqualTo(inputTokens + outputTokens); - - var summary = runtime.GetAssertionSummary(); - await Assert.That(summary.Matched).IsEqualTo(1); - await Assert.That(summary.Unmatched).IsEqualTo(0); - await Assert.That(summary.InputTokens).IsEqualTo(inputTokens); - await Assert.That(summary.OutputTokens).IsEqualTo(outputTokens); - await Assert.That(summary.TotalTokens).IsEqualTo(inputTokens + outputTokens); - await Assert.That(summary.Events[0].ScenarioId).IsEqualTo("blue-whale"); - await Assert.That(summary.Events[0].Usage?.InputTokens).IsEqualTo(inputTokens); - await Assert.That(summary.Events[0].Usage?.OutputTokens).IsEqualTo(outputTokens); - await Assert.That(summary.Events[0].Usage?.TotalTokens).IsEqualTo( - inputTokens + outputTokens - ); - } - - [Test] - public async Task CompleteChatAsync_CarriesConfiguredReasoningTokensAsync() - { - const string reasoningModel = "gpt-5-nano"; - const string userMessage = "Use reasoning."; - const string assistantMessage = "reasoned answer"; - const int reasoningTokens = 17; - var inputTokens = LlmTckTokenCounter.CountTextTokens(userMessage); - var visibleOutputTokens = LlmTckTokenCounter.CountTextTokens(assistantMessage); - - var runtime = new LlmTckRuntime(); - await runtime.ConfigureAsync( - new LlmTckConfigurationBuilder() - .AddReasoningChatModel(reasoningModel, reasoningTokens) - .AddChatScenario( - "reasoning-answer", - scenario => scenario - .ForModel(reasoningModel) - .WhenUserContains("reasoning") - .Responds(assistantMessage) - ) - .Build() - ); - - var result = await runtime.CompleteChatAsync( - new LlmTckChatRequest - { - ModelId = reasoningModel, - Messages = [new LlmTckMessage { Role = "user", Content = userMessage }], - } - ); - - await Assert.That(result.IsSuccess).IsTrue(); - await Assert.That(result.Usage.InputTokens).IsEqualTo(inputTokens); - await Assert.That(result.Usage.OutputTokens) - .IsEqualTo(visibleOutputTokens + reasoningTokens); - await Assert.That(result.Usage.ReasoningTokens).IsEqualTo(reasoningTokens); - await Assert.That(result.Usage.TotalTokens) - .IsEqualTo(inputTokens + visibleOutputTokens + reasoningTokens); - - var summary = runtime.GetAssertionSummary(); - await Assert.That(summary.OutputTokens).IsEqualTo(visibleOutputTokens + reasoningTokens); - await Assert.That(summary.ReasoningTokens).IsEqualTo(reasoningTokens); - await Assert.That(summary.Events[0].Usage?.ReasoningTokens).IsEqualTo(reasoningTokens); - } - - [Test] - public async Task CompleteChatAsync_TracksPromptCacheReadAndWriteTokensAsync() - { - var cacheableSystemPrompt = CreatePromptWithAtLeastTokens(1100); - var runtime = new LlmTckRuntime(); - await runtime.ConfigureAsync( - new LlmTckConfigurationBuilder() - .AddChatScenario( - "prompt-cache", - scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) - .WhenUserContains("cache turn") - .Responds("first") - .Responds("second") - ) - .Build() - ); - - var first = await runtime.CompleteChatAsync( - CreateCacheRequest(cacheableSystemPrompt, "cache turn one") - ); - var second = await runtime.CompleteChatAsync( - CreateCacheRequest(cacheableSystemPrompt, "cache turn two") - ); - - await Assert.That(first.IsSuccess).IsTrue(); - await Assert.That(first.Usage.CachedInputTokens).IsEqualTo(0); - await Assert.That(first.Usage.CacheCreationInputTokens).IsGreaterThanOrEqualTo(1024); - await Assert.That(second.IsSuccess).IsTrue(); - await Assert.That(second.Usage.CachedInputTokens).IsGreaterThanOrEqualTo(1024); - await Assert.That(second.Usage.CacheCreationInputTokens).IsEqualTo(0); - await Assert.That(second.Usage.TotalTokens) - .IsEqualTo(second.Usage.InputTokens + second.Usage.OutputTokens); - - var summary = runtime.GetAssertionSummary(); - await Assert.That(summary.CachedInputTokens).IsEqualTo(second.Usage.CachedInputTokens); - await Assert.That(summary.CacheCreationInputTokens) - .IsEqualTo(first.Usage.CacheCreationInputTokens); - - await runtime.ResetAsync(); - - var afterReset = await runtime.CompleteChatAsync( - CreateCacheRequest(cacheableSystemPrompt, "cache turn one") - ); - - await Assert.That(afterReset.IsSuccess).IsTrue(); - await Assert.That(afterReset.Usage.CachedInputTokens).IsEqualTo(0); - await Assert.That(afterReset.Usage.CacheCreationInputTokens) - .IsGreaterThanOrEqualTo(1024); - } - - [Test] - public async Task CompleteChatAsync_ReportsUnmatchedRequestAsync() - { - var runtime = new LlmTckRuntime(); - await runtime.ConfigureAsync(LlmTckConfiguration.CreateDefault()); - - var result = await runtime.CompleteChatAsync( - new LlmTckChatRequest - { - ModelId = LlmTckKnownModelIds.Gpt41Mini, - Messages = - [ - new LlmTckMessage - { - Role = "user", - Content = "No configured scenario matches this.", - }, - ], - } - ); - - await Assert.That(result.IsSuccess).IsFalse(); - await Assert.That(result.StatusCode).IsEqualTo(404); - await Assert.That(result.ErrorCode).IsEqualTo("llm_tck_unmatched_request"); - - var summary = runtime.GetAssertionSummary(); - await Assert.That(summary.Unmatched).IsEqualTo(1); - } - - [Test] - public async Task CompleteChatAsync_ReportsAuthModelErrorsExhaustionAndScriptedErrorsAsync() - { - var runtime = new LlmTckRuntime(); - await runtime.ConfigureAsync( - new LlmTckConfigurationBuilder() - .RequireBearerToken("test-key") - .AddModel("other-chat", LlmTckModelKind.Chat) - .AddChatScenario( - "other-model", - scenario => scenario - .ForModel("other-chat") - .WhenUserContains("largest animal") - .Responds("wrong model") - ) - .AddChatScenario( - "tenant-only", - scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) - .RequireBearerToken("test-key") - .WhenUserContains("tenant") - .Responds("tenant response") - ) - .AddChatScenario( - "single-use", - scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) - .WhenUserContains("largest animal") - .Responds("blue whale") - ) - .AddChatScenario( - "scripted-error", - scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) - .WhenUserContains("blocked fixture") - .Fails(400, "content_filter", "Scripted refusal.") - ) - .Build() - ); - - var globalAuth = await runtime.CompleteChatAsync(_largestAnimalRequest, bearerToken: "wrong-key"); - var unknownModel = await runtime.CompleteChatAsync( - _largestAnimalRequest with { ModelId = "missing-chat" }, - bearerToken: "test-key" - ); - var tenantResponse = await runtime.CompleteChatAsync( - _largestAnimalRequest with - { - Messages = [new LlmTckMessage { Role = "user", Content = "tenant request" }], - }, - bearerToken: "test-key" - ); - var first = await runtime.CompleteChatAsync(_largestAnimalRequest, bearerToken: "test-key"); - var exhausted = await runtime.CompleteChatAsync(_largestAnimalRequest, bearerToken: "test-key"); - var scriptedError = await runtime.CompleteChatAsync( - _largestAnimalRequest with - { - Messages = [new LlmTckMessage { Role = "user", Content = "blocked fixture" }], - }, - bearerToken: "test-key" - ); - - await Assert.That(globalAuth.StatusCode).IsEqualTo(401); - await Assert.That(unknownModel.ErrorCode).IsEqualTo("llm_tck_unknown_model"); - await Assert.That(tenantResponse.Content).IsEqualTo("tenant response"); - await Assert.That(first.Content).IsEqualTo("blue whale"); - await Assert.That(exhausted.StatusCode).IsEqualTo(409); - await Assert.That(exhausted.ErrorCode).IsEqualTo("llm_tck_scenario_exhausted"); - await Assert.That(scriptedError.StatusCode).IsEqualTo(400); - await Assert.That(scriptedError.ErrorCode).IsEqualTo("content_filter"); - - var summary = runtime.GetAssertionSummary(); - await Assert.That(summary.AuthFailed).IsEqualTo(1); - await Assert.That(summary.ModelNotFound).IsEqualTo(1); - await Assert.That(summary.Matched).IsEqualTo(2); - await Assert.That(summary.ScenarioExhausted).IsEqualTo(1); - await Assert.That(summary.ErrorsReturned).IsEqualTo(1); - } - - [Test] - public async Task CompleteChatAsync_UsesExactMatchAndRejectsShapeDriftAsync() - { - var runtime = new LlmTckRuntime(); - await runtime.ConfigureAsync( - new LlmTckConfigurationBuilder() - .AddChatScenario( - "exact-contract", - scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) - .WithExactMatch( - new LlmTckMessage { Role = "system", Content = "Return JSON only." }, - new LlmTckMessage { Role = "user", Content = "Give me the invoice total." } - ) - .Responds("""{"total":42.50}""") - ) - .Build() - ); - - var roleDrift = await runtime.CompleteChatAsync( - new LlmTckChatRequest - { - ModelId = LlmTckKnownModelIds.Gpt41Mini, - Messages = - [ - new LlmTckMessage { Role = "user", Content = "Return JSON only." }, - new LlmTckMessage { Role = "user", Content = "Give me the invoice total." }, - ], - } - ); - var countDrift = await runtime.CompleteChatAsync( - new LlmTckChatRequest - { - ModelId = LlmTckKnownModelIds.Gpt41Mini, - Messages = [new LlmTckMessage { Role = "system", Content = "Return JSON only." }], - } - ); - var exact = await runtime.CompleteChatAsync( - new LlmTckChatRequest - { - ModelId = LlmTckKnownModelIds.Gpt41Mini, - Messages = - [ - new LlmTckMessage { Role = "system", Content = "Return JSON only." }, - new LlmTckMessage { Role = "user", Content = "Give me the invoice total." }, - ], - } - ); - - await Assert.That(roleDrift.ErrorCode).IsEqualTo("llm_tck_unmatched_request"); - await Assert.That(countDrift.ErrorCode).IsEqualTo("llm_tck_unmatched_request"); - await Assert.That(exact.Content).IsEqualTo("""{"total":42.50}"""); - await Assert.That(runtime.GetAssertionSummary().Unmatched).IsEqualTo(2); - } - - [Test] - public async Task Modalities_ReturnDeterministicDefaultsAndUseBearerTokenAsync() - { - var runtime = new LlmTckRuntime(); - await runtime.ConfigureAsync( - new LlmTckConfigurationBuilder() - .RequireBearerToken("test-key") - .AddModel("text-model", LlmTckModelKind.Chat) - .WithDefaultEmbeddingVector(0.1f, 0.2f, 0.3f, 0.4f) - .WithDefaultVideo([0, 0, 0, 24, 102, 116, 121, 112], "video/test") - .WithDefaultTranscriptionText("hello from audio") - .WithDefaultTranslationText("hello translated audio") - .Build() - ); - - var unauthorized = await runtime.CreateEmbeddingAsync( - LlmTckKnownModelIds.TextEmbedding3Small, - ["input"], - bearerToken: "wrong-key" - ); - var embeddings = await runtime.CreateEmbeddingAsync( - LlmTckKnownModelIds.TextEmbedding3Small, - ["first", "second"], - bearerToken: "test-key" - ); - var image = await runtime.GenerateImageAsync( - LlmTckKnownModelIds.GptImage1, - "a compatibility test image", - bearerToken: "test-key" - ); - var audio = await runtime.GenerateAudioAsync( - LlmTckKnownModelIds.Gpt4OMiniTts, - "hello", - bearerToken: "test-key" - ); - var video = await runtime.GenerateVideoAsync( - LlmTckKnownModelIds.Sora2, - "generate a compatibility clip", - bearerToken: "test-key" - ); - var transcription = await runtime.TranscribeAudioAsync( - LlmTckKnownModelIds.Gpt4OMiniTts, - "fixture.wav", - bearerToken: "test-key" - ); - var translation = await runtime.TranslateAudioAsync( - LlmTckKnownModelIds.Gpt4OMiniTts, - "fixture.wav", - bearerToken: "test-key" - ); - - await Assert.That(unauthorized.StatusCode).IsEqualTo(401); - await Assert.That(embeddings.Vectors).Count().IsEqualTo(2); - await Assert.That(embeddings.Vectors[0]).IsEquivalentTo([0.1f, 0.2f, 0.3f, 0.4f]); - await Assert.That(image.DataUri).StartsWith("data:image/png;base64,"); - await Assert.That(audio.Bytes.Length).IsGreaterThan(0); - await Assert.That(video.Bytes).IsEquivalentTo((byte[])[0, 0, 0, 24, 102, 116, 121, 112]); - await Assert.That(video.MediaType).IsEqualTo("video/test"); - await Assert.That(video.Usage.InputTokens) - .IsEqualTo(LlmTckTokenCounter.CountTextTokens("generate a compatibility clip")); - await Assert.That(video.Usage.TotalTokens).IsEqualTo(video.Usage.InputTokens); - await Assert.That(transcription.Text).IsEqualTo("hello from audio"); - await Assert.That(translation.Text).IsEqualTo("hello translated audio"); - - var summary = runtime.GetAssertionSummary(); - await Assert.That(summary.AuthFailed).IsEqualTo(1); - await Assert.That(summary.Matched).IsEqualTo(6); - } - - [Test] - public async Task Modalities_RejectUnknownOrWrongKindModelsAsync() - { - var runtime = new LlmTckRuntime(); - await runtime.ConfigureAsync(LlmTckConfiguration.CreateDefault()); - - var missingEmbedding = await runtime.CreateEmbeddingAsync("missing-embedding", ["input"]); - var wrongKindImage = await runtime.GenerateImageAsync(LlmTckKnownModelIds.Gpt41Mini, "prompt"); - var missingAudio = await runtime.GenerateAudioAsync("missing-audio", "input"); - var wrongKindTranscription = await runtime.TranscribeAudioAsync(LlmTckKnownModelIds.Gpt41Mini, "fixture.wav"); - var wrongKindTranslation = await runtime.TranslateAudioAsync(LlmTckKnownModelIds.Gpt41Mini, "fixture.wav"); - var wrongKindVideo = await runtime.GenerateVideoAsync(LlmTckKnownModelIds.Gpt41Mini, "prompt"); - - await Assert.That(missingEmbedding.IsSuccess).IsFalse(); - await Assert.That(missingEmbedding.StatusCode).IsEqualTo(404); - await Assert.That(missingEmbedding.ErrorCode).IsEqualTo("llm_tck_unknown_model"); - await Assert.That(wrongKindImage.IsSuccess).IsFalse(); - await Assert.That(wrongKindImage.ErrorCode).IsEqualTo("llm_tck_unknown_model"); - await Assert.That(missingAudio.IsSuccess).IsFalse(); - await Assert.That(missingAudio.ErrorCode).IsEqualTo("llm_tck_unknown_model"); - await Assert.That(wrongKindTranscription.IsSuccess).IsFalse(); - await Assert.That(wrongKindTranscription.ErrorCode).IsEqualTo("llm_tck_unknown_model"); - await Assert.That(wrongKindTranslation.IsSuccess).IsFalse(); - await Assert.That(wrongKindTranslation.ErrorCode).IsEqualTo("llm_tck_unknown_model"); - await Assert.That(wrongKindVideo.IsSuccess).IsFalse(); - await Assert.That(wrongKindVideo.ErrorCode).IsEqualTo("llm_tck_unknown_model"); - - var summary = runtime.GetAssertionSummary(); - await Assert.That(summary.ModelNotFound).IsEqualTo(6); - await Assert.That(summary.Matched).IsEqualTo(0); - } - - [Test] - public async Task Modalities_ReportAuthFailuresForEveryProviderKindAsync() - { - var runtime = new LlmTckRuntime(); - await runtime.ConfigureAsync( - new LlmTckConfigurationBuilder() - .RequireBearerToken("test-key") - .Build() - ); - - var image = await runtime.GenerateImageAsync(LlmTckKnownModelIds.GptImage1, "prompt", bearerToken: "wrong-key"); - var audio = await runtime.GenerateAudioAsync(LlmTckKnownModelIds.Gpt4OMiniTts, "input", bearerToken: "wrong-key"); - var transcription = await runtime.TranscribeAudioAsync( - LlmTckKnownModelIds.Gpt4OMiniTts, - "fixture.wav", - bearerToken: "wrong-key" - ); - var translation = await runtime.TranslateAudioAsync( - LlmTckKnownModelIds.Gpt4OMiniTts, - "fixture.wav", - bearerToken: "wrong-key" - ); - var video = await runtime.GenerateVideoAsync( - LlmTckKnownModelIds.Sora2, - "prompt", - bearerToken: "wrong-key" - ); - - await Assert.That(image.StatusCode).IsEqualTo(401); - await Assert.That(image.ErrorCode).IsEqualTo("invalid_api_key"); - await Assert.That(audio.StatusCode).IsEqualTo(401); - await Assert.That(audio.ErrorCode).IsEqualTo("invalid_api_key"); - await Assert.That(transcription.StatusCode).IsEqualTo(401); - await Assert.That(transcription.ErrorCode).IsEqualTo("invalid_api_key"); - await Assert.That(translation.StatusCode).IsEqualTo(401); - await Assert.That(translation.ErrorCode).IsEqualTo("invalid_api_key"); - await Assert.That(video.StatusCode).IsEqualTo(401); - await Assert.That(video.ErrorCode).IsEqualTo("invalid_api_key"); - await Assert.That(runtime.GetAssertionSummary().AuthFailed).IsEqualTo(5); - } - - [Test] - public async Task FaultSimulation_ReturnsContentFilterAcrossRuntimeModalitiesAsync() - { - var runtime = new LlmTckRuntime(); - await runtime.ConfigureAsync( - new LlmTckConfigurationBuilder() - .SimulateContentFilter("blocked-term") - .Build() - ); - - var chat = await runtime.CompleteChatAsync( - _largestAnimalRequest with - { - Messages = [new LlmTckMessage { Role = "user", Content = "blocked-term" }], - } - ); - var embeddings = await runtime.CreateEmbeddingAsync(LlmTckKnownModelIds.TextEmbedding3Small, ["blocked-term"]); - var image = await runtime.GenerateImageAsync(LlmTckKnownModelIds.GptImage1, "blocked-term"); - var audio = await runtime.GenerateAudioAsync(LlmTckKnownModelIds.Gpt4OMiniTts, "blocked-term"); - var transcription = await runtime.TranscribeAudioAsync( - LlmTckKnownModelIds.Gpt4OMiniTts, - "fixture.wav", - prompt: "blocked-term" - ); - var translation = await runtime.TranslateAudioAsync( - LlmTckKnownModelIds.Gpt4OMiniTts, - "fixture.wav", - prompt: "blocked-term" - ); - var video = await runtime.GenerateVideoAsync(LlmTckKnownModelIds.Sora2, "blocked-term"); - - await AssertFaultAsync(chat.StatusCode, chat.ErrorCode, chat.ErrorMessage, "content_filter"); - await AssertFaultAsync( - embeddings.StatusCode, - embeddings.ErrorCode, - embeddings.ErrorMessage, - "content_filter" - ); - await AssertFaultAsync(image.StatusCode, image.ErrorCode, image.ErrorMessage, "content_filter"); - await AssertFaultAsync(audio.StatusCode, audio.ErrorCode, audio.ErrorMessage, "content_filter"); - await AssertFaultAsync( - transcription.StatusCode, - transcription.ErrorCode, - transcription.ErrorMessage, - "content_filter" - ); - await AssertFaultAsync( - translation.StatusCode, - translation.ErrorCode, - translation.ErrorMessage, - "content_filter" - ); - await AssertFaultAsync(video.StatusCode, video.ErrorCode, video.ErrorMessage, "content_filter"); - await Assert.That(runtime.GetAssertionSummary().ErrorsReturned).IsEqualTo(7); - } - - [Test] - public async Task FaultSimulation_RateLimitCountsRequestsAcrossModalitiesAndResetAsync() - { - var runtime = new LlmTckRuntime(); - await runtime.ConfigureAsync( - new LlmTckConfigurationBuilder() - .SimulateRateLimitAfter(2) - .AddChatScenario( - "rate-limit-reset", - scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) - .WhenUserContains("largest animal") - .Responds("blue whale") - ) - .Build() - ); - - var first = await runtime.CompleteChatAsync(_largestAnimalRequest); - var second = await runtime.CreateEmbeddingAsync(LlmTckKnownModelIds.TextEmbedding3Small, ["first"]); - var third = await runtime.GenerateImageAsync(LlmTckKnownModelIds.GptImage1, "after limit"); - - await Assert.That(first.IsSuccess).IsTrue(); - await Assert.That(second.IsSuccess).IsTrue(); - await AssertFaultAsync( - third.StatusCode, - third.ErrorCode, - third.ErrorMessage, - "too_many_requests" - ); - await Assert.That(runtime.GetAssertionSummary().ErrorsReturned).IsEqualTo(1); - - await runtime.ResetAsync(); - var afterReset = await runtime.CompleteChatAsync(_largestAnimalRequest); - - await Assert.That(afterReset.IsSuccess).IsTrue(); - await Assert.That(runtime.GetAssertionSummary().Matched).IsEqualTo(1); - await Assert.That(runtime.GetAssertionSummary().ErrorsReturned).IsEqualTo(0); - } - - [Test] - public async Task ConfigureAsync_SnapshotsMutableConfigurationAsync() - { - var runtime = new LlmTckRuntime(); - var configuration = new LlmTckConfigurationBuilder() - .AddChatScenario( - "snapshot-blue-whale", - scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) - .WhenUserContains("largest animal") - .Responds("blue whale", "blue ", "whale") - ) - .Build(); - - await runtime.ConfigureAsync(configuration); - - configuration.Models.Clear(); - configuration.ChatScenarios[0].Match.Messages.Clear(); - configuration.ChatScenarios[0].Responses[0].StreamChunks[0] = "mutated"; - configuration.DefaultEmbeddingVector[0] = 99f; - - var chat = await runtime.CompleteChatAsync( - new LlmTckChatRequest - { - ModelId = LlmTckKnownModelIds.Gpt41Mini, - Messages = [new LlmTckMessage { Role = "user", Content = "largest animal" }], - } - ); - var embeddings = await runtime.CreateEmbeddingAsync(LlmTckKnownModelIds.TextEmbedding3Small, ["input"]); - - await Assert.That(chat.IsSuccess).IsTrue(); - await Assert.That(chat.StreamChunks).IsEquivalentTo(["blue ", "whale"]); - await Assert.That(embeddings.Vectors[0][0]).IsEqualTo(0.125f); - } - - [Test] - public async Task ResetAsync_ClearsEventsAndScenarioPositionsWithoutReplacingConfigurationAsync() - { - var runtime = new LlmTckRuntime(); - await runtime.ConfigureAsync( - new LlmTckConfigurationBuilder() - .AddChatScenario( - "resettable", - scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) - .WhenUserContains("largest animal") - .Responds("blue whale") - ) - .Build() - ); - - var beforeReset = await runtime.CompleteChatAsync(_largestAnimalRequest); - - await runtime.ResetAsync(); - - var afterReset = await runtime.CompleteChatAsync(_largestAnimalRequest); - - await Assert.That(beforeReset.Content).IsEqualTo("blue whale"); - await Assert.That(afterReset.Content).IsEqualTo("blue whale"); - await Assert.That(runtime.GetAssertionSummary().Matched).IsEqualTo(1); - } - - [Test] - public async Task ResetAsync_PreventsInFlightRequestFromAppendingEventsOrCacheToNewGenerationAsync() - { - var cacheableSystemPrompt = CreatePromptWithAtLeastTokens(1100); - var request = CreateCacheRequest(cacheableSystemPrompt, "generation reset"); - var runtime = new LlmTckRuntime(); - await runtime.ConfigureAsync( - new LlmTckConfigurationBuilder() - .AddChatScenario( - "generation-reset", - scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) - .WhenUserContains("generation reset") - .Responds("fresh generation") - .DelaysBy(1500) - ) - .Build() - ); - - var supersededTask = runtime.CompleteChatAsync(request); - await Assert.That(supersededTask.IsCompleted).IsFalse(); - - await runtime.ResetAsync(); - - var afterResetSummary = runtime.GetAssertionSummary(); - await Assert.That(afterResetSummary.TotalEvents).IsEqualTo(0); - await Assert.That(afterResetSummary.TotalTokens).IsEqualTo(0); - - var freshTask = runtime.CompleteChatAsync(request); - var superseded = await supersededTask; - var fresh = await freshTask; - - await Assert.That(superseded.IsSuccess).IsFalse(); - await Assert.That(superseded.StatusCode).IsEqualTo(409); - await Assert.That(superseded.ErrorCode).IsEqualTo("llm_tck_request_superseded"); - await Assert.That(fresh.IsSuccess).IsTrue(); - await Assert.That(fresh.Content).IsEqualTo("fresh generation"); - await Assert.That(fresh.Usage.CachedInputTokens).IsEqualTo(0); - await Assert.That(fresh.Usage.CacheCreationInputTokens).IsGreaterThanOrEqualTo(1024); - - var finalSummary = runtime.GetAssertionSummary(); - await Assert.That(finalSummary.TotalEvents).IsEqualTo(1); - await Assert.That(finalSummary.Matched).IsEqualTo(1); - await Assert.That(finalSummary.CachedInputTokens).IsEqualTo(0); - await Assert.That(finalSummary.CacheCreationInputTokens) - .IsEqualTo(fresh.Usage.CacheCreationInputTokens); - await Assert.That(finalSummary.TotalTokens).IsEqualTo(fresh.Usage.TotalTokens); - } - - [Test] - public async Task RequestScopeStartedBeforeReset_CannotAppendAnEventAfterResetAsync() - { - var runtime = new LlmTckRuntime(); - await runtime.ConfigureAsync( - new LlmTckConfigurationBuilder() - .AddChatScenario( - "scope-reset", - scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) - .WhenUserContains("scope reset") - .Responds("must not survive reset") - ) - .Build() - ); - - using var requestScope = ((ILlmTckRuntimeRequestScope)runtime).BeginRequest( - "request-before-reset" - ); - await runtime.ResetAsync(); - var result = await runtime.CompleteChatAsync( - new LlmTckChatRequest - { - ModelId = LlmTckKnownModelIds.Gpt41Mini, - Messages = [new LlmTckMessage { Role = "user", Content = "scope reset" }], - } - ); - - await Assert.That(result.IsSuccess).IsFalse(); - await Assert.That(result.ErrorCode).IsEqualTo("llm_tck_request_superseded"); - await Assert.That(runtime.GetAssertionSummary().TotalEvents).IsEqualTo(0); - } - - [Test] - public async Task NonChatRequestScopeStartedBeforeReset_CannotConsumeNewFaultBudgetAsync() - { - var runtime = new LlmTckRuntime(); - await runtime.ConfigureAsync( - new LlmTckConfigurationBuilder().SimulateRateLimitAfter(1).Build() - ); - - LlmTckEmbeddingResult stale; - using (((ILlmTckRuntimeRequestScope)runtime).BeginRequest("embedding-before-reset")) - { - await runtime.ResetAsync(); - stale = await runtime.CreateEmbeddingAsync( - LlmTckKnownModelIds.TextEmbedding3Small, - ["stale"] - ); - } - - var fresh = await runtime.CreateEmbeddingAsync( - LlmTckKnownModelIds.TextEmbedding3Small, - ["fresh"] - ); - - await Assert.That(stale.IsSuccess).IsFalse(); - await Assert.That(stale.StatusCode).IsEqualTo(409); - await Assert.That(stale.ErrorCode).IsEqualTo("llm_tck_request_superseded"); - await Assert.That(fresh.IsSuccess).IsTrue(); - var summary = runtime.GetAssertionSummary(); - await Assert.That(summary.TotalEvents).IsEqualTo(1); - await Assert.That(summary.Matched).IsEqualTo(1); - await Assert.That(summary.ErrorsReturned).IsEqualTo(0); - } - - [Test] - public async Task RuntimeEventJournal_RetainsLatestFiveHundredWhileTotalsRemainCumulativeAsync() - { - const int TotalRequests = 501; - var runtime = new LlmTckRuntime(); - await runtime.ConfigureAsync(LlmTckConfiguration.CreateDefault()); - - for (var index = 0; index < TotalRequests; index++) - { - var result = await runtime.CompleteChatAsync( - new LlmTckChatRequest - { - RequestId = $"request-{index}", - ModelId = "missing-chat-model", - Messages = [new LlmTckMessage { Role = "user", Content = "retention" }], - } - ); - - await Assert.That(result.ErrorCode).IsEqualTo("llm_tck_unknown_model"); - } - - var summary = runtime.GetAssertionSummary(); - - await Assert.That(summary.TotalEvents).IsEqualTo(TotalRequests); - await Assert.That(summary.ModelNotFound).IsEqualTo(TotalRequests); - await Assert.That(summary.Events.Count).IsEqualTo(500); - await Assert.That(summary.Events[0].RequestId).IsEqualTo("request-1"); - await Assert.That(summary.Events[^1].RequestId).IsEqualTo("request-500"); - } - - [Test] - public async Task RuntimeEventJournal_BoundsExceptionalMetadataAndMarksTruncationAsync() - { - const int MetadataCharacterBudget = 4096; - var oversizedModelId = new string('x', MetadataCharacterBudget + 1024); - var runtime = new LlmTckRuntime(); - await runtime.ConfigureAsync(LlmTckConfiguration.CreateDefault()); - - var result = await runtime.CompleteChatAsync( - new LlmTckChatRequest - { - ModelId = oversizedModelId, - Messages = [], - } - ); - - await Assert.That(result.IsSuccess).IsFalse(); - var runtimeEvent = runtime.GetAssertionSummary().Events.Single(); - - await Assert.That(runtimeEvent.PayloadTruncated).IsTrue(); - await Assert.That(runtimeEvent.ModelId.Length).IsEqualTo(MetadataCharacterBudget); - await Assert.That(runtimeEvent.Message.Length).IsEqualTo(MetadataCharacterBudget); - await Assert.That(runtimeEvent.ModelId.Length).IsLessThan(oversizedModelId.Length); - } - - [Test] - public async Task RuntimeEventJournal_BoundsEmptyMessageFanoutByPerItemCostAsync() - { - const int SourceMessageCount = 33_000; - var messages = Enumerable - .Range(0, SourceMessageCount) - .Select(_ => new LlmTckMessage { Role = string.Empty, Content = string.Empty }) - .ToList(); - var runtime = new LlmTckRuntime(); - await runtime.ConfigureAsync( - new LlmTckConfigurationBuilder().AddDefaultOpenAiModels().Build() - ); - - var result = await runtime.CompleteChatAsync( - new LlmTckChatRequest - { - ModelId = LlmTckKnownModelIds.Gpt41Mini, - Messages = messages, - } - ); - - await Assert.That(result.ErrorCode).IsEqualTo("llm_tck_unmatched_request"); - var runtimeEvent = runtime.GetAssertionSummary().Events.Single(); - await Assert.That(runtimeEvent.PayloadTruncated).IsTrue(); - await Assert.That(runtimeEvent.Messages.Count).IsLessThan(SourceMessageCount); - await Assert.That(runtimeEvent.Messages.Count).IsGreaterThan(0); - } - - [Test] - public async Task CancelledDelayedChat_DoesNotConsumeScenarioResponseAsync() - { - var runtime = new LlmTckRuntime(); - await runtime.ConfigureAsync( - new LlmTckConfigurationBuilder() - .AddChatScenario( - "delayed-blue-whale", - scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) - .WhenUserContains("largest animal") - .Responds("blue whale") - .DelaysBy(500) - ) - .Build() - ); - - using var cts = new CancellationTokenSource(); - var pending = runtime.CompleteChatAsync( - new LlmTckChatRequest - { - ModelId = LlmTckKnownModelIds.Gpt41Mini, - Messages = [new LlmTckMessage { Role = "user", Content = "largest animal" }], - }, - cancellationToken: cts.Token - ); - cts.Cancel(); - var cancelled = false; - try - { - await pending; - } - catch (OperationCanceledException) when (cts.IsCancellationRequested) - { - cancelled = true; - } - - await Assert.That(cancelled).IsTrue(); - - var result = await runtime.CompleteChatAsync( - new LlmTckChatRequest - { - ModelId = LlmTckKnownModelIds.Gpt41Mini, - Messages = [new LlmTckMessage { Role = "user", Content = "largest animal" }], - } - ); - - await Assert.That(result.IsSuccess).IsTrue(); - await Assert.That(result.Content).IsEqualTo("blue whale"); - await Assert.That(runtime.GetAssertionSummary().Matched).IsEqualTo(1); - } - - private static LlmTckChatRequest CreateCacheRequest( - string systemPrompt, - string userPrompt - ) - { - return new() - { - ModelId = LlmTckKnownModelIds.Gpt41Mini, - PromptCachePolicy = LlmTckPromptCachePolicy.OpenAiCompatible, - Messages = - [ - new LlmTckMessage { Role = "system", Content = systemPrompt }, - new LlmTckMessage { Role = "user", Content = userPrompt }, - ], - }; - } - - private static string CreatePromptWithAtLeastTokens(int minimumTokens) - { - var builder = new StringBuilder("cacheable fixture"); - while (LlmTckTokenCounter.CountTextTokens(builder.ToString()) < minimumTokens) - { - builder.Append(" stable-prefix"); - } - - return builder.ToString(); - } - - private static async Task AssertFaultAsync( - int statusCode, - string? errorCode, - string? errorMessage, - string expectedCode - ) - { - await Assert.That(statusCode).IsEqualTo( - string.Equals(expectedCode, "too_many_requests", StringComparison.Ordinal) - ? 429 - : 400 - ); - await Assert.That(errorCode).IsEqualTo(expectedCode); - await Assert.That(errorMessage).Contains(expectedCode); - } -} diff --git a/tests/ManagedCode.LlmTck.Tests/Runtime/LlmTckRuntimeToolJournalTests.cs b/tests/ManagedCode.LlmTck.Tests/Runtime/LlmTckRuntimeToolJournalTests.cs deleted file mode 100644 index fd7656c..0000000 --- a/tests/ManagedCode.LlmTck.Tests/Runtime/LlmTckRuntimeToolJournalTests.cs +++ /dev/null @@ -1,59 +0,0 @@ -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Scenarios; - -namespace ManagedCode.LlmTck.Tests.Runtime; - -public sealed class LlmTckRuntimeToolJournalTests -{ - [Test] - public async Task OfferedToolsRemainStructuredAndIndependentFromTheCallerAsync() - { - var tools = new List - { - new() { Name = "search", Description = "Find a document", ParametersJson = "{\"type\":\"object\"}" }, - new() { Name = "read", Description = null } - }; - var runtime = new LlmTckRuntime(); - await runtime.CompleteChatAsync(new LlmTckChatRequest { ModelId = "missing", Tools = tools }); - tools.Clear(); - var entry = runtime.GetAssertionSummary().Events.Single(); - await Assert.That(entry.PayloadTruncated).IsFalse(); - await Assert.That(entry.Tools.Select(tool => tool.Name)).IsEquivalentTo(["search", "read"]); - await Assert.That(entry.Tools[0].Description).IsEqualTo("Find a document"); - await Assert.That(entry.Tools[0].ParametersJson).IsEqualTo("{\"type\":\"object\"}"); - await Assert.That(entry.Tools[1].Description).IsNull(); - await Assert.That(() => ((IList)entry.Tools).Clear()).Throws(); - } - - [Test] - public async Task EmptyToolFanoutIsBoundedByTheJournalBudgetAsync() - { - const int sourceCount = 33000; - var runtime = new LlmTckRuntime(); - await runtime.CompleteChatAsync(new LlmTckChatRequest - { - ModelId = "missing", - Tools = Enumerable.Range(0, sourceCount).Select(_ => new LlmTckToolDefinition - { Name = string.Empty, ParametersJson = string.Empty }).ToList() - }); - var entry = runtime.GetAssertionSummary().Events.Single(); - await Assert.That(entry.PayloadTruncated).IsTrue(); - await Assert.That(entry.Tools.Count).IsGreaterThan(0).And.IsLessThan(sourceCount); - await Assert.That(entry.Tools.Count * 256).IsLessThanOrEqualTo(8 * 1024 * 1024); - } - [Test] - public async Task OversizedToolSchemaIsTruncatedAndDoesNotEscapeThePayloadBudgetAsync() - { - const int budget = 8 * 1024 * 1024; - var runtime = new LlmTckRuntime(); - await runtime.CompleteChatAsync(new LlmTckChatRequest - { - ModelId = "missing", - Tools = [new() { Name = "search", ParametersJson = "{\"type\":\"object\",\"description\":\"" + string.Concat(Enumerable.Repeat("word ", budget / 5 + 1)) + "\"}" }] - }); - var entry = runtime.GetAssertionSummary().Events.Single(); - await Assert.That(entry.PayloadTruncated).IsTrue(); - await Assert.That(entry.Tools.Single().Name).IsEqualTo("search"); - await Assert.That(entry.Tools.Single().ParametersJson.Length).IsGreaterThan(0).And.IsLessThan(budget); - } -} diff --git a/tests/ManagedCode.LlmTck.Tests/Runtime/LlmTckTokenCounterTests.cs b/tests/ManagedCode.LlmTck.Tests/Runtime/LlmTckTokenCounterTests.cs deleted file mode 100644 index 45223f7..0000000 --- a/tests/ManagedCode.LlmTck.Tests/Runtime/LlmTckTokenCounterTests.cs +++ /dev/null @@ -1,25 +0,0 @@ -using ManagedCode.LlmTck.Runtime; - -namespace ManagedCode.LlmTck.Tests.Runtime; - -public sealed class LlmTckTokenCounterTests -{ - [Test] - [Arguments("hello,world", 3)] - [Arguments("Text tokenization is the process of splitting a string into a list of tokens.", 16)] - [Arguments("antidisestablishmentarianism", 6)] - public async Task CountTextTokens_UsesTiktokenO200kBaseCountsAsync( - string text, - int expectedTokens - ) - { - await Assert.That(LlmTckTokenCounter.CountTextTokens(text)).IsEqualTo(expectedTokens); - } - - [Test] - public async Task CountTextTokens_ReturnsZeroForMissingTextAsync() - { - await Assert.That(LlmTckTokenCounter.CountTextTokens(null)).IsEqualTo(0); - await Assert.That(LlmTckTokenCounter.CountTextTokens(string.Empty)).IsEqualTo(0); - } -} diff --git a/tests/ManagedCode.LlmTck.Tests/TestSupport/TestChatClient.cs b/tests/ManagedCode.LlmTck.Tests/TestSupport/TestChatClient.cs deleted file mode 100644 index 142f018..0000000 --- a/tests/ManagedCode.LlmTck.Tests/TestSupport/TestChatClient.cs +++ /dev/null @@ -1,55 +0,0 @@ -using Microsoft.Extensions.AI; - -namespace ManagedCode.LlmTck.Tests.TestSupport; - -internal sealed class TestChatClient : IChatClient -{ - public Func, ChatOptions?, CancellationToken, Task>? - GetResponseAsyncCallback - { - get; - init; - } - - public Func< - IEnumerable, - ChatOptions?, - CancellationToken, - IAsyncEnumerable - >? GetStreamingResponseAsyncCallback - { - get; - init; - } - - public Task GetResponseAsync( - IEnumerable messages, - ChatOptions? options = null, - CancellationToken cancellationToken = default - ) - { - return GetResponseAsyncCallback?.Invoke(messages, options, cancellationToken) - ?? throw new InvalidOperationException("GetResponseAsyncCallback was not configured."); - } - - public IAsyncEnumerable GetStreamingResponseAsync( - IEnumerable messages, - ChatOptions? options = null, - CancellationToken cancellationToken = default - ) - { - return GetStreamingResponseAsyncCallback?.Invoke(messages, options, cancellationToken) - ?? throw new InvalidOperationException( - "GetStreamingResponseAsyncCallback was not configured." - ); - } - - public object? GetService(Type serviceType, object? serviceKey = null) - { - return serviceType.IsInstanceOfType(this) ? this : null; - } - - public void Dispose() - { - } -} diff --git a/tests/ManagedCode.LlmTck.Tests/Anthropic/AnthropicEndpointTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Anthropic/AnthropicEndpointTests.cs similarity index 91% rename from tests/ManagedCode.LlmTck.Tests/Anthropic/AnthropicEndpointTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/Anthropic/AnthropicEndpointTests.cs index aa9235f..48c2f55 100644 --- a/tests/ManagedCode.LlmTck.Tests/Anthropic/AnthropicEndpointTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Anthropic/AnthropicEndpointTests.cs @@ -2,11 +2,11 @@ using System.Net.Http.Json; using System.Text; using System.Text.Json; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Tests.TestSupport; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Tests.TestSupport; -namespace ManagedCode.LlmTck.Tests.Anthropic; +namespace ManagedCode.LlmTestServer.Tests.Anthropic; public sealed class AnthropicEndpointTests { @@ -15,9 +15,9 @@ public sealed class AnthropicEndpointTests [Test] public async Task MessagesEndpoint_ReturnsAnthropicMessageShapeAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options + using var host = await LlmTestServerTestHost.StartAsync(options => options .RequireBearerToken("test-key") - .AddModel("claude-test", LlmTckModelKind.Chat) + .AddModel("claude-test", LlmTestServerModelKind.Chat) .AddChatScenario( "anthropic-blue-whale", scenario => scenario @@ -65,8 +65,8 @@ await Assert.That( public async Task MessagesEndpoint_WithCacheControl_ReturnsAnthropicCacheUsageAsync() { var cacheableSystemPrompt = CreatePromptWithAtLeastTokens(1100); - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("claude-cache-test", LlmTckModelKind.Chat) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("claude-cache-test", LlmTestServerModelKind.Chat) .AddChatScenario( "anthropic-cache", scenario => scenario @@ -111,8 +111,8 @@ await Assert.That(secondUsage.GetProperty("cache_read_input_tokens").GetInt32()) [Test] public async Task MessagesEndpoint_StreamsAnthropicServerSentEventsAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("claude-test", LlmTckModelKind.Chat) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("claude-test", LlmTestServerModelKind.Chat) .AddChatScenario( "anthropic-stream", scenario => scenario @@ -164,7 +164,7 @@ await Assert.That(response.Content.Headers.ContentType?.MediaType) [Test] public async Task MessagesEndpoint_RequiresAnthropicVersionHeaderAsync() { - using var host = await LlmTckTestHost.StartAsync(); + using var host = await LlmTestServerTestHost.StartAsync(); using var client = host.GetTestClient(); var response = await client.PostAsJsonAsync( @@ -188,7 +188,7 @@ await Assert.That(payload.GetProperty("error").GetProperty("type").GetString()) [Test] public async Task MessagesEndpoint_AcceptsAnthropicApiKeyHeaderAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options + using var host = await LlmTestServerTestHost.StartAsync(options => options .RequireBearerToken("test-key") .AddChatScenario( "anthropic-api-key", @@ -238,7 +238,7 @@ string userPrompt private static string CreatePromptWithAtLeastTokens(int minimumTokens) { var builder = new StringBuilder("cacheable fixture"); - while (LlmTckTokenCounter.CountTextTokens(builder.ToString()) < minimumTokens) + while (LlmTestServerTokenCounter.CountTextTokens(builder.ToString()) < minimumTokens) { builder.Append(" stable-prefix"); } diff --git a/tests/ManagedCode.LlmTestServer.Tests/Anthropic/AnthropicNativeWebSearchBoundaryIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Anthropic/AnthropicNativeWebSearchBoundaryIntegrationTests.cs new file mode 100644 index 0000000..528855f --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Anthropic/AnthropicNativeWebSearchBoundaryIntegrationTests.cs @@ -0,0 +1,102 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json.Nodes; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Scenarios; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.Anthropic; + +public sealed partial class AnthropicNativeWebSearchIntegrationTests +{ + [Test] + [Arguments("numericId")] + [Arguments("longId")] + [Arguments("emptyQuery")] + [Arguments("wrongResultShape")] + [Arguments("unknownError")] + [Arguments("nullOpaque")] + [Arguments("nativeUser")] + [Arguments("unknownCitation")] + public async Task MalformedNativeHistory_Returns400AndPreservesQueueAsync(string defect) + { + using var host = await LlmTestServerTestHost.StartAsync(); using var http = host.GetTestClient(); Headers(http); + var control = new LlmTestServerClient(http); await control.ConfigureAsync(Configuration(2)); + var first = await CompleteAsync(http); + var content = JsonNode.Parse(first.GetProperty("content").GetRawText())!; + var role = "assistant"; + switch (defect) + { + case "numericId": content[0]!["id"] = 123; break; + case "longId": content[0]!["id"] = new string('x', 129); break; + case "emptyQuery": content[0]!["input"]!["query"] = ""; break; + case "wrongResultShape": content[1]!["content"] = "not a result"; break; + case "unknownError": content[1]!["content"] = new JsonObject { ["type"] = "web_search_tool_result_error", ["error_code"] = "invented" }; break; + case "nullOpaque": content[1]!["content"]![0]!["encrypted_content"] = null; break; + case "nativeUser": role = "user"; break; + case "unknownCitation": content[2]!["citations"]![0]!["type"] = "invented"; break; + } + var body = Request(); ((JsonArray)body["messages"]!).Add(new JsonObject { ["role"] = role, ["content"] = content }); + ((JsonArray)body["messages"]!).Add(new JsonObject { ["role"] = "user", ["content"] = "follow" }); + using var rejected = await http.PostAsJsonAsync("/anthropic/v1/messages", body); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + await AssertContentAsync((await CompleteAsync(http)).GetProperty("content")); + } + + [Test] + [Arguments("/openai/v1/responses")] + [Arguments("/openai/v1/chat/completions")] + public async Task AnthropicNativeDeclarations_DoNotLeakIntoOpenAiNamespacesAsync(string path) + { + using var host = await LlmTestServerTestHost.StartAsync(); using var http = host.GetTestClient(); Headers(http); + await new LlmTestServerClient(http).ConfigureAsync(Configuration()); + var body = Request(); body["input"] = "start"; + using var rejected = await http.PostAsJsonAsync(path, body); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + await AssertContentAsync((await CompleteAsync(http)).GetProperty("content")); + } + + [Test] + public async Task ProviderAuthenticationFailure_DoesNotConsumeNativeFixtureAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); using var http = host.GetTestClient(); Headers(http); + var configuration = Configuration(); configuration = configuration with { RequiredBearerToken = "provider-key" }; + var control = new LlmTestServerClient(http, "provider-key"); await control.ConfigureAsync(configuration); + http.DefaultRequestHeaders.Remove("x-api-key"); http.DefaultRequestHeaders.Add("x-api-key", "wrong-key"); + using var rejected = await http.PostAsJsonAsync("/anthropic/v1/messages", Request()); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.Unauthorized); + http.DefaultRequestHeaders.Remove("x-api-key"); http.DefaultRequestHeaders.Add("x-api-key", "provider-key"); + await AssertContentAsync((await CompleteAsync(http)).GetProperty("content")); + } + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task UnicodeDomainPaths_AllowColonAndAtSignAndApplyBlockingWithoutConsumingFixtureAsync(bool blocked) + { + using var host = await LlmTestServerTestHost.StartAsync(); using var http = host.GetTestClient(); Headers(http); + const string domainPath = "example.com/джерело:версія/@author"; + const string sourceUrl = "https://example.com/джерело:версія/@author/article"; + var items = Items(); + items[1] = ((LlmTestServerResponseServerToolResultItem)items[1]) with + { Result = new LlmTestServerWebSearchResults { Results = [new() { Url = sourceUrl, Title = _title, EncryptedContent = _opaque }] } }; + items[2] = ((LlmTestServerResponseMessageItem)items[2]) with + { Citations = [new LlmTestServerAnthropicWebSearchCitation { Url = sourceUrl, Title = _title, EncryptedIndex = _index, CitedText = "Джерело" }] }; + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(config => config.UseChatModel(_model).UseChatScenario("unicode-domain-path", scenario => + scenario.ForModel(_model).RespondsWithItems(items))); + var body = Request(); var tool = body["tools"]![0]!; + tool[blocked ? "blocked_domains" : "allowed_domains"] = new JsonArray(domainPath); + if (blocked) + { + using var rejected = await http.PostAsJsonAsync("/anthropic/v1/messages", body); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + tool["blocked_domains"] = new JsonArray("example.com/інше:значення/@author"); + } + var response = await CompleteAsync(http, body); + await Assert.That(response.GetProperty("content")[1].GetProperty("content")[0].GetProperty("url").GetString()).IsEqualTo(sourceUrl); + await Assert.That(response.GetProperty("content")[2].GetProperty("citations")[0].GetProperty("url").GetString()).IsEqualTo(sourceUrl); + await Assert.That(response.GetProperty("content")[2].GetProperty("citations")[0].GetProperty("cited_text").GetString()).IsEqualTo("Джерело"); + await Assert.That(response.GetProperty("usage").GetProperty("server_tool_use").GetProperty("web_search_requests").GetInt32()).IsEqualTo(1); + } + +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Anthropic/AnthropicNativeWebSearchHistoryIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Anthropic/AnthropicNativeWebSearchHistoryIntegrationTests.cs new file mode 100644 index 0000000..a08924d --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Anthropic/AnthropicNativeWebSearchHistoryIntegrationTests.cs @@ -0,0 +1,89 @@ +using System.Text.Json.Nodes; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Scenarios; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.Anthropic; + +public sealed partial class AnthropicNativeWebSearchIntegrationTests +{ + [Test] + public async Task MixedClientFunctionRound_ExplicitNativeHistoryPreservesPairOpaqueDataAndLogicalTurnsAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); using var http = host.GetTestClient(); Headers(http); + var control = new LlmTestServerClient(http); + var call = new LlmTestServerToolCall { Id = "toolu_client", Name = "client_fixture", ArgumentsJson = "{}" }; + await control.ConfigureAsync(config => config.UseChatModel(_model).UseConversation("anthropic-round", flow => flow.ForModel(_model) + .Step("search", step => step.WhenTurn(1).WhenLatestUserEquals("start").RespondsWithItems( + new LlmTestServerResponseFunctionCallItem { Id = "function_fixture", Call = call }, Items()[0], Items()[1], Items()[2]).Then("final")) + .Step("final", step => step.WhenTurn(2).WhenToolResult(call.Name, call.Id).WhenLatestTextContains("client result").Responds("Complete client round.").Complete()))); + using var scoped = control.ForConversation("anthropic-round"); + var body = Request(); ((JsonArray)body["tools"]!).Add(new JsonObject { ["name"] = call.Name, ["input_schema"] = new JsonObject { ["type"] = "object" } }); + var first = await CompleteAsync(scoped.HttpClient, body); + await Assert.That(first.GetProperty("content")[0].GetProperty("type").GetString()).IsEqualTo("tool_use"); + await Assert.That(first.GetProperty("content")[0].GetProperty("id").GetString()).IsEqualTo(call.Id); + var messages = (JsonArray)body["messages"]!; + messages.Add(new JsonObject { ["role"] = "assistant", ["content"] = JsonNode.Parse(first.GetProperty("content").GetRawText()) }); + messages.Add(new JsonObject + { + ["role"] = "user", + ["content"] = new JsonArray(new JsonObject + { ["type"] = "tool_result", ["tool_use_id"] = call.Id, ["content"] = "client result" }) + }); + var final = await CompleteAsync(scoped.HttpClient, body); + await Assert.That(final.GetProperty("content")[0].GetProperty("text").GetString()).IsEqualTo("Complete client round."); + await Assert.That((await scoped.Control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(2); + var trace = (await scoped.Control.GetRequestsAsync()).Requests[0]; + var history = trace.Messages.Where(message => message.Role == "assistant").SelectMany(message => message.OutputItems).ToArray(); + await Assert.That(history.Length).IsEqualTo(4); + await Assert.That(((LlmTestServerResponseServerToolUseItem)history[1]).Id).IsEqualTo(_callId); + await Assert.That(((LlmTestServerWebSearchResults)((LlmTestServerResponseServerToolResultItem)history[2]).Result).Results.Single().EncryptedContent).IsEqualTo(_opaque); + await Assert.That(trace.Messages.Single(message => message.Role == "tool").ToolCallId).IsEqualTo(call.Id); + } + + [Test] + public async Task OpaqueResultsAndCitationMetadata_DoNotInflateResponseOrHistoricalInputTokensAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); using var http = host.GetTestClient(); Headers(http); + var config = Configuration(4); var changed = Items(); + var url = "https://example.com/" + new string('a', 1500); + changed[1] = ((LlmTestServerResponseServerToolResultItem)changed[1]) with + { + Result = new LlmTestServerWebSearchResults + { Results = [new() { Url = url, Title = new string('t', 1500), EncryptedContent = new string('e', 10000), PageAge = new string('p', 1500) }] } + }; + changed[2] = ((LlmTestServerResponseMessageItem)changed[2]) with + { + Citations = [new LlmTestServerAnthropicWebSearchCitation + { Url = url, Title = new string('t', 1500), CitedText = new string('c', 1500), EncryptedIndex = new string('i', 10000) }] + }; + config.ChatScenarios[0].Responses[1] = config.ChatScenarios[0].Responses[1] with { OutputItems = [.. changed] }; + var control = new LlmTestServerClient(http); await control.ConfigureAsync(config); + var first = await CompleteAsync(http); var second = await CompleteAsync(http); + await Assert.That(second.GetProperty("usage").GetProperty("output_tokens").GetInt32()).IsEqualTo(first.GetProperty("usage").GetProperty("output_tokens").GetInt32()); + var inputs = new List(); + foreach (var previous in new[] { first, second }) + { + var body = Request(); ((JsonArray)body["messages"]!).Add(new JsonObject { ["role"] = "assistant", ["content"] = JsonNode.Parse(previous.GetProperty("content").GetRawText()) }); + ((JsonArray)body["messages"]!).Add(new JsonObject { ["role"] = "user", ["content"] = "follow" }); + var result = await CompleteAsync(http, body); var usage = result.GetProperty("usage"); + inputs.Add(usage.GetProperty("input_tokens").GetInt32() + (usage.TryGetProperty("cache_read_input_tokens", out var read) ? read.GetInt32() : 0) + (usage.TryGetProperty("cache_creation_input_tokens", out var write) ? write.GetInt32() : 0)); + } + await Assert.That(inputs[1]).IsEqualTo(inputs[0]); + var historical = (await control.GetRequestsAsync()).Requests[0].Messages.Where(message => message.Role == "assistant").SelectMany(message => message.OutputItems).ToArray(); + await Assert.That(((LlmTestServerWebSearchResults)((LlmTestServerResponseServerToolResultItem)historical[1]).Result).Results.Single().EncryptedContent.Length).IsEqualTo(10000); + } + + [Test] + public async Task CompletedNativeGraph_ReplaysExactContentWithoutAdvancingAgainAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); using var http = host.GetTestClient(); Headers(http); + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(config => config.UseChatModel(_model).UseConversation("anthropic-replay", flow => flow.ForModel(_model) + .Step("search", step => step.RespondsWithItems(Items()).Complete()))); + var first = await CompleteAsync(http); var replay = await CompleteAsync(http); + await Assert.That(replay.GetProperty("content").GetRawText()).IsEqualTo(first.GetProperty("content").GetRawText()); + await Assert.That((await control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); + await Assert.That((await control.GetRequestsAsync()).Requests.Count(trace => trace.IsReplay)).IsEqualTo(1); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Anthropic/AnthropicNativeWebSearchIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Anthropic/AnthropicNativeWebSearchIntegrationTests.cs new file mode 100644 index 0000000..012d8f0 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Anthropic/AnthropicNativeWebSearchIntegrationTests.cs @@ -0,0 +1,92 @@ +using System.Net.Http.Json; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Configuration; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Scenarios; + +namespace ManagedCode.LlmTestServer.Tests.Anthropic; + +public sealed partial class AnthropicNativeWebSearchIntegrationTests +{ + private const string _model = LlmTestServerKnownModelIds.ClaudeSonnet55; + private const string _callId = "srvtoolu_fixture"; + private const string _query = "Paris fixture facts"; + private const string _answer = "Paris fixture."; + private const string _url = "https://example.com/source"; + private const string _title = "Fixture source"; + private const string _opaque = "opaque-source-content"; + private const string _index = "opaque-citation-index"; + private static readonly JsonSerializerOptions _json = new(JsonSerializerDefaults.Web) + { Converters = { new JsonStringEnumConverter(JsonNamingPolicy.CamelCase) } }; + + private static LlmTestServerResponseItem[] Items(string? error = null, bool empty = false) + { + return [new LlmTestServerResponseServerToolUseItem + { Id = _callId, Input = new LlmTestServerWebSearchInput { Query = _query } }, + new LlmTestServerResponseServerToolResultItem + { ToolUseId = _callId, Result = error is null ? new LlmTestServerWebSearchResults + { Results = empty ? [] : [new() { Url = _url, Title = _title, EncryptedContent = _opaque, PageAge = "2 days ago" }] } + : new LlmTestServerWebSearchErrorResult { ErrorCode = error } }, + new LlmTestServerResponseMessageItem + { Content = _answer, StreamChunks = ["Paris ", "fixture."], Citations = error is not null || empty ? [] : + [new LlmTestServerAnthropicWebSearchCitation { Url = _url, Title = _title, EncryptedIndex = _index, CitedText = "Scripted source excerpt" }] }]; + } + + private static LlmTestServerConfiguration Configuration(int responses = 1, string? error = null, bool empty = false) + { + return new LlmTestServerClientConfigurationBuilder().UseChatModel(_model).UseChatScenario("anthropic-native", scenario => + { scenario.ForModel(_model); for (var index = 0; index < responses; index++) { scenario.RespondsWithItems(Items(error, empty)); } }).Build(); + } + + private static JsonObject Request(bool stream = false, string input = "start") + { + return new JsonObject + { + ["model"] = _model, + ["max_tokens"] = 256, + ["tool_choice"] = new JsonObject { ["type"] = "auto" }, + ["stream"] = stream, + ["messages"] = new JsonArray(new JsonObject { ["role"] = "user", ["content"] = input }), + ["tools"] = new JsonArray(new JsonObject { ["type"] = "web_search_20250305", ["name"] = "web_search" }), + }; + } + + private static void Headers(HttpClient http) + { + http.DefaultRequestHeaders.Add("anthropic-version", "2023-06-01"); + http.DefaultRequestHeaders.Add("x-api-key", "provider-key"); + } + + private static async Task CompleteAsync(HttpClient http, JsonObject? body = null) + { + using var response = await http.PostAsJsonAsync("/anthropic/v1/messages", body ?? Request()); + response.EnsureSuccessStatusCode(); + return await response.Content.ReadFromJsonAsync(); + } + + private static async Task AssertContentAsync(JsonElement content) + { + await Assert.That(content.GetArrayLength()).IsEqualTo(3); + await Assert.That(content[0].GetProperty("type").GetString()).IsEqualTo("server_tool_use"); + await Assert.That(content[0].GetProperty("id").GetString()).IsEqualTo(_callId); + await Assert.That(content[0].GetProperty("name").GetString()).IsEqualTo("web_search"); + await Assert.That(content[0].GetProperty("input").GetProperty("query").GetString()).IsEqualTo(_query); + await Assert.That(content[1].GetProperty("type").GetString()).IsEqualTo("web_search_tool_result"); + await Assert.That(content[1].GetProperty("tool_use_id").GetString()).IsEqualTo(_callId); + var source = content[1].GetProperty("content")[0]; + await Assert.That(source.GetProperty("type").GetString()).IsEqualTo("web_search_result"); + await Assert.That(source.GetProperty("url").GetString()).IsEqualTo(_url); + await Assert.That(source.GetProperty("title").GetString()).IsEqualTo(_title); + await Assert.That(source.GetProperty("encrypted_content").GetString()).IsEqualTo(_opaque); + await Assert.That(source.GetProperty("page_age").GetString()).IsEqualTo("2 days ago"); + await Assert.That(content[2].GetProperty("text").GetString()).IsEqualTo(_answer); + var citation = content[2].GetProperty("citations")[0]; + await Assert.That(citation.GetProperty("type").GetString()).IsEqualTo("web_search_result_location"); + await Assert.That(citation.GetProperty("url").GetString()).IsEqualTo(_url); + await Assert.That(citation.GetProperty("encrypted_index").GetString()).IsEqualTo(_index); + await Assert.That(citation.GetProperty("cited_text").GetString()).IsEqualTo("Scripted source excerpt"); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Anthropic/AnthropicNativeWebSearchLifecycleIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Anthropic/AnthropicNativeWebSearchLifecycleIntegrationTests.cs new file mode 100644 index 0000000..3852003 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Anthropic/AnthropicNativeWebSearchLifecycleIntegrationTests.cs @@ -0,0 +1,164 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using System.Text.Json.Nodes; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Scenarios; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.Anthropic; + +public sealed partial class AnthropicNativeWebSearchIntegrationTests +{ + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task NativeCancellationOrReset_RollsBackGraphReservationAsync(bool reset) + { + using var host = await LlmTestServerTestHost.StartAsync(); using var http = host.GetTestClient(); Headers(http); + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(config => config.UseChatModel(_model).UseConversation("anthropic-cancel", flow => flow.ForModel(_model) + .Step("search", step => step.RespondsWithItems(Items()).DelaysBy(800).Complete()))); + using var cancellation = new CancellationTokenSource(); + var pending = http.PostAsJsonAsync("/anthropic/v1/messages", Request(), cancellation.Token); + var busy = false; + for (var attempt = 0; attempt < 200; attempt++) + { + if ((await control.GetConversationsAsync()).Any(state => state.IsBusy)) { busy = true; break; } + if (pending.IsCompleted) { break; } + await Task.Delay(2); + } + await Assert.That(busy).IsTrue(); + if (reset) + { + await control.ResetAsync(); using var superseded = await pending; + await Assert.That(superseded.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + } + else + { + cancellation.Cancel(); var cancelled = false; + try { using var response = await pending; } catch (OperationCanceledException) when (cancellation.IsCancellationRequested) { cancelled = true; } + await Assert.That(cancelled).IsTrue(); + } + await Assert.That((await control.GetConversationsAsync()).Count).IsEqualTo(0); + await AssertContentAsync((await CompleteAsync(http)).GetProperty("content")); + await Assert.That((await control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); + } + + [Test] + public async Task NativeOpaquePayload_ConsumesConversationByteBudgetEvenThoughItCostsNoTokensAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); using var http = host.GetTestClient(); Headers(http); + var items = Items(); items[1] = ((LlmTestServerResponseServerToolResultItem)items[1]) with + { Result = new LlmTestServerWebSearchResults { Results = [new() { Url = _url, Title = _title, EncryptedContent = new string('x', 10000) }] } }; + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(config => config.UseChatModel(_model).UseConversationCapacity(4, 3000) + .UseConversation("anthropic-budget", flow => flow.ForModel(_model).Step("search", step => step.RespondsWithItems(items).Complete()))); + using var rejected = await http.PostAsJsonAsync("/anthropic/v1/messages", Request()); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + await Assert.That((await control.GetConversationsAsync()).Count).IsEqualTo(0); + await control.ConfigureAsync(Configuration()); await CompleteAsync(http); + } + + [Test] + [Arguments("max")] + [Arguments("allowed")] + [Arguments("blocked")] + [Arguments("location")] + public async Task NativeOptions_PartitionPromptCacheAndSurviveTypedExportAsync(string option) + { + using var host = await LlmTestServerTestHost.StartAsync(); using var http = host.GetTestClient(); Headers(http); + var control = new LlmTestServerClient(http); await control.ConfigureAsync(Configuration(3)); + var body = Request(input: string.Join(' ', Enumerable.Repeat("stable native Anthropic cache reference", 1600))); + body["system"] = new JsonArray(new JsonObject { ["type"] = "text", ["text"] = string.Join(' ', Enumerable.Repeat("long shared fixture prefix", 1600)), ["cache_control"] = new JsonObject { ["type"] = "ephemeral" } }); + var first = await CompleteAsync(http, body); + await Assert.That(first.GetProperty("usage").TryGetProperty("cache_read_input_tokens", out var read) ? read.GetInt32() : 0).IsEqualTo(0); + var tool = body["tools"]![0]!; + switch (option) + { + case "max": tool["max_uses"] = 2; break; + case "allowed": tool["allowed_domains"] = new JsonArray("example.com/source*"); break; + case "blocked": tool["blocked_domains"] = new JsonArray("blocked.example"); break; + case "location": tool["user_location"] = new JsonObject { ["type"] = "approximate", ["city"] = "Paris" }; break; + } + var changed = await CompleteAsync(http, body); + await Assert.That(changed.GetProperty("usage").TryGetProperty("cache_read_input_tokens", out read) ? read.GetInt32() : 0).IsEqualTo(0); + var repeated = await CompleteAsync(http, body); + await Assert.That(repeated.GetProperty("usage").GetProperty("cache_read_input_tokens").GetInt32()).IsGreaterThanOrEqualTo(1024); + var trace = (await control.ExportRequestsAsync()).Evidence.Requests[0]; + await Assert.That(trace.Usage!.CachedInputTokens).IsGreaterThanOrEqualTo(1024); + var options = trace.NativeTools.Single().AnthropicWebSearchOptions!; + switch (option) + { + case "max": await Assert.That(options.MaxUses).IsEqualTo(2); break; + case "allowed": await Assert.That(options.AllowedDomains!.Single()).IsEqualTo("example.com/source*"); break; + case "blocked": await Assert.That(options.BlockedDomains!.Single()).IsEqualTo("blocked.example"); break; + case "location": await Assert.That(options.UserLocation!.City).IsEqualTo("Paris"); break; + } + } + + [Test] + public async Task SessionsAndNamedConversations_IsolateNativeGraphExecutionAndResetAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); using var http = host.GetTestClient(); Headers(http); + var control = new LlmTestServerClient(http); + await using var first = await control.CreateSessionAsync(config => config.UseChatModel(_model).UseChatScenario("session-native", scenario => scenario.ForModel(_model).RespondsWithItems(Items()))); + await using var second = await control.CreateSessionAsync(config => config.UseChatModel(_model).UseChatScenario("session-native", scenario => scenario.ForModel(_model).RespondsWithItems(Items()))); + await CompleteAsync(first.HttpClient); await CompleteAsync(second.HttpClient); + await Assert.That((await first.Control.GetRequestsAsync()).Requests.Count).IsEqualTo(1); + await Assert.That((await second.Control.GetRequestsAsync()).Requests.Count).IsEqualTo(1); + await first.Control.ResetAsync(); + await Assert.That((await first.Control.GetRequestsAsync()).Requests.Count).IsEqualTo(0); + await Assert.That((await second.Control.GetRequestsAsync()).Requests.Count).IsEqualTo(1); + } + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task CacheOnlyRequest_DoesNotExecuteNativeGraphBeforeOrdinaryCompletionAsync(bool streaming) + { + using var host = await LlmTestServerTestHost.StartAsync(); using var http = host.GetTestClient(); Headers(http); + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(config => config.UseChatModel(_model).UseConversation("anthropic-cache-only", flow => flow.ForModel(_model) + .Step("native", step => step.WhenTurn(1).RespondsWithItems(Items()).Complete()))); + using var conversation = control.ForConversation("anthropic-cache-only"); + var body = Request(streaming); body["max_tokens"] = 0; + using var response = await conversation.HttpClient.PostAsJsonAsync("/anthropic/v1/messages", body); + response.EnsureSuccessStatusCode(); + if (!streaming) + { + var message = await response.Content.ReadFromJsonAsync(); + await Assert.That(message.GetProperty("content").GetArrayLength()).IsEqualTo(0); + await Assert.That(message.GetProperty("stop_reason").GetString()).IsEqualTo("max_tokens"); + var usage = message.GetProperty("usage"); + await Assert.That(usage.GetProperty("output_tokens").GetInt32()).IsEqualTo(0); + await Assert.That(usage.TryGetProperty("server_tool_use", out var server) ? server.GetProperty("web_search_requests").GetInt32() : 0).IsEqualTo(0); + } + else + { + var frames = new List(); + foreach (var line in (await response.Content.ReadAsStringAsync()).Split('\n')) + { + if (line.StartsWith("data: ", StringComparison.Ordinal)) { frames.Add(JsonSerializer.Deserialize(line[6..])); } + } + await Assert.That(frames.Count(frame => frame.GetProperty("type").GetString() == "content_block_start")).IsEqualTo(0); + var start = frames.Single(frame => frame.GetProperty("type").GetString() == "message_start").GetProperty("message"); + await Assert.That(start.GetProperty("content").GetArrayLength()).IsEqualTo(0); + var delta = frames.Single(frame => frame.GetProperty("type").GetString() == "message_delta"); + await Assert.That(delta.GetProperty("delta").GetProperty("stop_reason").GetString()).IsEqualTo("max_tokens"); + await Assert.That(delta.GetProperty("usage").GetProperty("output_tokens").GetInt32()).IsEqualTo(0); + foreach (var usage in new[] { start.GetProperty("usage"), delta.GetProperty("usage") }) + { + await Assert.That(usage.TryGetProperty("server_tool_use", out var server) ? server.GetProperty("web_search_requests").GetInt32() : 0).IsEqualTo(0); + } + await Assert.That(frames.Count(frame => frame.GetProperty("type").GetString() == "message_stop")).IsEqualTo(1); + } + await Assert.That((await conversation.Control.GetConversationsAsync()).Sum(state => state.ExecutedSteps)).IsEqualTo(0); + var completed = await CompleteAsync(conversation.HttpClient); + await AssertContentAsync(completed.GetProperty("content")); + await Assert.That(completed.GetProperty("usage").GetProperty("server_tool_use").GetProperty("web_search_requests").GetInt32()).IsEqualTo(1); + var state = (await conversation.Control.GetConversationsAsync()).Single(); + await Assert.That(state.ExecutedSteps).IsEqualTo(1); + await Assert.That(state.IsTerminal).IsTrue(); + } + +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Anthropic/AnthropicNativeWebSearchSdkIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Anthropic/AnthropicNativeWebSearchSdkIntegrationTests.cs new file mode 100644 index 0000000..56997d9 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Anthropic/AnthropicNativeWebSearchSdkIntegrationTests.cs @@ -0,0 +1,174 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using global::Anthropic; +using global::Anthropic.Core; +using global::Anthropic.Exceptions; +using global::Anthropic.Models.Messages; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Scenarios; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.Anthropic; + +public sealed class AnthropicNativeWebSearchSdkIntegrationTests +{ + private static readonly string _model = JsonSerializer.SerializeToElement(Model.ClaudeSonnet5_5).GetString()!; + private const string _key = "anthropic-sdk-key"; + private const string _url = "https://example.com/sdk-source"; + private const string _call = "srvtoolu_sdk"; + + [Test] + public async Task OfficialSdk_ParsesNativeJsonSseAndAssistantHistoryAsync(CancellationToken cancellationToken) + { + using var host = await LlmTestServerTestHost.StartAsync(cancellationToken: cancellationToken); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(config => config.UseChatModel(_model) + .UseChatScenario("anthropic-sdk", scenario => scenario.ForModel(_model) + .RespondsWithItems(Items()).RespondsWithItems(Items()).Responds("SDK history complete.")), cancellationToken); + using var sdk = new AnthropicClient(new ClientOptions + { HttpClient = http, BaseUrl = new Uri(http.BaseAddress!, "/anthropic").ToString(), ApiKey = _key, MaxRetries = 0, ResponseValidation = true }); + var options = Options(); + var first = await sdk.Messages.Create(options, cancellationToken); + AssertContent(first.Content); + await Assert.That(first.Usage.ServerToolUse!.WebSearchRequests).IsEqualTo(1L); + + var blocks = new JsonArray(); var input = string.Empty; var stopped = new List(); + var messageStopped = false; var citationDeltas = 0; var searchCount = -1L; + await foreach (var update in sdk.Messages.CreateStreaming(options, cancellationToken)) + { + switch (update.Value) + { + case RawContentBlockStartEvent start: + await Assert.That(start.Index).IsEqualTo((long)blocks.Count); + blocks.Add(JsonNode.Parse(start.ContentBlock.Json.GetRawText())); + break; + case RawContentBlockDeltaEvent delta: + switch (delta.Delta.Value) + { + case InputJsonDelta json: input += json.PartialJson; break; + case TextDelta text: + var index = checked((int)delta.Index); + blocks[index]!["text"] = (blocks[index]!["text"]?.GetValue() ?? string.Empty) + text.Text; + break; + case CitationsDelta citation: + citationDeltas++; + var citationIndex = checked((int)delta.Index); + blocks[citationIndex]!["citations"] ??= new JsonArray(); + ((JsonArray)blocks[citationIndex]!["citations"]!).Add(JsonNode.Parse(citation.Citation.Json.GetRawText())); + break; + } + break; + case RawContentBlockStopEvent stop: stopped.Add(stop.Index); break; + case RawMessageDeltaEvent delta: + searchCount = delta.Usage.ServerToolUse!.WebSearchRequests; + break; + case RawMessageStopEvent: messageStopped = true; break; + } + } + blocks[0]!["input"] = JsonNode.Parse(input); + await Assert.That(messageStopped && stopped.SequenceEqual(new long[] { 0, 1, 2 }) && citationDeltas == 1 && searchCount == 1).IsTrue(); + var streamed = JsonSerializer.Deserialize>(blocks.ToJsonString())!; + AssertContent(streamed); + await Assert.That(JsonNode.DeepEquals(blocks, JsonNode.Parse(JsonSerializer.Serialize(first.Content)))).IsTrue(); + + var history = JsonSerializer.Deserialize>(JsonSerializer.Serialize(first.Content))!; + var follow = options with { Messages = [.. options.Messages, new() { Role = Role.Assistant, Content = history }, new() { Role = Role.User, Content = "SDK follow" }] }; + var continued = await sdk.Messages.Create(follow, cancellationToken); + await Assert.That(((TextBlock)continued.Content.Single().Value!).Text).IsEqualTo("SDK history complete."); + var traces = (await control.ExportRequestsAsync(cancellationToken: cancellationToken)).Evidence.Requests; + await Assert.That(traces.Count).IsEqualTo(3); + var stream = traces.Single(trace => trace.IsStreaming); + await Assert.That(stream.TerminalMarkerObserved == true && stream.NativeTools.Count == 1 && stream.ResponseToolCalls.Count == 0).IsTrue(); + var restored = traces.Single(trace => trace.Messages.Any(message => message.Content == "SDK follow")); + await Assert.That(restored.Messages.Single(message => message.Role == "assistant").OutputItems.Count).IsEqualTo(3); + } + + [Test] + [Arguments("invalid_tool_input")] + [Arguments("unavailable")] + [Arguments("max_uses_exceeded")] + [Arguments("too_many_requests")] + [Arguments("query_too_long")] + [Arguments("request_too_large")] + public async Task OfficialSdk_ParsesTypedNativeErrorsAndNextQueuedSuccessAsync(string code, CancellationToken cancellationToken) + { + using var host = await LlmTestServerTestHost.StartAsync(cancellationToken: cancellationToken); + using var http = host.GetTestClient(); var control = new LlmTestServerClient(http); + await control.ConfigureAsync(config => config.UseChatModel(_model) + .UseChatScenario("anthropic-sdk-errors", scenario => scenario.ForModel(_model) + .RespondsWithItems(Items(code)).RespondsWithItems(Items())), cancellationToken); + using var sdk = new AnthropicClient(new ClientOptions + { HttpClient = http, BaseUrl = new Uri(http.BaseAddress!, "/anthropic").ToString(), ApiKey = _key, MaxRetries = 0, ResponseValidation = true }); + var request = Options(code == "max_uses_exceeded" ? 0 : 1); + var result = await sdk.Messages.Create(request, cancellationToken); + var returned = (WebSearchToolResultBlock)result.Content[1].Value!; + await Assert.That(returned.Content.TryPickError(out var error)).IsTrue(); + await Assert.That(error!.ErrorCode.Json.GetString()).IsEqualTo(code); + await Assert.That(result.Usage.ServerToolUse!.WebSearchRequests).IsEqualTo(code == "max_uses_exceeded" ? 0L : 1L); + AssertContent((await sdk.Messages.Create(Options(), cancellationToken)).Content); + await Assert.That((await control.GetRequestsAsync(cancellationToken: cancellationToken)).Requests.Count).IsEqualTo(2); + } + + [Test] + public async Task OfficialSdk_RejectsProgrammaticCallerAndPreservesQueuedResponseAsync(CancellationToken cancellationToken) + { + using var host = await LlmTestServerTestHost.StartAsync(cancellationToken: cancellationToken); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(config => config.UseChatModel(_model) + .UseChatScenario("anthropic-sdk-caller", scenario => scenario.ForModel(_model) + .RespondsWithItems(Items()).RespondsWithItems(Items())), cancellationToken); + using var sdk = new AnthropicClient(new ClientOptions + { HttpClient = http, BaseUrl = new Uri(http.BaseAddress!, "/anthropic").ToString(), ApiKey = _key, MaxRetries = 0, ResponseValidation = true }); + var options = Options(); + var first = await sdk.Messages.Create(options, cancellationToken); + var history = JsonSerializer.Deserialize>(JsonSerializer.Serialize(first.Content))!; + var call = (ServerToolUseBlockParam)history[0].Value!; + history[0] = call with { Caller = new ServerToolCaller { ToolID = "toolu_programmatic" } }; + var unsupported = options with + { Messages = [.. options.Messages, new() { Role = Role.Assistant, Content = history }, new() { Role = Role.User, Content = "unsupported caller" }] }; + var rejected = false; + try { _ = await sdk.Messages.Create(unsupported, cancellationToken); } + catch (AnthropicBadRequestException) { rejected = true; } + await Assert.That(rejected).IsTrue(); + AssertContent((await sdk.Messages.Create(options, cancellationToken)).Content); + var traces = (await control.GetRequestsAsync(cancellationToken: cancellationToken)).Requests; + await Assert.That(traces.Count).IsEqualTo(3); + await Assert.That(traces.Count(trace => trace.StatusCode == 400)).IsEqualTo(1); + } + + private static MessageCreateParams Options(long maxUses = 1) + { + return new() + { + Model = Model.ClaudeSonnet5_5, + MaxTokens = 256, + ToolChoice = new ToolChoiceAuto(), + Messages = [new() { Role = Role.User, Content = "SDK fixture" }], + Tools = [new WebSearchTool20250305 + { MaxUses = maxUses, AllowedDomains = ["example.com/sdk*"], UserLocation = new() { Country = "US", Timezone = "America/New_York" } }], + }; + } + + private static LlmTestServerResponseItem[] Items(string? error = null) + { + return + [new LlmTestServerResponseServerToolUseItem { Id = _call, Input = new LlmTestServerWebSearchInput { Query = "SDK fixture query" } }, + new LlmTestServerResponseServerToolResultItem { ToolUseId = _call, + Result = error is null ? new LlmTestServerWebSearchResults { Results = [new() { Url = _url, Title = "SDK source", EncryptedContent = "opaque-sdk-source" }] } + : new LlmTestServerWebSearchErrorResult { ErrorCode = error } }, + new LlmTestServerResponseMessageItem { Content = "SDK answer.", StreamChunks = ["SDK ", "answer."], + Citations = error is null ? [new LlmTestServerAnthropicWebSearchCitation { Url = _url, EncryptedIndex = "opaque-sdk-index", CitedText = "SDK source excerpt", Title = "SDK source" }] : [] }]; + } + + private static void AssertContent(IReadOnlyList content) + { + if (content.Count != 3 || content[0].Value is not ServerToolUseBlock call || call.ID != _call + || call.Input["query"].GetString() != "SDK fixture query" || content[1].Value is not WebSearchToolResultBlock returned + || returned.ToolUseID != _call || !returned.Content.TryPickWebSearchResultBlocks(out var results) + || results.Single().EncryptedContent != "opaque-sdk-source" || content[2].Value is not TextBlock text || text.Text != "SDK answer." + || text.Citations!.Single().Value is not CitationsWebSearchResultLocation citation || citation.EncryptedIndex != "opaque-sdk-index") + { throw new InvalidOperationException("Official Anthropic SDK must parse typed native calls, results and citations."); } + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Anthropic/AnthropicNativeWebSearchValidationIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Anthropic/AnthropicNativeWebSearchValidationIntegrationTests.cs new file mode 100644 index 0000000..1d19a32 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Anthropic/AnthropicNativeWebSearchValidationIntegrationTests.cs @@ -0,0 +1,192 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using System.Text.Json.Nodes; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Scenarios; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.Anthropic; + +public sealed partial class AnthropicNativeWebSearchIntegrationTests +{ + [Test] + [Arguments("negativeMax")] + [Arguments("largeMax")] + [Arguments("fractionMax")] + [Arguments("bothDomains")] + [Arguments("scheme")] + [Arguments("wildcardHost")] + [Arguments("queryDomain")] + [Arguments("emptyLocation")] + [Arguments("wrongCountry")] + [Arguments("wrongTimezone")] + [Arguments("cache_control")] + [Arguments("defer_loading")] + [Arguments("strict")] + [Arguments("allowed_callers")] + [Arguments("newVersion")] + [Arguments("wrongName")] + [Arguments("model")] + public async Task InvalidNativeDeclaration_Returns400WithoutConsumingFixtureAsync(string defect) + { + using var host = await LlmTestServerTestHost.StartAsync(); using var http = host.GetTestClient(); Headers(http); + await new LlmTestServerClient(http).ConfigureAsync(Configuration()); + var body = Request(); var tool = body["tools"]![0]!; + switch (defect) + { + case "negativeMax": tool["max_uses"] = -1; break; + case "largeMax": tool["max_uses"] = 129; break; + case "fractionMax": tool["max_uses"] = 1.5; break; + case "bothDomains": tool["allowed_domains"] = new JsonArray(); tool["blocked_domains"] = new JsonArray(); break; + case "scheme": tool["allowed_domains"] = new JsonArray("https://example.com"); break; + case "wildcardHost": tool["allowed_domains"] = new JsonArray("*.example.com"); break; + case "queryDomain": tool["allowed_domains"] = new JsonArray("example.com?query=1"); break; + case "emptyLocation": tool["user_location"] = new JsonObject { ["type"] = "approximate" }; break; + case "wrongCountry": tool["user_location"] = new JsonObject { ["type"] = "approximate", ["country"] = "ZZ" }; break; + case "wrongTimezone": tool["user_location"] = new JsonObject { ["type"] = "approximate", ["timezone"] = "Mars/Fixture" }; break; + case "cache_control": tool[defect] = new JsonObject { ["type"] = "ephemeral" }; break; + case "defer_loading": case "strict": tool[defect] = true; break; + case "allowed_callers": tool[defect] = new JsonArray("direct"); break; + case "newVersion": tool["type"] = "web_search_20260209"; break; + case "wrongName": tool["name"] = "other"; break; + case "model": body["model"] = "gpt-4.1-mini"; break; + } + using var rejected = await http.PostAsJsonAsync("/anthropic/v1/messages", body); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + await AssertContentAsync((await CompleteAsync(http)).GetProperty("content")); + } + + [Test] + [Arguments(false, "missing")] + [Arguments(false, "none")] + [Arguments(false, "allowed")] + [Arguments(false, "blocked")] + [Arguments(false, "budget")] + [Arguments(true, "missing")] + [Arguments(true, "none")] + [Arguments(true, "allowed")] + [Arguments(true, "blocked")] + [Arguments(true, "budget")] + public async Task FixtureMismatch_DoesNotConsumeQueueOrGraphAsync(bool graph, string mismatch) + { + using var host = await LlmTestServerTestHost.StartAsync(); using var http = host.GetTestClient(); Headers(http); + var control = new LlmTestServerClient(http); + if (graph) + { + await control.ConfigureAsync(config => config.UseChatModel(_model).UseConversation("anthropic-native", flow => flow.ForModel(_model) + .Step("search", step => step.RespondsWithItems(Items()).Complete()))); + } + else { await control.ConfigureAsync(Configuration()); } + var body = Request(); var tool = body["tools"]![0]!; + switch (mismatch) + { + case "missing": body["tools"] = new JsonArray(); break; + case "none": body["tool_choice"] = new JsonObject { ["type"] = "none" }; break; + case "allowed": tool["allowed_domains"] = new JsonArray("other.example"); break; + case "blocked": tool["blocked_domains"] = new JsonArray("example.com/source*"); break; + case "budget": tool["max_uses"] = 0; break; + } + using var rejected = await http.PostAsJsonAsync("/anthropic/v1/messages", body); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + await Assert.That((await control.GetConversationsAsync()).Count).IsEqualTo(0); + await AssertContentAsync((await CompleteAsync(http)).GetProperty("content")); + if (graph) { await Assert.That((await control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); } + } + + [Test] + [Arguments("orphan")] + [Arguments("pending")] + [Arguments("duplicateResult")] + [Arguments("wrongPair")] + [Arguments("invalidError")] + [Arguments("emptyOpaque")] + [Arguments("foreignCitation")] + [Arguments("nullInput")] + public async Task InvalidConfiguration_IsAtomicAndLeavesOldFixtureUsableAsync(string defect) + { + using var host = await LlmTestServerTestHost.StartAsync(); using var http = host.GetTestClient(); Headers(http); + await new LlmTestServerClient(http).ConfigureAsync(Configuration()); + var config = JsonSerializer.SerializeToNode(Configuration(), _json)!; + var items = (JsonArray)config["chatScenarios"]![0]!["responses"]![0]!["outputItems"]!; + switch (defect) + { + case "orphan": items.RemoveAt(0); break; + case "pending": items.RemoveAt(1); break; + case "duplicateResult": items.Insert(2, items[1]!.DeepClone()); break; + case "wrongPair": items[1]!["toolUseId"] = "srvtoolu_other"; break; + case "invalidError": items[1]!["result"] = new JsonObject { ["kind"] = "web_search_error", ["errorCode"] = "invented" }; break; + case "emptyOpaque": items[1]!["result"]!["results"]![0]!["encryptedContent"] = ""; break; + case "foreignCitation": items[2]!["citations"]![0]!["url"] = "https://other.example/source"; break; + case "nullInput": items[0]!["input"] = null; break; + } + using var rejected = await http.PostAsJsonAsync("/admin-api/configure", config); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + await AssertContentAsync((await CompleteAsync(http)).GetProperty("content")); + } + + [Test] + [Arguments("example.com/source")] + [Arguments("example.com/sour*")] + [Arguments("example.com")] + public async Task AllowedDomainPathsAndNullBlockedDomains_ArePreservedAsync(string domain) + { + using var host = await LlmTestServerTestHost.StartAsync(); using var http = host.GetTestClient(); Headers(http); + var control = new LlmTestServerClient(http); await control.ConfigureAsync(Configuration()); + var body = Request(); var tool = body["tools"]![0]!; + tool["allowed_domains"] = new JsonArray(domain, domain); tool["blocked_domains"] = null; tool["max_uses"] = 128; + tool["user_location"] = new JsonObject { ["type"] = "approximate", ["city"] = "Paris", ["country"] = "FR", ["timezone"] = "Europe/Paris" }; + await CompleteAsync(http, body); + var options = (await control.GetRequestsAsync()).Requests.Single().NativeTools.Single().AnthropicWebSearchOptions!; + await Assert.That(options.AllowedDomains!.Count).IsEqualTo(2); + await Assert.That(options.AllowedDomains[0]).IsEqualTo(domain); + await Assert.That(options.UserLocation!.City).IsEqualTo("Paris"); + } + [Test] + public async Task ForcedNativeChoices_Return400WithoutConsumingPlainTextOrNativeFixturesAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); using var http = host.GetTestClient(); Headers(http); + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(config => config.UseChatModel(_model).UseChatScenario("forced-native", scenario => + scenario.ForModel(_model).Responds("queued plain text").RespondsWithItems(Items()))); + foreach (var choice in new[] { new JsonObject { ["type"] = "any" }, new JsonObject { ["type"] = "tool", ["name"] = "web_search" } }) + { + var forced = Request(); forced["tool_choice"] = choice; + using var rejected = await http.PostAsJsonAsync("/anthropic/v1/messages", forced); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + } + var plain = await CompleteAsync(http); + await Assert.That(plain.GetProperty("content")[0].GetProperty("text").GetString()).IsEqualTo("queued plain text"); + await AssertContentAsync((await CompleteAsync(http)).GetProperty("content")); + var traces = (await control.GetRequestsAsync()).Requests; + await Assert.That(traces.Count).IsEqualTo(4); + await Assert.That(traces.Count(trace => trace.StatusCode == 400)).IsEqualTo(2); + await Assert.That(traces.Count(trace => trace.StatusCode == 200)).IsEqualTo(2); + } + + [Test] + public async Task MaxUses_RequiresExhaustionAtTheExactOrderedErrorAndCountsOnlyPerformedSearchesAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); using var http = host.GetTestClient(); Headers(http); + var first = Items(); var second = Items(); var exhausted = Items("max_uses_exceeded"); + second[0] = ((LlmTestServerResponseServerToolUseItem)second[0]) with { Id = "srvtoolu_second" }; + second[1] = ((LlmTestServerResponseServerToolResultItem)second[1]) with { ToolUseId = "srvtoolu_second" }; + exhausted[0] = ((LlmTestServerResponseServerToolUseItem)exhausted[0]) with { Id = "srvtoolu_exhausted" }; + exhausted[1] = ((LlmTestServerResponseServerToolResultItem)exhausted[1]) with { ToolUseId = "srvtoolu_exhausted" }; + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(config => config.UseChatModel(_model).UseChatScenario("ordered-max-uses", scenario => scenario.ForModel(_model) + .RespondsWithItems(first[0], first[1], second[0], second[1], exhausted[0], exhausted[1], first[2]))); + foreach (var invalidBudget in new[] { 1, 3 }) + { + var invalid = Request(); invalid["tools"]![0]!["max_uses"] = invalidBudget; + using var rejected = await http.PostAsJsonAsync("/anthropic/v1/messages", invalid); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + } + var body = Request(); body["tools"]![0]!["max_uses"] = 2; + var response = await CompleteAsync(http, body); + await Assert.That(response.GetProperty("content").GetArrayLength()).IsEqualTo(7); + await Assert.That(response.GetProperty("content")[5].GetProperty("content").GetProperty("error_code").GetString()).IsEqualTo("max_uses_exceeded"); + await Assert.That(response.GetProperty("usage").GetProperty("server_tool_use").GetProperty("web_search_requests").GetInt32()).IsEqualTo(2); + } + +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Anthropic/AnthropicNativeWebSearchWireIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Anthropic/AnthropicNativeWebSearchWireIntegrationTests.cs new file mode 100644 index 0000000..9cad750 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Anthropic/AnthropicNativeWebSearchWireIntegrationTests.cs @@ -0,0 +1,127 @@ +using System.Net.Http.Json; +using System.Text.Json; +using System.Text.Json.Nodes; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Hosting; +using ManagedCode.LlmTestServer.Scenarios; +using ManagedCode.LlmTestServer.Tests.TestSupport; +using Microsoft.Extensions.DependencyInjection; + +namespace ManagedCode.LlmTestServer.Tests.Anthropic; + +public sealed partial class AnthropicNativeWebSearchIntegrationTests +{ + [Test] + public async Task OrderedNativeJson_PreservesOpaqueSourcesCitationsAndPublicEvidenceAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); using var http = host.GetTestClient(); Headers(http); + var control = new LlmTestServerClient(http); await control.ConfigureAsync(Configuration()); + var response = await CompleteAsync(http); await AssertContentAsync(response.GetProperty("content")); + await Assert.That(response.GetProperty("stop_reason").GetString()).IsEqualTo("end_turn"); + await Assert.That(response.GetProperty("usage").GetProperty("server_tool_use").GetProperty("web_search_requests").GetInt32()).IsEqualTo(1); + var exported = await control.ExportRequestsAsync(); var trace = exported.Evidence.Requests.Single(); + await Assert.That(trace.ResponseToolCalls.Count).IsEqualTo(0); + await Assert.That(trace.NativeTools.Count).IsEqualTo(1); + await Assert.That(trace.ResponseOutputItems.Count).IsEqualTo(3); + await Assert.That(((LlmTestServerWebSearchResults)((LlmTestServerResponseServerToolResultItem)trace.ResponseOutputItems[1]).Result).Results[0].EncryptedContent).IsEqualTo(_opaque); + await Assert.That(((LlmTestServerAnthropicWebSearchCitation)((LlmTestServerResponseMessageItem)trace.ResponseOutputItems[2]).Citations.Single()).EncryptedIndex).IsEqualTo(_index); + var lines = (await control.ExportRequestsNdjsonAsync()).Split('\n', StringSplitOptions.RemoveEmptyEntries); + using var record = JsonDocument.Parse(lines[1]); + await Assert.That(record.RootElement.GetProperty("request").GetProperty("responseOutputItems")[0].GetProperty("kind").GetString()).IsEqualTo("server_tool_use"); + } + + [Test] + [Arguments("invalid_tool_input")] + [Arguments("unavailable")] + [Arguments("max_uses_exceeded")] + [Arguments("too_many_requests")] + [Arguments("query_too_long")] + [Arguments("request_too_large")] + [Arguments("")] + public async Task NativeSearchErrorsAndEmptyResults_AreSuccessfulTypedMessagesAsync(string code) + { + using var host = await LlmTestServerTestHost.StartAsync(); using var http = host.GetTestClient(); Headers(http); + await new LlmTestServerClient(http).ConfigureAsync(Configuration(error: code.Length == 0 ? null : code, empty: code.Length == 0)); + var body = Request(); if (code == "max_uses_exceeded") { body["tools"]![0]!["max_uses"] = 0; } + var response = await CompleteAsync(http, body); var result = response.GetProperty("content")[1].GetProperty("content"); + if (code.Length == 0) { await Assert.That(result.GetArrayLength()).IsEqualTo(0); } + else + { + await Assert.That(result.GetProperty("type").GetString()).IsEqualTo("web_search_tool_result_error"); + await Assert.That(result.GetProperty("error_code").GetString()).IsEqualTo(code); + } + await Assert.That(response.GetProperty("content")[2].GetProperty("text").GetString()).IsEqualTo(_answer); + await Assert.That(response.GetProperty("usage").GetProperty("server_tool_use").GetProperty("web_search_requests").GetInt32()).IsEqualTo(code == "max_uses_exceeded" ? 0 : 1); + } + + [Test] + public async Task NativeSse_ReconstructsTheSameOrderedContentAndTerminalUsageAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); using var http = host.GetTestClient(); Headers(http); + var config = Configuration(2); + var control = new LlmTestServerClient(http); await control.ConfigureAsync(config); + var baseline = await CompleteAsync(http); + using var response = await http.PostAsJsonAsync("/anthropic/v1/messages", Request(true)); response.EnsureSuccessStatusCode(); + var blocks = new JsonArray(); var input = ""; var stopped = new List(); var messageStopped = false; var usageCount = -1; + foreach (var line in (await response.Content.ReadAsStringAsync()).Split('\n')) + { + if (!line.StartsWith("data: ", StringComparison.Ordinal)) { continue; } + var frame = JsonNode.Parse(line[6..])!; var type = frame["type"]!.GetValue(); + if (type == "content_block_start") + { + await Assert.That(frame["index"]!.GetValue()).IsEqualTo(blocks.Count); + blocks.Add(frame["content_block"]!.DeepClone()); + } + if (type == "content_block_delta") + { + var index = frame["index"]!.GetValue(); var delta = frame["delta"]!; + switch (delta["type"]!.GetValue()) + { + case "input_json_delta": input += delta["partial_json"]!.GetValue(); break; + case "text_delta": blocks[index]!["text"] = (blocks[index]!["text"]?.GetValue() ?? "") + delta["text"]!.GetValue(); break; + case "citations_delta": + blocks[index]!["citations"] ??= new JsonArray(); + ((JsonArray)blocks[index]!["citations"]!).Add(delta["citation"]!.DeepClone()); break; + } + } + if (type == "content_block_stop") { stopped.Add(frame["index"]!.GetValue()); } + if (type == "message_delta") + { + await Assert.That(frame["delta"]!["stop_reason"]!.GetValue()).IsEqualTo("end_turn"); + usageCount = frame["usage"]!["server_tool_use"]!["web_search_requests"]!.GetValue(); + } + if (type == "message_stop") { messageStopped = true; } + } + blocks[0]!["input"] = JsonNode.Parse(input); + await AssertContentAsync(JsonSerializer.SerializeToElement(blocks)); + await Assert.That(JsonNode.DeepEquals(blocks, JsonNode.Parse(baseline.GetProperty("content").GetRawText()))).IsTrue(); + await Assert.That(string.Join(',', stopped)).IsEqualTo("0,1,2"); + await Assert.That(messageStopped).IsTrue(); await Assert.That(usageCount).IsEqualTo(1); + await Assert.That((await control.GetRequestsAsync()).Requests[0].TerminalMarkerObserved).IsTrue(); + } + [Test] + public async Task TruncatedNativePair_PublicSnapshotsRetainAnHonestCallPrefixWithoutThrowingAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(configureServices: services => + services.Configure(options => options.MaxPreviewBytes = 1024)); + using var http = host.GetTestClient(); Headers(http); + var config = Configuration(); var items = Items(); + items[1] = ((LlmTestServerResponseServerToolResultItem)items[1]) with + { Result = new LlmTestServerWebSearchResults { Results = [new() { Url = _url, Title = _title, EncryptedContent = new string('x', 10000) }] } }; + config.ChatScenarios[0].Responses[0] = config.ChatScenarios[0].Responses[0] with { OutputItems = [.. items] }; + var control = new LlmTestServerClient(http); await control.ConfigureAsync(config); + var response = await CompleteAsync(http); + await Assert.That(response.GetProperty("content")[1].GetProperty("content")[0].GetProperty("encrypted_content").GetString()!.Length).IsEqualTo(10000); + var snapshot = await control.GetRequestsAsync(); var trace = snapshot.Requests.Single(); + await Assert.That(trace.RetentionTruncated).IsTrue(); + await Assert.That(trace.ResponseOutputItems.Count).IsEqualTo(1); + await Assert.That(((LlmTestServerResponseServerToolUseItem)trace.ResponseOutputItems.Single()).Id).IsEqualTo(_callId); + var exported = await control.ExportRequestsAsync(); + await Assert.That(exported.Evidence.Requests.Single().ResponseOutputItems.Count).IsEqualTo(1); + await Assert.That(exported.Evidence.RetentionTruncated).IsTrue(); + var lines = (await control.ExportRequestsNdjsonAsync()).Split('\n', StringSplitOptions.RemoveEmptyEntries); + using var record = JsonDocument.Parse(lines[1]); + await Assert.That(record.RootElement.GetProperty("request").GetProperty("responseOutputItems").GetArrayLength()).IsEqualTo(1); + } + +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Aspire/AspireAuthorizationIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Aspire/AspireAuthorizationIntegrationTests.cs new file mode 100644 index 0000000..3780f6d --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Aspire/AspireAuthorizationIntegrationTests.cs @@ -0,0 +1,52 @@ +using System.Net; +using System.Net.Http.Headers; +using Aspire.Hosting.Testing; +using ManagedCode.LlmTestServer.Aspire; + +namespace ManagedCode.LlmTestServer.Tests.AspireIntegration; + +public sealed class AspireAuthorizationIntegrationTests +{ + [Test] + [Arguments(null)] + [Arguments("")] + [Arguments("test-server-key")] + [Timeout(120_000)] + public async Task PackagedServer_EnforcesCanonicalApiKeyConfigurationAsync( + string? apiKey, CancellationToken cancellationToken) + { + var builder = DistributedApplicationTestingBuilder.Create([]); + var resource = builder.AddLlmTestServer("authorization"); + if (!string.IsNullOrEmpty(apiKey)) + { + resource.WithApiKey(apiKey); + } + else if (apiKey is not null) + { + resource.WithEnvironment("LlmTestServer__RequiredBearerToken", apiKey); + } + + await using var app = await builder.BuildAsync(cancellationToken); + await app.StartAsync(cancellationToken); + await app.ResourceNotifications.WaitForResourceHealthyAsync("authorization", cancellationToken); + using var http = app.CreateHttpClient("authorization", "http"); + foreach (var path in new[] { "/admin-api/models", "/openai/v1/models", "/groq/openai/v1/models", "/openrouter/api/v1/models", "/deepseek/models" }) + { + http.DefaultRequestHeaders.Authorization = null; + using var missing = await http.GetAsync(path, cancellationToken); + await Assert.That(missing.StatusCode).IsEqualTo(string.IsNullOrEmpty(apiKey) + ? HttpStatusCode.OK : HttpStatusCode.Unauthorized); + + http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", "wrong-token"); + using var rejected = await http.GetAsync(path, cancellationToken); + await Assert.That(rejected.StatusCode).IsEqualTo(string.IsNullOrEmpty(apiKey) + ? HttpStatusCode.OK : HttpStatusCode.Unauthorized); + + http.DefaultRequestHeaders.Authorization = string.IsNullOrEmpty(apiKey) + ? null : new AuthenticationHeaderValue("Bearer", apiKey); + using var accepted = await http.GetAsync(path, cancellationToken); + await Assert.That(accepted.StatusCode).IsEqualTo(HttpStatusCode.OK); + await Assert.That(await accepted.Content.ReadAsStringAsync(cancellationToken)).Contains("gpt-4.1-mini"); + } + } +} diff --git a/tests/ManagedCode.LlmTck.Tests/Aspire/AspireIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Aspire/AspireIntegrationTests.cs similarity index 87% rename from tests/ManagedCode.LlmTck.Tests/Aspire/AspireIntegrationTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/Aspire/AspireIntegrationTests.cs index 884e8a9..41da996 100644 --- a/tests/ManagedCode.LlmTck.Tests/Aspire/AspireIntegrationTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Aspire/AspireIntegrationTests.cs @@ -7,9 +7,9 @@ using Azure.AI.Inference; using Azure.Core.Pipeline; using global::Aspire.Hosting.Testing; -using ManagedCode.LlmTck.Aspire; -using ManagedCode.LlmTck.Client; -using ManagedCode.LlmTck.Control; +using ManagedCode.LlmTestServer.Aspire; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Control; using Microsoft.Extensions.AI; using OpenAI; using ExtensionsChatMessage = Microsoft.Extensions.AI.ChatMessage; @@ -18,13 +18,13 @@ using FoundryEmbeddingClient = Azure.AI.Inference.EmbeddingsClient; using OpenAiChatClient = OpenAI.Chat.ChatClient; using OpenAiUserChatMessage = OpenAI.Chat.UserChatMessage; -using ProviderRoutes = ManagedCode.LlmTck.Providers.LlmTckProviderRouteNamespaces; +using ProviderRoutes = ManagedCode.LlmTestServer.Providers.LlmTestServerProviderRouteNamespaces; -namespace ManagedCode.LlmTck.Tests.AspireIntegration; +namespace ManagedCode.LlmTestServer.Tests.AspireIntegration; public sealed class AspireIntegrationTests { - private const string _resourceName = "llm-tck"; + private const string _resourceName = "llm-test-server"; private const string _apiKey = "test-key"; private const string _chatModel = "gpt-4.1-mini"; private const string _embeddingModel = "text-embedding-3-small"; @@ -32,19 +32,9 @@ public sealed class AspireIntegrationTests private const string _audioModel = "gpt-4o-mini-tts"; private const int _parallelResourceCount = 10; - [Test] - public async Task AddLlmTckContainer_RemainsExplicitContainerOptInAsync() - { - var builder = DistributedApplicationTestingBuilder.Create([]); - var llmTck = builder.AddLlmTckContainer().WithApiKey(_apiKey); - - await Assert.That(llmTck.Resource).IsTypeOf(); - await Assert.That(llmTck.GetHttpEndpoint().ToString()).IsNotEmpty(); - } - [Test] [Timeout(360_000)] - public async Task AddLlmTck_BuildsAppHostInTestAndSupportsConfiguredClientsAsync( + public async Task AddLlmTestServer_BuildsAppHostInTestAndSupportsConfiguredClientsAsync( CancellationToken cancellationToken ) { @@ -52,11 +42,11 @@ CancellationToken cancellationToken timeout.CancelAfter(TimeSpan.FromMinutes(5)); var builder = DistributedApplicationTestingBuilder.Create([]); - var llmTck = builder - .AddLlmTck() + var llmTestServer = builder + .AddLlmTestServer() .WithApiKey(_apiKey); - var llmTckEndpointExpression = llmTck.GetHttpEndpoint().ToString(); - var openAiEndpointExpression = llmTck.GetOpenAiEndpoint(); + var llmTestServerEndpointExpression = llmTestServer.GetHttpEndpoint().ToString(); + var openAiEndpointExpression = llmTestServer.GetOpenAiEndpoint(); await using var app = await builder.BuildAsync(timeout.Token); await app.StartAsync(timeout.Token); @@ -66,7 +56,7 @@ CancellationToken cancellationToken httpClient.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", _apiKey); - var controlClient = new LlmTckClient(httpClient, _apiKey); + var controlClient = new LlmTestServerClient(httpClient, _apiKey); await controlClient.ConfigureAsync( options => options .RequireBearerToken(_apiKey) @@ -94,10 +84,10 @@ await controlClient.ConfigureAsync( using var imageGenerator = controlClient.CreateImageGenerator(_imageModel); var root = await httpClient.GetAsync("/", timeout.Token); - var adminPage = await httpClient.GetStringAsync(LlmTckControlRoutes.Admin, timeout.Token); + var adminPage = await httpClient.GetStringAsync(LlmTestServerControlRoutes.Admin, timeout.Token); var models = await httpClient.GetFromJsonAsync("/openai/v1/models", timeout.Token); var adminModels = await httpClient.GetFromJsonAsync( - LlmTckControlRoutes.Models, + LlmTestServerControlRoutes.Models, timeout.Token ); var chat = await chatClient.GetResponseAsync( @@ -122,7 +112,7 @@ [new ExtensionsChatMessage(ExtensionsChatRole.User, "stream from aspire sdk")], ); var openAiEndpointValue = await openAiEndpointExpression.GetValueAsync(timeout.Token) - ?? throw new InvalidOperationException("LLM TCK OpenAI endpoint was not allocated."); + ?? throw new InvalidOperationException("LLM Test Server OpenAI endpoint was not allocated."); var openAiEndpoint = new Uri(openAiEndpointValue); var openAiChatClient = new OpenAiChatClient( _chatModel, @@ -210,15 +200,15 @@ [new OpenAiUserChatMessage("hello from azure aspire sdk")], ); var assertions = await controlClient.GetAssertionsAsync(timeout.Token); var adminAssertions = await httpClient.GetFromJsonAsync( - LlmTckControlRoutes.Assertions, + LlmTestServerControlRoutes.Assertions, timeout.Token ); - await Assert.That(llmTckEndpointExpression).IsNotEmpty(); + await Assert.That(llmTestServerEndpointExpression).IsNotEmpty(); await Assert.That(root.StatusCode).IsEqualTo(HttpStatusCode.OK); - await Assert.That(adminPage).Contains("LLM TCK request dashboard"); - await Assert.That(adminPage).Contains(LlmTckControlRoutes.Models); - await Assert.That(adminPage).Contains(LlmTckControlRoutes.Assertions); + await Assert.That(adminPage).Contains("LLM Test Server request dashboard"); + await Assert.That(adminPage).Contains(LlmTestServerControlRoutes.Models); + await Assert.That(adminPage).Contains(LlmTestServerControlRoutes.Assertions); await Assert.That(adminPage).Contains("Total tokens"); await Assert.That(models.GetProperty("data").GetArrayLength()).IsGreaterThanOrEqualTo(4); await Assert.That(adminModels.GetArrayLength()).IsGreaterThanOrEqualTo(4); @@ -243,7 +233,7 @@ [new OpenAiUserChatMessage("hello from azure aspire sdk")], [Test] [Timeout(600_000)] - public async Task AddLlmTck_StartsTenDistinctResourcesAndHealthEndpointsStayAliveAsync( + public async Task AddLlmTestServer_StartsTenDistinctResourcesAndHealthEndpointsStayAliveAsync( CancellationToken cancellationToken ) { @@ -259,7 +249,7 @@ CancellationToken cancellationToken foreach (var resourceName in resourceNames) { builder - .AddLlmTck(resourceName) + .AddLlmTestServer(resourceName) .WithApiKey($"{_apiKey}-{resourceName}"); } @@ -296,7 +286,7 @@ await app.ResourceNotifications.WaitForResourceHealthyAsync( async Task ProbeHealthAsync(string resourceName) { - using var httpClient = app.CreateHttpClient(resourceName, LlmTckResource.HttpEndpointName); + using var httpClient = app.CreateHttpClient(resourceName, LlmTestServerResource.HttpEndpointName); var health = await httpClient.GetAsync("/health", timeout.Token); return new HealthProbeResult( diff --git a/tests/ManagedCode.LlmTck.Tests/Azure/AzureSdkCompatibilityTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Azure/AzureSdkCompatibilityTests.cs similarity index 87% rename from tests/ManagedCode.LlmTck.Tests/Azure/AzureSdkCompatibilityTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/Azure/AzureSdkCompatibilityTests.cs index bd867bd..980575f 100644 --- a/tests/ManagedCode.LlmTck.Tests/Azure/AzureSdkCompatibilityTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Azure/AzureSdkCompatibilityTests.cs @@ -5,14 +5,14 @@ using Azure; using Azure.AI.Inference; using Azure.Core.Pipeline; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.Tests.TestSupport; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Tests.TestSupport; using FoundryChatClient = Azure.AI.Inference.ChatCompletionsClient; using FoundryEmbeddingClient = Azure.AI.Inference.EmbeddingsClient; using OpenAiUserChatMessage = OpenAI.Chat.UserChatMessage; -using ProviderRoutes = ManagedCode.LlmTck.Providers.LlmTckProviderRouteNamespaces; +using ProviderRoutes = ManagedCode.LlmTestServer.Providers.LlmTestServerProviderRouteNamespaces; -namespace ManagedCode.LlmTck.Tests.Azure; +namespace ManagedCode.LlmTestServer.Tests.Azure; public sealed class AzureSdkCompatibilityTests { @@ -21,10 +21,10 @@ public sealed class AzureSdkCompatibilityTests [Test] public async Task AzureOpenAiClient_CanUseDeploymentChatAndEmbeddingsAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options + using var host = await LlmTestServerTestHost.StartAsync(options => options .RequireBearerToken("test-key") - .AddModel("azure-chat", LlmTckModelKind.Chat) - .AddModel("azure-embedding", LlmTckModelKind.Embedding) + .AddModel("azure-chat", LlmTestServerModelKind.Chat) + .AddModel("azure-embedding", LlmTestServerModelKind.Embedding) .WithDefaultEmbeddingVector(0.25f, 0.5f) .AddChatScenario( "azure-sdk-blue-whale", @@ -76,10 +76,10 @@ [new OpenAiUserChatMessage("stream from azure sdk")], [Test] public async Task AzureAiInferenceClients_CanUseFoundryChatAndEmbeddingsAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options + using var host = await LlmTestServerTestHost.StartAsync(options => options .RequireBearerToken("test-key") - .AddModel("foundry-chat", LlmTckModelKind.Chat) - .AddModel("foundry-embedding", LlmTckModelKind.Embedding) + .AddModel("foundry-chat", LlmTestServerModelKind.Chat) + .AddModel("foundry-embedding", LlmTestServerModelKind.Embedding) .WithDefaultEmbeddingVector(0.75f, 0.875f) .AddChatScenario( "foundry-sdk-blue-whale", @@ -126,11 +126,11 @@ public async Task AzureAiInferenceClients_CanUseFoundryChatAndEmbeddingsAsync() [Test] public async Task AzureOpenAiDeploymentRoutes_UseApiKeyAndDeploymentModelForModalitiesAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options + using var host = await LlmTestServerTestHost.StartAsync(options => options .RequireBearerToken("test-key") - .AddModel("azure-embedding", LlmTckModelKind.Embedding) - .AddModel("azure-image", LlmTckModelKind.Image) - .AddModel("azure-audio", LlmTckModelKind.Audio) + .AddModel("azure-embedding", LlmTestServerModelKind.Embedding) + .AddModel("azure-image", LlmTestServerModelKind.Image) + .AddModel("azure-audio", LlmTestServerModelKind.Audio) .WithDefaultEmbeddingVector(0.125f, 0.25f)); using var httpClient = host.GetTestClient(); httpClient.DefaultRequestHeaders.Add("api-key", "test-key"); diff --git a/tests/ManagedCode.LlmTck.Tests/Azure/AzureV1SdkCompatibilityTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Azure/AzureV1SdkCompatibilityTests.cs similarity index 77% rename from tests/ManagedCode.LlmTck.Tests/Azure/AzureV1SdkCompatibilityTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/Azure/AzureV1SdkCompatibilityTests.cs index 5d714d7..8ce0460 100644 --- a/tests/ManagedCode.LlmTck.Tests/Azure/AzureV1SdkCompatibilityTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Azure/AzureV1SdkCompatibilityTests.cs @@ -2,15 +2,15 @@ using System.ClientModel.Primitives; using System.Net.Http.Json; using System.Text.Json; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.Providers; -using ManagedCode.LlmTck.Tests.TestSupport; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Providers; +using ManagedCode.LlmTestServer.Tests.TestSupport; using OpenAI; using OpenAI.Chat; using OpenAI.Images; using OpenAI.Responses; -namespace ManagedCode.LlmTck.Tests.Azure; +namespace ManagedCode.LlmTestServer.Tests.Azure; // The official stable SDK still marks its Responses surface experimental. #pragma warning disable OPENAI001 @@ -19,13 +19,13 @@ public sealed class AzureV1SdkCompatibilityTests [Test] public async Task AzureOpenAiV1Images_UseOfficialOpenAiSdkAndReturnImageBytesAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options + using var host = await LlmTestServerTestHost.StartAsync(options => options .RequireBearerToken("test-key") - .AddModel("gpt-image-1", LlmTckModelKind.Image)); + .AddModel("gpt-image-1", LlmTestServerModelKind.Image)); using var http = host.GetTestClient(); var client = new OpenAIClient(new ApiKeyCredential("test-key"), new OpenAIClientOptions { - Endpoint = new Uri(http.BaseAddress!, LlmTckProviderRouteNamespaces.AzureOpenAI + "/openai/v1/"), + Endpoint = new Uri(http.BaseAddress!, LlmTestServerProviderRouteNamespaces.AzureOpenAI + "/openai/v1/"), Transport = new HttpClientPipelineTransport(http), }); var imageClient = client.GetImageClient("gpt-image-1"); @@ -42,14 +42,14 @@ public async Task AzureOpenAiV1Images_UseOfficialOpenAiSdkAndReturnImageBytesAsy } [Test] - [Arguments(LlmTckProviderRouteNamespaces.AzureOpenAI)] - [Arguments(LlmTckProviderRouteNamespaces.MicrosoftFoundry)] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry)] public async Task V1ChatAndEmbeddings_UseOfficialSdkWithoutApiVersionAsync(string provider) { - using var host = await LlmTckTestHost.StartAsync(options => options + using var host = await LlmTestServerTestHost.StartAsync(options => options .RequireBearerToken("test-key") - .AddModel("gpt-4.1-mini", LlmTckModelKind.Chat) - .AddModel("text-embedding-3-small", LlmTckModelKind.Embedding) + .AddModel("gpt-4.1-mini", LlmTestServerModelKind.Chat) + .AddModel("text-embedding-3-small", LlmTestServerModelKind.Embedding) .WithDefaultEmbeddingVector(0.25f, 0.5f) .AddChatScenario("v1-chat", scenario => scenario .ForModel("gpt-4.1-mini").WhenUserContains("whale") @@ -75,19 +75,19 @@ public async Task V1ChatAndEmbeddings_UseOfficialSdkWithoutApiVersionAsync(strin } [Test] - [Arguments(LlmTckProviderRouteNamespaces.AzureOpenAI)] - [Arguments(LlmTckProviderRouteNamespaces.MicrosoftFoundry)] - [Arguments(LlmTckProviderRouteNamespaces.OpenAI)] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry)] + [Arguments(LlmTestServerProviderRouteNamespaces.OpenAI)] public async Task V1Responses_UseOfficialSdkAndStreamAsync(string provider) { - using var host = await LlmTckTestHost.StartAsync(options => options + using var host = await LlmTestServerTestHost.StartAsync(options => options .RequireBearerToken("test-key") - .AddModel("gpt-4.1-mini", LlmTckModelKind.Chat) + .AddModel("gpt-4.1-mini", LlmTestServerModelKind.Chat) .AddChatScenario("v1-response", scenario => scenario .ForModel("gpt-4.1-mini").WhenUserContains("whale") .Responds("blue whale").Responds("blue whale", "blue ", "whale"))); using var http = host.GetTestClient(); - var apiRoot = provider == LlmTckProviderRouteNamespaces.OpenAI ? "/v1/" : "/openai/v1/"; + var apiRoot = provider == LlmTestServerProviderRouteNamespaces.OpenAI ? "/v1/" : "/openai/v1/"; var client = new ResponsesClient(new ApiKeyCredential("test-key"), new ResponsesClientOptions { Endpoint = new Uri(http.BaseAddress!, provider.TrimStart('/') + apiRoot), @@ -114,13 +114,13 @@ public async Task V1Responses_UseOfficialSdkAndStreamAsync(string provider) } [Test] - [Arguments(LlmTckProviderRouteNamespaces.AzureOpenAI)] - [Arguments(LlmTckProviderRouteNamespaces.MicrosoftFoundry)] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry)] public async Task V1Responses_AcceptsApiKeyAndRejectsMissingCredentialsAsync(string provider) { - using var host = await LlmTckTestHost.StartAsync(options => options + using var host = await LlmTestServerTestHost.StartAsync(options => options .RequireBearerToken("test-key") - .AddModel("gpt-4.1-mini", LlmTckModelKind.Chat) + .AddModel("gpt-4.1-mini", LlmTestServerModelKind.Chat) .AddChatScenario("v1-auth", scenario => scenario.ForModel("gpt-4.1-mini") .WhenUserContains("whale").Responds("blue whale"))); using var client = host.GetTestClient(); diff --git a/tests/ManagedCode.LlmTck.Tests/Azure/LlmTckResponsesContinuationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Azure/LlmTestServerResponsesContinuationTests.cs similarity index 53% rename from tests/ManagedCode.LlmTck.Tests/Azure/LlmTckResponsesContinuationTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/Azure/LlmTestServerResponsesContinuationTests.cs index 0e78d98..112791a 100644 --- a/tests/ManagedCode.LlmTck.Tests/Azure/LlmTckResponsesContinuationTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Azure/LlmTestServerResponsesContinuationTests.cs @@ -1,19 +1,19 @@ +using System.Net.Http.Json; #pragma warning disable OPENAI001 // Official Responses client adapter is still marked experimental. using System.ClientModel; using System.ClientModel.Primitives; -using ManagedCode.LlmTck.Configuration; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.Providers; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Tests.TestSupport; +using ManagedCode.LlmTestServer.Configuration; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Providers; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Tests.TestSupport; using Microsoft.Extensions.AI; -using Microsoft.Extensions.DependencyInjection; using OpenAI.Responses; -namespace ManagedCode.LlmTck.Tests.Azure; +namespace ManagedCode.LlmTestServer.Tests.Azure; -public sealed class LlmTckResponsesContinuationTests +public sealed class LlmTestServerResponsesContinuationTests { private const string _key = "continuation-test-key"; private const string _prompt = "review-file-continuation"; @@ -21,28 +21,29 @@ public sealed class LlmTckResponsesContinuationTests private const string _toolName = "read_document"; [Test] - [Arguments(LlmTckProviderRouteNamespaces.OpenAI, false)] - [Arguments(LlmTckProviderRouteNamespaces.OpenAI, true)] - [Arguments(LlmTckProviderRouteNamespaces.AzureOpenAI, false)] - [Arguments(LlmTckProviderRouteNamespaces.AzureOpenAI, true)] - [Arguments(LlmTckProviderRouteNamespaces.MicrosoftFoundry, false)] - [Arguments(LlmTckProviderRouteNamespaces.MicrosoftFoundry, true)] + [Arguments(LlmTestServerProviderRouteNamespaces.OpenAI, false)] + [Arguments(LlmTestServerProviderRouteNamespaces.OpenAI, true)] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI, false)] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI, true)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry, false)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry, true)] public async Task OfficialMeaiToolLoop_RestoresPreviousResponseContextAsync(string provider, bool streaming) { - using var host = await LlmTckTestHost.StartAsync(options => options.RequireBearerToken(_key) - .AddModel(LlmTckKnownModelIds.Gpt41Mini, LlmTckModelKind.Chat) - .AddChatScenario(_prompt, scenario => scenario.ForModel(LlmTckKnownModelIds.Gpt41Mini) + using var host = await LlmTestServerTestHost.StartAsync(options => options.RequireBearerToken(_key) + .AddModel(LlmTestServerKnownModelIds.Gpt41Mini, LlmTestServerModelKind.Chat) + .AddChatScenario(_prompt, scenario => scenario.ForModel(LlmTestServerKnownModelIds.Gpt41Mini) .WhenUserContains(_prompt).CallsTool(_toolName, "{}", "call_document").Responds("reviewed"))); using var http = host.GetTestClient(); - var apiRoot = provider == LlmTckProviderRouteNamespaces.OpenAI ? "/v1/" : "/openai/v1/"; + http.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", _key); + var apiRoot = provider == LlmTestServerProviderRouteNamespaces.OpenAI ? "/v1/" : "/openai/v1/"; var sdk = new ResponsesClient(new ApiKeyCredential(_key), new ResponsesClientOptions { Endpoint = new Uri(http.BaseAddress!, provider.TrimStart('/') + apiRoot), Transport = new HttpClientPipelineTransport(http) }); - using var client = sdk.AsIChatClient(LlmTckKnownModelIds.Gpt41Mini).AsBuilder().UseFunctionInvocation().Build(); + using var client = sdk.AsIChatClient(LlmTestServerKnownModelIds.Gpt41Mini).AsBuilder().UseFunctionInvocation().Build(); var chatOptions = new ChatOptions { Tools = [AIFunctionFactory.Create(() => _evidence, _toolName)] }; - var picture = new LlmTckConfiguration().DefaultImageDataUri; + var picture = new LlmTestServerConfiguration().DefaultImageDataUri; ChatMessage[] input = [new(ChatRole.User, [new TextContent(_prompt), new DataContent(Convert.FromBase64String(picture[(picture.IndexOf(',') + 1)..]), "image/png")])]; if (streaming) @@ -55,49 +56,50 @@ public async Task OfficialMeaiToolLoop_RestoresPreviousResponseContextAsync(stri { await Assert.That((await client.GetResponseAsync(input, chatOptions)).Text).IsEqualTo("reviewed"); } - var events = host.Services.GetRequiredService().GetAssertionSummary().Events; + var events = (await http.GetFromJsonAsync("/admin-api/assertions"))!.Events; await Assert.That(events.Count).IsEqualTo(2); - await Assert.That(events.All(item => item.Kind == LlmTckEventKind.Matched)).IsTrue(); + await Assert.That(events.All(item => item.Kind == LlmTestServerEventKind.Matched)).IsTrue(); await Assert.That(events.Last().Messages.Any(message => message.Role == "user" && message.Content.Contains(_prompt, StringComparison.Ordinal) && message.Content.Contains(picture, StringComparison.Ordinal))).IsTrue(); await Assert.That(events.Last().Messages.Single(message => message.Role == "tool").Content).Contains(_evidence); } [Test] - [Arguments(LlmTckProviderRouteNamespaces.OpenAI, false)] - [Arguments(LlmTckProviderRouteNamespaces.OpenAI, true)] - [Arguments(LlmTckProviderRouteNamespaces.AzureOpenAI, false)] - [Arguments(LlmTckProviderRouteNamespaces.AzureOpenAI, true)] - [Arguments(LlmTckProviderRouteNamespaces.MicrosoftFoundry, false)] - [Arguments(LlmTckProviderRouteNamespaces.MicrosoftFoundry, true)] + [Arguments(LlmTestServerProviderRouteNamespaces.OpenAI, false)] + [Arguments(LlmTestServerProviderRouteNamespaces.OpenAI, true)] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI, false)] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI, true)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry, false)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry, true)] public async Task OfficialMeaiContinuation_AllowsModelChangeAndPreservesHistoryAsync(string provider, bool streaming) { - using var host = await LlmTckTestHost.StartAsync(options => options.RequireBearerToken(_key) - .AddModel(LlmTckKnownModelIds.Gpt41Mini, LlmTckModelKind.Chat) - .AddModel(LlmTckKnownModelIds.Gpt41, LlmTckModelKind.Chat) - .AddChatScenario("initial-model", scenario => scenario.ForModel(LlmTckKnownModelIds.Gpt41Mini) + using var host = await LlmTestServerTestHost.StartAsync(options => options.RequireBearerToken(_key) + .AddModel(LlmTestServerKnownModelIds.Gpt41Mini, LlmTestServerModelKind.Chat) + .AddModel(LlmTestServerKnownModelIds.Gpt41, LlmTestServerModelKind.Chat) + .AddChatScenario("initial-model", scenario => scenario.ForModel(LlmTestServerKnownModelIds.Gpt41Mini) .WhenUserContains(_prompt).Responds("initial reply")) - .AddChatScenario("updated-model", scenario => scenario.ForModel(LlmTckKnownModelIds.Gpt41) + .AddChatScenario("updated-model", scenario => scenario.ForModel(LlmTestServerKnownModelIds.Gpt41) .WhenUserContains(_prompt).Responds("updated reply"))); using var http = host.GetTestClient(); - var apiRoot = provider == LlmTckProviderRouteNamespaces.OpenAI ? "/v1/" : "/openai/v1/"; + http.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", _key); + var apiRoot = provider == LlmTestServerProviderRouteNamespaces.OpenAI ? "/v1/" : "/openai/v1/"; var sdk = new ResponsesClient(new ApiKeyCredential(_key), new ResponsesClientOptions { Endpoint = new Uri(http.BaseAddress!, provider.TrimStart('/') + apiRoot), Transport = new HttpClientPipelineTransport(http) }); - using var client = sdk.AsIChatClient(LlmTckKnownModelIds.Gpt41Mini); + using var client = sdk.AsIChatClient(LlmTestServerKnownModelIds.Gpt41Mini); var first = await client.GetResponseAsync(_prompt); await Assert.That(first.ConversationId).IsNotNull(); - var options = new ChatOptions { ModelId = LlmTckKnownModelIds.Gpt41, ConversationId = first.ConversationId }; + var options = new ChatOptions { ModelId = LlmTestServerKnownModelIds.Gpt41, ConversationId = first.ConversationId }; var result = streaming ? await client.GetStreamingResponseAsync("continue on the updated model", options).ToChatResponseAsync() : await client.GetResponseAsync("continue on the updated model", options); await Assert.That(result.Text).IsEqualTo("updated reply"); - await Assert.That(result.ModelId).IsEqualTo(LlmTckKnownModelIds.Gpt41); - var events = host.Services.GetRequiredService().GetAssertionSummary().Events; + await Assert.That(result.ModelId).IsEqualTo(LlmTestServerKnownModelIds.Gpt41); + var events = (await http.GetFromJsonAsync("/admin-api/assertions"))!.Events; await Assert.That(events.Count).IsEqualTo(2); - await Assert.That(events.All(item => item.Kind == LlmTckEventKind.Matched)).IsTrue(); + await Assert.That(events.All(item => item.Kind == LlmTestServerEventKind.Matched)).IsTrue(); await Assert.That(events.Last().Messages.Any(message => message.Role == "user" && message.Content == _prompt)).IsTrue(); await Assert.That(events.Last().Messages.Any(message => message.Role == "assistant" && message.Content == "initial reply")).IsTrue(); } diff --git a/tests/ManagedCode.LlmTck.Tests/Azure/V1AudioTranscriptionCompatibilityTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Azure/V1AudioTranscriptionCompatibilityTests.cs similarity index 78% rename from tests/ManagedCode.LlmTck.Tests/Azure/V1AudioTranscriptionCompatibilityTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/Azure/V1AudioTranscriptionCompatibilityTests.cs index 8a733a5..e363b93 100644 --- a/tests/ManagedCode.LlmTck.Tests/Azure/V1AudioTranscriptionCompatibilityTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Azure/V1AudioTranscriptionCompatibilityTests.cs @@ -3,16 +3,16 @@ using System.Net; using System.Net.Http.Headers; using System.Text.Json; -using ManagedCode.LlmTck.Client; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.Providers; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Tests.TestSupport; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Providers; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Tests.TestSupport; using Microsoft.Extensions.AI; using OpenAI; using OpenAI.Audio; -namespace ManagedCode.LlmTck.Tests.Azure; +namespace ManagedCode.LlmTestServer.Tests.Azure; #pragma warning disable OPENAI001 // Official transcription usage types are experimental. #pragma warning disable MEAI001 // Official speech usage modality counters are experimental. @@ -20,21 +20,21 @@ public sealed class V1AudioTranscriptionCompatibilityTests { private const string _whisperModel = "whisper-1"; [Test] - [Arguments(LlmTckProviderRouteNamespaces.AzureOpenAI, false)] - [Arguments(LlmTckProviderRouteNamespaces.AzureOpenAI, true)] - [Arguments(LlmTckProviderRouteNamespaces.MicrosoftFoundry, false)] - [Arguments(LlmTckProviderRouteNamespaces.MicrosoftFoundry, true)] - [Arguments(LlmTckProviderRouteNamespaces.OpenAI, false)] - [Arguments(LlmTckProviderRouteNamespaces.OpenAI, true)] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI, false)] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI, true)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry, false)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry, true)] + [Arguments(LlmTestServerProviderRouteNamespaces.OpenAI, false)] + [Arguments(LlmTestServerProviderRouteNamespaces.OpenAI, true)] public async Task OfficialSdk_PreservesTranscriptionUsageAsync(string provider, bool tokens) { - using var host = await LlmTckTestHost.StartAsync(); + using var host = await LlmTestServerTestHost.StartAsync(); using var http = host.GetTestClient(); - var control = LlmTckClient.Create(http); + var control = LlmTestServerClient.Create(http); await control.ConfigureAsync(configuration => configuration.RequireBearerToken("test-key") .UseAudioModel(_whisperModel).UseTranscriptionText("blue whale") - .UseTranscriptionUsage(tokens ? LlmTckTranscriptionUsage.ForTokens(12, 3, 4) - : LlmTckTranscriptionUsage.ForDuration(0.25))); + .UseTranscriptionUsage(tokens ? LlmTestServerTranscriptionUsage.ForTokens(12, 3, 4) + : LlmTestServerTranscriptionUsage.ForDuration(0.25))); var sdk = new OpenAIClient(new ApiKeyCredential("test-key"), new OpenAIClientOptions { Endpoint = Endpoint(http, provider), @@ -80,14 +80,14 @@ await Assert.That(((AudioTranscriptionDurationUsage)raw.Usage).Duration) } [Test] - [Arguments(LlmTckProviderRouteNamespaces.AzureOpenAI)] - [Arguments(LlmTckProviderRouteNamespaces.MicrosoftFoundry)] - [Arguments(LlmTckProviderRouteNamespaces.OpenAI)] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry)] + [Arguments(LlmTestServerProviderRouteNamespaces.OpenAI)] public async Task OfficialSdk_PreservesMissingTranscriptionUsageAsync(string provider) { - using var host = await LlmTckTestHost.StartAsync(); + using var host = await LlmTestServerTestHost.StartAsync(); using var http = host.GetTestClient(); - var control = LlmTckClient.Create(http); + var control = LlmTestServerClient.Create(http); await control.ConfigureAsync(configuration => configuration .RequireBearerToken("test-key") .UseAudioModel(_whisperModel) @@ -119,14 +119,14 @@ await control.ConfigureAsync(configuration => configuration } [Test] - [Arguments(LlmTckProviderRouteNamespaces.AzureOpenAI)] - [Arguments(LlmTckProviderRouteNamespaces.MicrosoftFoundry)] - [Arguments(LlmTckProviderRouteNamespaces.OpenAI)] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry)] + [Arguments(LlmTestServerProviderRouteNamespaces.OpenAI)] public async Task Streaming_PreservesFinalUsageAsync(string provider) { - using var host = await LlmTckTestHost.StartAsync(configuration => configuration - .RequireBearerToken("test-key").AddModel("gpt-4o-transcribe", LlmTckModelKind.Audio) - .WithDefaultTranscriptionUsage(LlmTckTranscriptionUsage.ForTokens(12, 0, 4))); + using var host = await LlmTestServerTestHost.StartAsync(configuration => configuration + .RequireBearerToken("test-key").AddModel("gpt-4o-transcribe", LlmTestServerModelKind.Audio) + .WithDefaultTranscriptionUsage(LlmTestServerTranscriptionUsage.ForTokens(12, 0, 4))); using var http = host.GetTestClient(); http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", "test-key"); using var form = AudioForm("gpt-4o-transcribe"); @@ -142,13 +142,13 @@ await Assert.That(payload.RootElement.GetProperty("usage").GetProperty("total_to } [Test] - [Arguments(LlmTckProviderRouteNamespaces.AzureOpenAI)] - [Arguments(LlmTckProviderRouteNamespaces.MicrosoftFoundry)] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry)] public async Task V1Audio_EnforcesAuthAndVersionAndSupportsVerboseJsonAsync(string provider) { - using var host = await LlmTckTestHost.StartAsync(configuration => configuration.RequireBearerToken("test-key") - .AddModel(_whisperModel, LlmTckModelKind.Audio) - .WithDefaultTranscriptionUsage(LlmTckTranscriptionUsage.ForDuration(0.25))); + using var host = await LlmTestServerTestHost.StartAsync(configuration => configuration.RequireBearerToken("test-key") + .AddModel(_whisperModel, LlmTestServerModelKind.Audio) + .WithDefaultTranscriptionUsage(LlmTestServerTranscriptionUsage.ForDuration(0.25))); using var http = host.GetTestClient(); var route = new Uri(Endpoint(http, provider), "audio/transcriptions"); using (var form = AudioForm(_whisperModel)) @@ -181,7 +181,7 @@ public async Task V1Audio_EnforcesAuthAndVersionAndSupportsVerboseJsonAsync(stri private static Uri Endpoint(HttpClient http, string provider) { - return new(http.BaseAddress!, provider.TrimStart('/') + (provider == LlmTckProviderRouteNamespaces.OpenAI ? "/v1/" : "/openai/v1/")); + return new(http.BaseAddress!, provider.TrimStart('/') + (provider == LlmTestServerProviderRouteNamespaces.OpenAI ? "/v1/" : "/openai/v1/")); } private static MultipartFormDataContent AudioForm(string model) diff --git a/tests/ManagedCode.LlmTck.Tests/Bedrock/BedrockEndpointTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Bedrock/BedrockEndpointTests.cs similarity index 92% rename from tests/ManagedCode.LlmTck.Tests/Bedrock/BedrockEndpointTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/Bedrock/BedrockEndpointTests.cs index b684aa4..8f3aa64 100644 --- a/tests/ManagedCode.LlmTck.Tests/Bedrock/BedrockEndpointTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Bedrock/BedrockEndpointTests.cs @@ -1,11 +1,11 @@ using System.Net.Http.Json; using System.Text; using System.Text.Json; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Tests.TestSupport; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Tests.TestSupport; -namespace ManagedCode.LlmTck.Tests.Bedrock; +namespace ManagedCode.LlmTestServer.Tests.Bedrock; public sealed class BedrockEndpointTests { @@ -16,9 +16,9 @@ public sealed class BedrockEndpointTests [Arguments("invoke-with-response-stream")] public async Task StreamingEndpoints_RejectInvalidRequestsWithoutConsumingFixtureAsync(string operation) { - using var host = await LlmTckTestHost.StartAsync(options => options + using var host = await LlmTestServerTestHost.StartAsync(options => options .RequireBearerToken("test-key") - .AddModel("amazon.nova-lite-v1:0", LlmTckModelKind.Chat) + .AddModel("amazon.nova-lite-v1:0", LlmTestServerModelKind.Chat) .AddChatScenario("stream-validation", scenario => scenario.ForModel("amazon.nova-lite-v1:0") .WhenUserContains("whale").Responds("blue whale"))); using var client = host.GetTestClient(); @@ -65,8 +65,8 @@ await Assert.That(delta.GetProperty("contentBlockDelta").GetProperty("delta").Ge [Test] public async Task ConverseEndpoint_ReturnsBedrockConverseShapeAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("amazon.nova-lite-v1:0", LlmTckModelKind.Chat) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("amazon.nova-lite-v1:0", LlmTestServerModelKind.Chat) .AddChatScenario( "bedrock-converse", scenario => scenario @@ -130,8 +130,8 @@ await Assert.That(payload.GetProperty("metrics").GetProperty("latencyMs").GetInt public async Task ConverseEndpoint_WithCachePoint_ReturnsBedrockCacheUsageAsync() { var cacheableSystemPrompt = CreatePromptWithAtLeastTokens(1100); - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("amazon.nova-cache-v1:0", LlmTckModelKind.Chat) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("amazon.nova-cache-v1:0", LlmTestServerModelKind.Chat) .AddChatScenario( "bedrock-cache", scenario => scenario @@ -174,8 +174,8 @@ await Assert.That(secondUsage.TryGetProperty("cacheWriteInputTokens", out _)) [Test] public async Task ConverseStreamEndpoint_ReturnsBedrockEventStreamShapeAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("amazon.nova-lite-v1:0", LlmTckModelKind.Chat) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("amazon.nova-lite-v1:0", LlmTestServerModelKind.Chat) .AddChatScenario( "bedrock-converse-stream", scenario => scenario @@ -242,8 +242,8 @@ await Assert.That(streamUsage.GetProperty("totalTokens").GetInt32()) [Test] public async Task InvokeEndpoint_ReturnsTitanTextShapeForChatModelAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("amazon.titan-text-premier-v1:0", LlmTckModelKind.Chat) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("amazon.titan-text-premier-v1:0", LlmTestServerModelKind.Chat) .AddChatScenario( "bedrock-titan-text", scenario => scenario @@ -277,8 +277,8 @@ await Assert.That(payload.GetProperty("results")[0].GetProperty("completionReaso [Test] public async Task InvokeModelWithResponseStreamEndpoint_ReturnsChunkBytesAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("amazon.titan-text-premier-v1:0", LlmTckModelKind.Chat) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("amazon.titan-text-premier-v1:0", LlmTestServerModelKind.Chat) .AddChatScenario( "bedrock-titan-stream", scenario => scenario @@ -313,8 +313,8 @@ await Assert.That(response.Headers.GetValues("x-amzn-bedrock-content-type").Sing [Test] public async Task InvokeEndpoint_ReturnsTitanEmbeddingShapeForEmbeddingModelAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("amazon.titan-embed-text-v2:0", LlmTckModelKind.Embedding) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("amazon.titan-embed-text-v2:0", LlmTestServerModelKind.Embedding) .WithDefaultEmbeddingVector(0.125f, 0.25f, 0.5f)); using var client = host.GetTestClient(); @@ -342,8 +342,8 @@ await Assert.That( [Test] public async Task InvokeEndpoint_ReturnsImageShapeForImageModelAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("stability.stable-image-ultra-v1:1", LlmTckModelKind.Image) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("stability.stable-image-ultra-v1:1", LlmTestServerModelKind.Image) .WithDefaultImageDataUri("data:image/png;base64,ZmFrZS1wbmc=")); using var client = host.GetTestClient(); @@ -431,7 +431,7 @@ string userPrompt private static string CreatePromptWithAtLeastTokens(int minimumTokens) { var builder = new StringBuilder("cacheable fixture"); - while (LlmTckTokenCounter.CountTextTokens(builder.ToString()) < minimumTokens) + while (LlmTestServerTokenCounter.CountTextTokens(builder.ToString()) < minimumTokens) { builder.Append(" stable-prefix"); } diff --git a/tests/ManagedCode.LlmTestServer.Tests/Bedrock/BedrockOfficialSdkIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Bedrock/BedrockOfficialSdkIntegrationTests.cs new file mode 100644 index 0000000..c5f19ec --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Bedrock/BedrockOfficialSdkIntegrationTests.cs @@ -0,0 +1,222 @@ +using System.Net; +using System.Text; +using System.Text.Json; +using Amazon.BedrockRuntime; +using Amazon.BedrockRuntime.Model; +using Amazon.Runtime; +using Amazon.Runtime.Documents; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Providers; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.Bedrock; + +public sealed class BedrockOfficialSdkIntegrationTests +{ + private const string _model = "amazon.nova-lite-v1:0"; + private const string _titan = "amazon.titan-text-express-v1"; + + [Test] + public async Task Converse_OfficialSdkDeserializesMessageStopReasonAndUsageAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(options => options.AddModel(_model, LlmTestServerModelKind.Chat) + .AddChatScenario("sdk-converse", scenario => scenario.ForModel(_model).WhenUserContains("whale").Responds("blue whale"))); + using var http = Transport(host.GetTestServer().CreateHandler(), host.GetTestServer().BaseAddress); + using var sdk = Sdk(http); + var response = await sdk.ConverseAsync(new ConverseRequest + { + ModelId = _model, + Messages = [User("whale")], + System = [new SystemContentBlock { Text = "Answer briefly." }], + InferenceConfig = new InferenceConfiguration { MaxTokens = 64, Temperature = 0, TopP = 1, StopSequences = ["END"] }, + }); + await Assert.That(response.HttpStatusCode).IsEqualTo(HttpStatusCode.OK); + await Assert.That(response.Output.Message.Role).IsEqualTo(ConversationRole.Assistant); + await Assert.That(response.Output.Message.Content.Single().Text).IsEqualTo("blue whale"); + await Assert.That(response.StopReason.Value).IsEqualTo("end_turn"); + await Assert.That(response.Usage.InputTokens ?? 0).IsGreaterThan(0); + await Assert.That(response.Usage.OutputTokens).IsEqualTo(2); + await Assert.That(response.Usage.TotalTokens).IsEqualTo(response.Usage.InputTokens + response.Usage.OutputTokens); + var trace = (await new LlmTestServerClient(http).GetRequestsAsync()).Requests.Single(); + await Assert.That(trace.OperationId).IsEqualTo(LlmTestServerProviderOperationIds.Bedrock.Converse); + await Assert.That(trace.Messages.Single(message => message.Role == "system").Content).IsEqualTo("Answer briefly."); + } + + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task InvokeModel_OfficialSdkReadsTitanJsonAndAmazonEventStreamAsync(bool streaming) + { + using var host = await LlmTestServerTestHost.StartAsync(options => options.AddModel(_titan, LlmTestServerModelKind.Chat) + .AddChatScenario("sdk-invoke", scenario => scenario.ForModel(_titan).WhenUserContains("whale").Responds("blue whale", "blue ", "whale"))); + using var http = Transport(host.GetTestServer().CreateHandler(), host.GetTestServer().BaseAddress); + using var sdk = Sdk(http); + if (streaming) + { + using var response = await sdk.InvokeModelWithResponseStreamAsync(new InvokeModelWithResponseStreamRequest + { ModelId = _titan, ContentType = "application/json", Accept = "application/json", Body = InvokeBody() }); + var chunks = new List(); + response.Body.ChunkReceived += (_, args) => + { + using var payload = JsonDocument.Parse(args.EventStreamEvent.Bytes.ToArray()); + chunks.Add(payload.RootElement.GetProperty("outputText").GetString()!); + }; + await response.Body.StartProcessingAsync(); + await Assert.That(response.HttpStatusCode).IsEqualTo(HttpStatusCode.OK); + await Assert.That(response.ContentType).IsEqualTo("application/json"); + await Assert.That(string.Join('|', chunks)).IsEqualTo("blue |whale"); + } + else + { + var response = await sdk.InvokeModelAsync(new InvokeModelRequest + { ModelId = _titan, ContentType = "application/json", Accept = "application/json", Body = InvokeBody() }); + using var payload = await JsonDocument.ParseAsync(response.Body); + await Assert.That(response.HttpStatusCode).IsEqualTo(HttpStatusCode.OK); + await Assert.That(payload.RootElement.GetProperty("inputTextTokenCount").GetInt32()).IsGreaterThan(0); + var result = payload.RootElement.GetProperty("results")[0]; + await Assert.That(result.GetProperty("outputText").GetString()).IsEqualTo("blue whale"); + await Assert.That(result.GetProperty("tokenCount").GetInt32()).IsEqualTo(2); + await Assert.That(result.GetProperty("completionReason").GetString()).IsEqualTo("FINISHED"); + } + var trace = (await new LlmTestServerClient(http).GetRequestsAsync()).Requests.Single(); + await Assert.That(trace.OperationId).IsEqualTo(streaming ? LlmTestServerProviderOperationIds.Bedrock.InvokeModelWithResponseStream : LlmTestServerProviderOperationIds.Bedrock.InvokeModel); + await Assert.That(trace.IsStreaming).IsEqualTo(streaming); + await Assert.That(trace.StatusCode).IsEqualTo(200); + } + + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task OfficialSdk_ClientToolRoundUsesExactCallIdArgumentsAndGraphTurnAsync(bool streaming) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = Transport(host.GetTestServer().CreateHandler(), host.GetTestServer().BaseAddress); + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(config => config.UseChatModel(_model).UseConversation("bedrock-sdk-round", graph => graph.ForModel(_model) + .Step("tool", step => step.WhenLatestUserEquals("weather").WhenTurn(1).CallsTool("weather", "{\"city\":\"Paris\"}", "toolu_weather").Then("final")) + .Step("final", step => step.WhenTurn(2).WhenToolResult("weather", "toolu_weather").WhenLatestTextContains("21C").Responds("Paris is 21C.").Complete()))); + using var scoped = control.ForConversation("bedrock-sdk-round"); using var sdk = Sdk(scoped.HttpClient); + var tools = new ToolConfiguration + { + Tools = [new Tool { ToolSpec = new ToolSpecification + { Name = "weather", Description = "Scripted weather", InputSchema = new ToolInputSchema { Json = new Document(new Dictionary + { ["type"] = "object", ["properties"] = new Document(new Dictionary { ["city"] = new Document(new Dictionary { ["type"] = "string" }) }), ["required"] = new Document(new List { "city" }) }) } } }], + }; + ToolUseBlock call; + if (streaming) + { + using var first = await sdk.ConverseStreamAsync(new ConverseStreamRequest { ModelId = _model, Messages = [User("weather")], ToolConfig = tools }); + var id = ""; var name = ""; var arguments = ""; var stop = ""; + first.Stream.ContentBlockStartReceived += (_, args) => + { id = args.EventStreamEvent.Start.ToolUse.ToolUseId; name = args.EventStreamEvent.Start.ToolUse.Name; }; + first.Stream.ContentBlockDeltaReceived += (_, args) => arguments += args.EventStreamEvent.Delta.ToolUse.Input; + first.Stream.MessageStopReceived += (_, args) => stop = args.EventStreamEvent.StopReason; + await first.Stream.StartProcessingAsync(); + await Assert.That(stop).IsEqualTo("tool_use"); + await Assert.That(JsonDocument.Parse(arguments).RootElement.GetProperty("city").GetString()).IsEqualTo("Paris"); + call = new ToolUseBlock { ToolUseId = id, Name = name, Input = new Document(new Dictionary { ["city"] = "Paris" }) }; + } + else + { + var first = await sdk.ConverseAsync(new ConverseRequest { ModelId = _model, Messages = [User("weather")], ToolConfig = tools }); + await Assert.That(first.StopReason.Value).IsEqualTo("tool_use"); + call = first.Output.Message.Content.Single().ToolUse; + await Assert.That(call.Input.AsDictionary()["city"].AsString()).IsEqualTo("Paris"); + } + await Assert.That(call.ToolUseId).IsEqualTo("toolu_weather"); await Assert.That(call.Name).IsEqualTo("weather"); + var final = await sdk.ConverseAsync(new ConverseRequest + { + ModelId = _model, + ToolConfig = tools, + Messages = [User("weather"), + new Message { Role = ConversationRole.Assistant, Content = [new ContentBlock { ToolUse = call }] }, + new Message { Role = ConversationRole.User, Content = [new ContentBlock { ToolResult = new ToolResultBlock + { ToolUseId = call.ToolUseId, Status = ToolResultStatus.Success, Content = [new ToolResultContentBlock { Text = "21C" }] } }] }], + }); + await Assert.That(final.Output.Message.Content.Single().Text).IsEqualTo("Paris is 21C."); + await Assert.That(final.StopReason.Value).IsEqualTo("end_turn"); + await Assert.That((await scoped.Control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(2); + var traces = (await scoped.Control.GetRequestsAsync()).Requests; + await Assert.That(traces.Count).IsEqualTo(2); + await Assert.That(traces[0].Messages.Single(message => message.Role == "tool").ToolCallId).IsEqualTo(call.ToolUseId); + await Assert.That(traces[1].ResponseToolCalls.Single().ArgumentsJson).IsEqualTo("{\"city\":\"Paris\"}"); + } + + [Test] + [Arguments("converse")] + [Arguments("invoke")] + [Arguments("stream")] + public async Task OfficialSdk_ReceivesScripted429AndRecoversOnExplicitNextAttemptAsync(string operation) + { + var model = operation == "converse" ? _model : _titan; + using var host = await LlmTestServerTestHost.StartAsync(options => options.AddModel(model, LlmTestServerModelKind.Chat) + .AddChatScenario("sdk-error", scenario => scenario.ForModel(model).Fails(429, "too_many_requests", "Retry the fixture.").Responds("blue whale"))); + using var http = Transport(host.GetTestServer().CreateHandler(), host.GetTestServer().BaseAddress); using var sdk = Sdk(http); + AmazonServiceException? observed = null; + try { await SendAsync(); } catch (AmazonServiceException exception) { observed = exception; } + await Assert.That(observed).IsNotNull(); + await Assert.That(observed!.StatusCode).IsEqualTo(HttpStatusCode.TooManyRequests); + await Assert.That(observed.Message).Contains("Retry the fixture."); + await SendAsync(); + var traces = (await new LlmTestServerClient(http).GetRequestsAsync()).Requests; + await Assert.That(traces.Count).IsEqualTo(2); + await Assert.That(traces[1].StatusCode).IsEqualTo(429); await Assert.That(traces[0].StatusCode).IsEqualTo(200); + async Task SendAsync() + { + if (operation == "converse") + { + var response = await sdk.ConverseAsync(new ConverseRequest { ModelId = model, Messages = [User("whale")] }); + await Assert.That(response.Output.Message.Content.Single().Text).IsEqualTo("blue whale"); + } + else if (operation == "invoke") + { + var response = await sdk.InvokeModelAsync(new InvokeModelRequest { ModelId = model, ContentType = "application/json", Body = InvokeBody() }); + using var payload = await JsonDocument.ParseAsync(response.Body); + await Assert.That(payload.RootElement.GetProperty("results")[0].GetProperty("outputText").GetString()).IsEqualTo("blue whale"); + } + else + { + using var response = await sdk.InvokeModelWithResponseStreamAsync(new InvokeModelWithResponseStreamRequest { ModelId = model, ContentType = "application/json", Body = InvokeBody() }); + var chunks = new List(); response.Body.ChunkReceived += (_, args) => + { using var payload = JsonDocument.Parse(args.EventStreamEvent.Bytes.ToArray()); chunks.Add(payload.RootElement.GetProperty("outputText").GetString()!); }; + await response.Body.StartProcessingAsync(); await Assert.That(string.Concat(chunks)).IsEqualTo("blue whale"); + } + } + } + + private static Message User(string text) + { + return new() { Role = ConversationRole.User, Content = [new ContentBlock { Text = text }] }; + } + + private static MemoryStream InvokeBody() + { + return new(Encoding.UTF8.GetBytes("{\"inputText\":\"whale\",\"textGenerationConfig\":{\"maxTokenCount\":64,\"temperature\":0,\"topP\":1}}")); + } + + private static HttpClient Transport(HttpMessageHandler handler, Uri address) + { + return new(new CanonicalUriHandler(handler)) { BaseAddress = address }; + } + + private static AmazonBedrockRuntimeClient Sdk(HttpClient http) + { + return new(new BasicAWSCredentials("test-access-key", "test-secret-key"), + new AmazonBedrockRuntimeConfig { ServiceURL = new Uri(http.BaseAddress!, LlmTestServerProviderRouteNamespaces.Bedrock).ToString(), AuthenticationRegion = "us-east-1", HttpClientFactory = new TestHttpClientFactory(http), MaxErrorRetry = 0 }); + } + + // ASP.NET TestHost requires a canonical Uri when AWS disables URI canonicalization for signing. + private sealed class CanonicalUriHandler(HttpMessageHandler inner) : DelegatingHandler(inner) + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { request.RequestUri = new Uri(request.RequestUri!.AbsoluteUri); return base.SendAsync(request, cancellationToken); } + } + private sealed class TestHttpClientFactory(HttpClient client) : HttpClientFactory + { + public override HttpClient CreateHttpClient(IClientConfig clientConfig) + { + return client; + } + } +} diff --git a/tests/ManagedCode.LlmTck.Tests/Bedrock/BedrockSdkCompatibilityTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Bedrock/BedrockSdkCompatibilityTests.cs similarity index 85% rename from tests/ManagedCode.LlmTck.Tests/Bedrock/BedrockSdkCompatibilityTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/Bedrock/BedrockSdkCompatibilityTests.cs index ce86501..a3fe04a 100644 --- a/tests/ManagedCode.LlmTck.Tests/Bedrock/BedrockSdkCompatibilityTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Bedrock/BedrockSdkCompatibilityTests.cs @@ -1,19 +1,19 @@ using Amazon.BedrockRuntime; using Amazon.BedrockRuntime.Model; using Amazon.Runtime; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.Providers; -using ManagedCode.LlmTck.Tests.TestSupport; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Providers; +using ManagedCode.LlmTestServer.Tests.TestSupport; -namespace ManagedCode.LlmTck.Tests.Bedrock; +namespace ManagedCode.LlmTestServer.Tests.Bedrock; public sealed class BedrockSdkCompatibilityTests { [Test] public async Task ConverseStream_OfficialSdkReadsTextAndUsageAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("amazon.nova-lite-v1:0", LlmTckModelKind.Chat) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("amazon.nova-lite-v1:0", LlmTestServerModelKind.Chat) .AddChatScenario("aws-sdk", scenario => scenario.ForModel("amazon.nova-lite-v1:0") .WhenUserContains("whale").Responds("blue whale", "blue ", "whale"))); using var http = new HttpClient(new CanonicalUriHandler(host.GetTestServer().CreateHandler())) @@ -23,7 +23,7 @@ public async Task ConverseStream_OfficialSdkReadsTextAndUsageAsync() using var sdk = new AmazonBedrockRuntimeClient(new BasicAWSCredentials("test-access-key", "test-secret-key"), new AmazonBedrockRuntimeConfig { - ServiceURL = new Uri(http.BaseAddress!, LlmTckProviderRouteNamespaces.Bedrock).ToString(), + ServiceURL = new Uri(http.BaseAddress!, LlmTestServerProviderRouteNamespaces.Bedrock).ToString(), AuthenticationRegion = "us-east-1", HttpClientFactory = new TestHttpClientFactory(http), MaxErrorRetry = 0, diff --git a/tests/ManagedCode.LlmTestServer.Tests/Client/LlmTestServerClientConfigurationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Client/LlmTestServerClientConfigurationTests.cs new file mode 100644 index 0000000..7a6b1c6 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Client/LlmTestServerClientConfigurationTests.cs @@ -0,0 +1,219 @@ +using System.Net.Http.Json; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Configuration; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Tests.TestSupport; +using Microsoft.Extensions.AI; + +namespace ManagedCode.LlmTestServer.Tests.Client; + +public sealed class LlmTestServerClientConfigurationTests +{ + [Test] + public async Task ClientBuilder_ExposesOfficialModelConvenienceMethodsAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(config => config.UseDefaultOpenAiModels().UseReasoningChatModel("gpt-5-nano", 11)); + var configuration = new LlmTestServerConfiguration + { + Models = (await http.GetFromJsonAsync>("/admin-api/models"))!, + }; + + await Assert.That(configuration.Models.Any(model => model.Id == LlmTestServerKnownModelIds.Gpt41Mini)) + .IsTrue(); + await Assert.That(configuration.Models.Any(model => model.Id == LlmTestServerKnownModelIds.TextEmbedding3Small)) + .IsTrue(); + await Assert.That(configuration.Models.Any(model => model.Id == LlmTestServerKnownModelIds.GptImage1)) + .IsTrue(); + await Assert.That(configuration.Models.Any(model => model.Id == LlmTestServerKnownModelIds.Gpt4OMiniTts)) + .IsTrue(); + await Assert.That(configuration.Models.Single(model => model.Id == "gpt-5-nano").ReasoningTokens) + .IsEqualTo(11); + } + + [Test] + public async Task ClientBuilder_ExposesKnownOpenAiModelConvenienceMethodsAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(config => config.UseKnownOpenAiModels(reasoningTokens: 19)); + var configuration = new LlmTestServerConfiguration + { + Models = (await http.GetFromJsonAsync>("/admin-api/models"))!, + }; + + await Assert.That(HasModel(configuration, LlmTestServerKnownModelIds.Gpt55, LlmTestServerModelKind.Chat)) + .IsTrue(); + await Assert.That(HasModel(configuration, LlmTestServerKnownModelIds.Gpt54Nano, LlmTestServerModelKind.Chat)) + .IsTrue(); + await Assert.That(HasModel(configuration, LlmTestServerKnownModelIds.TextEmbedding3Large, LlmTestServerModelKind.Embedding)) + .IsTrue(); + await Assert.That(HasModel(configuration, LlmTestServerKnownModelIds.GptImage2, LlmTestServerModelKind.Image)) + .IsTrue(); + await Assert.That(HasModel(configuration, LlmTestServerKnownModelIds.Tts1Hd, LlmTestServerModelKind.Audio)) + .IsTrue(); + await Assert.That(HasModel(configuration, LlmTestServerKnownModelIds.Sora2Pro, LlmTestServerModelKind.Video)) + .IsTrue(); + await Assert.That(configuration.Models.Single(model => model.Id == LlmTestServerKnownModelIds.Gpt55).ReasoningTokens) + .IsEqualTo(19); + } + + [Test] + public async Task ConfigureAsync_WithFluentClientApi_LoadsDatasetAndFixturesAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var httpClient = host.GetTestClient(); + var control = LlmTestServerClient.Create(httpClient, "test-key"); + + await control.ConfigureAsync( + server => server + .RequireBearerToken("test-key") + .UseChatModel("dataset-chat") + .UseEmbeddingModel("dataset-embedding") + .UseImageModel("dataset-image") + .UseAudioModel("dataset-audio") + .UseVideoModel("dataset-video") + .UseEmbeddingVector(0.9f, 0.8f) + .UseImageDataUri("data:image/png;base64,Zm9v") + .UseAudio([1, 2, 3, 4], "audio/test") + .UseVideo([0, 0, 0, 24, 102, 116, 121, 112], "video/test") + .UseTranscriptionText("dataset transcript") + .UseTranslationText("dataset translation") + .UseDataset( + "support-flow", + dataset => dataset + .AddChatScenario( + "dataset-blue", + scenario => scenario + .ForModel("dataset-chat") + .WhenUserContains("support") + .Responds("dataset answer", "dataset ", "answer") + .Responds("dataset answer", "dataset ", "answer") + ) + ), + resetFirst: true + ); + + using var chatClient = control.CreateChatClient("dataset-chat"); + using var embeddingGenerator = control.CreateEmbeddingGenerator("dataset-embedding"); + using var imageGenerator = control.CreateImageGenerator("dataset-image"); + + var chat = await chatClient.GetResponseAsync( + [new ChatMessage(ChatRole.User, "support request")] + ); + var streamChunks = new List(); + await foreach ( + var update in chatClient.GetStreamingResponseAsync( + [new ChatMessage(ChatRole.User, "support request")] + ) + ) + { + streamChunks.Add(update.Text); + } + + var embeddings = await embeddingGenerator.GenerateAsync(["alpha"]); + var image = await imageGenerator.GenerateAsync( + new ImageGenerationRequest { Prompt = "fixture prompt" } + ); + var audio = await control.GenerateAudioAsync("dataset-audio", "say this"); + var videoId = await control.GenerateVideoAsync("dataset-video", "make a fixture video"); + var transcription = await control.TranscribeAudioAsync( + "dataset-audio", + [82, 73, 70, 70], + "fixture.wav" + ); + var translation = await control.TranslateAudioAsync( + "dataset-audio", + [82, 73, 70, 70], + "fixture.wav" + ); + var assertions = await control.GetAssertionsAsync(); + + await Assert.That(chat.Text).IsEqualTo("dataset answer"); + await Assert.That(string.Concat(streamChunks)).IsEqualTo("dataset answer"); + await Assert.That(embeddings).Count().IsEqualTo(1); + await Assert.That(embeddings[0].Vector.ToArray()).IsEquivalentTo([0.9f, 0.8f]); + await Assert.That(image.Contents).Count().IsEqualTo(1); + await Assert.That(audio.Bytes).IsEquivalentTo((byte[])[1, 2, 3, 4]); + await Assert.That(audio.MediaType).IsEqualTo("audio/test"); + await Assert.That(videoId).IsEqualTo("video_llm_test_server"); + await Assert.That(transcription).IsEqualTo("dataset transcript"); + await Assert.That(translation).IsEqualTo("dataset translation"); + await Assert.That(assertions.Matched).IsGreaterThanOrEqualTo(8); + } + + [Test] + public async Task IndividualModelPresets_ServeEveryConfiguredModalityAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + await new LlmTestServerClient(http).ConfigureAsync(config => + { + config.UseGpt55(7).UseGpt55Pro(7).UseGpt54(7).UseGpt54Pro(7).UseGpt54Mini(7).UseGpt54Nano(7) + .UseGpt5(7).UseGpt5Pro(7).UseGpt5Mini(7).UseGpt5Nano(7).UseGpt41().UseGpt41Mini().UseGpt4OMini() + .UseTextEmbedding3Small().UseTextEmbedding3Large().UseTextEmbeddingAda002() + .UseGptImage2().UseGptImage15().UseGptImage1().UseGptImage1Mini() + .UseGpt4OMiniTts().UseTts1().UseTts1Hd().UseSora2().UseSora2Pro(); + string[] chatModels = [LlmTestServerKnownModelIds.Gpt55, LlmTestServerKnownModelIds.Gpt55Pro, + LlmTestServerKnownModelIds.Gpt54, LlmTestServerKnownModelIds.Gpt54Pro, LlmTestServerKnownModelIds.Gpt54Mini, + LlmTestServerKnownModelIds.Gpt54Nano, LlmTestServerKnownModelIds.Gpt5, LlmTestServerKnownModelIds.Gpt5Pro, + LlmTestServerKnownModelIds.Gpt5Mini, LlmTestServerKnownModelIds.Gpt5Nano, LlmTestServerKnownModelIds.Gpt41, + LlmTestServerKnownModelIds.Gpt41Mini, LlmTestServerKnownModelIds.Gpt4OMini]; + foreach (var id in chatModels) + { + config.UseChatScenario(id, scenario => scenario.ForModel(id).Responds("preset response")); + } + }); + var models = (await http.GetFromJsonAsync>("/admin-api/models"))!; + await Assert.That(models.Count).IsEqualTo(25); + foreach (var model in models) + { + var route = model.Kind switch + { + LlmTestServerModelKind.Chat => "chat/completions", + LlmTestServerModelKind.Embedding => "embeddings", + LlmTestServerModelKind.Image => "images/generations", + LlmTestServerModelKind.Audio => "audio/speech", + LlmTestServerModelKind.Video => "videos", + _ => throw new InvalidOperationException("Unexpected model kind returned by server.") + }; + using var response = await http.PostAsJsonAsync("/openai/v1/" + route, new + { + model = model.Id, + input = "preset input", + prompt = "preset image", + voice = "alloy", + messages = new[] { new { role = "user", content = "preset input" } } + }); + if (!response.IsSuccessStatusCode) + { + throw new HttpRequestException($"Model {model.Id} returned {response.StatusCode}: {await response.Content.ReadAsStringAsync()}"); + } + if (model.Kind == LlmTestServerModelKind.Chat) + { + var json = await response.Content.ReadFromJsonAsync(); + await Assert.That(json.GetProperty("choices")[0].GetProperty("message").GetProperty("content").GetString()).IsEqualTo("preset response"); + if (model.ReasoningTokens > 0) + { + await Assert.That(json.GetProperty("usage").GetProperty("completion_tokens_details").GetProperty("reasoning_tokens").GetInt32()).IsEqualTo(7); + } + } + else + { + await Assert.That((await response.Content.ReadAsByteArrayAsync()).Length).IsGreaterThan(0); + } + } + } + + private static bool HasModel( + LlmTestServerConfiguration configuration, + string id, + LlmTestServerModelKind kind + ) + { + return configuration.Models.Any(model => model.Id == id && model.Kind == kind); + } +} diff --git a/tests/ManagedCode.LlmTck.Tests/Cohere/CohereEndpointTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Cohere/CohereEndpointTests.cs similarity index 89% rename from tests/ManagedCode.LlmTck.Tests/Cohere/CohereEndpointTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/Cohere/CohereEndpointTests.cs index 3a7c15c..f7e8c81 100644 --- a/tests/ManagedCode.LlmTck.Tests/Cohere/CohereEndpointTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Cohere/CohereEndpointTests.cs @@ -1,9 +1,9 @@ using System.Net.Http.Json; using System.Text.Json; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.Tests.TestSupport; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Tests.TestSupport; -namespace ManagedCode.LlmTck.Tests.Cohere; +namespace ManagedCode.LlmTestServer.Tests.Cohere; public sealed class CohereEndpointTests { @@ -12,8 +12,8 @@ public sealed class CohereEndpointTests [Test] public async Task ChatEndpoint_ReturnsCohereChatShapeAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("command-test", LlmTckModelKind.Chat) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("command-test", LlmTestServerModelKind.Chat) .AddChatScenario( "cohere-blue-whale", scenario => scenario @@ -68,8 +68,8 @@ await Assert.That(billedUnits.GetProperty("input_tokens").GetInt32()) [Test] public async Task ChatEndpoint_StreamsCohereServerSentEventsAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("command-test", LlmTckModelKind.Chat) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("command-test", LlmTestServerModelKind.Chat) .AddChatScenario( "cohere-stream", scenario => scenario @@ -116,8 +116,8 @@ await Assert.That(response.Content.Headers.ContentType?.MediaType) [Test] public async Task EmbedEndpoint_ReturnsCohereEmbeddingShapeForTextsAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("embed-v4.0", LlmTckModelKind.Embedding) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("embed-v4.0", LlmTestServerModelKind.Embedding) .WithDefaultEmbeddingVector(0.25f, 0.5f)); using var client = host.GetTestClient(); @@ -145,8 +145,8 @@ await Assert.That(payload.GetProperty("embeddings").GetProperty("float")[0].GetA [Test] public async Task EmbedEndpoint_ReturnsCohereEmbeddingShapeForInputsAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("embed-v4.0", LlmTckModelKind.Embedding) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("embed-v4.0", LlmTestServerModelKind.Embedding) .WithDefaultEmbeddingVector(0.125f, 0.25f)); using var client = host.GetTestClient(); diff --git a/tests/ManagedCode.LlmTestServer.Tests/Conversations/ConversationHttpBoundaries.cs b/tests/ManagedCode.LlmTestServer.Tests/Conversations/ConversationHttpBoundaries.cs new file mode 100644 index 0000000..ed37c48 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Conversations/ConversationHttpBoundaries.cs @@ -0,0 +1,122 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json.Nodes; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.Conversations; + +public sealed partial class ConversationHttpIntegrationTests +{ + [Test] + [Arguments(0, 1000000)] + [Arguments(4, 0)] + [Arguments(4, 1024)] + public async Task ConversationCapacity_RejectsWithoutCreatingPartialStateAsync(int count, long bytes) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + await root.ConfigureAsync(config => config.UseConversationCapacity(count, bytes).UseConversation("capacity", graph => graph + .Step("only", step => step.Responds("accepted").Complete()))); + using var conversation = root.ForConversation("capacity-42"); + await AssertErrorAsync(conversation.HttpClient, [User("hello")], "conversation_capacity_exceeded"); + await AssertErrorAsync(conversation.HttpClient, [User("hello")], "conversation_capacity_exceeded"); + await Assert.That((await root.GetConversationsAsync()).Count).IsEqualTo(0); + await root.ConfigureAsync(config => config.UseConversationCapacity(4, 1000000).UseConversation("capacity", graph => graph + .Step("only", step => step.Responds("accepted").Complete()))); + await AssertChatAsync(conversation.HttpClient, [User("hello")], "accepted"); + } + + [Test] + public async Task ConversationCountLimit_PreservesExistingStateAndConfigureReleasesCapacityAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + await root.ConfigureAsync(config => config.UseConversationCapacity(1, 1000000).UseConversation("bounded", graph => graph + .Step("first", step => step.Responds("first answer").Then("second")) + .Step("second", step => step.Responds("second answer").Complete()))); + using var first = root.ForConversation("first"); + using var second = root.ForConversation("second"); + await AssertChatAsync(first.HttpClient, [User("hello")], "first answer"); + await AssertErrorAsync(second.HttpClient, [User("hello")], "conversation_capacity_exceeded"); + await AssertChatAsync(first.HttpClient, [User("hello"), Assistant("first answer"), User("continue")], "second answer"); + await Assert.That((await root.GetConversationsAsync()).Single().ConversationId).IsEqualTo("first"); + await root.ConfigureAsync(config => config.UseConversationCapacity(1, 1000000).UseConversation("bounded", graph => graph + .Step("only", step => step.Responds("configured again").Complete()))); + await Assert.That((await root.GetConversationsAsync()).Count).IsEqualTo(0); + await AssertChatAsync(second.HttpClient, [User("hello")], "configured again"); + } + + [Test] + [Arguments("missingStart")] + [Arguments("duplicateStep")] + [Arguments("unknownNext")] + [Arguments("duplicateNext")] + [Arguments("terminalAndNext")] + [Arguments("nonterminalNoNext")] + [Arguments("emptySteps")] + [Arguments("negativeSteps")] + [Arguments("tooManyExecutions")] + [Arguments("unknownExhaustion")] + [Arguments("invalidRole")] + [Arguments("zeroTurn")] + [Arguments("invalidArguments")] + [Arguments("invalidPointer")] + [Arguments("invalidPath")] + [Arguments("invalidExpectedJson")] + [Arguments("mixedResponseQueue")] + [Arguments("longScenarioId")] + [Arguments("longStepId")] + [Arguments("emptyStepId")] + [Arguments("negativeCapacity")] + [Arguments("negativeBytes")] + public async Task MalformedConversation_RejectsConfigurationAndPreservesServingFixtureAsync(string defect) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + await root.ConfigureAsync(config => config.UseConversation("preserved", graph => graph + .Step("only", step => step.Responds("preserved response").Complete()))); + var config = JsonNode.Parse(""" + {"models":[{"id":"gpt-4.1-mini","kind":0}],"chatScenarios":[ + {"id":"invalid","modelId":"gpt-4.1-mini","conversation":{"startStepId":"only","maxSteps":64,"steps":[ + {"id":"only","terminal":true,"response":{"content":"invalid replacement"},"match":{},"nextSteps":[]} + ]}} + ]} + """)!.AsObject(); + var scenario = config["chatScenarios"]![0]!; + var graph = scenario["conversation"]!; + var step = graph["steps"]![0]!; + switch (defect) + { + case "missingStart": graph["startStepId"] = "missing"; break; + case "duplicateStep": graph["steps"]!.AsArray().Add(step.DeepClone()); break; + case "unknownNext": step["terminal"] = false; step["nextSteps"] = new JsonArray("missing"); break; + case "duplicateNext": step["terminal"] = false; step["nextSteps"] = new JsonArray("only", "only"); break; + case "terminalAndNext": step["nextSteps"] = new JsonArray("only"); break; + case "nonterminalNoNext": step["terminal"] = false; break; + case "emptySteps": graph["steps"] = new JsonArray(); break; + case "negativeSteps": graph["maxSteps"] = -1; break; + case "tooManyExecutions": graph["maxSteps"] = 4097; break; + case "unknownExhaustion": graph["exhaustion"] = 99; break; + case "invalidRole": step["match"]!["latestRole"] = "banana"; break; + case "zeroTurn": step["match"]!["turnIndex"] = 0; break; + case "invalidArguments": step["match"]!["toolArgumentsJson"] = "{"; break; + case "invalidPointer": step["match"]!["jsonFields"] = JsonNode.Parse("""[{"path":"/bad~2escape","equalsJson":"null"}]"""); break; + case "invalidPath": step["match"]!["jsonFields"] = JsonNode.Parse("""[{"path":"$..unsupported","equalsJson":"null"}]"""); break; + case "invalidExpectedJson": step["match"]!["jsonFields"] = JsonNode.Parse("""[{"path":"$.value","equalsJson":"{"}]"""); break; + case "mixedResponseQueue": scenario["responses"] = JsonNode.Parse("""[{"content":"mixed"}]"""); break; + case "longScenarioId": scenario["id"] = new string('s', 129); break; + case "longStepId": step["id"] = new string('s', 129); break; + case "emptyStepId": step["id"] = ""; break; + case "negativeCapacity": config["maxConversations"] = -1; break; + case "negativeBytes": config["maxConversationBytes"] = -1; break; + } + using var rejected = await http.PostAsJsonAsync("/admin-api/configure", config); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + await AssertChatAsync(http, [User("still available")], "preserved response"); + await Assert.That((await root.GetConversationsAsync()).Single().ScenarioId).IsEqualTo("preserved"); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Conversations/ConversationHttpDiagnostics.cs b/tests/ManagedCode.LlmTestServer.Tests/Conversations/ConversationHttpDiagnostics.cs new file mode 100644 index 0000000..db716cf --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Conversations/ConversationHttpDiagnostics.cs @@ -0,0 +1,88 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Conversations; +using ManagedCode.LlmTestServer.Scenarios; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.Conversations; + +public sealed partial class ConversationHttpIntegrationTests +{ + [Test] + public async Task ExplainUnauthorizedScenario_DoesNotSelectFallbackOrConsumeResponseAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + await root.ConfigureAsync(config => config + .UseConversation("protected", graph => graph.RequireBearerToken("scenario-key") + .Step("only", step => step.WhenLatestUserEquals("hello").Responds("protected answer").Complete())) + .UseChatScenario("fallback", scenario => scenario.Responds("fallback answer"))); + using var conversation = root.ForConversation("protected-42"); + var explanation = await conversation.Control.ExplainMatchAsync(Explain([User("hello")])); + await Assert.That(explanation.SelectedScenarioId).IsEqualTo("protected"); + await Assert.That(explanation.SelectedStepId).IsNull(); + await Assert.That(explanation.Candidates.Single(candidate => candidate.ScenarioId == "protected").Mismatches) + .Contains("authorization"); + await Assert.That(explanation.Candidates.Single(candidate => candidate.ScenarioId == "fallback").IsSelected).IsFalse(); + await Assert.That((await root.GetConversationsAsync()).Count).IsEqualTo(0); + await AssertErrorAsync(conversation.HttpClient, [User("hello")], "invalid_api_key", HttpStatusCode.Unauthorized); + conversation.HttpClient.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", "scenario-key"); + await AssertChatAsync(conversation.HttpClient, [User("hello")], "protected answer"); + } + + [Test] + public async Task ExplainCompletedConversation_ReportsCompletedStateAndDoesNotRestartAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + await root.ConfigureAsync(config => config.UseConversation("completed", graph => graph.ReplaySuccessfulRequests(false) + .Step("only", step => step.Responds("finished").Complete()))); + using var conversation = root.ForConversation("completed-42"); + await AssertChatAsync(conversation.HttpClient, [User("hello")], "finished"); + var next = new object[] { User("hello"), Assistant("finished"), User("continue") }; + var explanation = await conversation.Control.ExplainMatchAsync(Explain(next)); + await Assert.That(explanation.SelectedStepId).IsNull(); + await Assert.That(explanation.Candidates.Any(candidate => candidate.Mismatches.Contains("state.completed", StringComparer.Ordinal))).IsTrue(); + await Assert.That((await root.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); + await AssertErrorAsync(conversation.HttpClient, next, "conversation_completed"); + } + + [Test] + public async Task ExplainStoredContinuation_RestoresHistoryWithoutAdvancingConversationAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + await root.ConfigureAsync(config => config.UseConversation("continued", graph => graph + .Step("first", step => step.WhenLatestUserEquals("begin").WhenTurn(1).Responds("ready").Then("last")) + .Step("last", step => step.WhenLatestUserEquals("finish").WhenTurn(2).Responds("done").Complete()))); + using var conversation = root.ForConversation("response-chain"); + using var initial = await conversation.HttpClient.PostAsJsonAsync("/openai/v1/responses", new { model = "gpt-4.1-mini", input = "begin" }); + initial.EnsureSuccessStatusCode(); + var id = (await initial.Content.ReadFromJsonAsync()).GetProperty("id").GetString(); + var request = new LlmTestServerMatchExplanationRequest + { + Request = new LlmTestServerChatRequest + { + PreviousResponseId = id, + HistoryNamespace = "openai", + Messages = [new LlmTestServerMessage { Role = "user", Content = "finish" }], + }, + }; + var explanation = await conversation.Control.ExplainMatchAsync(request); + await Assert.That(explanation.TurnIndex).IsEqualTo(2); + await Assert.That(explanation.SelectedStepId).IsEqualTo("last"); + await Assert.That((await root.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); + var repeated = await conversation.Control.ExplainMatchAsync(request); + await Assert.That(repeated.SelectedStepId).IsEqualTo("last"); + using var final = await conversation.HttpClient.PostAsJsonAsync("/openai/v1/responses", new + { model = "gpt-4.1-mini", input = "finish", previous_response_id = id }); + final.EnsureSuccessStatusCode(); + await Assert.That(await final.Content.ReadAsStringAsync()).Contains("done"); + await Assert.That((await root.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(2); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Conversations/ConversationHttpIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Conversations/ConversationHttpIntegrationTests.cs new file mode 100644 index 0000000..631511a --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Conversations/ConversationHttpIntegrationTests.cs @@ -0,0 +1,183 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Conversations; +using ManagedCode.LlmTestServer.Scenarios; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.Conversations; + +public sealed partial class ConversationHttpIntegrationTests +{ + [Test] + [Arguments("approve", "approved", "cancel")] + [Arguments("cancel", "cancelled", "approve")] + public async Task Branches_InspectLatestUserInsteadOfHistoricalApprovalAsync(string action, string answer, string unvisited) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + await root.ConfigureAsync(config => config.UseConversation("order", graph => graph + .Step("review", step => step.WhenLatestUserEquals("review").WhenTurn(1).Responds("choose").Then("approve", "cancel")) + .Step("approve", step => step.WhenLatestUserEquals("approve").WhenTurn(2).Responds("approved").Complete()) + .Step("cancel", step => step.WhenLatestUserEquals("cancel").WhenTurn(2).Responds("cancelled").Complete()))); + using var conversation = root.ForConversation("order-42"); + var initial = new object[] { User("approve from a historical message"), User("review") }; + await AssertChatAsync(conversation.HttpClient, initial, "choose"); + var messages = new object[] { User("approve"), User("review"), Assistant("choose"), User(action) }; + var explanation = await conversation.Control.ExplainMatchAsync(Explain(messages)); + await Assert.That(explanation.SelectedStepId).IsEqualTo(action); + await Assert.That(explanation.TurnIndex).IsEqualTo(2); + await Assert.That((await root.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); + await AssertChatAsync(conversation.HttpClient, messages, answer); + var state = (await root.GetConversationsAsync()).Single(); + await Assert.That(state.ConversationId).IsEqualTo("order-42"); + await Assert.That(state.IsTerminal).IsTrue(); + await Assert.That(state.ExecutedSteps).IsEqualTo(2); + await Assert.That(state.UnvisitedSteps).IsEquivalentTo(new[] { unvisited }); + await Assert.That(state.NextSteps.Count).IsEqualTo(0); + } + + [Test] + public async Task SuccessfulRetry_ReplaysWithoutAdvancingAndRejectsChangedStaleTurnAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + await root.ConfigureAsync(config => config.UseConversation("retry", graph => graph + .Step("start", step => step.WhenLatestUserEquals("begin").Responds("ready", "rea", "dy").Then("finish")) + .Step("finish", step => step.WhenLatestUserEquals("finish").Responds("done").Complete()))); + using var conversation = root.ForConversation("retry-safe"); + var first = new object[] { User("begin") }; + await AssertChatAsync(conversation.HttpClient, first, "ready"); + using var replay = await SendChatAsync(conversation.HttpClient, first, stream: true); + replay.EnsureSuccessStatusCode(); + var events = await replay.Content.ReadAsStringAsync(); + await Assert.That(events).Contains("rea"); + await Assert.That(events).Contains("dy"); + await Assert.That(events).Contains("[DONE]"); + await Assert.That((await root.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); + await AssertErrorAsync(conversation.HttpClient, [User("finish")], "conversation_turn_not_advanced"); + await AssertChatAsync(conversation.HttpClient, [User("begin"), Assistant("ready"), User("finish")], "done"); + await Assert.That((await root.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(2); + } + + [Test] + public async Task ScriptedFailure_TransitionsToRecoveryWithoutRequiringAssistantTurnAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + await root.ConfigureAsync(config => config.UseConversation("recover", graph => graph + .Step("unavailable", step => step.WhenLatestUserEquals("hello").WhenTurn(1) + .Fails(503, "fixture_unavailable", "Retry the same request").Then("recovered")) + .Step("recovered", step => step.WhenLatestUserEquals("hello").WhenTurn(1).Responds("available").Complete()))); + using var conversation = root.ForConversation("recover-42"); + await AssertErrorAsync(conversation.HttpClient, [User("hello")], "fixture_unavailable", HttpStatusCode.ServiceUnavailable); + var failed = (await root.GetConversationsAsync()).Single(); + await Assert.That(failed.ExecutedSteps).IsEqualTo(1); + await Assert.That(failed.LastTurnIndex).IsEqualTo(1); + await Assert.That(failed.NextSteps).IsEquivalentTo(new[] { "recovered" }); + await AssertChatAsync(conversation.HttpClient, [User("hello")], "available"); + var recovered = (await root.GetConversationsAsync()).Single(); + await Assert.That(recovered.ExecutedSteps).IsEqualTo(2); + await Assert.That(recovered.LastTurnIndex).IsEqualTo(1); + } + + [Test] + [Arguments(LlmTestServerConversationExhaustion.Reject)] + [Arguments(LlmTestServerConversationExhaustion.RepeatLastResponse)] + [Arguments(LlmTestServerConversationExhaustion.Restart)] + public async Task TerminalPolicy_ControlsRequestsAfterCompletionAsync(LlmTestServerConversationExhaustion policy) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + await root.ConfigureAsync(config => config.UseConversation("terminal", graph => graph + .ReplaySuccessfulRequests(false).OnExhaustion(policy) + .Step("only", step => step.WhenLatestUserEquals("begin").Responds("completed").Complete()))); + using var conversation = root.ForConversation("terminal-42"); + await AssertChatAsync(conversation.HttpClient, [User("begin")], "completed"); + object[] next = [User("begin"), Assistant("completed"), User(policy == LlmTestServerConversationExhaustion.Restart ? "begin" : "different")]; + if (policy == LlmTestServerConversationExhaustion.Reject) + { await AssertErrorAsync(conversation.HttpClient, next, "conversation_completed"); } + else + { await AssertChatAsync(conversation.HttpClient, next, "completed"); } + var state = (await root.GetConversationsAsync()).Single(); + await Assert.That(state.ExecutedSteps).IsEqualTo(1); + await Assert.That(state.IsTerminal).IsTrue(); + await Assert.That(state.LastTurnIndex).IsEqualTo(policy == LlmTestServerConversationExhaustion.Restart ? 2 : 1); + } + + [Test] + public async Task CyclicConversation_StopsAtConfiguredExecutionBudgetWithoutConsumingRejectedTurnAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + await root.ConfigureAsync(config => config.UseConversation("cycle", graph => graph.WithMaxSteps(2) + .Step("loop", step => step.WhenLatestUserContains("next").Responds("again").Then("loop")))); + using var conversation = root.ForConversation("cycle-42"); + await AssertChatAsync(conversation.HttpClient, [User("next one")], "again"); + await AssertChatAsync(conversation.HttpClient, [User("next one"), Assistant("again"), User("next two")], "again"); + object[] exhausted = [User("next one"), Assistant("again"), User("next two"), Assistant("again"), User("next three")]; + await AssertErrorAsync(conversation.HttpClient, exhausted, "conversation_exhausted"); + await AssertErrorAsync(conversation.HttpClient, exhausted, "conversation_exhausted"); + await Assert.That((await root.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(2); + await root.ResetAsync(); + await Assert.That((await root.GetConversationsAsync()).Count).IsEqualTo(0); + await AssertChatAsync(conversation.HttpClient, [User("next fresh")], "again"); + } + + private static object User(string content) + { + return new { role = "user", content }; + } + + private static object Assistant(string content) + { + return new { role = "assistant", content }; + } + + private static Task SendChatAsync(HttpClient http, object[] messages, bool stream = false, object[]? tools = null, + object? responseFormat = null) + { + var body = new Dictionary { ["model"] = "gpt-4.1-mini", ["messages"] = messages, ["stream"] = stream }; + if (tools is not null) { body["tools"] = tools; } + if (responseFormat is not null) { body["response_format"] = responseFormat; } + return http.PostAsJsonAsync("/openai/v1/chat/completions", body); + } + + private static async Task AssertChatAsync(HttpClient http, object[] messages, string content, object[]? tools = null) + { + using var response = await SendChatAsync(http, messages, tools: tools); + response.EnsureSuccessStatusCode(); + var json = await response.Content.ReadFromJsonAsync(); + await Assert.That(json.GetProperty("choices")[0].GetProperty("message").GetProperty("content").GetString()).IsEqualTo(content); + } + + private static async Task AssertErrorAsync(HttpClient http, object[] messages, string code, + HttpStatusCode status = HttpStatusCode.Conflict, object[]? tools = null) + { + using var response = await SendChatAsync(http, messages, tools: tools); + await Assert.That(response.StatusCode).IsEqualTo(status); + var json = await response.Content.ReadFromJsonAsync(); + await Assert.That(json.GetProperty("error").GetProperty("code").GetString()).IsEqualTo(code); + } + + private static LlmTestServerMatchExplanationRequest Explain(object[] messages) + { + return new() + { + Request = new LlmTestServerChatRequest + { + Messages = messages.Select(message => + { + var value = JsonSerializer.SerializeToElement(message); + return new LlmTestServerMessage { Role = value.GetProperty("role").GetString()!, Content = value.GetProperty("content").GetString()! }; + }).ToList(), + }, + }; + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Conversations/ConversationHttpTools.cs b/tests/ManagedCode.LlmTestServer.Tests/Conversations/ConversationHttpTools.cs new file mode 100644 index 0000000..0116ac1 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Conversations/ConversationHttpTools.cs @@ -0,0 +1,72 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.Conversations; + +public sealed partial class ConversationHttpIntegrationTests +{ + [Test] + [Arguments("$.forecast[0].temperature", "21", "{\"forecast\":[{\"temperature\":21}]}")] + [Arguments("/a~1b/~0value/0", "\"sunny\"", "{\"a/b\":{\"~value\":[\"sunny\"]}}")] + [Arguments("$", "{\"ok\":true}", "{\"ok\":true}")] + public async Task ToolResult_RequiresMatchingCallNameIdArgumentsAndJsonFieldsAsync(string path, string expectedJson, string validResult) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + await root.ConfigureAsync(config => config.UseConversation("weather", graph => graph + .Step("ask", step => step.WhenLatestUserEquals("weather").CallsTool("weather", "{\"city\":\"Paris\",\"units\":\"C\"}", "call_paris").Then("answer")) + .Step("answer", step => step.WhenToolResult("weather", "call_paris").WithToolArgumentsJson("{\"units\":\"C\",\"city\":\"Paris\"}") + .WhenLatestJson(path, expectedJson).WhenTurn(2).Responds("Paris forecast accepted").Complete()))); + using var conversation = root.ForConversation("weather-42"); + object[] tools = [new { type = "function", function = new { name = "weather", parameters = new { type = "object" } } }]; + using var start = await SendChatAsync(conversation.HttpClient, [User("weather")], tools: tools); + start.EnsureSuccessStatusCode(); + var wire = await start.Content.ReadFromJsonAsync(); + var call = wire.GetProperty("choices")[0].GetProperty("message").GetProperty("tool_calls")[0]; + await Assert.That(call.GetProperty("id").GetString()).IsEqualTo("call_paris"); + await Assert.That(call.GetProperty("function").GetProperty("name").GetString()).IsEqualTo("weather"); + object[] History(string name, string id, string arguments, string result, string resultId) + { + return [ + User("weather"), + new { role = "assistant", content = "", tool_calls = new[] { new { id, type = "function", function = new { name, arguments } } } }, + new { role = "tool", content = result, tool_call_id = resultId }, + ]; + } + + await AssertErrorAsync(conversation.HttpClient, History("other", "call_paris", "{\"city\":\"Paris\",\"units\":\"C\"}", validResult, "call_paris"), "conversation_step_mismatch", tools: tools); + await AssertErrorAsync(conversation.HttpClient, History("weather", "call_other", "{\"city\":\"Paris\",\"units\":\"C\"}", validResult, "call_other"), "conversation_step_mismatch", tools: tools); + await AssertErrorAsync(conversation.HttpClient, History("weather", "call_paris", "{\"city\":\"London\",\"units\":\"C\"}", validResult, "call_paris"), "conversation_step_mismatch", tools: tools); + await AssertErrorAsync(conversation.HttpClient, History("weather", "call_paris", "{\"city\":\"Paris\",\"units\":\"C\"}", "{}", "call_paris"), "conversation_step_mismatch", tools: tools); + await Assert.That((await root.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); + await AssertChatAsync(conversation.HttpClient, History("weather", "call_paris", "{\"units\":\"C\",\"city\":\"Paris\"}", validResult, "call_paris"), "Paris forecast accepted", tools); + var completed = (await root.GetConversationsAsync()).Single(); + await Assert.That(completed.IsTerminal).IsTrue(); + await Assert.That(completed.ExecutedSteps).IsEqualTo(2); + } + + [Test] + public async Task InvalidStructuredFixture_DoesNotCreateOrAdvanceConversationAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + await root.ConfigureAsync(config => config.UseConversation("plain", graph => graph + .Step("plain", step => step.WhenLatestUserEquals("hello").Responds("plain text").Complete()))); + using var conversation = root.ForConversation("schema-retry"); + using var invalid = await SendChatAsync(conversation.HttpClient, [User("hello")], responseFormat: new { type = "json_object" }); + await Assert.That(invalid.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + await Assert.That(await invalid.Content.ReadAsStringAsync()).Contains("llm_test_server_fixture_mismatch"); + await Assert.That((await root.GetConversationsAsync()).Count).IsEqualTo(0); + await AssertChatAsync(conversation.HttpClient, [User("hello")], "plain text"); + await Assert.That((await root.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); + using var invalidReplay = await SendChatAsync(conversation.HttpClient, [User("hello")], responseFormat: new { type = "json_object" }); + await Assert.That(invalidReplay.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + await AssertChatAsync(conversation.HttpClient, [User("hello")], "plain text"); + await Assert.That((await root.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Conversations/ConversationHttpValidation.cs b/tests/ManagedCode.LlmTestServer.Tests/Conversations/ConversationHttpValidation.cs new file mode 100644 index 0000000..310cb77 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Conversations/ConversationHttpValidation.cs @@ -0,0 +1,125 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using System.Text.Json.Nodes; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.Conversations; + +public sealed partial class ConversationHttpIntegrationTests +{ + [Test] + public async Task ExplainNullToolCall_ReturnsBadRequestWithoutConsumingConversationAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + await root.ConfigureAsync(config => config.UseConversation("validation", graph => graph + .Step("only", step => step.Responds("preserved answer").Complete()))); + var malformed = JsonNode.Parse(""" + {"providerNamespace":"/openai","request":{"modelId":"gpt-4.1-mini","messages":[ + {"role":"assistant","content":"","toolCalls":[null]} + ]}} + """); + using var rejected = await http.PostAsJsonAsync("/admin-api/match/explain", malformed); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + await Assert.That((await root.GetConversationsAsync()).Count).IsEqualTo(0); + await AssertChatAsync(http, [User("hello")], "preserved answer"); + await Assert.That((await root.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); + } + + [Test] + public async Task ExplainFreshToolFixture_ReportsMissingToolsAndMatchesActualRejectionAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + await root.ConfigureAsync(config => config.UseConversation("tool-validation", graph => graph + .Step("lookup", step => step.WhenLatestUserEquals("weather").CallsTool("weather", "{}", "call_weather").Complete()))); + using var conversation = root.ForConversation("fresh-tool-validation"); + var explanation = await conversation.Control.ExplainMatchAsync(Explain([User("weather")])); + await Assert.That(explanation.SelectedStepId).IsNull(); + var candidate = explanation.Candidates.Single(item => item.ScenarioId == "tool-validation" && item.StepId == "lookup"); + await Assert.That(candidate.Mismatches).Contains("fixture.mismatch"); + await Assert.That(candidate.IsMatch).IsFalse(); + await Assert.That(candidate.IsSelected).IsFalse(); + await Assert.That((await root.GetConversationsAsync()).Count).IsEqualTo(0); + await AssertErrorAsync(conversation.HttpClient, [User("weather")], "llm_test_server_fixture_mismatch"); + await Assert.That((await root.GetConversationsAsync()).Count).IsEqualTo(0); + object[] tools = [new { type = "function", function = new { name = "weather", parameters = new { type = "object" } } }]; + using var accepted = await SendChatAsync(conversation.HttpClient, [User("weather")], tools: tools); + accepted.EnsureSuccessStatusCode(); + var result = await accepted.Content.ReadFromJsonAsync(); + await Assert.That(result.GetProperty("choices")[0].GetProperty("message").GetProperty("tool_calls")[0].GetProperty("id").GetString()) + .IsEqualTo("call_weather"); + await Assert.That((await root.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); + } + + [Test] + public async Task ExplainOrderedBranches_DoesNotSkipFirstPredicateMatchWhenItsFixtureIsInvalidAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + await root.ConfigureAsync(config => config.UseConversation("ordered-validation", graph => graph + .Step("start", step => step.WhenLatestUserEquals("begin").Responds("ready").Then("invalid", "valid")) + .Step("invalid", step => step.WhenLatestUserEquals("continue").WhenTurn(2).CallsTool("weather", "{}").Complete()) + .Step("valid", step => step.WhenLatestUserEquals("continue").WhenTurn(2).Responds("must not skip ahead").Complete()))); + using var conversation = root.ForConversation("ordered-validation-42"); + await AssertChatAsync(conversation.HttpClient, [User("begin")], "ready"); + object[] messages = [User("begin"), Assistant("ready"), User("continue")]; + var explanation = await conversation.Control.ExplainMatchAsync(Explain(messages)); + await Assert.That(explanation.SelectedStepId).IsNull(); + var invalid = explanation.Candidates.Single(candidate => candidate.StepId == "invalid"); + await Assert.That(invalid.Mismatches).Contains("fixture.mismatch"); + await Assert.That(invalid.IsSelected).IsFalse(); + var shadowed = explanation.Candidates.Single(candidate => candidate.StepId == "valid"); + await Assert.That(shadowed.IsSelected).IsFalse(); + await Assert.That(shadowed.IsShadowed).IsTrue(); + await AssertErrorAsync(conversation.HttpClient, messages, "llm_test_server_fixture_mismatch"); + var state = (await root.GetConversationsAsync()).Single(); + await Assert.That(state.ExecutedSteps).IsEqualTo(1); + await Assert.That(state.NextSteps).IsEquivalentTo(new[] { "invalid", "valid" }); + } + + [Test] + [Arguments("nullContent")] + [Arguments("nullChunk")] + [Arguments("nullErrorCode")] + [Arguments("nullErrorMessage")] + [Arguments("zeroErrorStatus")] + [Arguments("successfulErrorStatus")] + [Arguments("outOfRangeErrorStatus")] + public async Task InvalidStepResponse_RejectsConfigurationAndPreservesOldServerBehaviorAsync(string defect) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + await root.ConfigureAsync(config => config.UseConversation("valid", graph => graph + .Step("only", step => step.Responds("valid response").Complete()))); + var config = JsonNode.Parse(""" + {"models":[{"id":"gpt-4.1-mini","kind":0}],"chatScenarios":[ + {"id":"invalid","modelId":"gpt-4.1-mini","conversation":{"startStepId":"only","steps":[ + {"id":"only","terminal":true,"response":{"content":"replacement"}} + ]}} + ]} + """)!; + var response = config["chatScenarios"]![0]!["conversation"]!["steps"]![0]!["response"]!; + switch (defect) + { + case "nullContent": response["content"] = null; break; + case "nullChunk": response["streamChunks"] = JsonNode.Parse("[null]"); break; + case "nullErrorCode": response["error"] = JsonNode.Parse("""{"statusCode":503,"code":null,"message":"unavailable"}"""); break; + case "nullErrorMessage": response["error"] = JsonNode.Parse("""{"statusCode":503,"code":"unavailable","message":null}"""); break; + case "zeroErrorStatus": response["error"] = JsonNode.Parse("""{"statusCode":0,"code":"unavailable","message":"unavailable"}"""); break; + case "successfulErrorStatus": response["error"] = JsonNode.Parse("""{"statusCode":200,"code":"unavailable","message":"unavailable"}"""); break; + case "outOfRangeErrorStatus": response["error"] = JsonNode.Parse("""{"statusCode":600,"code":"unavailable","message":"unavailable"}"""); break; + } + using var rejected = await http.PostAsJsonAsync("/admin-api/configure", config); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + await Assert.That((await root.GetConversationsAsync()).Count).IsEqualTo(0); + await AssertChatAsync(http, [User("still serving")], "valid response"); + await Assert.That((await root.GetConversationsAsync()).Single().ScenarioId).IsEqualTo("valid"); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Conversations/ConversationProviderIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Conversations/ConversationProviderIntegrationTests.cs new file mode 100644 index 0000000..0d5d5c7 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Conversations/ConversationProviderIntegrationTests.cs @@ -0,0 +1,243 @@ +using System.Diagnostics; +using System.Net; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Providers; +using ManagedCode.LlmTestServer.Sessions; +using ManagedCode.LlmTestServer.Tests.Providers; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.Conversations; + +public sealed class ConversationProviderIntegrationTests +{ + [Test] + public async Task EveryChatProvider_MaintainsItsOwnGraphWithinOneConversationAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(config => config.AddConversation("provider-isolation", graph => graph + .ReplaySuccessfulRequests(false).Step("first", step => step.WhenLatestUserEquals("same input").Responds("provider start").Complete()))); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + using var conversation = root.ForConversation("shared-id"); + foreach (var provider in ProviderFaultSimulationTests.GetProviderFaultCases()) + { + using var first = await provider.SendAsync(conversation.HttpClient, "same input"); + first.EnsureSuccessStatusCode(); + await Assert.That(await first.Content.ReadAsStringAsync()).Contains("provider start"); + using var exhausted = await provider.SendAsync(conversation.HttpClient, "same input"); + await Assert.That(exhausted.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + } + var states = await conversation.Control.GetConversationsAsync(); + await Assert.That(states.Count).IsEqualTo(13); + await Assert.That(states.Select(state => state.ProviderNamespace).Distinct().Count()).IsEqualTo(13); + await Assert.That(states.All(state => state.IsTerminal && state.ExecutedSteps == 1)).IsTrue(); + using var independent = root.ForConversation("another-id"); + using var fresh = await ProviderFaultSimulationTests.GetProviderFaultCases()[0].SendAsync(independent.HttpClient, "same input"); + fresh.EnsureSuccessStatusCode(); + await Assert.That((await independent.Control.GetConversationsAsync()).Single().ConversationId).IsEqualTo("another-id"); + } + + [Test] + public async Task IdenticalConversationIds_AreIsolatedByProviderCredentialAndSessionAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + await using var first = await root.CreateSessionAsync(config => config.UseConversation("isolated", graph => graph + .ReplaySuccessfulRequests(false).Step("one", step => step.Responds("first session").Complete()))); + await using var second = await root.CreateSessionAsync(config => config.UseConversation("isolated", graph => graph + .ReplaySuccessfulRequests(false).Step("one", step => step.Responds("second session").Complete()))); + using var firstConversation = first.Control.ForConversation("same-id"); + using var secondConversation = second.Control.ForConversation("same-id"); + foreach (var credential in new[] { "alpha-key", "beta-key" }) + { + using var accepted = await SendAsync(firstConversation.HttpClient, credential); + accepted.EnsureSuccessStatusCode(); + await Assert.That(await accepted.Content.ReadAsStringAsync()).Contains("first session"); + using var exhausted = await SendAsync(firstConversation.HttpClient, credential); + await Assert.That(exhausted.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + } + using var anotherSession = await SendAsync(secondConversation.HttpClient, "alpha-key"); + anotherSession.EnsureSuccessStatusCode(); + await Assert.That(await anotherSession.Content.ReadAsStringAsync()).Contains("second session"); + await Assert.That((await first.Control.GetConversationsAsync()).Count).IsEqualTo(2); + await Assert.That((await second.Control.GetConversationsAsync()).Count).IsEqualTo(1); + await Assert.That((await root.GetConversationsAsync()).Count).IsEqualTo(0); + firstConversation.Dispose(); + using var retained = first.Control.ForConversation("same-id"); + using var stillExhausted = await SendAsync(retained.HttpClient, "alpha-key"); + await Assert.That(stillExhausted.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + await first.Control.ResetAsync(); + using var reset = await SendAsync(retained.HttpClient, "alpha-key"); + reset.EnsureSuccessStatusCode(); + using var secondStillExhausted = await SendAsync(secondConversation.HttpClient, "alpha-key"); + await Assert.That(secondStillExhausted.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + } + + [Test] + public async Task CancelledStep_RollsBackBusyReservationAndPreservesOtherConversationsAsync() + { + using var watchdog = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + using var host = await LlmTestServerTestHost.StartAsync(config => config + .AddConversation("cancel", graph => graph.ReplaySuccessfulRequests(false) + .Step("delayed", step => step.WhenLatestUserEquals("same input").Responds("preserved").DelaysBy(60000).Complete())) + .AddConversation("independent", graph => graph.ReplaySuccessfulRequests(false) + .Step("ready", step => step.WhenLatestUserEquals("other input").Responds("independent state").Complete())), cancellationToken: watchdog.Token); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + using var first = root.ForConversation("cancelled"); + using var other = root.ForConversation("active"); + using (var established = await SendAsync(other.HttpClient, cancellationToken: watchdog.Token, input: "other input")) + { + established.EnsureSuccessStatusCode(); + await Assert.That(await established.Content.ReadAsStringAsync(watchdog.Token)).Contains("independent state"); + } + var independentBefore = (await other.Control.GetConversationsAsync(watchdog.Token)).Single(); + await Assert.That(independentBefore.ExecutedSteps).IsEqualTo(1); + await Assert.That(independentBefore.IsTerminal).IsTrue(); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(watchdog.Token); + var pending = SendAsync(first.HttpClient, cancellationToken: cancellation.Token); + try + { + await WaitUntilBusyAsync(first.Control, pending, watchdog.Token); + using var concurrent = await SendAsync(first.HttpClient, cancellationToken: watchdog.Token); + await Assert.That(concurrent.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + await Assert.That(await concurrent.Content.ReadAsStringAsync(watchdog.Token)).Contains("conversation_busy"); + cancellation.Cancel(); + var cancelled = false; + try { using var response = await pending; } + catch (OperationCanceledException) when (cancellation.IsCancellationRequested) { cancelled = true; } + await Assert.That(cancelled).IsTrue(); + while ((await first.Control.GetConversationsAsync(watchdog.Token)).Any(state => state.IsBusy)) + { await Task.Delay(10, watchdog.Token); } + // These observations prove rollback BEFORE any configuration mutation can clear state. + await Assert.That((await first.Control.GetConversationsAsync(watchdog.Token)).Count).IsEqualTo(0); + var selectable = await first.Control.ExplainMatchAsync(new() + { + ProviderNamespace = LlmTestServerProviderRouteNamespaces.OpenAI, + Request = new() { ModelId = LlmTestServerKnownModelIds.Gpt41Mini, Messages = [new() { Role = "user", Content = "same input" }] }, + }, watchdog.Token); + await Assert.That(selectable.SelectedScenarioId).IsEqualTo("cancel"); + await Assert.That(selectable.SelectedStepId).IsEqualTo("delayed"); + await Assert.That(selectable.Candidates.Single(candidate => candidate.IsSelected).IsMatch).IsTrue(); + var independentAfter = (await other.Control.GetConversationsAsync(watchdog.Token)).Single(); + await Assert.That(independentAfter.ExecutedSteps).IsEqualTo(independentBefore.ExecutedSteps); + await Assert.That(independentAfter.ScenarioId).IsEqualTo(independentBefore.ScenarioId); + await Assert.That(independentAfter.IsTerminal).IsTrue(); + await Assert.That(independentAfter.IsBusy).IsFalse(); + await Assert.That((await root.GetConversationsAsync(watchdog.Token)).Count).IsEqualTo(1); + await Assert.That((await root.GetAssertionsAsync(watchdog.Token)).Matched).IsEqualTo(1); + + // Configure deliberately resets both graphs only after their cancellation invariants were verified. + // Zero-delay fixtures keep explicit recovery fast; recovery alone is not the rollback proof. + await root.ConfigureAsync(config => config + .UseConversation("cancel", graph => graph.ReplaySuccessfulRequests(false) + .Step("delayed", step => step.WhenLatestUserEquals("same input").Responds("preserved").Complete())) + .UseConversation("independent", graph => graph.ReplaySuccessfulRequests(false) + .Step("ready", step => step.WhenLatestUserEquals("other input").Responds("independent state").Complete())), watchdog.Token); + var replies = await Task.WhenAll(SendAsync(first.HttpClient, cancellationToken: watchdog.Token), + SendAsync(other.HttpClient, cancellationToken: watchdog.Token, input: "other input")); + string[] expectedReplies = ["preserved", "independent state"]; + for (var index = 0; index < replies.Length; index++) + { + using var response = replies[index]; + response.EnsureSuccessStatusCode(); + await Assert.That(await response.Content.ReadAsStringAsync(watchdog.Token)).Contains(expectedReplies[index]); + } + var states = await root.GetConversationsAsync(watchdog.Token); + await Assert.That(states.Count).IsEqualTo(2); + await Assert.That(states.All(state => state.ExecutedSteps == 1 && !state.IsBusy)).IsTrue(); + } + finally + { + cancellation.Cancel(); + try { using var response = await pending.WaitAsync(TimeSpan.FromSeconds(60)); } + catch (OperationCanceledException) when (cancellation.IsCancellationRequested) { } + } + } + + [Test] + [Arguments(0)] + [Arguments(1)] + [Arguments(2)] + public async Task SessionMutation_InvalidatesPendingGraphAndClearsItsStateAsync(int mutation) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + await using var session = await root.CreateSessionAsync(config => config.UseConversation("pending", graph => graph + .Step("one", step => step.Responds("old graph").DelaysBy(500).Complete()))); + using var conversation = session.Control.ForConversation("pending-id"); + var pending = SendAsync(conversation.HttpClient); + await WaitUntilBusyAsync(conversation.Control); + if (mutation == 0) { await session.Control.ResetAsync(); } + else if (mutation == 1) + { + await session.Control.ConfigureAsync(config => config.UseConversation("replacement", graph => graph + .Step("one", step => step.Responds("new graph").Complete()))); + } + else + { + using var request = new HttpRequestMessage(HttpMethod.Delete, "/admin-api/sessions/" + session.Session.Id); + request.Headers.Add(LlmTestServerSessionHeaders.Id, session.Session.Id); + request.Headers.Add(LlmTestServerSessionHeaders.AccessKey, session.Session.AccessKey); + using var deleted = await http.SendAsync(request); + await Assert.That(deleted.StatusCode).IsEqualTo(HttpStatusCode.NoContent); + } + using var superseded = await pending; + await Assert.That(superseded.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + await Assert.That(await superseded.Content.ReadAsStringAsync()).Contains("llm_test_server_request_superseded"); + if (mutation != 2) + { + await Assert.That((await conversation.Control.GetConversationsAsync()).Count).IsEqualTo(0); + using var next = await SendAsync(conversation.HttpClient); + next.EnsureSuccessStatusCode(); + await Assert.That(await next.Content.ReadAsStringAsync()).Contains(mutation == 1 ? "new graph" : "old graph"); + await Assert.That((await conversation.Control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); + } + else + { + using var absent = await SendAsync(conversation.HttpClient); + await Assert.That(absent.StatusCode).IsEqualTo(HttpStatusCode.NotFound); + } + await Assert.That((await root.GetConversationsAsync()).Count).IsEqualTo(0); + } + + private static async Task SendAsync(HttpClient http, string? credential = null, string input = "same input", CancellationToken cancellationToken = default) + { + using var request = new HttpRequestMessage(HttpMethod.Post, "/openai/v1/chat/completions") + { + Content = JsonContent.Create(new { model = "gpt-4.1-mini", messages = new[] { new { role = "user", content = input } } }), + }; + if (credential is not null) { request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", credential); } + return await http.SendAsync(request, cancellationToken); + } + + private static async Task WaitUntilBusyAsync(LlmTestServerClient control, Task pending, CancellationToken cancellationToken) + { + while (true) + { + var observed = control.GetConversationsAsync(cancellationToken); + if (await Task.WhenAny(pending, observed) == pending) + { + await observed; + using var completed = await pending; + throw new InvalidOperationException($"The provider request completed with HTTP {(int)completed.StatusCode} before its busy reservation was observed."); + } + if ((await observed).Any(state => state.IsBusy)) { return; } + await Task.Delay(10, cancellationToken); + } + } + + private static async Task WaitUntilBusyAsync(LlmTestServerClient control) + { + var stopwatch = Stopwatch.StartNew(); + while (!(await control.GetConversationsAsync()).Any(state => state.IsBusy)) + { + if (stopwatch.Elapsed > TimeSpan.FromSeconds(5)) { throw new TimeoutException("The server did not reserve the conversation step."); } + await Task.Delay(10); + } + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Conversations/ConversationSdkIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Conversations/ConversationSdkIntegrationTests.cs new file mode 100644 index 0000000..0285d9d --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Conversations/ConversationSdkIntegrationTests.cs @@ -0,0 +1,125 @@ +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Net.Http.Json; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Tests.TestSupport; +using Microsoft.Extensions.AI; +using OpenAI; +using OpenAI.Chat; +using OpenAI.Responses; + +namespace ManagedCode.LlmTestServer.Tests.Conversations; + +#pragma warning disable OPENAI001 // Exercise the official experimental Responses SDK through HTTP. +public sealed class ConversationSdkIntegrationTests +{ + [Test] + [Arguments(false, false)] + [Arguments(false, true)] + [Arguments(true, false)] + [Arguments(true, true)] + public async Task OfficialChatSdk_CompletesToolResultGraphAsync(bool azure, bool streaming) + { + using var host = await LlmTestServerTestHost.StartAsync(config => config.AddConversation("weather", graph => graph + .Step("lookup", step => step.WhenLatestUserEquals("weather").WhenTurn(1) + .CallsTool("weather", """{"city":"Paris"}""", "call_weather").Then("result")) + .Step("result", step => step.WhenToolResult("weather", "call_weather").WhenTurn(2) + .WithToolArgumentsJson("""{"city":"Paris"}""").WhenLatestJsonValue("$.temperature", 22) + .Responds("Paris is 22 degrees", "Paris is ", "22 degrees").Complete()))); + using var http = host.GetTestClient(); + using var conversation = new LlmTestServerClient(http).ForConversation("sdk-weather"); + var transport = new HttpClientPipelineTransport(conversation.HttpClient); + var sdk = azure + ? new global::Azure.AI.OpenAI.AzureOpenAIClient(new Uri(http.BaseAddress!, "/azure-openai"), + new global::Azure.AzureKeyCredential("provider-key"), new() { Transport = transport }).GetChatClient(LlmTestServerKnownModelIds.Gpt41Mini) + : new global::OpenAI.Chat.ChatClient(LlmTestServerKnownModelIds.Gpt41Mini, new ApiKeyCredential("provider-key"), + new OpenAIClientOptions { Endpoint = new Uri(http.BaseAddress!, "/openai/v1"), Transport = transport }); + var options = new ChatCompletionOptions + { + Tools = { ChatTool.CreateFunctionTool("weather", functionParameters: BinaryData.FromString("""{"type":"object","properties":{"city":{"type":"string"}}}""")) }, + }; + List messages = [new UserChatMessage("weather")]; + var first = (await sdk.CompleteChatAsync(messages, options)).Value; + var call = first.ToolCalls.Single(); + await Assert.That(call.Id).IsEqualTo("call_weather"); + await Assert.That(call.FunctionArguments.ToString()).IsEqualTo("""{"city":"Paris"}"""); + messages.Add(new AssistantChatMessage(first)); + messages.Add(new ToolChatMessage(call.Id, """{"temperature":22}""")); + string result; + if (streaming) + { + result = string.Empty; + await foreach (var update in sdk.CompleteChatStreamingAsync(messages, options)) + { result += string.Concat(update.ContentUpdate.Select(part => part.Text)); } + } + else { result = (await sdk.CompleteChatAsync(messages, options)).Value.Content[0].Text; } + await Assert.That(result).IsEqualTo("Paris is 22 degrees"); + var state = (await conversation.Control.GetConversationsAsync()).Single(); + await Assert.That(state.ExecutedSteps).IsEqualTo(2); + await Assert.That(state.IsTerminal).IsTrue(); + var events = (await conversation.Control.GetAssertionsAsync()).Events; + await Assert.That(events.Select(item => item.StepId)).IsEquivalentTo(new string?[] { "lookup", "result" }); + await Assert.That(events.Select(item => item.TurnIndex)).IsEquivalentTo(new int?[] { 1, 2 }); + } + + [Test] + [Arguments("/openai/v1/", false)] + [Arguments("/openai/v1/", true)] + [Arguments("/azure-openai/openai/v1/", false)] + [Arguments("/azure-openai/openai/v1/", true)] + [Arguments("/microsoft-foundry/openai/v1/", false)] + [Arguments("/microsoft-foundry/openai/v1/", true)] + public async Task OfficialResponsesSdk_RestoresToolHistoryBeforeGraphMatchingAsync(string endpoint, bool streaming) + { + using var host = await LlmTestServerTestHost.StartAsync(config => config.AddConversation("document", graph => graph + .Step("read", step => step.WhenLatestUserEquals("review document").WhenTurn(1) + .CallsTool("read_document", "{}", "call_document").Then("review")) + .Step("review", step => step.WhenToolResult("read_document", "call_document").WhenTurn(2) + .WhenLatestTextContains("verified evidence").Responds("review complete", "review ", "complete").Complete()))); + using var http = host.GetTestClient(); + using var conversation = new LlmTestServerClient(http).ForConversation("responses-tool"); + var sdk = new ResponsesClient(new ApiKeyCredential("provider-key"), new ResponsesClientOptions + { Endpoint = new Uri(http.BaseAddress!, endpoint), Transport = new HttpClientPipelineTransport(conversation.HttpClient) }); + using var client = sdk.AsIChatClient(LlmTestServerKnownModelIds.Gpt41Mini).AsBuilder().UseFunctionInvocation().Build(); + var invocations = 0; + var options = new ChatOptions { Tools = [AIFunctionFactory.Create(() => { invocations++; return "verified evidence"; }, "read_document")] }; + var response = streaming + ? await client.GetStreamingResponseAsync("review document", options).ToChatResponseAsync() + : await client.GetResponseAsync("review document", options); + await Assert.That(response.Text).IsEqualTo("review complete"); + await Assert.That(invocations).IsEqualTo(1); + var states = await conversation.Control.GetConversationsAsync(); + await Assert.That(states.Single().ExecutedSteps).IsEqualTo(2); + var events = (await conversation.Control.GetAssertionsAsync()).Events; + await Assert.That(events.Last().Messages.Any(message => message.Role == "user" && message.Content == "review document")).IsTrue(); + await Assert.That(events.Last().Messages.Single(message => message.Role == "tool").Content).Contains("verified evidence"); + } + + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task ResponsesContinuation_ReplaysLatestLogicalTurnAcrossJsonAndSseAsync(bool replayStreaming) + { + using var host = await LlmTestServerTestHost.StartAsync(config => config.AddConversation("continuation", graph => graph + .Step("first", step => step.WhenLatestUserEquals("start").WhenTurn(1).Responds("first answer").Then("second")) + .Step("second", step => step.WhenLatestUserEquals("continue").WhenTurn(2).Responds("second answer", "second ", "answer").Complete()))); + using var http = host.GetTestClient(); + using var conversation = new LlmTestServerClient(http).ForConversation("continuation"); + using var first = await conversation.HttpClient.PostAsJsonAsync("/openai/v1/responses", new { model = LlmTestServerKnownModelIds.Gpt41Mini, input = "start" }); + first.EnsureSuccessStatusCode(); + var firstId = (await first.Content.ReadFromJsonAsync()).GetProperty("id").GetString(); + var body = new { model = LlmTestServerKnownModelIds.Gpt41Mini, input = "continue", previous_response_id = firstId }; + using var second = await conversation.HttpClient.PostAsJsonAsync("/openai/v1/responses", body); + second.EnsureSuccessStatusCode(); + var secondId = (await second.Content.ReadFromJsonAsync()).GetProperty("id").GetString(); + using var replay = await conversation.HttpClient.PostAsJsonAsync("/openai/v1/responses", new + { model = LlmTestServerKnownModelIds.Gpt41Mini, input = "continue", previous_response_id = firstId, stream = replayStreaming }); + replay.EnsureSuccessStatusCode(); + var replayText = await replay.Content.ReadAsStringAsync(); + await Assert.That(replayText).Contains(secondId!); + await Assert.That(replayText).Contains(replayStreaming ? "response.completed" : "second answer"); + await Assert.That((await conversation.Control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(2); + await Assert.That((await conversation.Control.GetAssertionsAsync()).Events.Last().IsReplay).IsTrue(); + } +} diff --git a/tests/ManagedCode.LlmTck.Tests/Decisions/CloudflareDecisionContractTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Decisions/CloudflareDecisionContractTests.cs similarity index 81% rename from tests/ManagedCode.LlmTck.Tests/Decisions/CloudflareDecisionContractTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/Decisions/CloudflareDecisionContractTests.cs index 10da430..c717b48 100644 --- a/tests/ManagedCode.LlmTck.Tests/Decisions/CloudflareDecisionContractTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Decisions/CloudflareDecisionContractTests.cs @@ -2,14 +2,13 @@ using System.Net.Http.Json; using System.Text.Json; using System.Text.Json.Nodes; -using ManagedCode.LlmTck.Cloudflare; -using ManagedCode.LlmTck.Decisions; -using ManagedCode.LlmTck.Hosting; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Tests.TestSupport; -using Microsoft.Extensions.DependencyInjection; +using ManagedCode.LlmTestServer.Cloudflare; +using ManagedCode.LlmTestServer.Decisions; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Tests.TestSupport; -namespace ManagedCode.LlmTck.Tests.Decisions; +namespace ManagedCode.LlmTestServer.Tests.Decisions; public sealed class CloudflareDecisionContractTests { @@ -24,12 +23,14 @@ public async Task PredicateCriteria_AcceptsNativeJsonScalarsWithoutWeakeningType { var body = CloudflareDecisionTestFixtures.Body(model); body["questions"]!["binary"]!["criteria"] = new JsonObject { ["true"] = JsonNode.Parse(nativeValue), ["false"] = false }; - var native = JsonSerializer.SerializeToElement(body); - await Assert.That(ManagedCode.LlmTck.SystemOne.SystemOneDecisionMapper.Validate(native, LlmTckDecisionProvider.TypeSafe).IsValid).IsFalse(); - using var host = await LlmTckTestHost.StartAsync(builder => builder.AddClef().AddClefFlash() + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddClef().AddClefFlash() .AddDecisionScenario(CloudflareDecisionTestFixtures.Scenario(model))); using var client = host.GetTestClient(); using var response = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor(model), body); await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.OK); + var typeSafeBody = body.DeepClone(); + typeSafeBody["model"] = "jev-latest"; + using var rejectedTypeSafe = await client.PostAsJsonAsync("/systemone/v1/systemone", typeSafeBody); + await Assert.That(rejectedTypeSafe.StatusCode).IsEqualTo(HttpStatusCode.UnprocessableEntity); var result = await response.Content.ReadFromJsonAsync(); await Assert.That(result!.Result!.Answers["binary"].Probability).IsEqualTo(0.75); } @@ -51,17 +52,17 @@ public async Task Images_PreserveNativeRepresentationScenarioInputAndRuntimeEvid images = JsonSerializer.SerializeToElement(body["images"]), }); var scenario = CloudflareDecisionTestFixtures.Scenario(model) with { ExpectedInput = expectedInput }; - using var host = await LlmTckTestHost.StartAsync(builder => builder.AddClef().AddClefFlash().AddDecisionScenario(scenario)); + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddClef().AddClefFlash().AddDecisionScenario(scenario)); using var client = host.GetTestClient(); using var response = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor(model), body); await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.OK); - var summary = host.Services.GetRequiredService().GetAssertionSummary(); + var summary = (await client.GetFromJsonAsync("/admin-api/assertions"))!; await Assert.That(summary.Matched).IsEqualTo(1); - var neutral = JsonSerializer.Deserialize(summary.Events.Single().Request!)!; + var neutral = JsonSerializer.Deserialize(summary.Events.Single().Request!)!; await Assert.That(neutral.Input).IsEqualTo(expectedInput); - var trace = host.Services.GetRequiredService().GetSnapshot().Single(); + var trace = (await client.GetFromJsonAsync>("/admin-api/requests"))!.Single(); var native = JsonSerializer.Deserialize(trace.RequestPreview!)!; await Assert.That(native.Images!.Count).IsEqualTo(1); await Assert.That(JsonSerializer.SerializeToElement(native.Images).GetRawText()) @@ -74,7 +75,7 @@ await Assert.That(JsonSerializer.SerializeToElement(native.Images).GetRawText()) public async Task ModelSelector_AllowsDocumentedSurroundingWhitespaceAsync(string model) { var body = CloudflareDecisionTestFixtures.Body(model); body["model"] = " \t" + model + "\r\n "; - using var host = await LlmTckTestHost.StartAsync(builder => builder.AddClef().AddClefFlash() + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddClef().AddClefFlash() .AddDecisionScenario(CloudflareDecisionTestFixtures.Scenario(model))); using var client = host.GetTestClient(); using var response = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor(model), body); @@ -87,7 +88,7 @@ public async Task ModelSelector_AllowsDocumentedSurroundingWhitespaceAsync(strin public async Task Instructions_AcceptWhitespaceAsNonemptyNativeContentAsync() { var body = CloudflareDecisionTestFixtures.Body("clef"); body["questions"]!["binary"]!["instructions"] = " \t\n"; - using var host = await LlmTckTestHost.StartAsync(builder => builder.AddClef() + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddClef() .AddDecisionScenario(CloudflareDecisionTestFixtures.Scenario("clef"))); using var client = host.GetTestClient(); using var response = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor("clef"), body); @@ -98,7 +99,7 @@ public async Task Instructions_AcceptWhitespaceAsNonemptyNativeContentAsync() public async Task PredicateCriteria_RejectsExplicitNullWithNativeErrorEnvelopeAsync() { var body = CloudflareDecisionTestFixtures.Body("clef"); body["questions"]!["binary"]!["criteria"] = null; - using var host = await LlmTckTestHost.StartAsync(); using var client = host.GetTestClient(); + using var host = await LlmTestServerTestHost.StartAsync(); using var client = host.GetTestClient(); using var response = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor("clef"), body); await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); var payload = await response.Content.ReadFromJsonAsync(); @@ -125,14 +126,14 @@ public async Task Choice_AllowsWhitespaceOpaqueValuesAndPreservesNativeConfidenc { ["choice"] = new() { - Kind = LlmTckDecisionKind.Choice, + Kind = LlmTestServerDecisionKind.Choice, Choice = " ", Confidence = 0.75, Probabilities = new(StringComparer.Ordinal) { [" "] = 0.75, ["Other"] = 0.25 }, }, }, }; - using var host = await LlmTckTestHost.StartAsync(builder => builder.AddClef().AddDecisionScenario(scenario)); + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddClef().AddDecisionScenario(scenario)); using var client = host.GetTestClient(); using var response = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor("clef"), body); await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.OK); @@ -161,7 +162,7 @@ public async Task Identifiers_RejectsEmptyQuestionOrChoiceValueWithNativeErrorAs ["criteria"] = new JsonObject { [""] = "Empty value", ["Other"] = "Other value" }, }, }; - using var host = await LlmTckTestHost.StartAsync(); + using var host = await LlmTestServerTestHost.StartAsync(); using var client = host.GetTestClient(); using var response = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor("clef"), body); await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); diff --git a/tests/ManagedCode.LlmTck.Tests/Decisions/CloudflareDecisionErrorTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Decisions/CloudflareDecisionErrorTests.cs similarity index 69% rename from tests/ManagedCode.LlmTck.Tests/Decisions/CloudflareDecisionErrorTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/Decisions/CloudflareDecisionErrorTests.cs index 1a913de..2c01644 100644 --- a/tests/ManagedCode.LlmTck.Tests/Decisions/CloudflareDecisionErrorTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Decisions/CloudflareDecisionErrorTests.cs @@ -1,9 +1,9 @@ using System.Net; using System.Net.Http.Json; -using ManagedCode.LlmTck.Cloudflare; -using ManagedCode.LlmTck.Tests.TestSupport; +using ManagedCode.LlmTestServer.Cloudflare; +using ManagedCode.LlmTestServer.Tests.TestSupport; -namespace ManagedCode.LlmTck.Tests.Decisions; +namespace ManagedCode.LlmTestServer.Tests.Decisions; public sealed class CloudflareDecisionErrorTests { @@ -18,7 +18,7 @@ public async Task ConfiguredFault_PreservesNativeNumericErrorCodeAndHttpStatusAs const string nativeDetail = "Explicit Cloudflare native failure."; var scenario = CloudflareDecisionTestFixtures.Scenario("clef") with { Error = new() { StatusCode = status, Code = code, Message = nativeDetail } }; - using var host = await LlmTckTestHost.StartAsync(builder => builder.AddClef().AddDecisionScenario(scenario)); + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddClef().AddDecisionScenario(scenario)); using var client = host.GetTestClient(); using var response = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor("clef"), CloudflareDecisionTestFixtures.Body("clef")); await Assert.That((int)response.StatusCode).IsEqualTo(status); @@ -32,7 +32,7 @@ public async Task ConfiguredFault_PreservesNativeNumericErrorCodeAndHttpStatusAs [Test] public async Task GenericRateLimit_UsesNativeCapacityCodeRatherThanHttpStatusAsCodeAsync() { - using var host = await LlmTckTestHost.StartAsync(builder => builder.AddClef() + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddClef() .AddDecisionScenario(CloudflareDecisionTestFixtures.Scenario("clef")).SimulateRateLimitAfter(0)); using var client = host.GetTestClient(); using var response = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor("clef"), CloudflareDecisionTestFixtures.Body("clef")); @@ -51,6 +51,15 @@ public async Task GenericRateLimit_UsesNativeCapacityCodeRatherThanHttpStatusAsC [Arguments(429, CloudflareDecisionErrors.Capacity)] public async Task DefaultFaultCodes_MatchDocumentedNativeWorkersAiMappingAsync(int status, int code) { - await Assert.That(CloudflareDecisionErrors.Create(status, "fault").Errors.Single().Code).IsEqualTo(code); + var scenario = CloudflareDecisionTestFixtures.Scenario("clef") with + { + Error = new() { StatusCode = status, Code = "configured_fault", Message = "fault" }, + }; + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddClef().AddDecisionScenario(scenario)); + using var http = host.GetTestClient(); + using var response = await http.PostAsJsonAsync(DecisionEndpointTests.PathFor("clef"), CloudflareDecisionTestFixtures.Body("clef")); + await Assert.That((int)response.StatusCode).IsEqualTo(status); + var payload = await response.Content.ReadFromJsonAsync(); + await Assert.That(payload!.Errors.Single().Code).IsEqualTo(code); } } diff --git a/tests/ManagedCode.LlmTck.Tests/Decisions/CloudflareDecisionImageTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Decisions/CloudflareDecisionImageTests.cs similarity index 80% rename from tests/ManagedCode.LlmTck.Tests/Decisions/CloudflareDecisionImageTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/Decisions/CloudflareDecisionImageTests.cs index 1321227..865ca39 100644 --- a/tests/ManagedCode.LlmTck.Tests/Decisions/CloudflareDecisionImageTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Decisions/CloudflareDecisionImageTests.cs @@ -2,10 +2,10 @@ using System.Net.Http.Json; using System.Text.Json; using System.Text.Json.Nodes; -using ManagedCode.LlmTck.Cloudflare; -using ManagedCode.LlmTck.Tests.TestSupport; +using ManagedCode.LlmTestServer.Cloudflare; +using ManagedCode.LlmTestServer.Tests.TestSupport; -namespace ManagedCode.LlmTck.Tests.Decisions; +namespace ManagedCode.LlmTestServer.Tests.Decisions; public sealed class CloudflareDecisionImageTests { @@ -24,7 +24,7 @@ public async Task InvalidImages_ReturnNativeNonSuccessRatherThanSilentlyDropping { var body = CloudflareDecisionTestFixtures.Body("clef"); body["images"] = InvalidImages(kind); - using var host = await LlmTckTestHost.StartAsync(); using var client = host.GetTestClient(); + using var host = await LlmTestServerTestHost.StartAsync(); using var client = host.GetTestClient(); using var response = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor("clef"), body); await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); var payload = await response.Content.ReadFromJsonAsync(); @@ -42,7 +42,7 @@ public async Task NativeImageBudgets_ReturnHttp413WithRequestTooLargeCodeAsync(s var body = CloudflareDecisionTestFixtures.Body("clef"); if (kind == "body") { body["state"] = new string('x', CloudflareDecisionImageValidation.MaxRequestBytes); } else { body["images"] = OversizedImages(kind); } - using var host = await LlmTckTestHost.StartAsync(); using var client = host.GetTestClient(); + using var host = await LlmTestServerTestHost.StartAsync(); using var client = host.GetTestClient(); using var response = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor("clef"), body); await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.RequestEntityTooLarge); var payload = await response.Content.ReadFromJsonAsync(); @@ -54,9 +54,16 @@ public async Task NativeImageBudgets_ReturnHttp413WithRequestTooLargeCodeAsync(s public async Task NoImages_PreservesOriginalStructuredStateScenarioInputAsync() { var body = CloudflareDecisionTestFixtures.Body("clef"); - var native = JsonSerializer.Deserialize(body.ToJsonString())!; - await Assert.That(CloudflareDecisionMapper.ToRequest(native).Input) - .IsEqualTo(JsonSerializer.Serialize(body["state"])); + var scenario = CloudflareDecisionTestFixtures.Scenario("clef") with + { + ExpectedInput = JsonSerializer.Serialize(body["state"]), + }; + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddClef().AddDecisionScenario(scenario)); + using var http = host.GetTestClient(); + using var response = await http.PostAsJsonAsync(DecisionEndpointTests.PathFor("clef"), body); + response.EnsureSuccessStatusCode(); + var payload = await response.Content.ReadFromJsonAsync(); + await Assert.That(payload!.Result!.Answers["binary"].Probability).IsEqualTo(0.75); } private static JsonNode? InvalidImages(string kind) diff --git a/tests/ManagedCode.LlmTck.Tests/Decisions/CloudflareDecisionLargeImageFixture.cs b/tests/ManagedCode.LlmTestServer.Tests/Decisions/CloudflareDecisionLargeImageFixture.cs similarity index 99% rename from tests/ManagedCode.LlmTck.Tests/Decisions/CloudflareDecisionLargeImageFixture.cs rename to tests/ManagedCode.LlmTestServer.Tests/Decisions/CloudflareDecisionLargeImageFixture.cs index 64d474f..f97a82a 100644 --- a/tests/ManagedCode.LlmTck.Tests/Decisions/CloudflareDecisionLargeImageFixture.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Decisions/CloudflareDecisionLargeImageFixture.cs @@ -1,4 +1,4 @@ -namespace ManagedCode.LlmTck.Tests.Decisions; +namespace ManagedCode.LlmTestServer.Tests.Decisions; // A valid generated 4001 x 4000 PNG exceeds Clef's documented 16-megapixel ceiling. internal static class CloudflareDecisionLargeImageFixture diff --git a/tests/ManagedCode.LlmTck.Tests/Decisions/CloudflareDecisionTestFixtures.cs b/tests/ManagedCode.LlmTestServer.Tests/Decisions/CloudflareDecisionTestFixtures.cs similarity index 87% rename from tests/ManagedCode.LlmTck.Tests/Decisions/CloudflareDecisionTestFixtures.cs rename to tests/ManagedCode.LlmTestServer.Tests/Decisions/CloudflareDecisionTestFixtures.cs index 479a48b..710e4e0 100644 --- a/tests/ManagedCode.LlmTck.Tests/Decisions/CloudflareDecisionTestFixtures.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Decisions/CloudflareDecisionTestFixtures.cs @@ -1,7 +1,7 @@ using System.Text.Json.Nodes; -using ManagedCode.LlmTck.Decisions; +using ManagedCode.LlmTestServer.Decisions; -namespace ManagedCode.LlmTck.Tests.Decisions; +namespace ManagedCode.LlmTestServer.Tests.Decisions; internal static class CloudflareDecisionTestFixtures { @@ -19,13 +19,13 @@ internal static JsonObject Body(string model) }; } - internal static LlmTckDecisionScenario Scenario(string model) + internal static LlmTestServerDecisionScenario Scenario(string model) { return new() { Id = "native-cloudflare-contract-" + model, ModelId = model, - Answers = new(StringComparer.Ordinal) { ["binary"] = new() { Kind = LlmTckDecisionKind.Predicate, Probability = 0.75 } }, + Answers = new(StringComparer.Ordinal) { ["binary"] = new() { Kind = LlmTestServerDecisionKind.Predicate, Probability = 0.75 } }, Usage = new() { InputTokens = 100, OutputTokens = 0, TotalTokens = 100 }, }; } diff --git a/tests/ManagedCode.LlmTck.Tests/Decisions/CloudflareNativeClientTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Decisions/CloudflareNativeClientTests.cs similarity index 69% rename from tests/ManagedCode.LlmTck.Tests/Decisions/CloudflareNativeClientTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/Decisions/CloudflareNativeClientTests.cs index da209db..74db23f 100644 --- a/tests/ManagedCode.LlmTck.Tests/Decisions/CloudflareNativeClientTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Decisions/CloudflareNativeClientTests.cs @@ -1,11 +1,11 @@ using System.Text.Json; -using ManagedCode.LlmTck.Client; -using ManagedCode.LlmTck.Cloudflare; -using ManagedCode.LlmTck.Decisions; -using ManagedCode.LlmTck.Scenarios; -using ManagedCode.LlmTck.Tests.TestSupport; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Cloudflare; +using ManagedCode.LlmTestServer.Decisions; +using ManagedCode.LlmTestServer.Scenarios; +using ManagedCode.LlmTestServer.Tests.TestSupport; -namespace ManagedCode.LlmTck.Tests.Decisions; +namespace ManagedCode.LlmTestServer.Tests.Decisions; public sealed class CloudflareNativeClientTests { @@ -23,10 +23,10 @@ public async Task TypedClientPreservesEmbeddedImagesProbabilitiesUsageAndCallerH body["images"] = new System.Text.Json.Nodes.JsonArray(CloudflareDecisionTestFixtures.Image(contentType, embedded)); var request = body.Deserialize()!; var scenario = CloudflareDecisionTestFixtures.Scenario(model) with { ExpectedInput = CloudflareDecisionMapper.ToRequest(request).Input }; - using var host = await LlmTckTestHost.StartAsync(builder => builder.RequireBearerToken(_credential) - .AddDecisionModel(model, LlmTckDecisionProvider.Cloudflare).AddDecisionScenario(scenario)); + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.RequireBearerToken(_credential) + .AddDecisionModel(model, LlmTestServerDecisionProvider.Cloudflare).AddDecisionScenario(scenario)); using var http = host.GetTestClient(); - var client = new LlmTckClient(http, _credential).CreateCloudflareDecisionClient(_account); + var client = new LlmTestServerClient(http, _credential).CreateCloudflareDecisionClient(_account); var response = await client.EvaluateAsync(request); var second = await client.EvaluateAsync(request); @@ -45,10 +45,10 @@ public async Task TypedClientPreservesNativeFailureEnvelopesAsync(int status, in { const string model = "clef"; var scenario = CloudflareDecisionTestFixtures.Scenario(model) with - { Error = new LlmTckScenarioError { StatusCode = status, Code = code.ToString(System.Globalization.CultureInfo.InvariantCulture), Message = "native failure" } }; - using var host = await LlmTckTestHost.StartAsync(builder => builder.AddDecisionModel(model, LlmTckDecisionProvider.Cloudflare).AddDecisionScenario(scenario)); + { Error = new LlmTestServerScenarioError { StatusCode = status, Code = code.ToString(System.Globalization.CultureInfo.InvariantCulture), Message = "native failure" } }; + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddDecisionModel(model, LlmTestServerDecisionProvider.Cloudflare).AddDecisionScenario(scenario)); using var http = host.GetTestClient(); - var client = new LlmTckClient(http).CreateCloudflareDecisionClient(_account); + var client = new LlmTestServerClient(http).CreateCloudflareDecisionClient(_account); var response = await client.EvaluateAsync(CloudflareDecisionTestFixtures.Body(model).Deserialize()!); @@ -60,12 +60,12 @@ public async Task TypedClientPreservesNativeFailureEnvelopesAsync(int status, in [Test] public async Task TypedClientRejectsRemoteImageUrlsUsingNativeErrorEnvelopeAsync() { - using var host = await LlmTckTestHost.StartAsync(); + using var host = await LlmTestServerTestHost.StartAsync(); using var http = host.GetTestClient(); var request = CloudflareDecisionTestFixtures.Body("clef").Deserialize()! with { Images = [new() { DataUrl = "https://example.org/clipping.png" }] }; - var response = await new LlmTckClient(http).CreateCloudflareDecisionClient(_account).EvaluateAsync(request); + var response = await new LlmTestServerClient(http).CreateCloudflareDecisionClient(_account).EvaluateAsync(request); await Assert.That(response.Success).IsFalse(); await Assert.That(response.Errors.Single().Code).IsEqualTo(CloudflareDecisionErrors.InvalidData); } diff --git a/tests/ManagedCode.LlmTck.Tests/Decisions/DecisionEndpointTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Decisions/DecisionEndpointTests.cs similarity index 79% rename from tests/ManagedCode.LlmTck.Tests/Decisions/DecisionEndpointTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/Decisions/DecisionEndpointTests.cs index 6a922ac..2f10249 100644 --- a/tests/ManagedCode.LlmTck.Tests/Decisions/DecisionEndpointTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Decisions/DecisionEndpointTests.cs @@ -2,16 +2,15 @@ using System.Net.Http.Json; using System.Text.Json; using System.Text.Json.Nodes; -using ManagedCode.LlmTck.Client; -using ManagedCode.LlmTck.Decisions; -using ManagedCode.LlmTck.Hosting; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.OpenAI; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Tests.TestSupport; -using Microsoft.Extensions.DependencyInjection; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Decisions; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.OpenAI; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Tests.TestSupport; -namespace ManagedCode.LlmTck.Tests.Decisions; +namespace ManagedCode.LlmTestServer.Tests.Decisions; public sealed class DecisionEndpointTests { @@ -28,7 +27,7 @@ public sealed class DecisionEndpointTests [Arguments("gpt-6-luna")] public async Task NativeDecisionEndpoints_PreserveAnswersAndUsageAsync(string model) { - using var host = await LlmTckTestHost.StartAsync(builder => builder.AddModel(model, LlmTckModelKind.Decision).AddDecisionScenario(Scenario(model))); + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddModel(model, LlmTestServerModelKind.Decision).AddDecisionScenario(Scenario(model))); using var client = host.GetTestClient(); using var response = await client.PostAsJsonAsync(PathFor(model), Body(model)); response.EnsureSuccessStatusCode(); @@ -53,10 +52,10 @@ public async Task NativeDecisionEndpoints_PreserveAnswersAndUsageAsync(string mo await Assert.That(payload.GetProperty("usage").GetProperty("output_tokens_details").GetProperty("reasoning_tokens").GetInt32()).IsEqualTo(0); } else { await Assert.That(score.GetProperty("legend").GetProperty("2").GetString()).IsEqualTo("high"); } - var summary = host.Services.GetRequiredService().GetAssertionSummary(); + var summary = (await client.GetFromJsonAsync("/admin-api/assertions"))!; await Assert.That(summary.Matched).IsEqualTo(1); await Assert.That(summary.Events.Single().ScenarioId).IsEqualTo("fixture-" + model); - var trace = host.Services.GetRequiredService().GetSnapshot().Single(); + var trace = (await client.GetFromJsonAsync>("/admin-api/requests"))!.Single(); await Assert.That(trace.ModelId).IsEqualTo(model); } @@ -74,9 +73,9 @@ public async Task NativeDecisionEndpoints_PreserveAnswersAndUsageAsync(string mo [Arguments("gpt-6-luna", 400)] public async Task NativeDecisionEndpoints_EnforceAuthAndFaultsAsync(string model, int status = 401) { - using var host = await LlmTckTestHost.StartAsync(builder => + using var host = await LlmTestServerTestHost.StartAsync(builder => { - builder.AddModel(model, LlmTckModelKind.Decision).AddDecisionScenario(Scenario(model)); + builder.AddModel(model, LlmTestServerModelKind.Decision).AddDecisionScenario(Scenario(model)); if (status == 401) { builder.RequireBearerToken(_key); } if (status == 429) { builder.SimulateRateLimitAfter(0); } if (status == 400) { builder.SimulateContentFilter("evidence"); } @@ -101,7 +100,7 @@ public async Task NativeDecisionEndpoints_EnforceAuthAndFaultsAsync(string model [Arguments("gpt-6-luna", "questions")] public async Task NativeDecisionEndpoints_RejectMissingRequiredFieldsAsync(string model, string property) { - using var host = await LlmTckTestHost.StartAsync(); + using var host = await LlmTestServerTestHost.StartAsync(); using var client = host.GetTestClient(); var body = Body(model); body.Remove(property); using var response = await client.PostAsJsonAsync(PathFor(model), body); @@ -111,9 +110,9 @@ public async Task NativeDecisionEndpoints_RejectMissingRequiredFieldsAsync(strin [Test] public async Task DecisionControlClient_ConfiguresFixturesAndMismatchIsObservableAsync() { - using var host = await LlmTckTestHost.StartAsync(); using var client = host.GetTestClient(); - var control = new LlmTckClient(client); - await control.ConfigureAsync(builder => builder.UseModel("jev-latest", LlmTckModelKind.Decision).UseDecisionScenario(Scenario("jev-latest"))); + using var host = await LlmTestServerTestHost.StartAsync(); using var client = host.GetTestClient(); + var control = new LlmTestServerClient(client); + await control.ConfigureAsync(builder => builder.UseModel("jev-latest", LlmTestServerModelKind.Decision).UseDecisionScenario(Scenario("jev-latest"))); var body = Body("jev-latest"); body["questions"]!["choice"]!["criteria"] = new JsonObject { ["another-id"] = "first", ["last-id"] = "last" }; using var mismatch = await client.PostAsJsonAsync(_systemOnePath, body); await Assert.That(mismatch.StatusCode).IsEqualTo(HttpStatusCode.Conflict); @@ -123,7 +122,7 @@ public async Task DecisionControlClient_ConfiguresFixturesAndMismatchIsObservabl await Assert.That(summary.ErrorsReturned).IsEqualTo(1); } - internal static LlmTckDecisionScenario Scenario(string model) + internal static LlmTestServerDecisionScenario Scenario(string model) { var firstChoice = model == "gpt-6-luna" ? OpenAiDecisionFixtureIds.ForChoiceValue("opaque-id") : "opaque-id"; var lastChoice = model == "gpt-6-luna" ? OpenAiDecisionFixtureIds.ForChoiceValue("other-id") : "other-id"; @@ -134,9 +133,9 @@ internal static LlmTckDecisionScenario Scenario(string model) Usage = new() { InputTokens = 100, CachedInputTokens = 4, OutputTokens = 10, TotalTokens = 110 }, Answers = new() { - [model == "gpt-6-luna" ? OpenAiDecisionFixtureIds.ForQuestionIndex(0) : "binary"] = new() { Kind = LlmTckDecisionKind.Predicate, Probability = 0.95 }, - [model == "gpt-6-luna" ? OpenAiDecisionFixtureIds.ForQuestionIndex(1) : "choice"] = new() { Kind = LlmTckDecisionKind.Choice, Choice = firstChoice, Confidence = model.StartsWith("clef", StringComparison.Ordinal) ? 0.95 : 0.9, Probabilities = new() { [firstChoice] = 0.95, [lastChoice] = 0.05 } }, - [model == "gpt-6-luna" ? OpenAiDecisionFixtureIds.ForQuestionIndex(2) : "score"] = new() { Kind = LlmTckDecisionKind.Score, Score = 1.75, Confidence = model is "jev-latest" or "kev-latest" ? 0.625 : 0.8, Probabilities = new() { ["0"] = 0.05, ["1"] = 0.15, ["2"] = 0.8 } }, + [model == "gpt-6-luna" ? OpenAiDecisionFixtureIds.ForQuestionIndex(0) : "binary"] = new() { Kind = LlmTestServerDecisionKind.Predicate, Probability = 0.95 }, + [model == "gpt-6-luna" ? OpenAiDecisionFixtureIds.ForQuestionIndex(1) : "choice"] = new() { Kind = LlmTestServerDecisionKind.Choice, Choice = firstChoice, Confidence = model.StartsWith("clef", StringComparison.Ordinal) ? 0.95 : 0.9, Probabilities = new() { [firstChoice] = 0.95, [lastChoice] = 0.05 } }, + [model == "gpt-6-luna" ? OpenAiDecisionFixtureIds.ForQuestionIndex(2) : "score"] = new() { Kind = LlmTestServerDecisionKind.Score, Score = 1.75, Confidence = model is "jev-latest" or "kev-latest" ? 0.625 : 0.8, Probabilities = new() { ["0"] = 0.05, ["1"] = 0.15, ["2"] = 0.8 } }, }, }; } diff --git a/tests/ManagedCode.LlmTck.Tests/Decisions/DecisionFixtureContractTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Decisions/DecisionFixtureContractTests.cs similarity index 74% rename from tests/ManagedCode.LlmTck.Tests/Decisions/DecisionFixtureContractTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/Decisions/DecisionFixtureContractTests.cs index 59bdfea..bd86f85 100644 --- a/tests/ManagedCode.LlmTck.Tests/Decisions/DecisionFixtureContractTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Decisions/DecisionFixtureContractTests.cs @@ -2,13 +2,10 @@ using System.Net.Http.Json; using System.Text.Json; using System.Text.Json.Nodes; -using ManagedCode.LlmTck.Cloudflare; -using ManagedCode.LlmTck.Configuration; -using ManagedCode.LlmTck.Decisions; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Tests.TestSupport; +using ManagedCode.LlmTestServer.Decisions; +using ManagedCode.LlmTestServer.Tests.TestSupport; -namespace ManagedCode.LlmTck.Tests.Decisions; +namespace ManagedCode.LlmTestServer.Tests.Decisions; public sealed class DecisionFixtureContractTests { @@ -29,7 +26,7 @@ public async Task NativeFixtures_RejectIncompleteOrInvalidNormalResponsesAsync(s if (defect == "mass") { answer.Probabilities[answer.Probabilities.Keys.Last()] = 0.95; } if (defect == "argmax") { scenario.Answers[key] = answer with { Choice = answer.Probabilities.Keys.Last() }; } if (defect == "max-confidence") { scenario.Answers[key] = answer with { Confidence = 0.5 }; } - using var host = await LlmTckTestHost.StartAsync(builder => builder.AddDecisionModel(model, Provider(model)).AddDecisionScenario(scenario)); + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddDecisionModel(model, Provider(model)).AddDecisionScenario(scenario)); using var client = host.GetTestClient(); using var response = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor(model), DecisionEndpointTests.Body(model)); await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.Conflict); } @@ -39,7 +36,7 @@ public async Task MalformedResponseFixture_ExplicitlyPreservesBadNativeProbabili { var scenario = DecisionEndpointTests.Scenario("gpt-6-luna") with { AllowMalformedResponse = true }; foreach (var key in scenario.Answers["1"].Probabilities.Keys) { scenario.Answers["1"].Probabilities[key] = 0.95; } - using var host = await LlmTckTestHost.StartAsync(builder => builder.AddOpenAiDecisionModel("gpt-6-luna").AddDecisionScenario(scenario)); + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddOpenAiDecisionModel("gpt-6-luna").AddDecisionScenario(scenario)); using var client = host.GetTestClient(); using var response = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor("gpt-6-luna"), DecisionEndpointTests.Body("gpt-6-luna")); response.EnsureSuccessStatusCode(); var probabilities = (await response.Content.ReadFromJsonAsync()).GetProperty("answers")[1].GetProperty("probabilities"); await Assert.That(probabilities.EnumerateArray().Sum(value => value.GetProperty("probability").GetDouble())).IsEqualTo(1.9); @@ -58,7 +55,7 @@ public async Task NativeFixtures_AcceptRoundedMassAndPreserveExplicitScoreAsync( Confidence = model == "clef" ? 0.3333 : 0, Probabilities = new() { ["0"] = 0.3333, ["1"] = 0.3333, ["2"] = model == "gpt-6-luna" ? 0.3333995 : 0.3333 } }; - using var host = await LlmTckTestHost.StartAsync(builder => builder.AddDecisionModel(model, Provider(model)).AddDecisionScenario(scenario)); + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddDecisionModel(model, Provider(model)).AddDecisionScenario(scenario)); using var client = host.GetTestClient(); using var response = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor(model), DecisionEndpointTests.Body(model)); response.EnsureSuccessStatusCode(); var payload = await response.Content.ReadFromJsonAsync(); if (model == "clef") { payload = payload.GetProperty("result"); } var score = model == "gpt-6-luna" ? payload.GetProperty("answers")[2] : payload.GetProperty("answers").GetProperty("score"); @@ -69,8 +66,8 @@ public async Task NativeFixtures_AcceptRoundedMassAndPreserveExplicitScoreAsync( public async Task OpenAiFixtures_PreserveSelectedValueAndIndependentConfidenceAsync() { var scenario = DecisionEndpointTests.Scenario("gpt-6-luna"); - scenario.Answers["1"] = scenario.Answers["1"] with { Choice = ManagedCode.LlmTck.OpenAI.OpenAiDecisionFixtureIds.ForChoiceValue("other-id"), Confidence = 0.4 }; - using var host = await LlmTckTestHost.StartAsync(builder => builder.AddOpenAiDecisionModel("gpt-6-luna").AddDecisionScenario(scenario)); + scenario.Answers["1"] = scenario.Answers["1"] with { Choice = ManagedCode.LlmTestServer.OpenAI.OpenAiDecisionFixtureIds.ForChoiceValue("other-id"), Confidence = 0.4 }; + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddOpenAiDecisionModel("gpt-6-luna").AddDecisionScenario(scenario)); using var client = host.GetTestClient(); using var response = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor("gpt-6-luna"), DecisionEndpointTests.Body("gpt-6-luna")); response.EnsureSuccessStatusCode(); var answer = (await response.Content.ReadFromJsonAsync()).GetProperty("answers")[1]; await Assert.That(answer.GetProperty("choice").GetString()).IsEqualTo("other-id"); @@ -82,7 +79,7 @@ public async Task OpenAiFixtures_PredicateDoesNotLeakOtherKindFieldsAsync() { var scenario = DecisionEndpointTests.Scenario("gpt-6-luna"); scenario.Answers["0"] = scenario.Answers["0"] with { Score = 1, Confidence = 0.9, Choice = "not-a-request-choice" }; - using var host = await LlmTckTestHost.StartAsync(builder => builder.AddOpenAiDecisionModel("gpt-6-luna").AddDecisionScenario(scenario)); + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddOpenAiDecisionModel("gpt-6-luna").AddDecisionScenario(scenario)); using var client = host.GetTestClient(); using var response = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor("gpt-6-luna"), DecisionEndpointTests.Body("gpt-6-luna")); response.EnsureSuccessStatusCode(); var answer = (await response.Content.ReadFromJsonAsync()).GetProperty("answers")[0]; await Assert.That(answer.EnumerateObject().Select(item => item.Name)).IsEquivalentTo(new[] { "type", "name", "probability" }); @@ -93,31 +90,31 @@ public async Task KevFixtures_RequireMaximalChoiceEvenForWideProbabilityDomainsA { var scenario = DecisionEndpointTests.Scenario("kev-latest"); var body = DecisionEndpointTests.Body("kev-latest"); var criteria = new JsonObject(); var probabilities = new Dictionary(); - for (var index = 0; index < ManagedCode.LlmTck.SystemOne.SystemOneDecisionMapper.MaxChoices; index++) + for (var index = 0; index < ManagedCode.LlmTestServer.SystemOne.SystemOneDecisionMapper.MaxChoices; index++) { var key = index.ToString(System.Globalization.CultureInfo.InvariantCulture); criteria[key] = key; probabilities[key] = 0; } probabilities["0"] = 0.495; probabilities["1"] = 0.505; body["questions"]!["choice"]!["criteria"] = criteria; scenario.Answers["choice"] = scenario.Answers["choice"] with { Choice = "0", Probabilities = probabilities }; - using var host = await LlmTckTestHost.StartAsync(builder => builder.AddKevLatest().AddDecisionScenario(scenario)); + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddKevLatest().AddDecisionScenario(scenario)); using var client = host.GetTestClient(); using var response = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor("kev-latest"), body); await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.Conflict); } [Test] - public async Task CloudflareMapper_AppliesNativeConfidencePolicyInDirectRuntimeAsync() + public async Task CloudflareEndpoint_RejectsMismatchedNativeConfidenceAsync() { var scenario = DecisionEndpointTests.Scenario("clef"); scenario.Answers["choice"] = scenario.Answers["choice"] with { Confidence = 0.9 }; - var runtime = new LlmTckRuntime(new LlmTckConfigurationBuilder().AddClef().AddDecisionScenario(scenario).Build()); - var body = DecisionEndpointTests.Body("clef").Deserialize()!; - var result = await runtime.DecideAsync(CloudflareDecisionMapper.ToRequest(body)); - await Assert.That(result.StatusCode).IsEqualTo(409); - await Assert.That(result.IsSuccess).IsFalse(); + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddClef().AddDecisionScenario(scenario)); + using var http = host.GetTestClient(); + using var response = await http.PostAsJsonAsync(DecisionEndpointTests.PathFor("clef"), DecisionEndpointTests.Body("clef")); + await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + await Assert.That((await response.Content.ReadFromJsonAsync()).GetProperty("success").GetBoolean()).IsFalse(); } - private static LlmTckDecisionProvider Provider(string model) + private static LlmTestServerDecisionProvider Provider(string model) { return model switch - { "kev-latest" => LlmTckDecisionProvider.Kev, "jev-latest" => LlmTckDecisionProvider.TypeSafe, "gpt-6-luna" => LlmTckDecisionProvider.OpenAI, _ => LlmTckDecisionProvider.Cloudflare }; + { "kev-latest" => LlmTestServerDecisionProvider.Kev, "jev-latest" => LlmTestServerDecisionProvider.TypeSafe, "gpt-6-luna" => LlmTestServerDecisionProvider.OpenAI, _ => LlmTestServerDecisionProvider.Cloudflare }; } } diff --git a/tests/ManagedCode.LlmTck.Tests/Decisions/DecisionInputTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Decisions/DecisionInputTests.cs similarity index 71% rename from tests/ManagedCode.LlmTck.Tests/Decisions/DecisionInputTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/Decisions/DecisionInputTests.cs index c4b6a74..13724e2 100644 --- a/tests/ManagedCode.LlmTck.Tests/Decisions/DecisionInputTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Decisions/DecisionInputTests.cs @@ -1,11 +1,10 @@ using System.Net; using System.Net.Http.Json; -using System.Text.Json; using System.Text.Json.Nodes; -using ManagedCode.LlmTck.OpenAI; -using ManagedCode.LlmTck.Tests.TestSupport; +using ManagedCode.LlmTestServer.OpenAI; +using ManagedCode.LlmTestServer.Tests.TestSupport; -namespace ManagedCode.LlmTck.Tests.Decisions; +namespace ManagedCode.LlmTestServer.Tests.Decisions; public sealed class DecisionInputTests { @@ -23,7 +22,7 @@ public sealed class DecisionInputTests public async Task OpenAiDecisionInput_RejectsUnsupportedMessageAndImageShapesAsync(string input) { var body = DecisionEndpointTests.Body("gpt-6-luna"); body["input"] = JsonNode.Parse(input); - using var host = await LlmTckTestHost.StartAsync(); using var client = host.GetTestClient(); + using var host = await LlmTestServerTestHost.StartAsync(); using var client = host.GetTestClient(); using var response = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor("gpt-6-luna"), body); await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); } @@ -39,7 +38,7 @@ public async Task OpenAiDecisionInput_AcceptsTextAndEmbeddedImagesAsync(string? var body = DecisionEndpointTests.Body("gpt-6-luna"); body["input"] = JsonNode.Parse("""[{"role":"user","type":"message","content":[{"type":"input_text","text":"evidence"},{"type":"input_image","image_url":"data:image/png;base64,YQ=="}]}]"""); body["input"]![0]!["content"]![1]!["detail"] = detail; - using var host = await LlmTckTestHost.StartAsync(builder => builder.AddOpenAiDecisionModel("gpt-6-luna").AddDecisionScenario(DecisionEndpointTests.Scenario("gpt-6-luna"))); + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddOpenAiDecisionModel("gpt-6-luna").AddDecisionScenario(DecisionEndpointTests.Scenario("gpt-6-luna"))); using var client = host.GetTestClient(); using var response = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor("gpt-6-luna"), body); await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.OK); } @@ -49,8 +48,15 @@ public async Task OpenAiDecisionInput_EnforcesAggregateImageLimitAsync() { var image = JsonNode.Parse("""{"type":"input_image","image_url":"data:image/png;base64,YQ=="}""")!; var input = new JsonArray(new JsonObject { ["role"] = "user", ["content"] = new JsonArray(Enumerable.Range(0, OpenAiDecisionInputValidation.MaxImages).Select(_ => image.DeepClone()).ToArray()) }); - await Assert.That(OpenAiDecisionInputValidation.IsValid(JsonSerializer.SerializeToElement(input))).IsTrue(); + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddOpenAiDecisionModel("gpt-6-luna") + .AddDecisionScenario(DecisionEndpointTests.Scenario("gpt-6-luna"))); + using var http = host.GetTestClient(); + var body = DecisionEndpointTests.Body("gpt-6-luna"); + body["input"] = input; + using var accepted = await http.PostAsJsonAsync("/openai/v1/decisions", body); + accepted.EnsureSuccessStatusCode(); input.Add(new JsonObject { ["role"] = "user", ["content"] = new JsonArray(image.DeepClone()) }); - await Assert.That(OpenAiDecisionInputValidation.IsValid(JsonSerializer.SerializeToElement(input))).IsFalse(); + using var rejected = await http.PostAsJsonAsync("/openai/v1/decisions", body); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); } } diff --git a/tests/ManagedCode.LlmTck.Tests/Decisions/DecisionQuestionTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Decisions/DecisionQuestionTests.cs similarity index 84% rename from tests/ManagedCode.LlmTck.Tests/Decisions/DecisionQuestionTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/Decisions/DecisionQuestionTests.cs index d8a83da..60c2bde 100644 --- a/tests/ManagedCode.LlmTck.Tests/Decisions/DecisionQuestionTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Decisions/DecisionQuestionTests.cs @@ -2,11 +2,11 @@ using System.Net.Http.Json; using System.Text.Json; using System.Text.Json.Nodes; -using ManagedCode.LlmTck.Decisions; -using ManagedCode.LlmTck.OpenAI; -using ManagedCode.LlmTck.Tests.TestSupport; +using ManagedCode.LlmTestServer.Decisions; +using ManagedCode.LlmTestServer.OpenAI; +using ManagedCode.LlmTestServer.Tests.TestSupport; -namespace ManagedCode.LlmTck.Tests.Decisions; +namespace ManagedCode.LlmTestServer.Tests.Decisions; public sealed class DecisionQuestionTests { @@ -18,7 +18,7 @@ public async Task OpenAiQuestions_PreserveUnnamedDuplicateAndEmptyNamesByPositio for (var index = 1; index < 4; index++) { scenario.Answers[OpenAiDecisionFixtureIds.ForQuestionIndex(index)] = new() { Probability = 0.1 * index }; } var body = DecisionEndpointTests.Body("gpt-6-luna"); body["questions"] = JsonNode.Parse("""[{"type":"predicate","instructions":""},{"type":"predicate","name":"same","instructions":" "},{"type":"predicate","name":"same","instructions":""},{"type":"predicate","name":"","instructions":""}]"""); - using var host = await LlmTckTestHost.StartAsync(builder => builder.AddOpenAiDecisionModel("gpt-6-luna").AddDecisionScenario(scenario)); + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddOpenAiDecisionModel("gpt-6-luna").AddDecisionScenario(scenario)); using var client = host.GetTestClient(); using var response = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor("gpt-6-luna"), body); response.EnsureSuccessStatusCode(); var answers = (await response.Content.ReadFromJsonAsync()).GetProperty("answers"); await Assert.That(answers[0].GetProperty("name").ValueKind).IsEqualTo(JsonValueKind.Null); @@ -32,11 +32,11 @@ public async Task OpenAiQuestions_PreserveUnnamedDuplicateAndEmptyNamesByPositio public async Task OpenAiQuestions_PreserveEmptyChoiceAndRepeatedScoreLabelsWithoutDescriptionsAsync() { var scenario = DecisionEndpointTests.Scenario("gpt-6-luna"); scenario.Answers.Remove("0"); - scenario.Answers["0"] = new() { Kind = LlmTckDecisionKind.Choice, Choice = OpenAiDecisionFixtureIds.ForChoiceValue(""), Confidence = 0.9, Probabilities = new() { [OpenAiDecisionFixtureIds.ForChoiceValue("")] = 0.9, [OpenAiDecisionFixtureIds.ForChoiceValue(" ")] = 0.1 } }; + scenario.Answers["0"] = new() { Kind = LlmTestServerDecisionKind.Choice, Choice = OpenAiDecisionFixtureIds.ForChoiceValue(""), Confidence = 0.9, Probabilities = new() { [OpenAiDecisionFixtureIds.ForChoiceValue("")] = 0.9, [OpenAiDecisionFixtureIds.ForChoiceValue(" ")] = 0.1 } }; scenario.Answers["1"] = scenario.Answers["2"]; scenario.Answers.Remove("2"); var body = DecisionEndpointTests.Body("gpt-6-luna"); body["questions"] = JsonNode.Parse("""[{"type":"choice","instructions":"","choices":[{"value":""},{"value":" "}]},{"type":"score","instructions":"","levels":[{"label":""},{"label":"same"},{"label":"same"}]}]"""); - using var host = await LlmTckTestHost.StartAsync(builder => builder.AddOpenAiDecisionModel("gpt-6-luna").AddDecisionScenario(scenario)); + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddOpenAiDecisionModel("gpt-6-luna").AddDecisionScenario(scenario)); using var client = host.GetTestClient(); using var response = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor("gpt-6-luna"), body); response.EnsureSuccessStatusCode(); var answers = (await response.Content.ReadFromJsonAsync()).GetProperty("answers"); await Assert.That(answers[0].GetProperty("choice").GetString()).IsEqualTo(""); @@ -61,7 +61,7 @@ public async Task OpenAiQuestions_EnforceDocumentedNativeFieldConstraintsAsync(s if (kind == "long-name") { body["questions"]![0]!["name"] = new string('a', OpenAiDecisionMapper.MaxQuestionTextLength + 1); } if (kind == "long-model") { body["model"] = new string('a', OpenAiDecisionMapper.MaxQuestionTextLength + 1); } if (kind == "safety-identifier") { body["safety_identifier"] = false; } - using var host = await LlmTckTestHost.StartAsync(); using var client = host.GetTestClient(); + using var host = await LlmTestServerTestHost.StartAsync(); using var client = host.GetTestClient(); using var response = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor("gpt-6-luna"), body); await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); } @@ -71,7 +71,7 @@ public async Task OpenAiQuestions_LeaveUndocumentedCountToFixtureMatchingAsync() { var scenario = DecisionEndpointTests.Scenario("gpt-6-luna"); scenario.Answers.Clear(); var body = DecisionEndpointTests.Body("gpt-6-luna"); body["questions"] = new JsonArray(); - using var host = await LlmTckTestHost.StartAsync(builder => builder.AddOpenAiDecisionModel("gpt-6-luna").AddDecisionScenario(scenario)); + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddOpenAiDecisionModel("gpt-6-luna").AddDecisionScenario(scenario)); using var client = host.GetTestClient(); using var response = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor("gpt-6-luna"), body); response.EnsureSuccessStatusCode(); await Assert.That((await response.Content.ReadFromJsonAsync()).GetProperty("answers").GetArrayLength()).IsEqualTo(0); @@ -86,7 +86,7 @@ public async Task OpenAiQuestions_PreserveChoiceValuesWithoutUndocumentedTextLim var scenario = DecisionEndpointTests.Scenario("gpt-6-luna"); scenario.Answers.Clear(); scenario.Answers[OpenAiDecisionFixtureIds.ForQuestionIndex(0)] = new() { - Kind = LlmTckDecisionKind.Choice, + Kind = LlmTestServerDecisionKind.Choice, Choice = choiceId, Confidence = 0.9, Probabilities = new() { [choiceId] = 0.9, [otherId] = 0.1 }, @@ -98,7 +98,7 @@ public async Task OpenAiQuestions_PreserveChoiceValuesWithoutUndocumentedTextLim ["instructions"] = "", ["choices"] = new JsonArray(new JsonObject { ["value"] = value }, new JsonObject { ["value"] = false }), }); - using var host = await LlmTckTestHost.StartAsync(builder => builder.AddOpenAiDecisionModel("gpt-6-luna").AddDecisionScenario(scenario)); + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddOpenAiDecisionModel("gpt-6-luna").AddDecisionScenario(scenario)); using var client = host.GetTestClient(); using var response = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor("gpt-6-luna"), body); response.EnsureSuccessStatusCode(); var answer = (await response.Content.ReadFromJsonAsync()).GetProperty("answers")[0]; await Assert.That(answer.GetProperty("choice").ValueEquals(value)).IsTrue(); diff --git a/tests/ManagedCode.LlmTestServer.Tests/Decisions/DecisionRuntimeTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Decisions/DecisionRuntimeTests.cs new file mode 100644 index 0000000..7630f8d --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Decisions/DecisionRuntimeTests.cs @@ -0,0 +1,114 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using ManagedCode.LlmTestServer.Configuration; +using ManagedCode.LlmTestServer.Decisions; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.Decisions; + +public sealed class DecisionRuntimeTests +{ + private const string _model = "jev-latest"; + private static readonly JsonSerializerOptions _configurationJson = new(JsonSerializerDefaults.Web) + { + NumberHandling = System.Text.Json.Serialization.JsonNumberHandling.AllowNamedFloatingPointLiterals, + }; + + [Test] + [Arguments("unknown", 404)] + [Arguments("unmatched", 404)] + [Arguments("error", 503)] + [Arguments("mismatch", 409)] + [Arguments("score-key", 409)] + [Arguments("score-high", 409)] + [Arguments("wrong-input", 404)] + public async Task DecisionFailures_AreObservableAsync(string kind, int status) + { + var builder = new LlmTestServerConfigurationBuilder(); + if (kind != "unknown") { builder.AddModel(_model, LlmTestServerModelKind.Decision); } + var scenario = DecisionEndpointTests.Scenario(_model); + if (kind == "error") { scenario = scenario with { Error = new() { StatusCode = status, Code = "configured", Message = "configured outage" } }; } + if (kind == "mismatch") { scenario.Answers.Remove("binary"); } + if (kind == "score-key") { scenario.Answers["score"] = scenario.Answers["score"] with { Probabilities = new() { ["invalid"] = 0.5 } }; } + if (kind == "score-high") { scenario.Answers["score"] = scenario.Answers["score"] with { Score = 100 }; } + if (kind == "wrong-input") { scenario = scenario with { ExpectedInput = "other" }; } + if (kind != "unmatched") { builder.AddDecisionScenario(scenario); } + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + using var configured = await http.PostAsJsonAsync("/admin-api/configure", builder.Build()); + configured.EnsureSuccessStatusCode(); + using var response = await http.PostAsJsonAsync(DecisionEndpointTests.PathFor(_model), DecisionEndpointTests.Body(_model)); + await Assert.That((int)response.StatusCode).IsEqualTo(status); + var summary = await http.GetFromJsonAsync("/admin-api/assertions"); + await Assert.That(summary!.TotalEvents).IsEqualTo(1); + } + + [Test] + public async Task DecisionEndpoint_PreservesFixturesAcrossRequestsAndResetAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddJevLatest() + .AddDecisionScenario(DecisionEndpointTests.Scenario(_model))); + using var http = host.GetTestClient(); + using var first = await http.PostAsJsonAsync(DecisionEndpointTests.PathFor(_model), DecisionEndpointTests.Body(_model)); + first.EnsureSuccessStatusCode(); + var original = await first.Content.ReadAsStringAsync(); + using var reset = await http.PostAsync("/admin-api/reset", null); + reset.EnsureSuccessStatusCode(); + using var second = await http.PostAsJsonAsync(DecisionEndpointTests.PathFor(_model), DecisionEndpointTests.Body(_model)); + second.EnsureSuccessStatusCode(); + var body = await second.Content.ReadFromJsonAsync(); + await Assert.That(body.GetProperty("answers").GetProperty("choice").GetProperty("probabilities").GetProperty("opaque-id").GetDouble()).IsEqualTo(0.95); + await Assert.That(original).Contains("opaque-id"); + } + + [Test] + [Arguments(-0.1)] + [Arguments(1.1)] + [Arguments(double.NaN)] + [Arguments(double.PositiveInfinity)] + public async Task DecisionFixtureValidation_RejectsInvalidProbabilityAsync(double value) + { + var scenario = DecisionEndpointTests.Scenario(_model); scenario.Answers["binary"] = new() { Probability = value }; + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var configuration = new LlmTestServerConfiguration { DecisionScenarios = [scenario] }; + var json = JsonSerializer.Serialize(configuration, _configurationJson); + using var content = new StringContent(json, System.Text.Encoding.UTF8, "application/json"); + using var response = await http.PostAsync("/admin-api/configure", content); + await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + } + + [Test] + [Arguments("score")] + [Arguments("choice")] + [Arguments("kind")] + public async Task DecisionFixtureValidation_RejectsMissingValuesAsync(string kind) + { + var scenario = DecisionEndpointTests.Scenario(_model); + scenario.Answers["binary"] = new() { Kind = kind switch { "score" => LlmTestServerDecisionKind.Score, "choice" => LlmTestServerDecisionKind.Choice, _ => (LlmTestServerDecisionKind)100 } }; + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var configuration = new LlmTestServerConfiguration { DecisionScenarios = [scenario] }; + var json = JsonSerializer.Serialize(configuration, _configurationJson); + using var content = new StringContent(json, System.Text.Encoding.UTF8, "application/json"); + using var response = await http.PostAsync("/admin-api/configure", content); + await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + } + + [Test] + public async Task OpenAiDecisions_EmitNativeRefusalAnswersAsync() + { + var scenario = DecisionEndpointTests.Scenario("gpt-6-luna"); scenario.Answers["0"] = new() { Refused = true }; + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddModel("gpt-6-luna", LlmTestServerModelKind.Decision).AddDecisionScenario(scenario)); + using var client = host.GetTestClient(); + using var response = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor("gpt-6-luna"), DecisionEndpointTests.Body("gpt-6-luna")); + await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.OK); + var payload = await response.Content.ReadFromJsonAsync(); + await Assert.That(payload.GetProperty("answers")[0].GetProperty("type").GetString()).IsEqualTo("refusal"); + await Assert.That(payload.GetProperty("answers")[0].TryGetProperty("probability", out _)).IsFalse(); + } + +} diff --git a/tests/ManagedCode.LlmTck.Tests/Decisions/DecisionValidationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Decisions/DecisionValidationTests.cs similarity index 73% rename from tests/ManagedCode.LlmTck.Tests/Decisions/DecisionValidationTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/Decisions/DecisionValidationTests.cs index 949a2d1..5119643 100644 --- a/tests/ManagedCode.LlmTck.Tests/Decisions/DecisionValidationTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Decisions/DecisionValidationTests.cs @@ -2,15 +2,13 @@ using System.Net.Http.Json; using System.Text.Json; using System.Text.Json.Nodes; -using ManagedCode.LlmTck.Cloudflare; -using ManagedCode.LlmTck.Configuration; -using ManagedCode.LlmTck.Decisions; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.OpenAI; -using ManagedCode.LlmTck.SystemOne; -using ManagedCode.LlmTck.Tests.TestSupport; +using ManagedCode.LlmTestServer.Cloudflare; +using ManagedCode.LlmTestServer.Decisions; +using ManagedCode.LlmTestServer.OpenAI; +using ManagedCode.LlmTestServer.SystemOne; +using ManagedCode.LlmTestServer.Tests.TestSupport; -namespace ManagedCode.LlmTck.Tests.Decisions; +namespace ManagedCode.LlmTestServer.Tests.Decisions; public sealed class DecisionValidationTests { @@ -33,9 +31,9 @@ public async Task SystemOneValidation_RejectsInvalidNativeShapeAsync(string kind case "criteria": body["questions"]!["choice"]!["criteria"] = new JsonArray(); break; default: var duplicate = """{"model":"jev-latest","state":"text","questions":{"x":{"type":"noul"},"x":{"type":"noul"}}}"""; - await Assert.That(SystemOneDecisionMapper.Validate(JsonSerializer.Deserialize(duplicate)).IsValid).IsFalse(); return; + await AssertRejectedAsync("/systemone/v1/systemone", duplicate, HttpStatusCode.UnprocessableEntity); return; } - await Assert.That(SystemOneDecisionMapper.Validate(JsonSerializer.SerializeToElement(body)).IsValid).IsFalse(); + await AssertRejectedAsync("/systemone/v1/systemone", body.ToJsonString(), HttpStatusCode.UnprocessableEntity); } [Test] @@ -52,7 +50,7 @@ public async Task OpenAiValidation_RejectsInvalidNativeShapeAsync(string kind) if (kind == "options") { body["questions"]![1]!["choices"] = new JsonArray(); } if (kind == "label") { body["questions"]![2]!["levels"]![0]!["label"] = true; } if (kind == "description") { body["questions"]![2]!["levels"]![0]!["description"] = false; } - await Assert.That(OpenAiDecisionMapper.Validate(JsonSerializer.SerializeToElement(body)).IsValid).IsFalse(); + await AssertRejectedAsync("/openai/v1/decisions", body.ToJsonString(), HttpStatusCode.BadRequest); } [Test] @@ -70,7 +68,7 @@ public async Task SystemOneValidation_AcceptsOptionalPredicateCriteriaAndJsonCon if (kind == "true-only") { body["questions"]!["binary"]!["criteria"] = new JsonObject { ["true"] = "yes" }; } if (kind == "false-only") { body["questions"]!["binary"]!["criteria"] = new JsonObject { ["false"] = new JsonArray("no") }; } if (kind == "object-instructions") { body["questions"]!["binary"]!["instructions"] = new JsonObject { ["task"] = "classify" }; } - using var host = await LlmTckTestHost.StartAsync(builder => builder.AddJevLatest().AddDecisionScenario(DecisionEndpointTests.Scenario("jev-latest"))); + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddJevLatest().AddDecisionScenario(DecisionEndpointTests.Scenario("jev-latest"))); using var client = host.GetTestClient(); using var response = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor("jev-latest"), body); await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.OK); @@ -94,7 +92,7 @@ public async Task CloudflareValidation_EnforcesNativeQuestionAndModelLimitsAsync if (kind == "choice") { body["questions"]!["choice"]!["criteria"] = new JsonObject { ["only"] = "one" }; } if (kind == "score") { body["questions"]!["score"]!["criteria"] = new JsonArray("one"); } if (kind == "model") { body["model"] = "clef-flash"; } - using var host = await LlmTckTestHost.StartAsync(); using var client = host.GetTestClient(); + using var host = await LlmTestServerTestHost.StartAsync(); using var client = host.GetTestClient(); using var response = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor("clef"), body); await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); await Assert.That((await response.Content.ReadFromJsonAsync()).GetProperty("success").GetBoolean()).IsFalse(); @@ -103,11 +101,8 @@ public async Task CloudflareValidation_EnforcesNativeQuestionAndModelLimitsAsync [Test] public async Task DecisionProfiles_EnforceJevScoreLimitAndKevNativeRoundingAsync() { - var builder = new LlmTckConfigurationBuilder().AddJevLatest().AddKevLatest().AddClef().AddClefFlash().AddOpenAiDecisionModel("gpt-6-luna"); - var profiles = builder.Build().Models.Where(model => model.Kind == LlmTckModelKind.Decision).ToArray(); - await Assert.That(profiles.Select(model => model.DecisionProvider)).IsEquivalentTo(new[] { LlmTckDecisionProvider.TypeSafe, LlmTckDecisionProvider.Kev, LlmTckDecisionProvider.Cloudflare, LlmTckDecisionProvider.Cloudflare, LlmTckDecisionProvider.OpenAI }); var fixture = DecisionEndpointTests.Scenario("kev-latest"); fixture.Answers["binary"] = fixture.Answers["binary"] with { Probability = 0.12345678 }; - using var host = await LlmTckTestHost.StartAsync(b => b.AddJevLatest().AddKevLatest().AddDecisionScenario(fixture)); using var client = host.GetTestClient(); + using var host = await LlmTestServerTestHost.StartAsync(b => b.AddJevLatest().AddKevLatest().AddDecisionScenario(fixture)); using var client = host.GetTestClient(); var body = DecisionEndpointTests.Body("jev-latest"); body["questions"]!["score"]!["criteria"] = new JsonArray(Enumerable.Range(0, SystemOneDecisionMapper.JevMaxScoreLevels + 1).Select(index => JsonValue.Create(index.ToString(System.Globalization.CultureInfo.InvariantCulture))).ToArray()); using var invalid = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor("jev-latest"), body); await Assert.That(invalid.StatusCode).IsEqualTo(HttpStatusCode.UnprocessableEntity); @@ -121,11 +116,11 @@ public async Task OpenAiDecisions_PreserveBooleanChoicesAndUserMessageArrayAsync { var scenario = DecisionEndpointTests.Scenario("gpt-6-luna"); scenario.Answers.Clear(); - scenario.Answers[OpenAiDecisionFixtureIds.ForQuestionIndex(0)] = new() { Kind = LlmTckDecisionKind.Choice, Choice = "true", Confidence = 0.9, Probabilities = new() { ["true"] = 0.9, ["false"] = 0.1 } }; + scenario.Answers[OpenAiDecisionFixtureIds.ForQuestionIndex(0)] = new() { Kind = LlmTestServerDecisionKind.Choice, Choice = "true", Confidence = 0.9, Probabilities = new() { ["true"] = 0.9, ["false"] = 0.1 } }; var body = DecisionEndpointTests.Body("gpt-6-luna"); body["input"] = JsonNode.Parse("""[{"role":"user","content":"evidence"}]"""); body["questions"] = JsonNode.Parse("""[{"type":"choice","name":"choice","instructions":"is urgent?","choices":[{"value":true,"description":"urgent"},{"value":false,"description":"routine"}]}]"""); - using var host = await LlmTckTestHost.StartAsync(builder => builder.AddOpenAiDecisionModel("gpt-6-luna").AddDecisionScenario(scenario)); + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddOpenAiDecisionModel("gpt-6-luna").AddDecisionScenario(scenario)); using var client = host.GetTestClient(); using var response = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor("gpt-6-luna"), body); response.EnsureSuccessStatusCode(); var payload = await response.Content.ReadFromJsonAsync(); var answer = payload.GetProperty("answers")[0]; @@ -134,11 +129,17 @@ public async Task OpenAiDecisions_PreserveBooleanChoicesAndUserMessageArrayAsync } [Test] - public async Task SystemOneQuestionDto_OmitsUndefinedOptionalPredicatePropertiesAsync() + public async Task SystemOneEndpoint_AcceptsOmittedOptionalPredicatePropertiesAsync() { - var json = JsonSerializer.SerializeToElement(new SystemOneDecisionQuestion { Type = SystemOneDecisionTypes.Predicate }); - await Assert.That(json.EnumerateObject().Count()).IsEqualTo(1); - await Assert.That(json.GetProperty("type").GetString()).IsEqualTo(SystemOneDecisionTypes.Predicate); + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddKevLatest() + .AddDecisionScenario(DecisionEndpointTests.Scenario("kev-latest"))); + using var http = host.GetTestClient(); + var body = DecisionEndpointTests.Body("kev-latest"); + body["questions"]!["binary"] = new JsonObject { ["type"] = SystemOneDecisionTypes.Predicate }; + using var response = await http.PostAsJsonAsync("/systemone/v1/systemone", body); + response.EnsureSuccessStatusCode(); + var payload = await response.Content.ReadFromJsonAsync(); + await Assert.That(payload.GetProperty("answers").GetProperty("binary").GetProperty("noul").GetDouble()).IsEqualTo(0.95); } [Test] @@ -148,14 +149,22 @@ public async Task OpenAiDecisions_PreserveDistinctBooleanAndStringChoiceValuesAs scenario.Answers.Clear(); var booleanId = OpenAiDecisionFixtureIds.ForChoiceValue(true); var stringId = OpenAiDecisionFixtureIds.ForChoiceValue("true"); - scenario.Answers[OpenAiDecisionFixtureIds.ForQuestionIndex(0)] = new() { Kind = LlmTckDecisionKind.Choice, Choice = booleanId, Confidence = 0.9, Probabilities = new() { [booleanId] = 0.9, [stringId] = 0.1 } }; + scenario.Answers[OpenAiDecisionFixtureIds.ForQuestionIndex(0)] = new() { Kind = LlmTestServerDecisionKind.Choice, Choice = booleanId, Confidence = 0.9, Probabilities = new() { [booleanId] = 0.9, [stringId] = 0.1 } }; var body = DecisionEndpointTests.Body("gpt-6-luna"); body["questions"] = JsonNode.Parse("""[{"type":"choice","name":"choice","instructions":"which?","choices":[{"value":true,"description":"boolean"},{"value":"true","description":"text"}]}]"""); - using var host = await LlmTckTestHost.StartAsync(builder => builder.AddOpenAiDecisionModel("gpt-6-luna").AddDecisionScenario(scenario)); + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddOpenAiDecisionModel("gpt-6-luna").AddDecisionScenario(scenario)); using var client = host.GetTestClient(); using var response = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor("gpt-6-luna"), body); response.EnsureSuccessStatusCode(); var answer = (await response.Content.ReadFromJsonAsync()).GetProperty("answers")[0]; await Assert.That(answer.GetProperty("choice").GetBoolean()).IsTrue(); await Assert.That(answer.GetProperty("probabilities")[0].GetProperty("value").ValueKind).IsEqualTo(JsonValueKind.True); await Assert.That(answer.GetProperty("probabilities")[1].GetProperty("value").GetString()).IsEqualTo("true"); } + private static async Task AssertRejectedAsync(string path, string body, HttpStatusCode status) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + using var content = new StringContent(body, System.Text.Encoding.UTF8, "application/json"); + using var response = await http.PostAsync(path, content); + await Assert.That(response.StatusCode).IsEqualTo(status); + } } diff --git a/tests/ManagedCode.LlmTck.Tests/Decisions/Jev114EndpointTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Decisions/Jev114EndpointTests.cs similarity index 84% rename from tests/ManagedCode.LlmTck.Tests/Decisions/Jev114EndpointTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/Decisions/Jev114EndpointTests.cs index f07cc58..d20b3e9 100644 --- a/tests/ManagedCode.LlmTck.Tests/Decisions/Jev114EndpointTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Decisions/Jev114EndpointTests.cs @@ -1,13 +1,12 @@ using System.Net.Http.Json; using System.Text.Json; using System.Text.Json.Nodes; -using ManagedCode.LlmTck.Decisions; -using ManagedCode.LlmTck.Hosting; -using ManagedCode.LlmTck.SystemOne; -using ManagedCode.LlmTck.Tests.TestSupport; -using Microsoft.Extensions.DependencyInjection; +using ManagedCode.LlmTestServer.Decisions; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.SystemOne; +using ManagedCode.LlmTestServer.Tests.TestSupport; -namespace ManagedCode.LlmTck.Tests.Decisions; +namespace ManagedCode.LlmTestServer.Tests.Decisions; public sealed class Jev114EndpointTests { @@ -22,7 +21,7 @@ public sealed class Jev114EndpointTests public async Task Jev114_JsonImagesAndBoxPreserveNativeAnswersAsync(bool boxOnly) { var scenario = Scenario(boxOnly); - using var host = await LlmTckTestHost.StartAsync(builder => builder.AddDecisionModel(_model, LlmTckDecisionProvider.TypeSafe).AddDecisionScenario(scenario)); + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddDecisionModel(_model, LlmTestServerDecisionProvider.TypeSafe).AddDecisionScenario(scenario)); using var client = host.GetTestClient(); client.DefaultRequestHeaders.Add(TypeSafeRequestValidation.ImagesHeader, TypeSafeRequestValidation.Enabled); client.DefaultRequestHeaders.Add(TypeSafeRequestValidation.LongContextHeader, TypeSafeRequestValidation.Enabled); @@ -42,7 +41,7 @@ public async Task Jev114_JsonImagesAndBoxPreserveNativeAnswersAsync(bool boxOnly var legend = payload.GetProperty("answers").GetProperty("rating").GetProperty("legend"); await Assert.That(legend.GetProperty("0").GetProperty("example").GetString()).IsEqualTo("photo-reference"); } - var trace = host.Services.GetRequiredService().GetSnapshot().Single(); + var trace = (await client.GetFromJsonAsync>("/admin-api/requests"))!.Single(); await Assert.That(trace.ModelId).IsEqualTo(_model); } @@ -65,7 +64,7 @@ public async Task Jev114_JsonImagesAndBoxPreserveNativeAnswersAsync(bool boxOnly [Arguments("box-criteria", 422)] public async Task Jev114_RejectsInvalidImageAndBoxRequestsAsync(string defect, int status) { - using var host = await LlmTckTestHost.StartAsync(); + using var host = await LlmTestServerTestHost.StartAsync(); using var client = host.GetTestClient(); if (defect != "missing-header") { client.DefaultRequestHeaders.Add(TypeSafeRequestValidation.ImagesHeader, TypeSafeRequestValidation.Enabled); } var body = Body(true); @@ -103,7 +102,7 @@ public async Task Jev114_RejectsInvalidImageAndBoxRequestsAsync(string defect, i public async Task Jev114_AliasReportsResolvedModelAsync() { var scenario = Scenario(true) with { ModelId = TypeSafeRequestValidation.PreviewModel, ResolvedModelId = _model }; - using var host = await LlmTckTestHost.StartAsync(builder => builder.AddDecisionModel(TypeSafeRequestValidation.PreviewModel, LlmTckDecisionProvider.TypeSafe).AddDecisionScenario(scenario)); + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddDecisionModel(TypeSafeRequestValidation.PreviewModel, LlmTestServerDecisionProvider.TypeSafe).AddDecisionScenario(scenario)); using var client = host.GetTestClient(); client.DefaultRequestHeaders.Add(TypeSafeRequestValidation.ImagesHeader, TypeSafeRequestValidation.Enabled); var body = Body(true); body["model"] = TypeSafeRequestValidation.PreviewModel; @@ -131,7 +130,7 @@ public async Task Jev114_ExplicitMalformedBoxFixtureSupportsConsumerProtocolTest { var scenario = Scenario(true) with { AllowMalformedResponse = true }; scenario.Answers["target"] = scenario.Answers["target"] with { Box = new() { Top = 0.8, Bottom = 0.2, Left = 0.1, Right = 0.9 } }; - using var host = await LlmTckTestHost.StartAsync(builder => builder.AddDecisionModel(_model, LlmTckDecisionProvider.TypeSafe).AddDecisionScenario(scenario)); + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddDecisionModel(_model, LlmTestServerDecisionProvider.TypeSafe).AddDecisionScenario(scenario)); using var client = host.GetTestClient(); client.DefaultRequestHeaders.Add(TypeSafeRequestValidation.ImagesHeader, TypeSafeRequestValidation.Enabled); using var response = await client.PostAsJsonAsync(_path, Body(true)); @@ -140,19 +139,19 @@ public async Task Jev114_ExplicitMalformedBoxFixtureSupportsConsumerProtocolTest await Assert.That(box.GetProperty("top").GetDouble()).IsEqualTo(0.8); } - internal static LlmTckDecisionScenario Scenario(bool boxOnly) + internal static LlmTestServerDecisionScenario Scenario(bool boxOnly) { - var scenario = new LlmTckDecisionScenario + var scenario = new LlmTestServerDecisionScenario { Id = "jev114-images", ModelId = _model, Usage = new() { InputTokens = 290, OutputTokens = 21, TotalTokens = 311 }, - Answers = new() { ["target"] = new() { Kind = LlmTckDecisionKind.Box, Box = new() { Top = 0.25, Bottom = 0.82, Left = 0.44, Right = 0.84 }, Confidence = 0.92 } }, + Answers = new() { ["target"] = new() { Kind = LlmTestServerDecisionKind.Box, Box = new() { Top = 0.25, Bottom = 0.82, Left = 0.44, Right = 0.84 }, Confidence = 0.92 } }, }; if (!boxOnly) { scenario.Answers["present"] = new() { Probability = 0.95 }; - scenario.Answers["rating"] = new() { Kind = LlmTckDecisionKind.Score, Score = 0.75, Confidence = 0.6, Probabilities = new() { ["0"] = 0.25, ["1"] = 0.75 } }; + scenario.Answers["rating"] = new() { Kind = LlmTestServerDecisionKind.Score, Score = 0.75, Confidence = 0.6, Probabilities = new() { ["0"] = 0.25, ["1"] = 0.75 } }; } return scenario; } diff --git a/tests/ManagedCode.LlmTck.Tests/Decisions/Jev114MultipartTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Decisions/Jev114MultipartTests.cs similarity index 85% rename from tests/ManagedCode.LlmTck.Tests/Decisions/Jev114MultipartTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/Decisions/Jev114MultipartTests.cs index 9f253fc..ed53ba5 100644 --- a/tests/ManagedCode.LlmTck.Tests/Decisions/Jev114MultipartTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Decisions/Jev114MultipartTests.cs @@ -3,12 +3,12 @@ using System.Net.Http.Json; using System.Text; using System.Text.Json; -using ManagedCode.LlmTck.Client; -using ManagedCode.LlmTck.Decisions; -using ManagedCode.LlmTck.SystemOne; -using ManagedCode.LlmTck.Tests.TestSupport; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Decisions; +using ManagedCode.LlmTestServer.SystemOne; +using ManagedCode.LlmTestServer.Tests.TestSupport; -namespace ManagedCode.LlmTck.Tests.Decisions; +namespace ManagedCode.LlmTestServer.Tests.Decisions; public sealed class Jev114MultipartTests { @@ -21,11 +21,11 @@ public sealed class Jev114MultipartTests [Arguments(true)] public async Task Jev114_TypedClientSupportsJsonAndMultipartImagesAsync(bool multipart) { - using var host = await LlmTckTestHost.StartAsync(builder => builder - .AddDecisionModel(TypeSafeRequestValidation.Jev114Model, LlmTckDecisionProvider.TypeSafe) + using var host = await LlmTestServerTestHost.StartAsync(builder => builder + .AddDecisionModel(TypeSafeRequestValidation.Jev114Model, LlmTestServerDecisionProvider.TypeSafe) .RequireBearerToken(_key).AddDecisionScenario(Jev114EndpointTests.Scenario(false))); using var http = host.GetTestClient(); - var client = new LlmTckClient(http, _key).CreateSystemOneClient(); + var client = new LlmTestServerClient(http, _key).CreateSystemOneClient(); var request = Request(multipart); var response = multipart ? await client.EvaluateMultipartAsync(request, [new() { PartName = _part, Data = Image(), ContentType = "IMAGE/PNG" }], enableImages: true, enableLongContext: true) @@ -48,7 +48,7 @@ public async Task Jev114_TypedClientSupportsJsonAndMultipartImagesAsync(bool mul [Arguments("malformed-json")] public async Task Jev114_MultipartRejectsNativePartViolationsAsync(string defect) { - using var host = await LlmTckTestHost.StartAsync(); + using var host = await LlmTestServerTestHost.StartAsync(); using var http = host.GetTestClient(); if (defect != "missing-header") { http.DefaultRequestHeaders.Add(TypeSafeRequestValidation.ImagesHeader, TypeSafeRequestValidation.Enabled); } var request = Request(true); @@ -68,7 +68,7 @@ public async Task Jev114_MultipartRejectsNativePartViolationsAsync(string defect [Arguments(true)] public async Task Jev114_MultipartEnforcesPhysicalBodyAndImageByteLimitsAsync(bool wholeBody) { - using var host = await LlmTckTestHost.StartAsync(); + using var host = await LlmTestServerTestHost.StartAsync(); using var http = host.GetTestClient(); http.DefaultRequestHeaders.Add(TypeSafeRequestValidation.ImagesHeader, TypeSafeRequestValidation.Enabled); using var content = new MultipartFormDataContent(); @@ -81,12 +81,12 @@ public async Task Jev114_MultipartEnforcesPhysicalBodyAndImageByteLimitsAsync(bo [Test] public async Task Jev114_MultipartInfersFormatAndPrefersPartOverBase64Async() { - using var host = await LlmTckTestHost.StartAsync(builder => builder - .AddDecisionModel(TypeSafeRequestValidation.Jev114Model, LlmTckDecisionProvider.TypeSafe).AddDecisionScenario(Jev114EndpointTests.Scenario(false))); + using var host = await LlmTestServerTestHost.StartAsync(builder => builder + .AddDecisionModel(TypeSafeRequestValidation.Jev114Model, LlmTestServerDecisionProvider.TypeSafe).AddDecisionScenario(Jev114EndpointTests.Scenario(false))); using var http = host.GetTestClient(); var request = Request(true); request.Assets!["photo-reference"] = request.Assets["photo-reference"] with { Format = null, DataBase64 = "ignored-invalid-base64" }; - var response = await new LlmTckClient(http).CreateSystemOneClient().EvaluateMultipartAsync(request, + var response = await new LlmTestServerClient(http).CreateSystemOneClient().EvaluateMultipartAsync(request, [new() { PartName = _part, Data = Image() }], enableImages: true); await Assert.That(response.Answers["target"].Box!.Right).IsEqualTo(0.84); } diff --git a/tests/ManagedCode.LlmTck.Tests/Decisions/SystemOneNativeContractTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Decisions/SystemOneNativeContractTests.cs similarity index 74% rename from tests/ManagedCode.LlmTck.Tests/Decisions/SystemOneNativeContractTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/Decisions/SystemOneNativeContractTests.cs index 93c76f4..eb60329 100644 --- a/tests/ManagedCode.LlmTck.Tests/Decisions/SystemOneNativeContractTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Decisions/SystemOneNativeContractTests.cs @@ -2,24 +2,24 @@ using System.Net.Http.Json; using System.Text.Json; using System.Text.Json.Nodes; -using ManagedCode.LlmTck.Decisions; -using ManagedCode.LlmTck.SystemOne; -using ManagedCode.LlmTck.Tests.TestSupport; +using ManagedCode.LlmTestServer.Decisions; +using ManagedCode.LlmTestServer.SystemOne; +using ManagedCode.LlmTestServer.Tests.TestSupport; -namespace ManagedCode.LlmTck.Tests.Decisions; +namespace ManagedCode.LlmTestServer.Tests.Decisions; public sealed class SystemOneNativeContractTests { [Test] public async Task Kev_DoesNotInheritTypeSafeImageUploadBodyLimitAsync() { - var scenario = new LlmTckDecisionScenario + var scenario = new LlmTestServerDecisionScenario { Id = "kev-large-text", ModelId = SystemOneDecisionMapper.KevDefaultModel, Answers = new() { ["present"] = new() { Probability = 0.75 } }, }; - using var host = await LlmTckTestHost.StartAsync(builder => builder.AddKevLatest().AddDecisionScenario(scenario)); + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddKevLatest().AddDecisionScenario(scenario)); using var client = host.GetTestClient(); using var content = new StringContent(JsonSerializer.Serialize(new { @@ -42,7 +42,30 @@ public async Task Kev_DoesNotInheritTypeSafeImageUploadBodyLimitAsync() [Arguments("1e16", "1e+16")] public async Task Kev_RenderedNumbersMatchNativePythonContentAsync(string json, string expected) { - await Assert.That(KevDecisionContent.Render(JsonSerializer.Deserialize(json))).IsEqualTo(expected); + var scenario = DecisionEndpointTests.Scenario("kev-latest"); + scenario.Answers.Clear(); + scenario.Answers["score"] = new() + { + Kind = LlmTestServerDecisionKind.Score, + Score = 0, + Confidence = 1, + Probabilities = new() { ["0"] = 1, ["1"] = 0 }, + }; + var body = DecisionEndpointTests.Body("kev-latest"); + body["questions"] = new JsonObject + { + ["score"] = new JsonObject + { + ["type"] = "score", + ["criteria"] = new JsonArray(JsonNode.Parse(json), "other"), + }, + }; + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddKevLatest().AddDecisionScenario(scenario)); + using var http = host.GetTestClient(); + using var response = await http.PostAsJsonAsync("/systemone/v1/systemone", body); + response.EnsureSuccessStatusCode(); + var payload = await response.Content.ReadFromJsonAsync(); + await Assert.That(payload.GetProperty("answers").GetProperty("score").GetProperty("legend").GetProperty("0").GetString()).IsEqualTo(expected); } [Test] @@ -57,7 +80,7 @@ public async Task Kev_AcceptsArbitraryJsonAndExplicitConfiguredDefaultAsync(stri body["questions"]!["binary"]!["criteria"] = new JsonObject { ["ignored"] = false, ["true"] = true, ["false"] = null }; body["questions"]!["choice"]!["criteria"]!["opaque-id"] = false; body["questions"]!["score"]!["criteria"] = new JsonArray(null, true, new JsonObject { ["list"] = new JsonArray(1, "x") }); - using var host = await LlmTckTestHost.StartAsync(builder => builder.AddKevLatest().AddDecisionScenario(DecisionEndpointTests.Scenario("kev-latest"))); + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddKevLatest().AddDecisionScenario(DecisionEndpointTests.Scenario("kev-latest"))); using var client = host.GetTestClient(); client.DefaultRequestHeaders.Add("x-typesafe-request-id", "native-request-id"); using var response = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor("kev-latest"), body); response.EnsureSuccessStatusCode(); var payload = await response.Content.ReadFromJsonAsync(); var legend = payload.GetProperty("answers").GetProperty("score").GetProperty("legend"); @@ -74,9 +97,9 @@ public async Task Kev_PreservesEmptyOpaqueNamesChoicesAndConfiguredTruncationMet { var scenario = DecisionEndpointTests.Scenario("kev-latest"); scenario.Answers.Clear(); scenario = scenario with { Metadata = new() { LatencyMilliseconds = 12.34, Truncated = true, StateTokens = 80, StateTokensUsed = 40 } }; - scenario.Answers[""] = new() { Kind = LlmTckDecisionKind.Choice, Choice = "", Confidence = 0.8, Probabilities = new() { [""] = 0.9, [" "] = 0.1 } }; + scenario.Answers[""] = new() { Kind = LlmTestServerDecisionKind.Choice, Choice = "", Confidence = 0.8, Probabilities = new() { [""] = 0.9, [" "] = 0.1 } }; var body = DecisionEndpointTests.Body("kev-latest"); body["questions"] = JsonNode.Parse("""{"":{"type":"choice","criteria":{"":null," ":42}}}"""); - using var host = await LlmTckTestHost.StartAsync(builder => builder.AddKevLatest().AddDecisionScenario(scenario)); + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddKevLatest().AddDecisionScenario(scenario)); using var client = host.GetTestClient(); using var response = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor("kev-latest"), body); response.EnsureSuccessStatusCode(); var payload = await response.Content.ReadFromJsonAsync(); await Assert.That(payload.GetProperty("answers").GetProperty("").GetProperty("choice").GetString()).IsEqualTo(""); @@ -91,7 +114,7 @@ public async Task Kev_PreservesEmptyOpaqueNamesChoicesAndConfiguredTruncationMet [Arguments(true)] public async Task SystemOne_ReturnsNativeValidationEnvelopeAndRequestIdAsync(bool configuredKev) { - using var host = await LlmTckTestHost.StartAsync(builder => { if (configuredKev) { builder.AddKevLatest(); } }); + using var host = await LlmTestServerTestHost.StartAsync(builder => { if (configuredKev) { builder.AddKevLatest(); } }); using var client = host.GetTestClient(); var body = DecisionEndpointTests.Body(configuredKev ? "kev-latest" : "jev-latest"); body.Remove("state"); using var response = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor("jev-latest"), body); @@ -106,7 +129,7 @@ public async Task SystemOne_ReturnsNativeValidationEnvelopeAndRequestIdAsync(boo [Test] public async Task Kev_AuthFailureCarriesBearerChallengeAndRequestIdAsync() { - using var host = await LlmTckTestHost.StartAsync(builder => builder.AddKevLatest().RequireBearerToken("test-token")); + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddKevLatest().RequireBearerToken("test-token")); using var client = host.GetTestClient(); using var response = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor("kev-latest"), DecisionEndpointTests.Body("kev-latest")); await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.Unauthorized); await Assert.That(response.Headers.WwwAuthenticate.Single().Scheme).IsEqualTo("Bearer"); @@ -118,7 +141,7 @@ public async Task SystemOne_PredicateOutputContainsOnlyNativeKindFieldsAsync() { var scenario = DecisionEndpointTests.Scenario("jev-latest"); scenario.Answers["binary"] = scenario.Answers["binary"] with { Confidence = 0.8, Choice = "unused", Score = 1 }; - using var host = await LlmTckTestHost.StartAsync(builder => builder.AddJevLatest().AddDecisionScenario(scenario)); + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddJevLatest().AddDecisionScenario(scenario)); using var client = host.GetTestClient(); using var response = await client.PostAsJsonAsync(DecisionEndpointTests.PathFor("jev-latest"), DecisionEndpointTests.Body("jev-latest")); response.EnsureSuccessStatusCode(); var binary = (await response.Content.ReadFromJsonAsync()).GetProperty("answers").GetProperty("binary"); await Assert.That(binary.EnumerateObject().Select(item => item.Name)).IsEquivalentTo(new[] { "type", "noul" }); diff --git a/tests/ManagedCode.LlmTck.Tests/Decisions/TypeSafeImageBoundaryFixtures.cs b/tests/ManagedCode.LlmTestServer.Tests/Decisions/TypeSafeImageBoundaryFixtures.cs similarity index 96% rename from tests/ManagedCode.LlmTck.Tests/Decisions/TypeSafeImageBoundaryFixtures.cs rename to tests/ManagedCode.LlmTestServer.Tests/Decisions/TypeSafeImageBoundaryFixtures.cs index fcad1a2..dd62de2 100644 --- a/tests/ManagedCode.LlmTck.Tests/Decisions/TypeSafeImageBoundaryFixtures.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Decisions/TypeSafeImageBoundaryFixtures.cs @@ -1,4 +1,4 @@ -namespace ManagedCode.LlmTck.Tests.Decisions; +namespace ManagedCode.LlmTestServer.Tests.Decisions; internal static class TypeSafeImageBoundaryFixtures { diff --git a/tests/ManagedCode.LlmTck.Tests/Decisions/TypeSafeVisionContractTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Decisions/TypeSafeVisionContractTests.cs similarity index 75% rename from tests/ManagedCode.LlmTck.Tests/Decisions/TypeSafeVisionContractTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/Decisions/TypeSafeVisionContractTests.cs index a79c912..a36b3c1 100644 --- a/tests/ManagedCode.LlmTck.Tests/Decisions/TypeSafeVisionContractTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Decisions/TypeSafeVisionContractTests.cs @@ -2,11 +2,11 @@ using System.Net.Http.Json; using System.Text; using System.Text.Json; -using ManagedCode.LlmTck.Decisions; -using ManagedCode.LlmTck.SystemOne; -using ManagedCode.LlmTck.Tests.TestSupport; +using ManagedCode.LlmTestServer.Decisions; +using ManagedCode.LlmTestServer.SystemOne; +using ManagedCode.LlmTestServer.Tests.TestSupport; -namespace ManagedCode.LlmTck.Tests.Decisions; +namespace ManagedCode.LlmTestServer.Tests.Decisions; public sealed class TypeSafeVisionContractTests { @@ -21,8 +21,8 @@ public sealed class TypeSafeVisionContractTests [Arguments(1569, 280, HttpStatusCode.RequestEntityTooLarge)] public async Task ImageSidesMatchDocumentedInclusiveLimitsAsync(int width, int height, HttpStatusCode status) { - using var host = await LlmTckTestHost.StartAsync(builder => builder - .AddDecisionModel(TypeSafeRequestValidation.Jev114Model, LlmTckDecisionProvider.TypeSafe) + using var host = await LlmTestServerTestHost.StartAsync(builder => builder + .AddDecisionModel(TypeSafeRequestValidation.Jev114Model, LlmTestServerDecisionProvider.TypeSafe) .AddDecisionScenario(Jev114EndpointTests.Scenario(true))); using var http = host.GetTestClient(); http.DefaultRequestHeaders.Add(TypeSafeRequestValidation.ImagesHeader, TypeSafeRequestValidation.Enabled); @@ -41,8 +41,8 @@ public async Task BoxAllowsNullableConfidenceAndOptionalInstructionsFromOpenApiA { var scenario = Jev114EndpointTests.Scenario(true); scenario.Answers["target"] = scenario.Answers["target"] with { Confidence = null }; - using var host = await LlmTckTestHost.StartAsync(builder => builder - .AddDecisionModel(TypeSafeRequestValidation.Jev114Model, LlmTckDecisionProvider.TypeSafe).AddDecisionScenario(scenario)); + using var host = await LlmTestServerTestHost.StartAsync(builder => builder + .AddDecisionModel(TypeSafeRequestValidation.Jev114Model, LlmTestServerDecisionProvider.TypeSafe).AddDecisionScenario(scenario)); using var http = host.GetTestClient(); http.DefaultRequestHeaders.Add(TypeSafeRequestValidation.ImagesHeader, TypeSafeRequestValidation.Enabled); var body = Jev114EndpointTests.Body(true); @@ -63,7 +63,7 @@ public async Task BoxAllowsNullableConfidenceAndOptionalInstructionsFromOpenApiA [Arguments(true)] public async Task PhysicalBodyLimitIncludesWhitespaceWithOrWithoutContentLengthAsync(bool unknownLength) { - using var host = await LlmTckTestHost.StartAsync(); + using var host = await LlmTestServerTestHost.StartAsync(); using var http = host.GetTestClient(); var body = Jev114EndpointTests.Body(true).ToJsonString(); var padded = Encoding.UTF8.GetBytes(new string(' ', TypeSafeRequestValidation.MaxRequestBytes) + body); @@ -76,9 +76,13 @@ public async Task PhysicalBodyLimitIncludesWhitespaceWithOrWithoutContentLengthA [Test] public async Task NativeSerializationDoesNotAddConfidenceToNoulAsync() { - var json = JsonSerializer.SerializeToElement(new SystemOneDecisionAnswer - { Type = SystemOneDecisionTypes.Predicate, Probability = 0.75 }, SystemOneNativeJson.Options); - await Assert.That(json.TryGetProperty("confidence", out _)).IsFalse(); + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddKevLatest() + .AddDecisionScenario(DecisionEndpointTests.Scenario("kev-latest"))); + using var http = host.GetTestClient(); + using var response = await http.PostAsJsonAsync(_path, DecisionEndpointTests.Body("kev-latest")); + response.EnsureSuccessStatusCode(); + var payload = await response.Content.ReadFromJsonAsync(); + await Assert.That(payload.GetProperty("answers").GetProperty("binary").TryGetProperty("confidence", out _)).IsFalse(); } private sealed class UnknownLengthStream(byte[] bytes) : MemoryStream(bytes) diff --git a/tests/ManagedCode.LlmTestServer.Tests/Determinism/DeterminismBoundaryIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Determinism/DeterminismBoundaryIntegrationTests.cs new file mode 100644 index 0000000..81fa3d3 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Determinism/DeterminismBoundaryIntegrationTests.cs @@ -0,0 +1,59 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using System.Text.Json.Nodes; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.Determinism; + +public sealed class DeterminismBoundaryIntegrationTests +{ + [Test] + [Arguments("null")] + [Arguments("empty")] + [Arguments("blank")] + [Arguments("oversized")] + public async Task InvalidSeedConfiguration_PreservesActiveFixtureAndClockAsync(string defect) + { + var fixedUtc = new DateTimeOffset(2001, 2, 3, 4, 5, 6, TimeSpan.Zero); + using var host = await LlmTestServerTestHost.StartAsync(config => config.WithDeterminism("original", fixedUtc) + .AddChatScenario("preserved", scenario => scenario.Responds("preserved"))); + using var http = host.GetTestClient(); + var configuration = JsonNode.Parse("""{"determinism":{"seed":"valid","fixedUtc":"2001-02-03T04:05:06Z"}}""")!; + configuration["determinism"]!["seed"] = defect switch + { "null" => null, "empty" => "", "blank" => " ", _ => new string('x', 257) }; + using var invalid = await http.PostAsJsonAsync("/admin-api/configure", configuration); + await Assert.That(invalid.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + using var actual = await http.PostAsJsonAsync("/openai/v1/chat/completions", new + { model = "gpt-4.1-mini", messages = new[] { new { role = "user", content = "hello" } } }); + actual.EnsureSuccessStatusCode(); + var result = await actual.Content.ReadFromJsonAsync(); + await Assert.That(result.GetProperty("created").GetInt64()).IsEqualTo(fixedUtc.ToUnixTimeSeconds()); + await Assert.That(result.GetProperty("choices")[0].GetProperty("message").GetProperty("content").GetString()).IsEqualTo("preserved"); + } + + [Test] + public async Task MaximumSeedAndOffsetClock_ProduceReproducibleSdkFixtureTimeAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var clock = new DateTimeOffset(2001, 2, 3, 6, 5, 6, TimeSpan.FromHours(2)); + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(config => config.UseDeterminism(new string('x', 256), clock) + .UseChatScenario("clock", scenario => scenario.Responds("stable"))); + async Task CallAsync() + { + using var response = await http.PostAsJsonAsync("/openai/v1/chat/completions", new + { model = "gpt-4.1-mini", messages = new[] { new { role = "user", content = "hello" } } }); + response.EnsureSuccessStatusCode(); + await Assert.That(response.Headers.Date).IsEqualTo(clock.ToUniversalTime()); + return await response.Content.ReadFromJsonAsync(); + } + var first = await CallAsync(); + await control.ResetAsync(); + var repeated = await CallAsync(); + await Assert.That(first.GetProperty("id").GetString()).IsEqualTo(repeated.GetProperty("id").GetString()); + await Assert.That(repeated.GetProperty("created").GetInt64()).IsEqualTo(clock.ToUnixTimeSeconds()); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Determinism/DeterminismIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Determinism/DeterminismIntegrationTests.cs new file mode 100644 index 0000000..e4d4dda --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Determinism/DeterminismIntegrationTests.cs @@ -0,0 +1,184 @@ +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Net.Http.Json; +using System.Text.Json; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Tests.TestSupport; +using OpenAI; +using OpenAI.Chat; + +namespace ManagedCode.LlmTestServer.Tests.Determinism; + +public sealed class DeterminismIntegrationTests +{ + private static readonly DateTimeOffset _fixedUtc = new(2001, 2, 3, 4, 5, 6, TimeSpan.Zero); + + [Test] + public async Task OfficialSdk_UsesFixedWireTimeAndResettableSeededIdsWithRealDiagnosticClocksAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(config => config.WithDeterminism("sdk-seed", _fixedUtc) + .AddChatScenario("clock", scenario => scenario.Responds("first").Responds("second"))); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + var sdk = new global::OpenAI.Chat.ChatClient("gpt-4.1-mini", new ApiKeyCredential("provider-key"), new OpenAIClientOptions + { Endpoint = new Uri(http.BaseAddress!, "/openai/v1"), Transport = new HttpClientPipelineTransport(http) }); + var started = DateTimeOffset.UtcNow.AddSeconds(-1); + var first = (await sdk.CompleteChatAsync([new UserChatMessage("clock")])).Value; + var second = (await sdk.CompleteChatAsync([new UserChatMessage("clock")])).Value; + await Assert.That(first.CreatedAt).IsEqualTo(_fixedUtc); + await Assert.That(second.CreatedAt).IsEqualTo(_fixedUtc); + await Assert.That(first.Id).IsNotEqualTo(second.Id); + var finished = DateTimeOffset.UtcNow.AddSeconds(1); + var events = (await control.GetAssertionsAsync()).Events; + await Assert.That(events.Count).IsEqualTo(2); + await Assert.That(events.All(item => item.Timestamp >= started && item.Timestamp <= finished)).IsTrue(); + var traces = (await http.GetFromJsonAsync("/admin-api/requests"))!; + await Assert.That(traces.Length).IsEqualTo(2); + await Assert.That(traces.All(trace => trace.StartedAt >= started && trace.CompletedAt <= finished && trace.DurationMilliseconds >= 0)).IsTrue(); + await control.ResetAsync(); + var afterReset = (await sdk.CompleteChatAsync([new UserChatMessage("clock")])).Value; + await Assert.That(afterReset.Id).IsEqualTo(first.Id); + await Assert.That(afterReset.CreatedAt).IsEqualTo(_fixedUtc); + await control.ConfigureAsync(config => config.UseDeterminism("sdk-seed", _fixedUtc) + .UseChatScenario("clock", scenario => scenario.Responds("first").Responds("second"))); + var afterConfigure = (await sdk.CompleteChatAsync([new UserChatMessage("clock")])).Value; + await Assert.That(afterConfigure.Id).IsEqualTo(first.Id); + await Assert.That(afterConfigure.Content[0].Text).IsEqualTo("first"); + using var meai = control.CreateChatClient("gpt-4.1-mini"); + var meaiResult = await meai.GetResponseAsync([new Microsoft.Extensions.AI.ChatMessage(Microsoft.Extensions.AI.ChatRole.User, "clock")]); + await Assert.That(meaiResult.CreatedAt).IsEqualTo(_fixedUtc); + await control.ResetAsync(); + await foreach (var update in meai.GetStreamingResponseAsync([new Microsoft.Extensions.AI.ChatMessage(Microsoft.Extensions.AI.ChatRole.User, "clock")])) + { await Assert.That(update.CreatedAt).IsEqualTo(_fixedUtc); } + } + + [Test] + [Arguments("/openai/v1/chat/completions")] + [Arguments("/groq/openai/v1/chat/completions")] + [Arguments("/mistral/v1/chat/completions")] + [Arguments("/openrouter/api/v1/chat/completions")] + [Arguments("/deepseek/v1/chat/completions")] + public async Task JsonAndSse_ShareSeededFixtureIdentityAndFixedTimestampAsync(string route) + { + using var host = await LlmTestServerTestHost.StartAsync(config => config.WithDeterminism("transport-seed", _fixedUtc) + .AddChatScenario("transport", scenario => scenario.Responds("stable reply", "stable ", "reply"))); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + var json = await ChatAsync(http, route); + var id = json.GetProperty("id").GetString(); + await Assert.That(json.GetProperty("created").GetInt64()).IsEqualTo(_fixedUtc.ToUnixTimeSeconds()); + await control.ResetAsync(); + using var stream = await http.PostAsJsonAsync(route, ChatBody(true)); + stream.EnsureSuccessStatusCode(); + var chunks = ReadSse(await stream.Content.ReadAsStringAsync()); + await Assert.That(chunks.Count).IsGreaterThan(1); + await Assert.That(chunks.All(chunk => chunk.GetProperty("id").GetString() == id + && chunk.GetProperty("created").GetInt64() == _fixedUtc.ToUnixTimeSeconds())).IsTrue(); + var text = string.Concat(chunks.SelectMany(chunk => chunk.GetProperty("choices").EnumerateArray()) + .Select(choice => choice.GetProperty("delta").TryGetProperty("content", out var content) ? content.GetString() : null)); + await Assert.That(text).IsEqualTo("stable reply"); + } + + [Test] + [Arguments("/openai/v1/chat/completions")] + [Arguments("/openai/v1/responses")] + public async Task ConversationReplay_PreservesWireIdentityAcrossJsonAndSseAsync(string route) + { + using var host = await LlmTestServerTestHost.StartAsync(config => config.WithDeterminism("replay-seed", _fixedUtc) + .AddConversation("replay", graph => graph.Step("answer", step => step.Responds("stable reply", "stable ", "reply").Complete()))); + using var http = host.GetTestClient(); + using var conversation = new LlmTestServerClient(http).ForConversation("fixed-replay"); + var responses = route.EndsWith("responses", StringComparison.Ordinal); + object Body(bool stream) + { + return responses ? new { model = "gpt-4.1-mini", input = "clock", stream } : ChatBody(stream); + } + + using var first = await conversation.HttpClient.PostAsJsonAsync(route, Body(false)); + first.EnsureSuccessStatusCode(); + var firstJson = await first.Content.ReadFromJsonAsync(); + var id = firstJson.GetProperty("id").GetString(); + await Assert.That(firstJson.GetProperty(responses ? "created_at" : "created").GetInt64()).IsEqualTo(_fixedUtc.ToUnixTimeSeconds()); + using var retry = await conversation.HttpClient.PostAsJsonAsync(route, Body(true)); + retry.EnsureSuccessStatusCode(); + var chunks = ReadSse(await retry.Content.ReadAsStringAsync()); + if (responses) + { + var completed = chunks.Single(chunk => chunk.GetProperty("type").GetString() == "response.completed").GetProperty("response"); + await Assert.That(completed.GetProperty("id").GetString()).IsEqualTo(id); + await Assert.That(completed.GetProperty("created_at").GetInt64()).IsEqualTo(_fixedUtc.ToUnixTimeSeconds()); + } + else + { + await Assert.That(chunks.All(chunk => chunk.GetProperty("id").GetString() == id + && chunk.GetProperty("created").GetInt64() == _fixedUtc.ToUnixTimeSeconds())).IsTrue(); + } + await Assert.That((await conversation.Control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); + await Assert.That((await conversation.Control.GetAssertionsAsync()).Events.Last().IsReplay).IsTrue(); + } + + [Test] + public async Task Sessions_ReproduceSameSeedWithoutSharingResponseCountersOrConfigurationAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + await using var first = await root.CreateSessionAsync(config => config.UseDeterminism("session-seed", _fixedUtc) + .UseChatScenario("seed", scenario => scenario.Responds("one").Responds("two"))); + await using var second = await root.CreateSessionAsync(config => config.UseDeterminism("session-seed", _fixedUtc) + .UseChatScenario("seed", scenario => scenario.Responds("one").Responds("two"))); + await using var anotherSeed = await root.CreateSessionAsync(config => config.UseDeterminism("different-seed", _fixedUtc) + .UseChatScenario("seed", scenario => scenario.Responds("one"))); + var a1 = await ChatAsync(first.HttpClient); + var a2 = await ChatAsync(first.HttpClient); + var b1 = await ChatAsync(second.HttpClient); + var b2 = await ChatAsync(second.HttpClient); + var different = await ChatAsync(anotherSeed.HttpClient); + await Assert.That(a1.GetProperty("id").GetString()).IsEqualTo(b1.GetProperty("id").GetString()); + await Assert.That(a2.GetProperty("id").GetString()).IsEqualTo(b2.GetProperty("id").GetString()); + await Assert.That(a1.GetProperty("id").GetString()).IsNotEqualTo(different.GetProperty("id").GetString()); + await first.Control.ResetAsync(); + var reset = await ChatAsync(first.HttpClient); + await Assert.That(reset.GetProperty("id").GetString()).IsEqualTo(a1.GetProperty("id").GetString()); + await Assert.That((await second.Control.GetAssertionsAsync()).Matched).IsEqualTo(2); + await Assert.That((await root.GetAssertionsAsync()).Matched).IsEqualTo(0); + } + + [Test] + public async Task ProviderNamespaces_AvoidWireIdCollisionsAndReproduceAfterResetAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(config => config.WithDeterminism("provider-seed", _fixedUtc) + .AddChatScenario("providers", scenario => scenario.Responds("one").Responds("two"))); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + var openAi = await ChatAsync(http, "/openai/v1/chat/completions"); + var groq = await ChatAsync(http, "/groq/openai/v1/chat/completions"); + await Assert.That(openAi.GetProperty("id").GetString()).IsNotEqualTo(groq.GetProperty("id").GetString()); + await control.ResetAsync(); + var repeatOpenAi = await ChatAsync(http, "/openai/v1/chat/completions"); + var repeatGroq = await ChatAsync(http, "/groq/openai/v1/chat/completions"); + await Assert.That(repeatOpenAi.GetProperty("id").GetString()).IsEqualTo(openAi.GetProperty("id").GetString()); + await Assert.That(repeatGroq.GetProperty("id").GetString()).IsEqualTo(groq.GetProperty("id").GetString()); + } + + private static object ChatBody(bool stream = false) + { + return new + { model = "gpt-4.1-mini", stream, messages = new[] { new { role = "user", content = "clock" } } }; + } + + private static async Task ChatAsync(HttpClient http, string route = "/openai/v1/chat/completions") + { + using var response = await http.PostAsJsonAsync(route, ChatBody()); + response.EnsureSuccessStatusCode(); + return await response.Content.ReadFromJsonAsync(); + } + + private static List ReadSse(string stream) + { + return stream.Split('\n') + .Where(line => line.StartsWith("data: ", StringComparison.Ordinal) && line != "data: [DONE]") + .Select(line => JsonSerializer.Deserialize(line[6..])).ToList(); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Determinism/NativeDeterminismIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Determinism/NativeDeterminismIntegrationTests.cs new file mode 100644 index 0000000..16a06d4 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Determinism/NativeDeterminismIntegrationTests.cs @@ -0,0 +1,176 @@ +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Text; +using System.Text.Json; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.Determinism; + +public sealed class NativeDeterminismIntegrationTests +{ + private static readonly DateTimeOffset _fixedUtc = new(2001, 2, 3, 4, 5, 6, TimeSpan.Zero); + + [Test] + [Arguments(0, false)] + [Arguments(1, false)] + [Arguments(1, true)] + [Arguments(2, false)] + [Arguments(3, false)] + public async Task NativeJsonResetAndGraphStreamingReplay_PreserveProviderIdentityAsync(int format, bool geminiArray) + { + var model = format switch { 0 => LlmTestServerKnownModelIds.ClaudeSonnet55, 1 => "gemini-2.5-flash", 2 => "command-r", _ => "llama3.2" }; + using var host = await LlmTestServerTestHost.StartAsync(config => config.WithDeterminism("native-seed", _fixedUtc) + .AddModel(model, LlmTestServerModelKind.Chat) + .AddConversation("native-clock", graph => graph.ForModel(model).Step("answer", step => step.Responds("stable reply", "stable ", "reply").Complete()))); + using var http = host.GetTestClient(); + http.DefaultRequestHeaders.Add("anthropic-version", "2023-06-01"); + var control = new LlmTestServerClient(http); + using var conversation = control.ForConversation("native-clock-id"); + string Route(bool stream) + { + return format switch + { + 0 => "/anthropic/v1/messages", + 1 => $"/gemini/v1beta/models/{model}:" + (stream ? "streamGenerateContent" + (geminiArray ? "" : "?alt=sse") : "generateContent"), + 2 => "/cohere/v2/chat", + _ => "/ollama/api/chat", + }; + } + + object Body(bool stream) + { + return format == 1 + ? new { contents = new[] { new { role = "user", parts = new[] { new { text = "clock" } } } } } + : new { model, max_tokens = 128, stream, messages = new[] { new { role = "user", content = "clock" } } }; + } + + async Task JsonAsync() + { + using var response = await conversation.HttpClient.PostAsJsonAsync(Route(false), Body(false)); + response.EnsureSuccessStatusCode(); + await Assert.That(response.Headers.Date).IsEqualTo(_fixedUtc); + return await response.Content.ReadFromJsonAsync(); + } + string? Identity(JsonElement value) + { + return format switch + { 1 => value.GetProperty("responseId").GetString(), 3 => value.GetProperty("created_at").GetString(), _ => value.GetProperty("id").GetString() }; + } + + var first = await JsonAsync(); + var id = Identity(first); + await Assert.That(id).IsNotNull(); + if (format == 3) { await Assert.That(first.GetProperty("created_at").GetDateTimeOffset()).IsEqualTo(_fixedUtc); } + await control.ResetAsync(); + var repeated = await JsonAsync(); + await Assert.That(Identity(repeated)).IsEqualTo(id); + using var streamed = await conversation.HttpClient.PostAsJsonAsync(Route(true), Body(true)); + streamed.EnsureSuccessStatusCode(); + var text = await streamed.Content.ReadAsStringAsync(); + var events = format == 3 ? ReadLines(text) : format == 1 && geminiArray ? ReadArray(text) : ReadSse(text); + await Assert.That(events.Count).IsGreaterThan(1); + if (format == 0) + { + var start = events.Single(item => item.GetProperty("type").GetString() == "message_start"); + await Assert.That(start.GetProperty("message").GetProperty("id").GetString()).IsEqualTo(id); + await Assert.That(events.Last().GetProperty("type").GetString()).IsEqualTo("message_stop"); + } + else if (format == 2) + { + await Assert.That(events.First().GetProperty("id").GetString()).IsEqualTo(id); + await Assert.That(events.Last().GetProperty("type").GetString()).IsEqualTo("message-end"); + } + else + { + await Assert.That(events.All(item => Identity(item) == id)).IsTrue(); + if (format == 3) + { + await Assert.That(events.All(item => item.GetProperty("created_at").GetDateTimeOffset() == _fixedUtc)).IsTrue(); + await Assert.That(events.Last().GetProperty("done").GetBoolean()).IsTrue(); + } + else { await Assert.That(events.Last().GetProperty("candidates")[0].GetProperty("finishReason").GetString()).IsEqualTo("STOP"); } + } + await Assert.That((await conversation.Control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); + await Assert.That((await conversation.Control.GetAssertionsAsync()).Events.Last().IsReplay).IsTrue(); + await Assert.That((await conversation.Control.GetRequestsAsync()).Requests[0].TerminalMarkerObserved == true).IsTrue(); + } + + [Test] + public async Task EmbeddingAndImageFixtures_ResetSeededIdsAndUseFixedImageTimestampsAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(config => config.WithDeterminism("media-seed", _fixedUtc) + .AddTextEmbedding3Small().AddGptImage1()); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + async Task PostAsync(string route, object body) + { + using var response = await http.PostAsJsonAsync(route, body); + response.EnsureSuccessStatusCode(); + return await response.Content.ReadFromJsonAsync(); + } + var embeddingBody = new { model = "text-embedding-3-small", input = "clock" }; + var embedding = await PostAsync("/openai/v1/embeddings", embeddingBody); + await control.ResetAsync(); + var repeatedEmbedding = await PostAsync("/openai/v1/embeddings", embeddingBody); + await Assert.That(repeatedEmbedding.GetProperty("id").GetString()).IsEqualTo(embedding.GetProperty("id").GetString()); + var imageBody = new { model = "gpt-image-1", prompt = "fixture" }; + var image = await PostAsync("/openai/v1/images/generations", imageBody); + await Assert.That(image.GetProperty("created").GetInt64()).IsEqualTo(_fixedUtc.ToUnixTimeSeconds()); + await control.ResetAsync(); + var repeatedImage = await PostAsync("/openai/v1/images/generations", imageBody); + await Assert.That(repeatedImage.GetProperty("created").GetInt64()).IsEqualTo(_fixedUtc.ToUnixTimeSeconds()); + await Assert.That(repeatedImage.GetProperty("data")[0].GetProperty("b64_json").GetString()).IsEqualTo(image.GetProperty("data")[0].GetProperty("b64_json").GetString()); + using var stream = await http.PostAsJsonAsync("/openai/v1/images/generations", new { model = "gpt-image-1", prompt = "fixture", stream = true, partial_images = 1 }); + stream.EnsureSuccessStatusCode(); + var events = ReadSse(await stream.Content.ReadAsStringAsync()); + await Assert.That(events.Count).IsEqualTo(2); + await Assert.That(events.All(item => item.GetProperty("created_at").GetInt64() == _fixedUtc.ToUnixTimeSeconds())).IsTrue(); + await Assert.That(events.Last().GetProperty("type").GetString()).IsEqualTo("image_generation.completed"); + } + + [Test] + public async Task GroqTranscription_ResetReproducesSeededRequestMetadataAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(config => config.WithDeterminism("groq-audio-seed", _fixedUtc) + .AddModel("whisper-large-v3", LlmTestServerModelKind.Audio).WithDefaultTranscriptionText("clock transcript")); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + async Task TranscribeAsync() + { + using var form = new MultipartFormDataContent(); + using var audio = new ByteArrayContent(Encoding.ASCII.GetBytes("RIFF....WAVE")); + audio.Headers.ContentType = new MediaTypeHeaderValue("audio/wav"); + form.Add(audio, "file", "fixture.wav"); + form.Add(new StringContent("whisper-large-v3"), "model"); + form.Add(new StringContent("json"), "response_format"); + using var response = await http.PostAsync("/groq/openai/v1/audio/transcriptions", form); + response.EnsureSuccessStatusCode(); + await Assert.That(response.Headers.Date).IsEqualTo(_fixedUtc); + return await response.Content.ReadFromJsonAsync(); + } + var first = await TranscribeAsync(); + await control.ResetAsync(); + var repeated = await TranscribeAsync(); + await Assert.That(repeated.GetProperty("text").GetString()).IsEqualTo("clock transcript"); + await Assert.That(repeated.GetProperty("x_groq").GetProperty("id").GetString()).IsEqualTo(first.GetProperty("x_groq").GetProperty("id").GetString()); + } + + private static List ReadSse(string stream) + { + return stream.Split('\n').Where(line => line.StartsWith("data: ", StringComparison.Ordinal)) + .Select(line => JsonSerializer.Deserialize(line[6..])).ToList(); + } + + private static List ReadLines(string stream) + { + return stream.Split('\n', StringSplitOptions.RemoveEmptyEntries).Select(line => JsonSerializer.Deserialize(line)).ToList(); + } + + private static List ReadArray(string stream) + { + using var document = JsonDocument.Parse(stream); + return document.RootElement.EnumerateArray().Select(item => item.Clone()).ToList(); + } +} diff --git a/tests/ManagedCode.LlmTck.Tests/Gemini/GeminiEndpointTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Gemini/GeminiEndpointTests.cs similarity index 90% rename from tests/ManagedCode.LlmTck.Tests/Gemini/GeminiEndpointTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/Gemini/GeminiEndpointTests.cs index 3b745ed..3e98f5a 100644 --- a/tests/ManagedCode.LlmTck.Tests/Gemini/GeminiEndpointTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Gemini/GeminiEndpointTests.cs @@ -1,11 +1,11 @@ using System.Net.Http.Json; using System.Text; using System.Text.Json; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Tests.TestSupport; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Tests.TestSupport; -namespace ManagedCode.LlmTck.Tests.Gemini; +namespace ManagedCode.LlmTestServer.Tests.Gemini; public sealed class GeminiEndpointTests { @@ -14,8 +14,8 @@ public sealed class GeminiEndpointTests [Test] public async Task GenerateContentEndpoint_ReturnsGeminiCandidateShapeAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("gemini-test", LlmTckModelKind.Chat) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("gemini-test", LlmTestServerModelKind.Chat) .AddChatScenario( "gemini-rings", scenario => scenario @@ -82,8 +82,8 @@ await Assert.That(usage.GetProperty("totalTokenCount").GetInt32()) public async Task GenerateContentEndpoint_ReturnsGeminiCachedContentTokenCountAsync() { var cacheableSystemPrompt = CreatePromptWithAtLeastTokens(2100); - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("gemini-cache-test", LlmTckModelKind.Chat) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("gemini-cache-test", LlmTestServerModelKind.Chat) .AddChatScenario( "gemini-cache", scenario => scenario @@ -124,8 +124,8 @@ await Assert.That(secondUsage.GetProperty("promptTokenCount").GetInt32()) [Test] public async Task StreamGenerateContentEndpoint_ReturnsGeminiServerSentEventsAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("gemini-test", LlmTckModelKind.Chat) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("gemini-test", LlmTestServerModelKind.Chat) .AddChatScenario( "gemini-stream", scenario => scenario @@ -170,8 +170,8 @@ await Assert.That(response.Content.Headers.ContentType?.MediaType) [Test] public async Task EmbedContentEndpoint_ReturnsGeminiEmbeddingShapeAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("gemini-embedding-001", LlmTckModelKind.Embedding) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("gemini-embedding-001", LlmTestServerModelKind.Embedding) .WithDefaultEmbeddingVector(0.125f, 0.25f)); using var client = host.GetTestClient(); @@ -207,8 +207,8 @@ await Assert.That(payload.GetProperty("usageMetadata").GetProperty("totalTokenCo [Test] public async Task PredictLongRunningVideoEndpoint_ReturnsGeminiOperationAndGeneratedFileAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("veo-3.1-generate-preview", LlmTckModelKind.Video) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("veo-3.1-generate-preview", LlmTestServerModelKind.Video) .WithDefaultVideo([0, 0, 0, 24, 102, 116, 121, 112], "video/mp4")); using var client = host.GetTestClient(); client.DefaultRequestHeaders.Add("x-goog-api-key", "test-key"); @@ -221,7 +221,7 @@ public async Task PredictLongRunningVideoEndpoint_ReturnsGeminiOperationAndGener { new { - prompt = "a deterministic tck video", + prompt = "a deterministic test server video", }, }, parameters = new @@ -238,7 +238,7 @@ public async Task PredictLongRunningVideoEndpoint_ReturnsGeminiOperationAndGener var hasStartResponse = startPayload.TryGetProperty("response", out _); await Assert.That(operationName) - .IsEqualTo("models/veo-3.1-generate-preview/operations/gen_llm_tck"); + .IsEqualTo("models/veo-3.1-generate-preview/operations/gen_llm_test_server"); await Assert.That(startPayload.GetProperty("done").GetBoolean()).IsFalse(); await Assert.That(hasStartResponse).IsFalse(); @@ -271,7 +271,7 @@ await Assert.That(operationUsage.GetProperty("totalTokenCount").GetInt32()) var filePayload = await fileMetadataResponse.Content.ReadFromJsonAsync(_jsonOptions); var downloadUri = filePayload.GetProperty("downloadUri").GetString(); - await Assert.That(filePayload.GetProperty("name").GetString()).IsEqualTo("files/video_llm_tck"); + await Assert.That(filePayload.GetProperty("name").GetString()).IsEqualTo("files/video_llm_test_server"); await Assert.That(filePayload.GetProperty("state").GetString()).IsEqualTo("ACTIVE"); await Assert.That(filePayload.GetProperty("source").GetString()).IsEqualTo("GENERATED"); await Assert.That( @@ -317,7 +317,7 @@ string userPrompt private static string CreatePromptWithAtLeastTokens(int minimumTokens) { var builder = new StringBuilder("cacheable fixture"); - while (LlmTckTokenCounter.CountTextTokens(builder.ToString()) < minimumTokens) + while (LlmTestServerTokenCounter.CountTextTokens(builder.ToString()) < minimumTokens) { builder.Append(" stable-prefix"); } diff --git a/tests/ManagedCode.LlmTestServer.Tests/Gemini/GeminiOfficialSdkIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Gemini/GeminiOfficialSdkIntegrationTests.cs new file mode 100644 index 0000000..160f6fb --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Gemini/GeminiOfficialSdkIntegrationTests.cs @@ -0,0 +1,509 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using Google.GenAI; +using Google.GenAI.Types; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Providers; +using ManagedCode.LlmTestServer.Scenarios; +using ManagedCode.LlmTestServer.Tests.TestSupport; +using GenAiType = Google.GenAI.Types.Type; + +namespace ManagedCode.LlmTestServer.Tests.Gemini; + +public sealed class GeminiOfficialSdkIntegrationTests +{ + // Current stable model codes: https://ai.google.dev/gemini-api/docs/models/gemini-3.8-flash + // and https://ai.google.dev/gemini-api/docs/models/gemini-embedding-2 (verified 2026-10-20). + private const string _chatModel = "gemini-3.8-flash"; + private const string _embeddingModel = "gemini-embedding-2"; + private const string _signature = "Z29vZ2xlLWZpeHR1cmUtc2lnbmF0dXJl"; + private const string _key = "gemini-sdk-fixture-key"; + + [Test] + public async Task GenerateContent_ParsesTypedJsonAndPreservesSystemInstructionAndUsageAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http, _key); + await control.ConfigureAsync(config => config.RequireBearerToken(_key).UseChatModel(_chatModel) + .UseChatScenario("google-json", scenario => scenario.ForModel(_chatModel).RespondsJson("{\"answer\":\"fixture\"}"))); + using var sdk = Sdk(http); + var response = await sdk.Models.GenerateContentAsync(_chatModel, "answer with JSON", new GenerateContentConfig + { + SystemInstruction = new() { Parts = [Part.FromText("Use the scripted response")] }, + ResponseMimeType = "application/json", + ResponseSchema = new() { Type = GenAiType.Object, Properties = new() { ["answer"] = new() { Type = GenAiType.String } }, Required = ["answer"] }, + }); + await Assert.That(response.Text).IsEqualTo("{\"answer\":\"fixture\"}"); + await Assert.That(response.ModelVersion).IsEqualTo(_chatModel); + await Assert.That(response.Candidates!.Single().FinishReason!.Value).IsEqualTo(FinishReason.Stop); + var trace = (await control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending })).Requests.Single(); + await Assert.That(trace.OperationId).IsEqualTo(LlmTestServerProviderOperationIds.Gemini.ModelsGenerateContent); + await Assert.That(trace.Messages[0].Content).IsEqualTo("Use the scripted response"); + await Assert.That(response.UsageMetadata!.PromptTokenCount).IsEqualTo(trace.Usage!.InputTokens); + await Assert.That(response.UsageMetadata.TotalTokenCount).IsEqualTo(trace.Usage.TotalTokens); + } + + [Test] + public async Task GenerateContentStream_ConsumesActualSseAndKeepsStableIdentityAndTerminalUsageAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http, _key); + await control.ConfigureAsync(config => config.RequireBearerToken(_key).UseChatModel(_chatModel) + .UseChatScenario("google-stream", scenario => scenario.ForModel(_chatModel).Responds("blue whale", "blue ", "whale"))); + using var sdk = Sdk(http); + var chunks = new List(); + await foreach (var chunk in sdk.Models.GenerateContentStreamAsync(_chatModel, "stream fixture")) { chunks.Add(chunk); } + await Assert.That(string.Concat(chunks.Select(chunk => chunk.Text))).IsEqualTo("blue whale"); + await Assert.That(chunks.Select(chunk => chunk.ResponseId).Distinct().Count()).IsEqualTo(1); + await Assert.That(chunks[^1].Candidates!.Single().FinishReason!.Value).IsEqualTo(FinishReason.Stop); + var trace = (await control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending })).Requests.Single(); + await Assert.That(trace.OperationId).IsEqualTo(LlmTestServerProviderOperationIds.Gemini.ModelsStreamGenerateContent); + await Assert.That(trace.ResponseContentType).Contains("text/event-stream"); + await Assert.That(trace.TerminalMarkerObserved).IsTrue(); + await Assert.That(chunks[^1].UsageMetadata!.TotalTokenCount).IsEqualTo(trace.Usage!.TotalTokens); + } + + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task FunctionRound_OfficialSdkPreservesCallIdentityArgumentsAndToolResultAsync(bool streaming) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http, _key); + await control.ConfigureAsync(config => config.RequireBearerToken(_key).UseChatModel(_chatModel) + .UseConversation("google-function", graph => graph.ForModel(_chatModel) + .Step("call", step => step.WhenLatestUserEquals("weather").CallsTools(new LlmTestServerToolCall { Name = "weather", ArgumentsJson = "{\"city\":\"Paris\"}", Id = "call_google_weather", ContinuationToken = _signature }).Then("answer")) + .Step("answer", step => step.WhenToolResult("weather", "call_google_weather") + .WithToolArgumentsJson("{\"city\":\"Paris\"}").WhenLatestJsonValue("$.temperature", 22) + .Responds("Paris is 22 C", "Paris is ", "22 C").Complete()))); + using var conversation = control.ForConversation("google-function"); + using var sdk = Sdk(conversation.HttpClient); + var options = new GenerateContentConfig + { + Tools = [new() { FunctionDeclarations = [new() { Name = "weather", Parameters = new() + { Type = GenAiType.Object, Properties = new() { ["city"] = new() { Type = GenAiType.String } }, Required = ["city"] } }] }], + ToolConfig = new() { FunctionCallingConfig = new() { Mode = FunctionCallingConfigMode.Any, AllowedFunctionNames = ["weather"] } }, + }; + List messages = [Text("user", "weather")]; + var first = await ResponsesAsync(sdk, messages, options, streaming); + var call = first.SelectMany(response => response.Candidates!).SelectMany(candidate => candidate.Content!.Parts!) + .Select(part => part.FunctionCall).Single(value => value is not null)!; + await Assert.That(call.Name).IsEqualTo("weather"); + await Assert.That(call.Id).IsEqualTo("call_google_weather"); + await Assert.That(JsonSerializer.SerializeToElement(call.Args!).GetProperty("city").GetString()).IsEqualTo("Paris"); + var signedContent = first.SelectMany(response => response.Candidates!).Select(candidate => candidate.Content!).Single(content => content.Parts!.Any(part => part.FunctionCall is not null)); + await Assert.That(Convert.ToBase64String(signedContent.Parts!.Single(part => part.FunctionCall is not null).ThoughtSignature!)).IsEqualTo(_signature); + messages.Add(signedContent); + messages.Add(new() + { + Role = "user", + Parts = [new() { FunctionResponse = new() + { Id = call.Id, Name = call.Name, Response = new() { ["temperature"] = 22 } } }] + }); + options.ToolConfig = new() { FunctionCallingConfig = new() { Mode = FunctionCallingConfigMode.Auto } }; + var completed = await ResponsesAsync(sdk, messages, options, streaming); + await Assert.That(string.Concat(completed.Select(response => response.Text))).IsEqualTo("Paris is 22 C"); + var trace = (await conversation.Control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending })).Requests[^1]; + await Assert.That(trace.Messages.Single(message => message.Role == "tool").ToolCallId).IsEqualTo(call.Id); + await Assert.That(trace.Messages.Single(message => message.Role == "tool").Content).IsEqualTo("{\"temperature\":22}"); + var state = (await conversation.Control.GetConversationsAsync()).Single(); + await Assert.That(state.ExecutedSteps).IsEqualTo(2); + await Assert.That(state.IsTerminal).IsTrue(); + } + + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task EmbedContent_OfficialSdkUsesBatchRouteForSingleAndMultipleInputsAsync(bool multiple) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http, _key); + var vector = Enumerable.Range(0, 256).Select(index => (float)index / 256).ToArray(); + await control.ConfigureAsync(config => config.RequireBearerToken(_key).UseEmbeddingModel(_embeddingModel).UseEmbeddingVector(vector)); + using var sdk = Sdk(http); + List inputs = [Text("user", "first embedding")]; + if (multiple) { inputs.Add(Text("user", "second embedding")); } + var response = await sdk.Models.EmbedContentAsync(_embeddingModel, inputs, new() { OutputDimensionality = 128 }); + await Assert.That(response.Embeddings!.Count).IsEqualTo(inputs.Count); + foreach (var embedding in response.Embeddings) + { await Assert.That(embedding.Values!).IsEquivalentTo(vector.Take(128).Select(value => (double)value)); } + var trace = (await control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending })).Requests.Single(); + await Assert.That(trace.OperationId).IsEqualTo(LlmTestServerProviderOperationIds.Gemini.ModelsBatchEmbedContents); + await Assert.That(trace.StatusCode).IsEqualTo(200); + await Assert.That(trace.Path).Contains(":batchEmbedContents"); + } + + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task EmbedContent_RejectsDimensionsBeyondFixtureAtomicallyAndAllowsExplicitRetryAsync(bool multiple) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http, _key); + var vector = Enumerable.Range(0, 128).Select(index => (float)index / 128).ToArray(); + await control.ConfigureAsync(config => config.RequireBearerToken(_key).UseEmbeddingModel(_embeddingModel).UseEmbeddingVector(vector)); + using var sdk = Sdk(http); + List inputs = [Text("user", "first embedding")]; + if (multiple) { inputs.Add(Text("user", "second embedding")); } + + ClientError? failure = null; + try { await sdk.Models.EmbedContentAsync(_embeddingModel, inputs, new() { OutputDimensionality = 256 }); } + catch (ClientError exception) { failure = exception; } + await Assert.That(failure?.StatusCode).IsEqualTo(409); + var rejectedSummary = await control.GetAssertionsAsync(); + await Assert.That(rejectedSummary.Matched).IsEqualTo(0); + await Assert.That(rejectedSummary.ErrorsReturned).IsEqualTo(1); + await Assert.That(rejectedSummary.Events.Single().Message).Contains("fewer coordinates"); + var rejected = (await control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending })).Requests.Single(); + await Assert.That(rejected.StatusCode).IsEqualTo(409); + await Assert.That(rejected.ErrorCode).IsEqualTo("409"); + await Assert.That(rejected.OperationId).IsEqualTo(LlmTestServerProviderOperationIds.Gemini.ModelsBatchEmbedContents); + + var response = await sdk.Models.EmbedContentAsync(_embeddingModel, inputs, new() { OutputDimensionality = 128 }); + await Assert.That(response.Embeddings!.Count).IsEqualTo(inputs.Count); + foreach (var embedding in response.Embeddings) + { + await Assert.That(embedding.Values!.Count).IsEqualTo(128); + await Assert.That(embedding.Values.SequenceEqual(vector.Select(value => (double)value))).IsTrue(); + } + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(1); + var evidence = await control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending }); + await Assert.That(evidence.Requests.Count).IsEqualTo(2); + await Assert.That(evidence.Requests[^1].StatusCode).IsEqualTo(200); + await Assert.That(evidence.Requests[^1].OperationId).IsEqualTo(LlmTestServerProviderOperationIds.Gemini.ModelsBatchEmbedContents); + } + + [Test] + [Arguments(400, false)] + [Arguments(400, true)] + [Arguments(429, false)] + [Arguments(429, true)] + [Arguments(500, false)] + [Arguments(500, true)] + public async Task OfficialSdk_ReportsScriptedProviderErrorsOnceThenExplicitRetryRecoversAsync(int status, bool streaming) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http, _key); + await control.ConfigureAsync(config => config.RequireBearerToken(_key).UseChatModel(_chatModel) + .UseChatScenario("google-error", scenario => scenario.ForModel(_chatModel) + .Fails(status, "fixture_provider_failure", "Scripted Google SDK failure").Responds("recovered"))); + using var sdk = Sdk(http); + ApiException? failure = null; + try { await ResponsesAsync(sdk, [Text("user", "error")], null, streaming); } + catch (ApiException exception) { failure = exception; } + await Assert.That(failure?.StatusCode).IsEqualTo(status); + await Assert.That(failure!.Message).Contains("Scripted Google SDK failure"); + await Assert.That((await control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending })).Requests.Count).IsEqualTo(1); + var response = await ResponsesAsync(sdk, [Text("user", "error")], null, streaming); + await Assert.That(string.Concat(response.Select(chunk => chunk.Text))).IsEqualTo("recovered"); + await Assert.That((await control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending })).Requests.Count).IsEqualTo(2); + } + + [Test] + public async Task AuthenticationAndInputFilter_DoNotConsumeSuccessfulFixtureAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http, _key); + await control.ConfigureAsync(config => config.RequireBearerToken(_key).UseChatModel(_chatModel) + .SimulateContentFilter("blocked_sdk_input") + .UseChatScenario("google-protected", scenario => scenario.ForModel(_chatModel).Responds("protected response"))); + using var wrong = Sdk(http, "wrong-key"); + ApiException? unauthorized = null; + try { await wrong.Models.GenerateContentAsync(_chatModel, "hello"); } + catch (ApiException exception) { unauthorized = exception; } + await Assert.That(unauthorized?.StatusCode).IsEqualTo(401); + using var sdk = Sdk(http); + ApiException? filtered = null; + try { await sdk.Models.GenerateContentAsync(_chatModel, "blocked_sdk_input"); } + catch (ApiException exception) { filtered = exception; } + await Assert.That(filtered?.StatusCode).IsEqualTo(400); + var response = await sdk.Models.GenerateContentAsync(_chatModel, "clean input"); + await Assert.That(response.Text).IsEqualTo("protected response"); + var evidence = await control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending }); + await Assert.That(evidence.Requests.Count).IsEqualTo(3); + await Assert.That(evidence.Requests[^1].StatusCode).IsEqualTo(200); + } + + [Test] + [Arguments(0)] + [Arguments(1)] + [Arguments(2)] + [Arguments(3)] + [Arguments(4)] + [Arguments(5)] + [Arguments(6)] + [Arguments(7)] + [Arguments(8)] + public async Task BatchEmbedding_InvalidShapeRejectsBeforeRuntimeThenOfficialSdkStillSucceedsAsync(int kind) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http, _key); + var vector = Enumerable.Range(0, 128).Select(index => (float)index / 128).ToArray(); + await control.ConfigureAsync(config => config.UseEmbeddingModel(_embeddingModel).UseEmbeddingVector(vector)); + var entry = new Dictionary + { + ["model"] = "models/" + _embeddingModel, + ["content"] = new { parts = new[] { new { text = "valid text" } } }, + }; + object request; + switch (kind) + { + case 0: request = new { requests = Array.Empty() }; break; + case 1: request = new { requests = Enumerable.Repeat(entry, 129).ToArray() }; break; + case 2: entry["model"] = "models/wrong-model"; request = new { requests = new[] { entry } }; break; + case 3: entry["content"] = new { parts = new[] { new { inlineData = new { mimeType = "image/png", data = "AA==" } } } }; request = new { requests = new[] { entry } }; break; + case 4: entry["outputDimensionality"] = 0; request = new { requests = new[] { entry } }; break; + case 5: entry["outputDimensionality"] = 3073; request = new { requests = new[] { entry } }; break; + case 6: entry["taskType"] = "RETRIEVAL_QUERY"; request = new { requests = new[] { entry } }; break; + case 7: entry["title"] = "unsupported title"; request = new { requests = new[] { entry } }; break; + default: entry["embedContentConfig"] = new { documentOcr = true }; request = new { requests = new[] { entry } }; break; + } + using var rejected = await http.PostAsJsonAsync("/gemini/v1beta/models/" + _embeddingModel + ":batchEmbedContents", request); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(0); + using var sdk = Sdk(http); + var response = await sdk.Models.EmbedContentAsync(_embeddingModel, "valid text"); + await Assert.That(response.Embeddings!.Single().Values!).IsEquivalentTo(vector.Select(value => (double)value)); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(1); + } + + [Test] + public async Task OfficialSdk_UnknownModelsExpose404ForGenerationAndEmbeddingAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http, _key); + await control.ConfigureAsync(config => config.UseChatModel(_chatModel).UseEmbeddingModel(_embeddingModel) + .UseEmbeddingVector(Enumerable.Repeat(0.25f, 128).ToArray()).UseChatScenario("known-google-model", scenario => scenario.ForModel(_chatModel).Responds("known model response"))); + using var sdk = Sdk(http); + ApiException? generation = null; + try { await sdk.Models.GenerateContentAsync("unconfigured-google-sdk-model", "unknown"); } + catch (ApiException exception) { generation = exception; } + await Assert.That(generation?.StatusCode).IsEqualTo(404); + ApiException? embedding = null; + try { await sdk.Models.EmbedContentAsync("unconfigured-google-sdk-model", "unknown"); } + catch (ApiException exception) { embedding = exception; } + await Assert.That(embedding?.StatusCode).IsEqualTo(404); + await Assert.That((await sdk.Models.GenerateContentAsync(_chatModel, "known")).Text).IsEqualTo("known model response"); + await Assert.That((await control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending })).Requests.Count).IsEqualTo(3); + } + + [Test] + [Arguments(false, false)] + [Arguments(false, true)] + [Arguments(true, false)] + [Arguments(true, true)] + public async Task FunctionContinuation_RejectsMissingOrChangedSignatureWithoutAdvancingAndRecoversAsync(bool streaming, bool tampered) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http, _key); + await control.ConfigureAsync(config => config.RequireBearerToken(_key).UseChatModel(_chatModel) + .UseConversation("signed-recovery", graph => graph.ForModel(_chatModel) + .Step("call", step => step.WhenLatestUserEquals("weather").CallsTools(SignedCall("call_signed")).Then("answer")) + .Step("answer", step => step.WhenToolResult("weather", "call_signed").Responds("signed recovery").Complete()))); + using var conversation = control.ForConversation("signed-recovery"); + using var sdk = Sdk(conversation.HttpClient); + var options = WeatherOptions(); + options.SystemInstruction = new() { Parts = [Part.FromText(string.Join(" ", Enumerable.Repeat("shared reference", 4096)))] }; + var first = await ResponsesAsync(sdk, [Text("user", "weather")], options, streaming); + var signed = first.SelectMany(response => response.Candidates!).Select(candidate => candidate.Content!) + .Single(content => content.Parts!.Any(part => part.FunctionCall is not null)); + var signedPart = signed.Parts!.Single(part => part.FunctionCall is not null); + var changed = signed with { Parts = [signedPart with { ThoughtSignature = tampered ? new byte[] { 1, 2, 3 } : null }] }; + List invalid = [Text("user", "weather"), changed, FunctionResult("call_signed")]; + options.ToolConfig = new() { FunctionCallingConfig = new() { Mode = FunctionCallingConfigMode.Auto } }; + ClientError? failure = null; + try { await ResponsesAsync(sdk, invalid, options, streaming); } + catch (ClientError exception) { failure = exception; } + await Assert.That(failure?.StatusCode).IsEqualTo(tampered ? 409 : 400); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(1); + var beforeRecovery = (await control.GetConversationsAsync()).Single(); + await Assert.That(beforeRecovery.ExecutedSteps).IsEqualTo(1); + await Assert.That(beforeRecovery.NextSteps.Single()).IsEqualTo("answer"); + var rejected = (await control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending })).Requests[^1]; + await Assert.That(rejected.StatusCode).IsEqualTo(tampered ? 409 : 400); + var recovered = await ResponsesAsync(sdk, [Text("user", "weather"), signed, FunctionResult("call_signed")], options, streaming); + await Assert.That(string.Concat(recovered.Select(response => response.Text))).IsEqualTo("signed recovery"); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(2); + var traces = (await control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending })).Requests; + await Assert.That(traces.Count).IsEqualTo(3); + await Assert.That(traces[^1].Messages.Single(message => message.ToolCalls.Count > 0).ToolCalls.Single().ContinuationToken).IsEqualTo(_signature); + await Assert.That(traces[^1].Usage!.CachedInputTokens).IsGreaterThan(0); + await Assert.That((await control.GetConversationsAsync()).Single().IsTerminal).IsTrue(); + } + + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task FunctionContinuation_PreservesParallelAndSequentialSignaturesAndIgnoresPreviousTurnAsync(bool streaming) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http, _key); + const string secondSignature = "c2Vjb25kLXN0ZXAtc2lnbmF0dXJl"; + await control.ConfigureAsync(config => config.RequireBearerToken(_key).UseChatModel(_chatModel) + .UseConversation("signed-steps", graph => graph.ForModel(_chatModel) + .Step("parallel", step => step.WhenLatestUserEquals("weather").CallsTools(SignedCall("parallel_first"), + new LlmTestServerToolCall { Id = "parallel_second", Name = "weather", ArgumentsJson = "{\"city\":\"Paris\"}" }).Then("sequential")) + .Step("sequential", step => step.WhenToolResult("weather", "parallel_second") + .CallsTools(SignedCall("sequential", secondSignature)).Then("answer")) + .Step("answer", step => step.WhenToolResult("weather", "sequential").Responds("all signatures preserved").Complete()))); + using var conversation = control.ForConversation("signed-steps"); + using var sdk = Sdk(conversation.HttpClient); + // Unsigned function history precedes a new ordinary user turn and is intentionally valid. + List messages = [Text("user", "old turn"), new() { Role = "model", Parts = [new() { FunctionCall = new() { Id = "old_call", Name = "weather", Args = new() { ["city"] = "Paris" } } }] }, + FunctionResult("old_call"), Text("user", "weather")]; + var options = WeatherOptions(); + var first = await ResponsesAsync(sdk, messages, options, streaming); + var parallel = first.SelectMany(response => response.Candidates!).Select(candidate => candidate.Content!) + .Single(content => content.Parts!.Any(part => part.FunctionCall is not null)); + var parts = parallel.Parts!.Where(part => part.FunctionCall is not null).ToList(); + await Assert.That(parts.Count).IsEqualTo(2); + await Assert.That(Convert.ToBase64String(parts[0].ThoughtSignature!)).IsEqualTo(_signature); + await Assert.That(parts[1].ThoughtSignature).IsNull(); + messages.Add(parallel); + messages.Add(FunctionResult("parallel_first")); + messages.Add(FunctionResult("parallel_second")); + var next = await ResponsesAsync(sdk, messages, options, streaming); + var sequential = next.SelectMany(response => response.Candidates!).Select(candidate => candidate.Content!) + .Single(content => content.Parts!.Any(part => part.FunctionCall is not null)); + await Assert.That(Convert.ToBase64String(sequential.Parts!.Single(part => part.FunctionCall is not null).ThoughtSignature!)).IsEqualTo(secondSignature); + messages.Add(sequential); + messages.Add(FunctionResult("sequential")); + options.ToolConfig = new() { FunctionCallingConfig = new() { Mode = FunctionCallingConfigMode.Auto } }; + var missingEarlier = messages.Select(content => ReferenceEquals(content, parallel) + ? content with { Parts = parts.Select((part, index) => index == 0 ? part with { ThoughtSignature = null } : part).ToList() } : content).ToList(); + ClientError? missing = null; + try { await ResponsesAsync(sdk, missingEarlier, options, streaming); } + catch (ClientError exception) { missing = exception; } + await Assert.That(missing?.StatusCode).IsEqualTo(400); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(2); + var final = await ResponsesAsync(sdk, messages, options, streaming); + await Assert.That(string.Concat(final.Select(response => response.Text))).IsEqualTo("all signatures preserved"); + await Assert.That((await control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(3); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(3); + } + + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task FunctionFixture_RequiresSignatureAndCountsItsRetentionBytesBeforeAdmissionAsync(bool oversized) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http, _key); + await control.ConfigureAsync(config => config.RequireBearerToken(_key).UseChatModel(_chatModel).UseConversationCapacity(8, 4096) + .UseConversation("signed-budget", graph => graph.ForModel(_chatModel) + .Step("call", step => step.WhenLatestUserEquals("weather").CallsTools(SignedCall("budget", oversized ? Convert.ToBase64String(new byte[6144]) : null)).Complete()))); + using var conversation = control.ForConversation("signed-budget"); + using var sdk = Sdk(conversation.HttpClient); + ClientError? rejected = null; + try { await sdk.Models.GenerateContentAsync(_chatModel, "weather", WeatherOptions()); } + catch (ClientError exception) { rejected = exception; } + await Assert.That(rejected?.StatusCode).IsEqualTo(409); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(0); + await Assert.That((await control.GetConversationsAsync()).Count).IsEqualTo(0); + var failureTrace = (await control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending })).Requests.Single(); + await Assert.That(failureTrace.ErrorCode).IsEqualTo("409"); + await Assert.That(failureTrace.RuntimeMessage).Contains(oversized ? "conversation state capacity" : "continuation metadata"); + await control.ConfigureAsync(config => config.RequireBearerToken(_key).UseChatModel(_chatModel).UseConversationCapacity(8, 4096) + .UseConversation("signed-budget", graph => graph.ForModel(_chatModel) + .Step("call", step => step.WhenLatestUserEquals("weather").CallsTools(SignedCall("budget")).Complete()))); + var valid = await sdk.Models.GenerateContentAsync(_chatModel, "weather", WeatherOptions()); + await Assert.That(Convert.ToBase64String(valid.Candidates!.Single().Content!.Parts!.Single().ThoughtSignature!)).IsEqualTo(_signature); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(1); + if (oversized) + { + var smallUsage = (await control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending })).Requests[^1].Usage!; + var largeSignature = Convert.ToBase64String(new byte[6144]); + await control.ConfigureAsync(config => config.RequireBearerToken(_key).UseChatModel(_chatModel).UseConversationCapacity(8, 32768) + .UseConversation("signed-budget", graph => graph.ForModel(_chatModel) + .Step("call", step => step.WhenLatestUserEquals("weather").CallsTools(SignedCall("budget", largeSignature)).Complete()))); + var large = await sdk.Models.GenerateContentAsync(_chatModel, "weather", WeatherOptions()); + await Assert.That(Convert.ToBase64String(large.Candidates!.Single().Content!.Parts!.Single().ThoughtSignature!)).IsEqualTo(largeSignature); + var largeUsage = (await control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending })).Requests[^1].Usage!; + await Assert.That(largeUsage.InputTokens).IsEqualTo(smallUsage.InputTokens); + await Assert.That(largeUsage.OutputTokens).IsEqualTo(smallUsage.OutputTokens); + } + } + + [Test] + [Arguments(0)] + [Arguments(1)] + [Arguments(2)] + public async Task FunctionContinuation_RejectsMalformedWireSignatureBeforeConsumingFixtureAsync(int kind) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http, _key); + await control.ConfigureAsync(config => config.UseChatModel(_chatModel) + .UseChatScenario("signature-shape", scenario => scenario.ForModel(_chatModel).Responds("not consumed"))); + var signature = kind switch { 0 => string.Empty, 1 => "not base64!", _ => Convert.ToBase64String(new byte[12289]) }; + using var rejected = await http.PostAsJsonAsync("/gemini/v1beta/models/" + _chatModel + ":generateContent", new + { + contents = new object[] { new { role = "user", parts = new[] { new { text = "weather" } } }, + new { role = "model", parts = new[] { new { functionCall = new { name = "weather", args = new { city = "Paris" } }, thoughtSignature = signature } } } } + }); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(0); + using var sdk = Sdk(http); + await Assert.That((await sdk.Models.GenerateContentAsync(_chatModel, "clean input")).Text).IsEqualTo("not consumed"); + } + + private static LlmTestServerToolCall SignedCall(string id, string? signature = _signature) + { + return new() { Id = id, Name = "weather", ArgumentsJson = "{\"city\":\"Paris\"}", ContinuationToken = signature }; + } + + private static GenerateContentConfig WeatherOptions() + { + return new() + { + Tools = [new() { FunctionDeclarations = [new() { Name = "weather", Parameters = new() + { Type = GenAiType.Object, Properties = new() { ["city"] = new() { Type = GenAiType.String } }, Required = ["city"] } }] }], + ToolConfig = new() { FunctionCallingConfig = new() { Mode = FunctionCallingConfigMode.Any } } + }; + } + + private static Content FunctionResult(string id) + { + return new() + { + Role = "user", + Parts = [new() { FunctionResponse = new() + { Id = id, Name = "weather", Response = new() { ["temperature"] = 22 } } }] + }; + } + + private static Google.GenAI.Client Sdk(HttpClient http, string apiKey = _key) + { + return new(apiKey: apiKey, enterprise: false, + httpOptions: new() { BaseUrl = new Uri(http.BaseAddress!, "/gemini").ToString(), ApiVersion = "v1beta", RetryOptions = new() { Attempts = 1 } }, + clientOptions: new() { HttpClientFactory = () => http }); + } + + private static Content Text(string role, string value) + { + return new() { Role = role, Parts = [Part.FromText(value)] }; + } + + private static async Task> ResponsesAsync(Google.GenAI.Client sdk, + List messages, GenerateContentConfig? options, bool streaming) + { + if (!streaming) { return [await sdk.Models.GenerateContentAsync(_chatModel, messages, options)]; } + var responses = new List(); + await foreach (var response in sdk.Models.GenerateContentStreamAsync(_chatModel, messages, options)) { responses.Add(response); } + return responses; + } +} diff --git a/tests/ManagedCode.LlmTck.Tests/Hosting/AdminPanelTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Hosting/AdminPanelTests.cs similarity index 61% rename from tests/ManagedCode.LlmTck.Tests/Hosting/AdminPanelTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/Hosting/AdminPanelTests.cs index cd93af7..2c966b7 100644 --- a/tests/ManagedCode.LlmTck.Tests/Hosting/AdminPanelTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Hosting/AdminPanelTests.cs @@ -3,17 +3,15 @@ using System.Net.Http.Json; using System.Text; using System.Text.Json; -using ManagedCode.LlmTck.Control; -using ManagedCode.LlmTck.Hosting; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.Providers; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Scenarios; -using ManagedCode.LlmTck.Tests.TestSupport; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Options; - -namespace ManagedCode.LlmTck.Tests.Hosting; +using ManagedCode.LlmTestServer.Control; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Hosting; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Providers; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.Hosting; public sealed class AdminPanelTests { @@ -22,21 +20,21 @@ public sealed class AdminPanelTests [Test] public async Task AdminPanel_ServesStyledHtmlShellAtRootAsync() { - using var host = await LlmTckTestHost.StartAsync(); + using var host = await LlmTestServerTestHost.StartAsync(); using var client = host.GetTestClient(); - var response = await client.GetAsync(LlmTckControlRoutes.Admin); + var response = await client.GetAsync(LlmTestServerControlRoutes.Admin); var body = await response.Content.ReadAsStringAsync(); response.EnsureSuccessStatusCode(); await Assert.That(response.Content.Headers.ContentType?.MediaType).IsEqualTo("text/html"); - await Assert.That(LlmTckControlRoutes.Admin).IsEqualTo("/"); + await Assert.That(LlmTestServerControlRoutes.Admin).IsEqualTo("/"); await Assert.That(body).Contains(""); await Assert.That(body).Contains(""""""); await Assert.That(body).Contains(""""""); - await Assert.That(body).Contains("LLM TCK request dashboard"); - await Assert.That(body).Contains(LlmTckControlRoutes.Models); - await Assert.That(body).Contains(LlmTckControlRoutes.Assertions); + await Assert.That(body).Contains("LLM Test Server request dashboard"); + await Assert.That(body).Contains(LlmTestServerControlRoutes.Models); + await Assert.That(body).Contains(LlmTestServerControlRoutes.Assertions); await Assert.That(body).Contains("Fixtures / models"); await Assert.That(body).Contains("Runtime actions"); await Assert.That(body).Contains("Provider APIs"); @@ -59,11 +57,11 @@ public async Task AdminPanel_ServesStyledHtmlShellAtRootAsync() [Test] public async Task AdminPanel_StaysReachableWhenBearerTokenIsRequiredAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options.RequireBearerToken("test-key")); + using var host = await LlmTestServerTestHost.StartAsync(options => options.RequireBearerToken("test-key")); using var client = host.GetTestClient(); - var page = await client.GetAsync(LlmTckControlRoutes.Admin); - var protectedModels = await client.GetAsync(LlmTckControlRoutes.Models); + var page = await client.GetAsync(LlmTestServerControlRoutes.Admin); + var protectedModels = await client.GetAsync(LlmTestServerControlRoutes.Models); // The shell renders without a token so operators can enter one; the data // endpoints it calls remain protected. @@ -71,98 +69,13 @@ public async Task AdminPanel_StaysReachableWhenBearerTokenIsRequiredAsync() await Assert.That(protectedModels.StatusCode).IsEqualTo(HttpStatusCode.Unauthorized); } - [Test] - public async Task AdminPanel_IsServedByBlazorServerSideRenderingAsync() - { - var repositoryRoot = FindRepositoryRoot(); - var hostingProject = await File.ReadAllTextAsync(Path.Combine( - repositoryRoot, - "src/ManagedCode.LlmTck.Hosting/ManagedCode.LlmTck.Hosting.csproj" - )); - var endpointSource = await File.ReadAllTextAsync(Path.Combine( - repositoryRoot, - "src/ManagedCode.LlmTck.Hosting/LlmTckEndpointRouteBuilderExtensions.cs" - )); - var razorComponentPath = Path.Combine( - repositoryRoot, - "src/ManagedCode.LlmTck.Hosting/LlmTckAdminPage.razor" - ); - var razorComponent = await File.ReadAllTextAsync(razorComponentPath); - var panelComponentPath = Path.Combine( - repositoryRoot, - "src/ManagedCode.LlmTck.Hosting/LlmTckAdminPanel.razor" - ); - var panelComponent = await File.ReadAllTextAsync(panelComponentPath); - - await Assert.That(File.Exists(razorComponentPath)).IsTrue(); - await Assert.That(File.Exists(panelComponentPath)).IsTrue(); - await Assert.That(hostingProject).Contains("Sdk=\"Microsoft.NET.Sdk.Razor\""); - await Assert.That(hostingProject).Contains("Microsoft.FluentUI.AspNetCore.Components"); - await Assert.That(endpointSource).Contains("MapRazorComponents"); - await Assert.That(endpointSource).Contains("MapStaticAssets"); - await Assert.That(endpointSource).Contains("StaticWebAssetsLoader.UseStaticWebAssets"); - await Assert.That(endpointSource).Contains("catch (DirectoryNotFoundException exception)"); - await Assert.That(endpointSource).Contains("Skipping LLM TCK static web assets runtime manifest"); - await Assert.That(endpointSource).Contains("AddInteractiveServerComponents"); - await Assert.That(endpointSource).Contains("AddHttpClient"); - await Assert.That(endpointSource).Contains("AddFluentUIComponents"); - await Assert.That(endpointSource).Contains("AddInteractiveServerRenderMode"); - await Assert.That(endpointSource).Contains("AddLlmTckProviderHttpTracing"); - await Assert.That(endpointSource).Contains("DisableAntiforgery"); - await Assert.That(razorComponent).Contains("@page \"/\""); - await Assert.That(razorComponent).Contains("InteractiveServerRenderMode"); - await Assert.That(razorComponent).Contains("LlmTckAdminPanel"); - await Assert.That(razorComponent).Contains("Prostir technical polish × Managed Code editorial warmth"); - await Assert.That(razorComponent).Contains("#a4f258"); - await Assert.That(razorComponent).Contains("#f9f7f5"); - await Assert.That(razorComponent).Contains("#db3276"); - await Assert.That(panelComponent).Contains("PeriodicTimer"); - await Assert.That(panelComponent).Contains("StateHasChanged"); - await Assert.That(panelComponent).Contains("ILlmTckProviderHttpTraceStore"); - await Assert.That(panelComponent).Contains("Provider requests"); - await Assert.That(panelComponent).Contains("Conversation"); - await Assert.That(panelComponent).Contains("Wire payloads"); - await Assert.That(panelComponent).Contains("Tokens & cache"); - await Assert.That(panelComponent).Contains("Latency"); - await Assert.That(panelComponent).Contains("data-testid=\"request-row\""); - await Assert.That(panelComponent).Contains(" options.AddChatScenario( + using var host = await LlmTestServerTestHost.StartAsync(options => options.AddChatScenario( "capture-scenario", scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) + .ForModel(LlmTestServerKnownModelIds.Gpt41Mini) .WhenUserContains("color") .Responds("blue whale") )); @@ -172,7 +85,7 @@ public async Task RuntimeEvents_CaptureIncomingRequestAndModelResponseAsync() "/openai/v1/chat/completions", new { - model = LlmTckKnownModelIds.Gpt41Mini, + model = LlmTestServerKnownModelIds.Gpt41Mini, messages = new[] { new { role = "user", content = "what color is the whale" } }, }, _jsonOptions @@ -180,7 +93,7 @@ public async Task RuntimeEvents_CaptureIncomingRequestAndModelResponseAsync() chat.EnsureSuccessStatusCode(); var summary = await client.GetFromJsonAsync( - LlmTckControlRoutes.Assertions, + LlmTestServerControlRoutes.Assertions, _jsonOptions ); var events = summary.GetProperty("events"); @@ -208,10 +121,10 @@ await Assert.That(lastEvent.GetProperty("usage").GetProperty("totalTokens").GetI public async Task ProviderRequestTrace_CapturesProviderMultiTurnPayloadAndRuntimeUsageAsync() { var longSystemMessage = $"{new string('x', 900)} END_OF_LONG_PROMPT"; - using var host = await LlmTckTestHost.StartAsync(options => options.AddChatScenario( + using var host = await LlmTestServerTestHost.StartAsync(options => options.AddChatScenario( "trace-multi-turn", scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) + .ForModel(LlmTestServerKnownModelIds.Gpt41Mini) .WhenUserContains("color") .Responds("blue whale") )); @@ -221,7 +134,7 @@ public async Task ProviderRequestTrace_CapturesProviderMultiTurnPayloadAndRuntim "/openai/v1/chat/completions", new { - model = LlmTckKnownModelIds.Gpt41Mini, + model = LlmTestServerKnownModelIds.Gpt41Mini, messages = new[] { new { role = "system", content = longSystemMessage }, @@ -234,17 +147,15 @@ public async Task ProviderRequestTrace_CapturesProviderMultiTurnPayloadAndRuntim ); response.EnsureSuccessStatusCode(); - var store = host.Services.GetRequiredService(); - var trace = store.GetSnapshot().Single(); - var runtime = host.Services.GetRequiredService(); - var runtimeEvent = runtime.GetAssertionSummary().Events.Single(); + var trace = (await GetTracesAsync(client)).Single(); + var runtimeEvent = (await client.GetFromJsonAsync(LlmTestServerControlRoutes.Assertions))!.Events.Single(); - await Assert.That(trace.ProviderId).IsEqualTo(LlmTckCompatibilityTags.OpenAI); + await Assert.That(trace.ProviderId).IsEqualTo(LlmTestServerCompatibilityTags.OpenAI); await Assert.That(trace.Method).IsEqualTo(HttpMethod.Post.Method); await Assert.That(trace.Path).IsEqualTo("/openai/v1/chat/completions"); await Assert.That(trace.OperationId) - .IsEqualTo(LlmTckProviderOperationIds.OpenAI.ChatCompletionsCreate); - await Assert.That(trace.ModelId).IsEqualTo(LlmTckKnownModelIds.Gpt41Mini); + .IsEqualTo(LlmTestServerProviderOperationIds.OpenAI.ChatCompletionsCreate); + await Assert.That(trace.ModelId).IsEqualTo(LlmTestServerKnownModelIds.Gpt41Mini); await Assert.That(trace.StatusCode).IsEqualTo((int)HttpStatusCode.OK); await Assert.That(trace.DurationMilliseconds).IsNotNull(); await Assert.That(trace.DurationMilliseconds!.Value).IsGreaterThanOrEqualTo(0d); @@ -264,44 +175,42 @@ await Assert.That(trace.OperationId) await Assert.That(trace.Usage!.TotalTokens).IsGreaterThan(0); await Assert.That(runtimeEvent.RequestId).IsEqualTo(trace.RequestId); await Assert.That(runtimeEvent.Messages.Count).IsEqualTo(4); - await Assert.That(response.Headers.GetValues(LlmTckProviderHttpTraceOptions.DefaultRequestIdHeaderName).Single()) + await Assert.That(response.Headers.GetValues(LlmTestServerProviderHttpTraceOptions.DefaultRequestIdHeaderName).Single()) .IsEqualTo(trace.RequestId); } [Test] public async Task ProviderRequestTrace_CorrelatesMultipartAudioModelAndOperationAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options + using var host = await LlmTestServerTestHost.StartAsync(options => options .WithDefaultTranscriptionText("deterministic transcript")); using var client = host.GetTestClient(); - using var content = CreateTranscriptionContent(LlmTckKnownModelIds.Gpt4OMiniTts); + using var content = CreateTranscriptionContent(LlmTestServerKnownModelIds.Gpt4OMiniTts); var response = await client.PostAsync("/openai/v1/audio/transcriptions", content); response.EnsureSuccessStatusCode(); - var trace = host.Services - .GetRequiredService() - .GetSnapshot() + var trace = (await GetTracesAsync(client)) .Single(); - await Assert.That(trace.ProviderId).IsEqualTo(LlmTckCompatibilityTags.OpenAI); - await Assert.That(trace.ModelId).IsEqualTo(LlmTckKnownModelIds.Gpt4OMiniTts); + await Assert.That(trace.ProviderId).IsEqualTo(LlmTestServerCompatibilityTags.OpenAI); + await Assert.That(trace.ModelId).IsEqualTo(LlmTestServerKnownModelIds.Gpt4OMiniTts); await Assert.That(trace.OperationId) - .IsEqualTo(LlmTckProviderOperationIds.OpenAI.AudioTranscriptionsCreate); - await Assert.That(trace.RuntimeEventKind).IsEqualTo(LlmTckEventKind.Matched); + .IsEqualTo(LlmTestServerProviderOperationIds.OpenAI.AudioTranscriptionsCreate); + await Assert.That(trace.RuntimeEventKind).IsEqualTo(LlmTestServerEventKind.Matched); await Assert.That(trace.RequestPreview).Contains("[multipart body summary;"); await Assert.That(trace.RequestPreview) - .Contains($"field model: {LlmTckKnownModelIds.Gpt4OMiniTts}"); + .Contains($"field model: {LlmTestServerKnownModelIds.Gpt4OMiniTts}"); await Assert.That(trace.ResponsePreview).Contains("deterministic transcript"); } [Test] public async Task ProviderRequestTrace_CapturesStreamingChunksAndRendersInspectorAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options.AddChatScenario( + using var host = await LlmTestServerTestHost.StartAsync(options => options.AddChatScenario( "trace-streaming", scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) + .ForModel(LlmTestServerKnownModelIds.Gpt41Mini) .WhenUserContains("stream response") .Responds("blue whale", "blue ", "whale") )); @@ -311,7 +220,7 @@ public async Task ProviderRequestTrace_CapturesStreamingChunksAndRendersInspecto "/openai/v1/chat/completions", new { - model = LlmTckKnownModelIds.Gpt41Mini, + model = LlmTestServerKnownModelIds.Gpt41Mini, stream = true, messages = new[] { new { role = "user", content = "stream response" } }, }, @@ -320,11 +229,9 @@ public async Task ProviderRequestTrace_CapturesStreamingChunksAndRendersInspecto var responseBody = await response.Content.ReadAsStringAsync(); response.EnsureSuccessStatusCode(); - var trace = host.Services - .GetRequiredService() - .GetSnapshot() + var trace = (await GetTracesAsync(client)) .Single(); - var page = await client.GetStringAsync(LlmTckControlRoutes.Admin); + var page = await client.GetStringAsync(LlmTestServerControlRoutes.Admin); var chunkSectionIndex = page.IndexOf("Stream chunks", StringComparison.Ordinal); var firstChunkIndex = page.IndexOf("blue ", chunkSectionIndex, StringComparison.Ordinal); var secondChunkIndex = page.IndexOf("whale", firstChunkIndex + "blue ".Length, StringComparison.Ordinal); @@ -333,7 +240,7 @@ public async Task ProviderRequestTrace_CapturesStreamingChunksAndRendersInspecto await Assert.That(responseBody).Contains("data: [DONE]"); await Assert.That(trace.IsStreaming).IsTrue(); await Assert.That(trace.OperationId) - .IsEqualTo(LlmTckProviderOperationIds.OpenAI.ChatCompletionsCreate); + .IsEqualTo(LlmTestServerProviderOperationIds.OpenAI.ChatCompletionsCreate); await Assert.That(trace.StreamChunks.Count).IsEqualTo(2); await Assert.That(trace.StreamChunks[0]).IsEqualTo("blue "); await Assert.That(trace.StreamChunks[1]).IsEqualTo("whale"); @@ -349,10 +256,10 @@ await Assert.That(trace.OperationId) [Test] public async Task ProviderRequestTrace_RedactsNestedCredentialsAndPreservesUsageFieldsAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options.AddChatScenario( + using var host = await LlmTestServerTestHost.StartAsync(options => options.AddChatScenario( "trace-redaction", scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) + .ForModel(LlmTestServerKnownModelIds.Gpt41Mini) .WhenUserContains("redact credentials") .Responds("safe response") )); @@ -362,7 +269,7 @@ public async Task ProviderRequestTrace_RedactsNestedCredentialsAndPreservesUsage "/openai/v1/chat/completions", new { - model = LlmTckKnownModelIds.Gpt41Mini, + model = LlmTestServerKnownModelIds.Gpt41Mini, max_tokens = 321, refresh_token = "refresh-secret-value", metadata = new @@ -377,9 +284,7 @@ public async Task ProviderRequestTrace_RedactsNestedCredentialsAndPreservesUsage ); response.EnsureSuccessStatusCode(); - var trace = host.Services - .GetRequiredService() - .GetSnapshot() + var trace = (await GetTracesAsync(client)) .Single(); var preview = trace.RequestPreview ?? throw new InvalidOperationException("Expected a captured JSON request preview."); @@ -400,104 +305,23 @@ await Assert.That(metadata.GetProperty("nested").GetProperty("session_token").Ge await Assert.That(preview).DoesNotContain("session-secret-value"); } - [Test] - public async Task ProviderRequestTraceStore_RetainsNewestEntriesWithinConfiguredBoundAsync() - { - const int MaxEntries = 3; - var store = new LlmTckProviderHttpTraceStore( - Options.Create(new LlmTckProviderHttpTraceOptions { MaxEntries = MaxEntries }) - ); - - for (var index = 1; index <= MaxEntries + 1; index++) - { - store.Start(new LlmTckProviderHttpTrace { RequestId = $"request-{index}" }); - } - - var snapshot = store.GetSnapshot(); - - await Assert.That(snapshot.Count).IsEqualTo(MaxEntries); - await Assert.That(snapshot[0].RequestId).IsEqualTo("request-4"); - await Assert.That(snapshot[1].RequestId).IsEqualTo("request-3"); - await Assert.That(snapshot[2].RequestId).IsEqualTo("request-2"); - await Assert.That(snapshot.Any(trace => trace.RequestId == "request-1")).IsFalse(); - } - - [Test] - public async Task ProviderRequestTraceStore_BoundsRetainedBytesAndTruncatesOversizedTraceAsync() - { - const int MaxRetainedBytes = 2048; - var store = new LlmTckProviderHttpTraceStore( - Options.Create( - new LlmTckProviderHttpTraceOptions - { - MaxEntries = 10, - MaxPreviewBytes = 512, - MaxRetainedBytes = MaxRetainedBytes, - } - ) - ); - - for (var index = 1; index <= 4; index++) - { - var requestId = $"request-{index}"; - store.Start( - new LlmTckProviderHttpTrace - { - RequestId = requestId, - ProviderId = LlmTckCompatibilityTags.OpenAI, - ProviderNamespace = LlmTckProviderRouteNamespaces.OpenAI, - Method = HttpMethod.Post.Method, - Path = "/openai", - } - ); - store.TryEnrich( - requestId, - new LlmTckProviderHttpTraceEnrichment - { - Messages = - [ - new LlmTckMessage - { - Role = "user", - Content = new string('m', 2000), - }, - ], - RuntimeResponse = new string('r', 2000), - } - ); - } - - var snapshot = store.GetSnapshot(); - - await Assert.That(store.RetainedByteCount).IsLessThanOrEqualTo(MaxRetainedBytes); - await Assert.That(snapshot.Count).IsEqualTo(1); - await Assert.That(snapshot[0].RequestId).IsEqualTo("request-4"); - await Assert.That(snapshot[0].RetentionTruncated).IsTrue(); - await Assert.That(snapshot[0].RetainedByteCount).IsLessThanOrEqualTo(MaxRetainedBytes); - await Assert.That(snapshot[0].Messages.Single().Content.Length).IsLessThan(2000); - await Assert.That(snapshot[0].RuntimeResponse).StartsWith(new string('r', 400)); - await Assert.That(snapshot[0].RuntimeResponse).EndsWith("…"); - } - [Test] public async Task ProviderRequestTrace_CapturesHttpOnlyAndValidationFailureRequestsAsync() { - using var host = await LlmTckTestHost.StartAsync(); + using var host = await LlmTestServerTestHost.StartAsync(); using var client = host.GetTestClient(); var models = await client.GetAsync("/openai/v1/models"); var invalid = await client.PostAsJsonAsync( "/openai/v1/chat/completions", - new { model = LlmTckKnownModelIds.Gpt41Mini }, + new { model = LlmTestServerKnownModelIds.Gpt41Mini }, _jsonOptions ); models.EnsureSuccessStatusCode(); await Assert.That(invalid.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); - var traces = host.Services - .GetRequiredService() - .GetSnapshot(); + var traces = (await GetTracesAsync(client)); var modelsTrace = traces.Single(trace => trace.Path.EndsWith("/models", StringComparison.Ordinal)); var invalidTrace = traces.Single(trace => trace.Path.EndsWith("/chat/completions", StringComparison.Ordinal)); @@ -514,7 +338,7 @@ public async Task ProviderFallbacks_TraceUnknownAndWrongMethodRequestsAndRenderB { const string FallbackModel = "fallback-diagnostic-model"; const string RequestPayload = "wrong method fallback payload"; - using var host = await LlmTckTestHost.StartAsync(); + using var host = await LlmTestServerTestHost.StartAsync(); using var client = host.GetTestClient(); var missing = await client.GetAsync("/openai/v1/diagnostics/not-supported"); @@ -528,9 +352,7 @@ public async Task ProviderFallbacks_TraceUnknownAndWrongMethodRequestsAndRenderB await Assert.That(wrongMethod.StatusCode).IsEqualTo(HttpStatusCode.MethodNotAllowed); await Assert.That(wrongMethod.Content.Headers.Allow.Single()).IsEqualTo("GET"); - var traces = host.Services - .GetRequiredService() - .GetSnapshot(); + var traces = (await GetTracesAsync(client)); var missingTrace = traces.Single(trace => trace.Path.EndsWith("/diagnostics/not-supported", StringComparison.Ordinal) ); @@ -543,23 +365,23 @@ public async Task ProviderFallbacks_TraceUnknownAndWrongMethodRequestsAndRenderB var previewRoot = previewDocument.RootElement; await Assert.That(traces.Count).IsEqualTo(2); - await Assert.That(missingTrace.ProviderId).IsEqualTo(LlmTckCompatibilityTags.OpenAI); + await Assert.That(missingTrace.ProviderId).IsEqualTo(LlmTestServerCompatibilityTags.OpenAI); await Assert.That(missingTrace.StatusCode).IsEqualTo((int)HttpStatusCode.NotFound); await Assert.That(missingTrace.TransportState) - .IsEqualTo(LlmTckProviderHttpTraceTransportState.Completed); + .IsEqualTo(LlmTestServerProviderHttpTraceTransportState.Completed); await Assert.That(missingTrace.OperationId).IsNull(); - await Assert.That(wrongMethodTrace.ProviderId).IsEqualTo(LlmTckCompatibilityTags.OpenAI); + await Assert.That(wrongMethodTrace.ProviderId).IsEqualTo(LlmTestServerCompatibilityTags.OpenAI); await Assert.That(wrongMethodTrace.Method).IsEqualTo(HttpMethod.Post.Method); await Assert.That(wrongMethodTrace.StatusCode) .IsEqualTo((int)HttpStatusCode.MethodNotAllowed); await Assert.That(wrongMethodTrace.TransportState) - .IsEqualTo(LlmTckProviderHttpTraceTransportState.Completed); + .IsEqualTo(LlmTestServerProviderHttpTraceTransportState.Completed); await Assert.That(wrongMethodTrace.OperationId).IsNull(); await Assert.That(wrongMethodTrace.ModelId).IsEqualTo(FallbackModel); await Assert.That(previewRoot.GetProperty("model").GetString()).IsEqualTo(FallbackModel); await Assert.That(previewRoot.GetProperty("request").GetString()).IsEqualTo(RequestPayload); - var page = await client.GetStringAsync(LlmTckControlRoutes.Admin); + var page = await client.GetStringAsync(LlmTestServerControlRoutes.Admin); await Assert.That(CountOccurrences(page, "data-testid=\"request-row\"")).IsEqualTo(2); await Assert.That(page).Contains($"data-request-id=\"{missingTrace.RequestId}\""); await Assert.That(page).Contains($"data-request-id=\"{wrongMethodTrace.RequestId}\""); @@ -571,10 +393,10 @@ await Assert.That(wrongMethodTrace.TransportState) public async Task AdminPanel_RendersScriptedErrorResponseAsProviderErrorAsync() { const string ErrorMessage = "Scripted provider refusal."; - using var host = await LlmTckTestHost.StartAsync(options => options.AddChatScenario( + using var host = await LlmTestServerTestHost.StartAsync(options => options.AddChatScenario( "provider-error-label", scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) + .ForModel(LlmTestServerKnownModelIds.Gpt41Mini) .WhenUserContains("trigger provider error") .Fails(400, "content_filter", ErrorMessage) )); @@ -584,21 +406,19 @@ public async Task AdminPanel_RendersScriptedErrorResponseAsProviderErrorAsync() "/openai/v1/chat/completions", new { - model = LlmTckKnownModelIds.Gpt41Mini, + model = LlmTestServerKnownModelIds.Gpt41Mini, messages = new[] { new { role = "user", content = "trigger provider error" } }, }, _jsonOptions ); await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); - var trace = host.Services - .GetRequiredService() - .GetSnapshot() + var trace = (await GetTracesAsync(client)) .Single(); - await Assert.That(trace.RuntimeEventKind).IsEqualTo(LlmTckEventKind.ErrorReturned); + await Assert.That(trace.RuntimeEventKind).IsEqualTo(LlmTestServerEventKind.ErrorReturned); await Assert.That(trace.RuntimeResponse).IsEqualTo(ErrorMessage); - var page = await client.GetStringAsync(LlmTckControlRoutes.Admin); + var page = await client.GetStringAsync(LlmTestServerControlRoutes.Admin); await Assert.That(page).Contains("
provider error"); await Assert.That(page).Contains(ErrorMessage); @@ -608,10 +428,10 @@ public async Task AdminPanel_RendersScriptedErrorResponseAsProviderErrorAsync() public async Task ProviderRequestTrace_CorrelatesCacheWriteThenCacheReadPerRequestAsync() { var cacheablePrefix = string.Join(' ', Enumerable.Repeat("stable-prefix", 1200)); - using var host = await LlmTckTestHost.StartAsync(options => options.AddChatScenario( + using var host = await LlmTestServerTestHost.StartAsync(options => options.AddChatScenario( "trace-cache", scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) + .ForModel(LlmTestServerKnownModelIds.Gpt41Mini) .WhenUserContains("color") .Responds("first response") .Responds("second response") @@ -619,7 +439,7 @@ public async Task ProviderRequestTrace_CorrelatesCacheWriteThenCacheReadPerReque using var client = host.GetTestClient(); var request = new { - model = LlmTckKnownModelIds.Gpt41Mini, + model = LlmTestServerKnownModelIds.Gpt41Mini, messages = new[] { new { role = "system", content = cacheablePrefix }, @@ -632,12 +452,10 @@ public async Task ProviderRequestTrace_CorrelatesCacheWriteThenCacheReadPerReque (await client.PostAsJsonAsync("/openai/v1/chat/completions", request, _jsonOptions)) .EnsureSuccessStatusCode(); - var traces = host.Services - .GetRequiredService() - .GetSnapshot(); + var traces = (await GetTracesAsync(client)); var second = traces[0]; var first = traces[1]; - var page = await client.GetStringAsync(LlmTckControlRoutes.Admin); + var page = await client.GetStringAsync(LlmTestServerControlRoutes.Admin); await Assert.That(traces.Count).IsEqualTo(2); await Assert.That(first.Usage).IsNotNull(); @@ -658,10 +476,10 @@ public async Task ProviderRequestTrace_CorrelatesCacheWriteThenCacheReadPerReque public async Task AdminPanel_RendersOneRequestRowWithFullConversationAndExchangeAsync() { var longSystemMessage = $"{new string('z', 900)} END_OF_LONG_PROMPT"; - using var host = await LlmTckTestHost.StartAsync(options => options.AddChatScenario( + using var host = await LlmTestServerTestHost.StartAsync(options => options.AddChatScenario( "dashboard-multi-turn", scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) + .ForModel(LlmTestServerKnownModelIds.Gpt41Mini) .WhenUserContains("color") .Responds("blue whale") )); @@ -670,7 +488,7 @@ public async Task AdminPanel_RendersOneRequestRowWithFullConversationAndExchange "/openai/v1/chat/completions", new { - model = LlmTckKnownModelIds.Gpt41Mini, + model = LlmTestServerKnownModelIds.Gpt41Mini, messages = new[] { new { role = "system", content = longSystemMessage }, @@ -683,7 +501,7 @@ public async Task AdminPanel_RendersOneRequestRowWithFullConversationAndExchange ); response.EnsureSuccessStatusCode(); - var page = await client.GetStringAsync(LlmTckControlRoutes.Admin); + var page = await client.GetStringAsync(LlmTestServerControlRoutes.Admin); var conversationIndex = page.IndexOf("data-testid=\"conversation\"", StringComparison.Ordinal); var responseIndex = page.IndexOf("data-response-card=\"true\"", StringComparison.Ordinal); var collapsedSystemIndex = page.IndexOf( @@ -694,7 +512,7 @@ public async Task AdminPanel_RendersOneRequestRowWithFullConversationAndExchange await Assert.That(CountOccurrences(page, "data-testid=\"request-row\"")).IsEqualTo(1); await Assert.That(CountOccurrences(page, "data-testid=\"conversation-turn\"")).IsEqualTo(5); await Assert.That(page).Contains("OpenAI"); - await Assert.That(page).Contains(LlmTckKnownModelIds.Gpt41Mini); + await Assert.That(page).Contains(LlmTestServerKnownModelIds.Gpt41Mini); await Assert.That(page).Contains("Request metadata"); await Assert.That(page).Contains("Tokens & cache"); await Assert.That(page).Contains("Wire payloads"); @@ -716,7 +534,7 @@ await Assert.That(page.IndexOf("END_OF_LONG_PROMPT", conversationIndex, StringCo [Test] public async Task AdminPanel_PrettyPrintsJsonWithoutUnicodeEscapeNoiseAsync() { - using var host = await LlmTckTestHost.StartAsync(); + using var host = await LlmTestServerTestHost.StartAsync(); using var client = host.GetTestClient(); var response = await client.PostAsJsonAsync( @@ -730,7 +548,7 @@ public async Task AdminPanel_PrettyPrintsJsonWithoutUnicodeEscapeNoiseAsync() ); await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.NotFound); - var page = await client.GetStringAsync(LlmTckControlRoutes.Admin); + var page = await client.GetStringAsync(LlmTestServerControlRoutes.Admin); await Assert.That(page).Contains("data-testid=\"overview-response-json\""); await Assert.That(page).Contains(""error""); @@ -742,11 +560,11 @@ public async Task AdminPanel_PrettyPrintsJsonWithoutUnicodeEscapeNoiseAsync() [Test] public async Task AdminPanel_LabelsNonChatJsonAsProviderResponseAsync() { - using var host = await LlmTckTestHost.StartAsync(); + using var host = await LlmTestServerTestHost.StartAsync(); using var client = host.GetTestClient(); (await client.GetAsync("/openai/v1/models")).EnsureSuccessStatusCode(); - var page = await client.GetStringAsync(LlmTckControlRoutes.Admin); + var page = await client.GetStringAsync(LlmTestServerControlRoutes.Admin); await Assert.That(page).Contains("provider response"); await Assert.That(page).DoesNotContain(">assistant"); @@ -755,15 +573,14 @@ public async Task AdminPanel_LabelsNonChatJsonAsProviderResponseAsync() [Test] public async Task ResetControlRoute_ClearsProviderRequestJournalAsync() { - using var host = await LlmTckTestHost.StartAsync(); + using var host = await LlmTestServerTestHost.StartAsync(); using var client = host.GetTestClient(); - var store = host.Services.GetRequiredService(); (await client.GetAsync("/openai/v1/models")).EnsureSuccessStatusCode(); - await Assert.That(store.GetSnapshot().Count).IsEqualTo(1); + await Assert.That((await GetTracesAsync(client)).Count).IsEqualTo(1); - (await client.PostAsync(LlmTckControlRoutes.Reset, content: null)).EnsureSuccessStatusCode(); - await Assert.That(store.GetSnapshot()).IsEmpty(); + (await client.PostAsync(LlmTestServerControlRoutes.Reset, content: null)).EnsureSuccessStatusCode(); + await Assert.That((await GetTracesAsync(client))).IsEmpty(); } private static int CountOccurrences(string value, string search) @@ -790,20 +607,8 @@ private static MultipartFormDataContent CreateTranscriptionContent(string model) return content; } - private static string FindRepositoryRoot() + private static async Task> GetTracesAsync(HttpClient client) { - var directory = new DirectoryInfo(AppContext.BaseDirectory); - - while (directory is not null) - { - if (File.Exists(Path.Combine(directory.FullName, "ManagedCode.LlmTck.slnx"))) - { - return directory.FullName; - } - - directory = directory.Parent; - } - - throw new DirectoryNotFoundException("Could not find ManagedCode.LlmTck.slnx."); + return await client.GetFromJsonAsync>(LlmTestServerControlRoutes.Requests) ?? []; } } diff --git a/tests/ManagedCode.LlmTestServer.Tests/Hosting/ConfigurationBoundaryIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Hosting/ConfigurationBoundaryIntegrationTests.cs new file mode 100644 index 0000000..2a635fc --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Hosting/ConfigurationBoundaryIntegrationTests.cs @@ -0,0 +1,53 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json.Nodes; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.Hosting; + +public sealed class ConfigurationBoundaryIntegrationTests +{ + [Test] + [Arguments("maxPromptCacheEntries", "-1")] + [Arguments("maxVideoJobs", "-1")] + [Arguments("maxVideoBytes", "-1")] + [Arguments("maxStoredChatResponses", "-1")] + [Arguments("maxStoredChatResponseBytes", "-1")] + [Arguments("models", "null")] + [Arguments("models", "[null]")] + [Arguments("models", "[{\"id\":\"\",\"kind\":0}]")] + [Arguments("models", "[{\"id\":\"gpt-4.1-mini\",\"kind\":0,\"reasoningTokens\":-1}]")] + [Arguments("chatScenarios", "null")] + [Arguments("chatScenarios", "[null]")] + [Arguments("chatScenarios", "[{\"id\":\"invalid\",\"match\":null}]")] + [Arguments("chatScenarios", "[{\"id\":\"invalid\",\"match\":{\"messages\":null}}]")] + [Arguments("chatScenarios", "[{\"id\":\"invalid\",\"responses\":null}]")] + [Arguments("chatScenarios", "[{\"id\":\"invalid\",\"responses\":[{\"delayMilliseconds\":-1}]}]")] + [Arguments("chatScenarios", "[{\"id\":\"invalid\",\"responses\":[{\"streamChunks\":null}]}]")] + [Arguments("chatScenarios", "[{\"id\":\"invalid\",\"responses\":[{\"toolCalls\":null}]}]")] + [Arguments("datasets", "null")] + [Arguments("datasets", "[null]")] + [Arguments("datasets", "[{\"id\":\"invalid\",\"chatScenarios\":null}]")] + [Arguments("decisionScenarios", "null")] + [Arguments("defaultEmbeddingVector", "null")] + [Arguments("defaultAudioBytes", "null")] + [Arguments("defaultVideoBytes", "null")] + [Arguments("faultSimulation", "null")] + [Arguments("faultSimulation", "{\"contentFilterTerms\":null}")] + [Arguments("chatScenarios", "[{\"id\":\"invalid\",\"responses\":[{\"toolCalls\":[{\"id\":\"call\",\"name\":\"weather\",\"argumentsJson\":\"{\"}]}]}]")] + public async Task InvalidConfiguration_Returns400AndPreservesActiveScenarioAsync(string field, string invalidJson) + { + using var host = await LlmTestServerTestHost.StartAsync(config => config + .AddChatScenario("preserved", scenario => scenario.Responds("still active"))); + using var http = host.GetTestClient(); + using var invalid = await http.PostAsJsonAsync("/admin-api/configure", new JsonObject { [field] = JsonNode.Parse(invalidJson) }); + await Assert.That(invalid.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + using var chat = await http.PostAsJsonAsync("/openai/v1/chat/completions", new + { + model = "gpt-4.1-mini", + messages = new[] { new { role = "user", content = "verify current configuration" } } + }); + chat.EnsureSuccessStatusCode(); + await Assert.That(await chat.Content.ReadAsStringAsync()).Contains("still active"); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Hosting/DashboardAnalyticsIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Hosting/DashboardAnalyticsIntegrationTests.cs new file mode 100644 index 0000000..33b8ec7 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Hosting/DashboardAnalyticsIntegrationTests.cs @@ -0,0 +1,330 @@ +using System.Globalization; +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using System.Text.RegularExpressions; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Conversations; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Scenarios; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.Hosting; + +public sealed class DashboardAnalyticsIntegrationTests +{ + [Test] + public async Task Analytics_ObservesCacheMissAndHitWithExplicitConversationCorrelationAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(config => config.AddReasoningChatModel(LlmTestServerKnownModelIds.Gpt5Mini, 17).AddChatScenario("analytics-cache", scenario => scenario + .ForModel(LlmTestServerKnownModelIds.Gpt5Mini) + .Responds("first answer").Responds("second answer").Responds("uncorrelated answer"))); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + var body = new + { + model = LlmTestServerKnownModelIds.Gpt5Mini, + messages = new[] + { + new { role = "system", content = string.Join(' ', Enumerable.Repeat("stable analytical prefix", 1200)) }, + new { role = "user", content = "explain the fixture" }, + }, + }; + http.DefaultRequestHeaders.Add(LlmTestServerConversationHeaders.Id, "analytics-cache-conversation"); + using var firstResponse = await http.PostAsJsonAsync("/openai/v1/chat/completions", body); + firstResponse.EnsureSuccessStatusCode(); + using var secondResponse = await http.PostAsJsonAsync("/openai/v1/chat/completions", body); + secondResponse.EnsureSuccessStatusCode(); + var wireUsage = (await secondResponse.Content.ReadFromJsonAsync()).GetProperty("usage"); + var snapshot = await control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending }); + var first = snapshot.Requests[0]; + var second = snapshot.Requests[1]; + await Assert.That(snapshot.Requests.Count).IsEqualTo(2); + await Assert.That(first.ConversationIdWasProvided).IsTrue(); + await Assert.That(second.ConversationIdWasProvided).IsTrue(); + await Assert.That(second.ConversationId).IsEqualTo("analytics-cache-conversation"); + await Assert.That(first.Usage).IsNotNull(); + await Assert.That(second.Usage).IsNotNull(); + await Assert.That(first.Usage!.CachedInputTokens).IsEqualTo(0); + await Assert.That(first.Usage.CacheCreationInputTokens).IsGreaterThanOrEqualTo(1024); + await Assert.That(second.Usage!.CachedInputTokens).IsGreaterThanOrEqualTo(1024); + await Assert.That(second.Usage.CacheCreationInputTokens).IsEqualTo(0); + await Assert.That(second.Usage.InputTokens).IsEqualTo(wireUsage.GetProperty("prompt_tokens").GetInt32()); + await Assert.That(second.Usage.CachedInputTokens).IsEqualTo(wireUsage.GetProperty("prompt_tokens_details").GetProperty("cached_tokens").GetInt32()); + await Assert.That(second.Usage.InputTokens >= second.Usage.CachedInputTokens + second.Usage.CacheCreationInputTokens).IsTrue(); + await Assert.That(second.Usage.ReasoningTokens).IsEqualTo(17); + await Assert.That(second.Usage.OutputTokens).IsEqualTo(wireUsage.GetProperty("completion_tokens").GetInt32()); + await Assert.That(second.Usage.OutputTokens > second.Usage.ReasoningTokens).IsTrue(); + await Assert.That(second.Usage.TotalTokens).IsEqualTo(second.Usage.InputTokens + second.Usage.OutputTokens); + await Assert.That(first.ResponseToolCalls).IsEmpty(); + await Assert.That(second.ResponseToolCalls).IsEmpty(); + var page = await http.GetStringAsync("/"); + await Assert.That(page).Contains(second.Usage.TotalTokens.ToString("N0", CultureInfo.InvariantCulture)); + await Assert.That(page).Contains(second.Usage.CachedInputTokens.ToString("N0", CultureInfo.InvariantCulture)); + await Assert.That(page).Contains("analytics-cache-conversation"); + await AssertMetricAsync(page, "input", second.Usage.InputTokens); + await AssertMetricAsync(page, "output", second.Usage.OutputTokens); + await AssertMetricAsync(page, "total", second.Usage.TotalTokens); + await AssertMetricAsync(page, "reasoning", 17); + await AssertMetricAsync(page, "cache-read", second.Usage.CachedInputTokens); + await AssertMetricAsync(page, "cache-write", 0); + await AssertMetricAsync(page, "requests", 1); + await AssertMetricAsync(page, "tool-calls", 0); + await AssertMetricAsync(page, "tool-results", 0); + + http.DefaultRequestHeaders.Remove(LlmTestServerConversationHeaders.Id); + using var uncorrelated = await http.PostAsJsonAsync("/openai/v1/chat/completions", body); + uncorrelated.EnsureSuccessStatusCode(); + var last = (await control.GetRequestsAsync()).Requests[0]; + await Assert.That(last.ConversationIdWasProvided).IsFalse(); + } + + [Test] + public async Task Analytics_DistinguishesReturnedToolCallsFromObservedToolResultsAsync() + { + var call = new LlmTestServerToolCall { Id = "call_analytics", Name = "read_fixture", ArgumentsJson = "{}" }; + using var host = await LlmTestServerTestHost.StartAsync(config => config.AddConversation("analytics-tools", graph => graph + .Step("call", step => step.WhenLatestUserEquals("read fixture").CallsTools(call).Then("result")) + .Step("result", step => step.WhenToolResult(call.Name, call.Id).Responds("fixture read").Complete()))); + using var http = host.GetTestClient(); + http.DefaultRequestHeaders.Add(LlmTestServerConversationHeaders.Id, "analytics-tool-conversation"); + var tools = new[] { new { type = "function", function = new { name = call.Name, parameters = new { type = "object" } } } }; + using var first = await http.PostAsJsonAsync("/openai/v1/chat/completions", new + { + model = LlmTestServerKnownModelIds.Gpt41Mini, + messages = new[] { new { role = "user", content = "read fixture" } }, + tools, + }); + first.EnsureSuccessStatusCode(); + var assistant = (await first.Content.ReadFromJsonAsync()).GetProperty("choices")[0].GetProperty("message"); + using var second = await http.PostAsJsonAsync("/openai/v1/chat/completions", new + { + model = LlmTestServerKnownModelIds.Gpt41Mini, + messages = new object[] + { + new { role = "user", content = "read fixture" }, + assistant, + new { role = "tool", tool_call_id = call.Id, content = "fixture contents" }, + }, + tools, + }); + second.EnsureSuccessStatusCode(); + var snapshot = await new LlmTestServerClient(http).GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending }); + await Assert.That(snapshot.Requests.Count).IsEqualTo(2); + await Assert.That(snapshot.Requests[0].ResponseToolCalls.Single().Id).IsEqualTo(call.Id); + await Assert.That(snapshot.Requests[0].Messages.Count(message => message.ToolCallId is not null)).IsEqualTo(0); + await Assert.That(snapshot.Requests[1].ResponseToolCalls).IsEmpty(); + var result = snapshot.Requests[1].Messages.Single(message => message.ToolCallId is not null); + await Assert.That(result.ToolCallId).IsEqualTo(call.Id); + await Assert.That(result.Content).IsEqualTo("fixture contents"); + await Assert.That(snapshot.Requests[1].Messages.SelectMany(message => message.ToolCalls).Single().Id).IsEqualTo(call.Id); + await Assert.That(snapshot.Requests.All(trace => trace.ConversationIdWasProvided)).IsTrue(); + await Assert.That(snapshot.Requests.Select(trace => trace.ConversationId).Distinct().Single()).IsEqualTo("analytics-tool-conversation"); + await Assert.That(snapshot.Requests.Sum(trace => trace.ResponseToolCalls.Count)).IsEqualTo(1); + await Assert.That(snapshot.Requests.Sum(trace => trace.Messages.Count(message => message.ToolCallId is not null))).IsEqualTo(1); + var page = await http.GetStringAsync("/"); + await AssertMetricAsync(page, "tool-calls", 0); + await AssertMetricAsync(page, "tool-results", 1); + await AssertMetricAsync(page, "tool-result-ids", 1); + } + + [Test] + public async Task Analytics_ModelDiscoveryKeepsUnknownUsageVisibleAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + http.DefaultRequestHeaders.Add(LlmTestServerConversationHeaders.Id, "analytics-discovery"); + using var response = await http.GetAsync("/openai/v1/models"); + response.EnsureSuccessStatusCode(); + var models = await response.Content.ReadFromJsonAsync(); + await Assert.That(models.GetProperty("data").GetArrayLength()).IsGreaterThan(0); + var trace = (await new LlmTestServerClient(http).GetRequestsAsync()).Requests.Single(); + await Assert.That(trace.ConversationIdWasProvided).IsTrue(); + await Assert.That(trace.ConversationId).IsEqualTo("analytics-discovery"); + await Assert.That(trace.Usage).IsNull(); + var page = await http.GetStringAsync("/"); + await AssertMetricAsync(page, "requests", 1); + await AssertMetricAsync(page, "usage-requests", 0); + foreach (var metric in new[] { "input", "output", "total", "reasoning", "cache-read", "cache-write", "cache-share", "cache-hit-rate" }) + { + var element = Regex.Match(WebUtility.HtmlDecode(page), "<[^>]*data-testid=\"analytics-" + metric + "\"[^>]*>[^<]*").Value; + await Assert.That(element).Contains("data-value=\"\""); + await Assert.That(element.Contains('—') || element.Contains("N/A", StringComparison.Ordinal)).IsTrue(); + } + var retention = Regex.Match(page, "<[^>]*data-testid=\"analytics-retention\"[^>]*>").Value; + await Assert.That(retention).Contains("data-incomplete=\"true\""); + await Assert.That(page).Contains("data-testid=\"analytics-flow-empty\""); + await Assert.That(page).DoesNotContain("data-testid=\"analytics-flow\""); + } + + [Test] + public async Task Analytics_ItemizesOpenAiInputByRoleAndReconcilesWithWireUsageAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(config => config.AddChatScenario("analytics-mix", scenario => scenario + .ForModel(LlmTestServerKnownModelIds.Gpt41Mini).Responds("Take the 14:30 train."))); + using var http = host.GetTestClient(); + using var response = await http.PostAsJsonAsync("/openai/v1/chat/completions", new + { + model = LlmTestServerKnownModelIds.Gpt41Mini, + messages = new object[] + { + new { role = "system", content = "You are a rail concierge. Use only the supplied tools and never invent timetables." }, + new { role = "user", content = "Find an afternoon train from Paris to Fontainebleau." }, + new + { + role = "assistant", + content = "Checking departures.", + tool_calls = new[] { new { id = "call_trains", type = "function", function = new { name = "search_trains", arguments = "{\"from\":\"Paris Gare de Lyon\",\"to\":\"Fontainebleau\"}" } } }, + }, + new { role = "tool", tool_call_id = "call_trains", content = "{\"departures\":[{\"time\":\"14:30\",\"platform\":\"L\"}]}" }, + new { role = "user", content = "Which one should I take?" }, + }, + tools = new[] + { + new { type = "function", function = new { name = "search_trains", description = "Search departures between two stations.", parameters = new { type = "object", properties = new { from = new { type = "string" }, to = new { type = "string" } } } } }, + }, + }); + response.EnsureSuccessStatusCode(); + var promptTokens = (await response.Content.ReadFromJsonAsync()).GetProperty("usage").GetProperty("prompt_tokens").GetInt32(); + var usage = (await new LlmTestServerClient(http).GetRequestsAsync()).Requests.Single().Usage!; + var composition = usage.InputComposition!; + await AssertItemizedAsync(composition, usage.InputTokens, promptTokens); + + var page = await http.GetStringAsync("/"); + await AssertMetricAsync(page, "mix-system", composition.SystemTokens); + await AssertMetricAsync(page, "mix-user", composition.UserTokens); + await AssertMetricAsync(page, "mix-assistant", composition.AssistantTokens); + await AssertMetricAsync(page, "mix-tool-calls", composition.ToolCallTokens); + await AssertMetricAsync(page, "mix-tool-results", composition.ToolResultTokens); + await AssertMetricAsync(page, "mix-tool-definitions", composition.ToolDefinitionTokens); + await Assert.That(page).DoesNotContain("data-testid=\"analytics-mix-not-itemized\""); + foreach (var key in new[] { "system", "tool-definitions", "user", "assistant", "tool-calls", "tool-results", "output" }) + { + await Assert.That(page).Contains("data-testid=\"analytics-flow-source\" data-key=\"" + key + "\""); + } + foreach (var key in new[] { "uncached", "response" }) + { + await Assert.That(page).Contains("data-testid=\"analytics-flow-target\" data-key=\"" + key + "\""); + } + } + + [Test] + public async Task Analytics_ItemizesAnthropicSystemToolUseAndToolResultBlocksAsync() + { + const string model = "claude-sonnet-4-5"; + using var host = await LlmTestServerTestHost.StartAsync(config => config.AddModel(model, LlmTestServerModelKind.Chat) + .AddChatScenario("analytics-anthropic-mix", scenario => scenario.ForModel(model).Responds("Paris is 18 °C and sunny."))); + using var http = host.GetTestClient(); + http.DefaultRequestHeaders.Add("anthropic-version", "2023-06-01"); + using var response = await http.PostAsJsonAsync("/anthropic/v1/messages", new + { + model, + max_tokens = 256, + system = "You are a weather assistant. Answer in one sentence.", + tools = new[] { new { name = "get_weather", description = "Current weather for a city.", input_schema = new { type = "object", properties = new { city = new { type = "string" } } } } }, + messages = new object[] + { + new { role = "user", content = "What is the weather in Paris?" }, + new { role = "assistant", content = new object[] { new { type = "text", text = "Let me check." }, new { type = "tool_use", id = "toolu_weather", name = "get_weather", input = new { city = "Paris" } } } }, + new { role = "user", content = new object[] { new { type = "tool_result", tool_use_id = "toolu_weather", content = "18 °C, sunny" } } }, + }, + }); + response.EnsureSuccessStatusCode(); + var wireInput = (await response.Content.ReadFromJsonAsync()).GetProperty("usage").GetProperty("input_tokens").GetInt32(); + var usage = (await new LlmTestServerClient(http).GetRequestsAsync()).Requests.Single().Usage!; + await Assert.That(usage.CachedInputTokens + usage.CacheCreationInputTokens).IsEqualTo(0); + await AssertItemizedAsync(usage.InputComposition!, usage.InputTokens, wireInput); + var page = await http.GetStringAsync("/"); + await AssertMetricAsync(page, "mix-system", usage.InputComposition!.SystemTokens); + await AssertMetricAsync(page, "mix-tool-results", usage.InputComposition.ToolResultTokens); + } + + [Test] + public async Task Insights_ChartRetainedTrafficByProviderModelOutcomeDeliveryAndContextAsync() + { + const string claude = "claude-sonnet-4-5"; + using var host = await LlmTestServerTestHost.StartAsync(config => config.AddModel(claude, LlmTestServerModelKind.Chat) + .AddChatScenario("insights-openai", scenario => scenario.ForModel(LlmTestServerKnownModelIds.Gpt41Mini).Responds("one").Responds("two")) + .AddChatScenario("insights-anthropic", scenario => scenario.ForModel(claude).Responds("three"))); + using var http = host.GetTestClient(); + http.DefaultRequestHeaders.Add("anthropic-version", "2023-06-01"); + var openAiMessages = new object[] + { + new { role = "system", content = "You answer in one word." }, + new { role = "user", content = "Say one." }, + }; + using (var single = await http.PostAsJsonAsync("/openai/v1/chat/completions", new { model = LlmTestServerKnownModelIds.Gpt41Mini, messages = openAiMessages })) + { + single.EnsureSuccessStatusCode(); + } + using (var streamed = await http.PostAsJsonAsync("/openai/v1/chat/completions", new { model = LlmTestServerKnownModelIds.Gpt41Mini, stream = true, messages = openAiMessages })) + { + streamed.EnsureSuccessStatusCode(); + await Assert.That(await streamed.Content.ReadAsStringAsync()).Contains("[DONE]"); + } + using (var anthropic = await http.PostAsJsonAsync("/anthropic/v1/messages", new + { + model = claude, + max_tokens = 64, + system = "You answer in one word.", + messages = new[] { new { role = "user", content = "Say three." } }, + })) + { + anthropic.EnsureSuccessStatusCode(); + } + using (var missing = await http.PostAsJsonAsync("/openai/v1/chat/completions", new { model = "gpt-missing", messages = openAiMessages })) + { + await Assert.That(missing.StatusCode).IsEqualTo(HttpStatusCode.NotFound); + } + + var journal = (await new LlmTestServerClient(http).GetRequestsAsync()).Requests; + await Assert.That(journal.Count).IsEqualTo(4); + var page = await http.GetStringAsync("/?view=insights"); + await Assert.That(page).Contains("data-testid=\"insights-panel\""); + foreach (var provider in journal.GroupBy(trace => trace.ProviderId)) + { + await AssertInsightAsync(page, "providers", provider.Key, provider.Count()); + } + foreach (var model in journal.GroupBy(trace => trace.ModelId!)) + { + await AssertInsightAsync(page, "models", model.Key, model.Count()); + } + await AssertInsightAsync(page, "outcomes", "success", journal.Count(trace => trace.StatusCode < 400)); + await AssertInsightAsync(page, "outcomes", "client-error", journal.Count(trace => trace.StatusCode is >= 400 and < 500)); + await AssertInsightAsync(page, "delivery", "stream", journal.Count(trace => trace.IsStreaming)); + await AssertInsightAsync(page, "delivery", "single", journal.Count(trace => !trace.IsStreaming)); + var itemized = journal.Select(trace => trace.Usage).Where(usage => usage?.InputComposition?.TotalTokens == usage?.InputTokens && usage is not null) + .Select(usage => usage!.InputComposition!).ToArray(); + await Assert.That(itemized.Length).IsGreaterThanOrEqualTo(3); + await AssertInsightAsync(page, "context-mix", "system", itemized.Sum(composition => composition.SystemTokens)); + await AssertInsightAsync(page, "context-mix", "user", itemized.Sum(composition => composition.UserTokens)); + await Assert.That(await http.GetStringAsync("/")).DoesNotContain("data-testid=\"insights-panel\""); + } + + private static async Task AssertInsightAsync(string page, string chart, string key, int expected) + { + var tag = Regex.Match(page, "<[^>]*data-testid=\"insight-" + chart + "-value\" data-key=\"" + Regex.Escape(WebUtility.HtmlEncode(key)) + "\"[^>]*>").Value; + await Assert.That(tag).Contains("data-value=\"" + expected.ToString(CultureInfo.InvariantCulture) + "\""); + } + + private static async Task AssertItemizedAsync(LlmTestServerInputTokenComposition composition, int inputTokens, int wireInputTokens) + { + await Assert.That(composition.SystemTokens).IsGreaterThan(0); + await Assert.That(composition.UserTokens).IsGreaterThan(0); + await Assert.That(composition.AssistantTokens).IsGreaterThan(0); + await Assert.That(composition.ToolCallTokens).IsGreaterThan(0); + await Assert.That(composition.ToolResultTokens).IsGreaterThan(0); + await Assert.That(composition.ToolDefinitionTokens).IsGreaterThan(0); + await Assert.That(composition.OtherTokens).IsEqualTo(0); + await Assert.That(composition.TotalTokens).IsEqualTo(inputTokens); + await Assert.That(inputTokens).IsEqualTo(wireInputTokens); + } + + private static async Task AssertMetricAsync(string page, string metric, int expected) + { + var tag = Regex.Match(page, "<[^>]*data-testid=\"analytics-" + metric + "\"[^>]*>").Value; + await Assert.That(tag).Contains("data-value=\"" + expected.ToString(CultureInfo.InvariantCulture) + "\""); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Hosting/PromptCacheCapacityIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Hosting/PromptCacheCapacityIntegrationTests.cs new file mode 100644 index 0000000..f73274a --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Hosting/PromptCacheCapacityIntegrationTests.cs @@ -0,0 +1,70 @@ +using System.Net.Http.Json; +using System.Text.Json; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.Hosting; + +public sealed class PromptCacheCapacityIntegrationTests +{ + [Test] + [Arguments(1)] + [Arguments(0)] + public async Task RepeatedMultiPrefixRequest_ReadsRetainedLongestPrefixBeforeCapacityEvictionAsync(int maximumEntries) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + var configuration = new LlmTestServerClientConfigurationBuilder().UseGpt41Mini() + .UseChatScenario("prefix-capacity", scenario => scenario + .Responds("first intact response") + .Responds("second intact response") + .Responds("third intact response")) + .Build() with + { MaxPromptCacheEntries = maximumEntries }; + await control.ConfigureAsync(configuration); + var request = new + { + model = LlmTestServerKnownModelIds.Gpt41Mini, + messages = new[] + { + new { role = "system", content = string.Join(' ', Enumerable.Repeat("System reference", 1200)) }, + new { role = "user", content = string.Join(' ', Enumerable.Repeat("User details", 1200)) }, + }, + }; + var responses = new List(); + foreach (var expectedText in new[] { "first intact response", "second intact response", "third intact response" }) + { + using var response = await http.PostAsJsonAsync("/openai/v1/chat/completions", request); + response.EnsureSuccessStatusCode(); + var body = await response.Content.ReadFromJsonAsync(); + await Assert.That(body.GetProperty("choices")[0].GetProperty("message").GetProperty("content").GetString()).IsEqualTo(expectedText); + responses.Add(body); + } + + var inputTokens = responses[0].GetProperty("usage").GetProperty("prompt_tokens").GetInt32(); + await Assert.That(inputTokens).IsGreaterThanOrEqualTo(2048); + await Assert.That(CachedTokens(responses[0])).IsEqualTo(0); + var expectedCachedTokens = maximumEntries == 0 ? 0 : inputTokens / 128 * 128; + await Assert.That(CachedTokens(responses[1])).IsEqualTo(expectedCachedTokens); + await Assert.That(CachedTokens(responses[2])).IsEqualTo(expectedCachedTokens); + var evidence = await control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending }); + await Assert.That(evidence.Requests.Count).IsEqualTo(3); + for (var index = 0; index < responses.Count; index++) + { + var usage = evidence.Requests[index].Usage!; + await Assert.That(evidence.Requests[index].StatusCode).IsEqualTo(200); + await Assert.That(usage.InputTokens).IsEqualTo(inputTokens); + await Assert.That(usage.CachedInputTokens).IsEqualTo(CachedTokens(responses[index])); + await Assert.That(usage.CacheCreationInputTokens).IsEqualTo(index == 0 ? expectedCachedTokens : 0); + await Assert.That(usage.TotalTokens).IsEqualTo(usage.InputTokens + usage.OutputTokens); + } + } + + private static int CachedTokens(JsonElement response) + { + return response.GetProperty("usage").GetProperty("prompt_tokens_details").GetProperty("cached_tokens").GetInt32(); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Hosting/PromptCacheNamespaceIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Hosting/PromptCacheNamespaceIntegrationTests.cs new file mode 100644 index 0000000..c605c78 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Hosting/PromptCacheNamespaceIntegrationTests.cs @@ -0,0 +1,89 @@ +using System.Net.Http.Json; +using System.Text.Json; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.Hosting; + +public sealed class PromptCacheNamespaceIntegrationTests +{ + private static readonly string[] _paths = + [ + "/openai/v1/chat/completions", + "/groq/openai/v1/chat/completions", + "/azure-openai/openai/deployments/gpt-4.1-mini/chat/completions?api-version=2024-10-21", + ]; + + [Test] + public async Task PromptCache_SeparatesProviderNamespacesBeforeWarmingEachIndependentlyAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(config => config.AddChatScenario("namespace-cache", scenario => scenario + .Responds("one").Responds("two").Responds("three").Responds("four").Responds("five").Responds("six"))); + using var http = host.GetTestClient(); + foreach (var path in _paths) + { + await Assert.That(await CachedAsync(http, path)).IsEqualTo(0); + } + foreach (var path in _paths) + { + await Assert.That(await CachedAsync(http, path)).IsGreaterThanOrEqualTo(1024); + } + var evidence = await new LlmTestServerClient(http).GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending }); + await Assert.That(evidence.Requests.Count).IsEqualTo(6); + await Assert.That(evidence.Requests.Select(trace => trace.ProviderNamespace).Distinct().Count()).IsEqualTo(3); + foreach (var trace in evidence.Requests.Take(3)) + { + await Assert.That(trace.Usage!.CachedInputTokens).IsEqualTo(0); + await Assert.That(trace.Usage.CacheCreationInputTokens).IsGreaterThanOrEqualTo(1024); + } + foreach (var trace in evidence.Requests.Skip(3)) + { + await Assert.That(trace.Usage!.CachedInputTokens).IsGreaterThanOrEqualTo(1024); + await Assert.That(trace.Usage.CacheCreationInputTokens).IsEqualTo(0); + } + } + + [Test] + public async Task PromptCache_SeparatesSessionsAndProvidersWithIdenticalKeysAndPrefixesAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + await using var first = await root.CreateSessionAsync(config => config.UseChatScenario("namespace-cache", scenario => scenario + .Responds("one").Responds("two").Responds("three").Responds("four"))); + await using var second = await root.CreateSessionAsync(config => config.UseChatScenario("namespace-cache", scenario => scenario + .Responds("one").Responds("two").Responds("three").Responds("four"))); + foreach (var session in new[] { first, second }) + { + await Assert.That(await CachedAsync(session.HttpClient, _paths[0])).IsEqualTo(0); + await Assert.That(await CachedAsync(session.HttpClient, _paths[1])).IsEqualTo(0); + } + foreach (var session in new[] { first, second }) + { + await Assert.That(await CachedAsync(session.HttpClient, _paths[0])).IsGreaterThanOrEqualTo(1024); + await Assert.That(await CachedAsync(session.HttpClient, _paths[1])).IsGreaterThanOrEqualTo(1024); + } + await Assert.That((await root.GetRequestsAsync()).Requests).IsEmpty(); + var firstEvidence = await first.Control.GetRequestsAsync(); + var secondEvidence = await second.Control.GetRequestsAsync(); + await Assert.That(firstEvidence.Requests.Count).IsEqualTo(4); + await Assert.That(secondEvidence.Requests.Count).IsEqualTo(4); + await Assert.That(firstEvidence.Requests.Select(trace => trace.SessionId).Distinct().Single()) + .IsNotEqualTo(secondEvidence.Requests.Select(trace => trace.SessionId).Distinct().Single()); + } + + private static async Task CachedAsync(HttpClient http, string path) + { + using var response = await http.PostAsJsonAsync(path, new + { + model = LlmTestServerKnownModelIds.Gpt41Mini, + prompt_cache_key = "same-cache-key", + messages = new[] { new { role = "user", content = string.Concat(Enumerable.Repeat("A stable shared provider prefix. ", 400)) } }, + }); + response.EnsureSuccessStatusCode(); + var json = await response.Content.ReadFromJsonAsync(); + return json.GetProperty("usage").GetProperty("prompt_tokens_details").GetProperty("cached_tokens").GetInt32(); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Hosting/RequestJournalIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Hosting/RequestJournalIntegrationTests.cs new file mode 100644 index 0000000..92588c1 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Hosting/RequestJournalIntegrationTests.cs @@ -0,0 +1,111 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Text.Json; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Hosting; +using ManagedCode.LlmTestServer.Providers; +using ManagedCode.LlmTestServer.Tests.TestSupport; +using Microsoft.Extensions.DependencyInjection; + +namespace ManagedCode.LlmTestServer.Tests.Hosting; + +public sealed class RequestJournalIntegrationTests +{ + [Test] + public async Task Journal_FiltersAndPagesActualProviderRequestsAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(options => options.AddChatScenario("requests", + scenario => scenario.Responds("first").Responds("second").Fails(429, "too_many_requests", "retry later"))); + using var http = host.GetTestClient(); + foreach (var path in new[] { "/openai/v1/chat/completions", "/mistral/v1/chat/completions", "/openai/v1/chat/completions" }) + { + using var response = await http.PostAsJsonAsync(path, new + { + model = "gpt-4.1-mini", + messages = new[] { new { role = "user", content = "journal" } }, + }); + } + using var page = await http.GetAsync("/admin-api/requests?provider=openai&model=gpt-4.1-mini&offset=0&limit=1"); + page.EnsureSuccessStatusCode(); + await Assert.That(page.Headers.GetValues("X-Total-Count").Single()).IsEqualTo("2"); + var entries = (await page.Content.ReadFromJsonAsync>())!; + await Assert.That(entries.Count).IsEqualTo(1); + var entry = entries.Single(); + await Assert.That(entry.StatusCode).IsEqualTo(429); + await Assert.That(entry.ProviderId).IsEqualTo(LlmTestServerCompatibilityTags.OpenAI); + await Assert.That(entry.ErrorCode).IsEqualTo("too_many_requests"); + + var matching = (await http.GetFromJsonAsync>( + $"/admin-api/requests?requestId={entry.RequestId}&status=429"))!; + await Assert.That(matching.Single().RequestId).IsEqualTo(entry.RequestId); + var next = (await http.GetFromJsonAsync>( + "/admin-api/requests?provider=openai&status=200&offset=0&limit=1"))!; + await Assert.That(next.Single().RuntimeResponse).IsEqualTo("first"); + var empty = (await http.GetFromJsonAsync>( + "/admin-api/requests?model=unknown&offset=10&limit=1"))!; + await Assert.That(empty).IsEmpty(); + using var reset = await http.PostAsync("/admin-api/reset", null); + reset.EnsureSuccessStatusCode(); + await Assert.That(await http.GetFromJsonAsync>("/admin-api/requests")).IsEmpty(); + } + + [Test] + [Arguments("offset=-1")] + [Arguments("limit=0")] + [Arguments("limit=501")] + [Arguments("status=99")] + [Arguments("status=600")] + [Arguments("offset=invalid")] + public async Task Journal_RejectsInvalidQueryParametersAsync(string query) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + using var response = await http.GetAsync("/admin-api/requests?" + query); + await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + } + + [Test] + public async Task Journal_RequiresTheConfiguredControlCredentialAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(options => options.RequireBearerToken("journal-key")); + using var http = host.GetTestClient(); + using var rejected = await http.GetAsync("/admin-api/requests"); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.Unauthorized); + http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", "journal-key"); + using var accepted = await http.GetAsync("/admin-api/requests"); + await Assert.That(accepted.StatusCode).IsEqualTo(HttpStatusCode.OK); + } + + [Test] + public async Task Journal_BoundsRetentionWithoutTruncatingActualProviderResponsesAsync() + { + var text = new string('x', 16_000); + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddChatScenario("large", scenario => scenario.Responds(text).Responds(text).Responds(text)), + configureServices: services => services.Configure(options => + { + options.MaxEntries = 2; + options.MaxPreviewBytes = 128; + options.MaxRetainedBytes = 8192; + })); + using var http = host.GetTestClient(); + for (var index = 0; index < 3; index++) + { + using var response = await http.PostAsJsonAsync("/openai/v1/chat/completions", new + { + model = "gpt-4.1-mini", + messages = new[] { new { role = "user", content = text } }, + }); + response.EnsureSuccessStatusCode(); + var body = await response.Content.ReadFromJsonAsync(); + await Assert.That(body.GetProperty("choices")[0].GetProperty("message").GetProperty("content").GetString()) + .IsEqualTo(text); + } + var entries = (await http.GetFromJsonAsync>("/admin-api/requests"))!; + await Assert.That(entries.Count).IsEqualTo(2); + await Assert.That(entries.Sum(entry => entry.RetainedByteCount)).IsLessThanOrEqualTo(8192); + await Assert.That(entries.All(entry => entry.RequestPreviewTruncated && entry.ResponsePreviewTruncated)).IsTrue(); + await Assert.That(entries.All(entry => entry.RetentionTruncated)).IsTrue(); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Hosting/RequestQueryExportIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Hosting/RequestQueryExportIntegrationTests.cs new file mode 100644 index 0000000..ec464a7 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Hosting/RequestQueryExportIntegrationTests.cs @@ -0,0 +1,216 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Text; +using System.Text.Json; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Hosting; +using ManagedCode.LlmTestServer.Tests.TestSupport; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Hosting.Server; +using Microsoft.AspNetCore.Hosting.Server.Features; +using Microsoft.Extensions.DependencyInjection; + +namespace ManagedCode.LlmTestServer.Tests.Hosting; + +public sealed class RequestQueryExportIntegrationTests +{ + [Test] + public async Task QueryExport_FiltersOrdersAndExportsTheSameRetainedRequestsAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(config => config.AddChatScenario("query", scenario => scenario.Responds("one").Fails(429, "retry", "later"))); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + using var conversation = control.ForConversation("query-conversation"); + var before = DateTimeOffset.UtcNow.AddSeconds(-1); + using var first = await conversation.HttpClient.PostAsJsonAsync("/openai/v1/chat/completions", new { model = "gpt-4.1-mini", messages = new[] { new { role = "user", content = "hello" } } }); + first.EnsureSuccessStatusCode(); + using var second = await conversation.HttpClient.PostAsJsonAsync("/openai/v1/chat/completions", new { model = "gpt-4.1-mini", messages = new[] { new { role = "user", content = "hello" } } }); + await Assert.That(second.StatusCode).IsEqualTo(HttpStatusCode.TooManyRequests); + var all = await control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending }); + await Assert.That(all.TotalObserved).IsEqualTo(2L); + await Assert.That(all.Requests.Select(entry => entry.Sequence).ToArray()).IsEquivalentTo(new long[] { 1, 2 }); + var query = new LlmTestServerRequestQuery + { + Provider = "openai", + Model = "gpt-4.1-mini", + Scenario = "query", + Conversation = "query-conversation", + Operation = all.Requests[0].OperationId, + Completed = true, + Streaming = false, + Success = false, + ErrorCode = "retry", + Status = 429, + From = before, + To = DateTimeOffset.UtcNow.AddSeconds(1) + }; + var selected = await control.GetRequestsAsync(query); + await Assert.That(selected.Requests.Single().Sequence).IsEqualTo(2L); + var completed = selected.Requests.Single(); + await Assert.That(completed.StartedMonotonicSeconds).IsNotNull(); + await Assert.That(completed.CompletedMonotonicSeconds).IsNotNull(); + await Assert.That(completed.CompletedMonotonicSeconds!.Value >= completed.StartedMonotonicSeconds!.Value).IsTrue(); + await Assert.That(completed.StartedMonotonicSeconds.Value >= all.Requests[0].CompletedMonotonicSeconds!.Value).IsTrue(); + var exported = await control.ExportRequestsAsync(query); + await Assert.That(exported.SchemaVersion).IsEqualTo(1); + await Assert.That(exported.Evidence.Requests.Single().RequestId).IsEqualTo(selected.Requests.Single().RequestId); + var lines = (await control.ExportRequestsNdjsonAsync(query)).Split('\n', StringSplitOptions.RemoveEmptyEntries); + await Assert.That(lines.Length).IsEqualTo(2); + using var metadata = JsonDocument.Parse(lines[0]); + using var record = JsonDocument.Parse(lines[1]); + await Assert.That(metadata.RootElement.GetProperty("type").GetString()).IsEqualTo("metadata"); + await Assert.That(record.RootElement.GetProperty("request").GetProperty("sequence").GetInt64()).IsEqualTo(2L); + await Assert.That((await control.GetRequestsAsync(query with { Step = "absent" })).Requests).IsEmpty(); + await Assert.That((await control.GetRequestsAsync(new() { Offset = 1, Limit = 1, Order = LlmTestServerRequestOrder.Ascending })).Requests.Single().Sequence).IsEqualTo(2L); + } + + [Test] + public async Task QueryExport_ReportsEvictionAndResetsEvidenceCountersAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(config => config.AddChatScenario("bounded", scenario => scenario.Responds("one").Responds("two")), + configureServices: services => services.Configure(options => { options.MaxEntries = 1; options.MaxPreviewBytes = 64; })); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + for (var index = 0; index < 2; index++) + { + using var response = await http.PostAsJsonAsync("/openai/v1/chat/completions", new { model = "gpt-4.1-mini", messages = new[] { new { role = "user", content = "hello" } } }); + response.EnsureSuccessStatusCode(); + } + var export = await control.ExportRequestsAsync(); + await Assert.That(export.Evidence.TotalObserved).IsEqualTo(2L); + await Assert.That(export.Evidence.EvictedCount).IsEqualTo(1L); + await Assert.That(export.Evidence.RetentionTruncated).IsTrue(); + await Assert.That(export.Evidence.FirstSequence).IsEqualTo(2L); + await control.ResetAsync(); + var reset = await control.ExportRequestsAsync(); + await Assert.That(reset.Evidence.TotalObserved).IsEqualTo(0L); + await Assert.That(reset.Evidence.EvictedCount).IsEqualTo(0L); + await Assert.That(reset.Evidence.Requests).IsEmpty(); + } + + [Test] + [Arguments("order=invalid")] + [Arguments("completed=invalid")] + [Arguments("streaming=invalid")] + [Arguments("success=invalid")] + [Arguments("from=invalid")] + [Arguments("to=invalid")] + [Arguments("from=2026-10-21T00:00:00Z&to=2026-10-20T00:00:00Z")] + [Arguments("limit=1&limit=2")] + [Arguments("provider=")] + [Arguments("format=xml")] + public async Task QueryExport_RejectsInvalidBoundariesThroughHttpAsync(string query) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + using var response = await http.GetAsync("/admin-api/requests/export?" + query); + await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + } + + [Test] + public async Task QueryExport_SelectsActiveRequestsAndConversationStepsAsync() + { + var builder = WebApplication.CreateBuilder(); + builder.WebHost.ConfigureKestrel(options => + { + options.Listen(IPAddress.Loopback, 0); + options.Limits.MinRequestBodyDataRate = null; + }); + builder.Services.AddLlmTestServer(config => config.AddConversation("step-query", graph => graph + .Step("finish", step => step.WhenLatestUserEquals("start").Responds("done").Complete()))); + await using var app = builder.Build(); + app.MapLlmTestServer(); + await app.StartAsync(); + var addresses = app.Services.GetRequiredService().Features.Get()!; + using var http = new HttpClient { BaseAddress = new Uri(addresses.Addresses.Single()) }; + var control = new LlmTestServerClient(http); + using var conversation = control.ForConversation("step-query-id"); + using var cancellation = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await control.GetRequestsAsync(cancellationToken: cancellation.Token); + using var body = new GatedJsonContent("""{"model":"gpt-4.1-mini","messages":[{"role":"user","content":"start"}]}"""); + var pending = conversation.HttpClient.PostAsync("/openai/v1/chat/completions", body, cancellation.Token); + try + { + LlmTestServerRequestSnapshot active = new(); + while (true) + { + active = await control.GetRequestsAsync(new() { Completed = false }, cancellation.Token); + if (active.Requests.Count > 0) { break; } + await Task.Delay(10, cancellation.Token); + } + await Assert.That(active.Requests.Count).IsEqualTo(1); + await Assert.That(active.Requests.Single().IsSuccess).IsNull(); + await Assert.That(active.Requests.Single().StartedMonotonicSeconds).IsNotNull(); + await Assert.That(active.Requests.Single().CompletedMonotonicSeconds).IsNull(); + await Assert.That(pending.IsCompleted).IsFalse(); + body.Release(); + using var response = await pending; + response.EnsureSuccessStatusCode(); + await Assert.That(await response.Content.ReadAsStringAsync(cancellation.Token)).Contains("done"); + var finished = await control.GetRequestsAsync(new() { Scenario = "step-query", Conversation = "step-query-id", Step = "finish", Completed = true, Success = true }, cancellation.Token); + await Assert.That(finished.Requests.Count).IsEqualTo(1); + await Assert.That(finished.Requests.Single().RequestId).IsEqualTo(active.Requests.Single().RequestId); + await Assert.That((await control.GetRequestsAsync(new() { Completed = false }, cancellation.Token)).Requests).IsEmpty(); + } + finally + { + body.Release(); + await cancellation.CancelAsync(); + try { using var cleanup = await pending; } + catch (OperationCanceledException) { } + catch (HttpRequestException) { } + await app.StopAsync(); + } + } + + private sealed class GatedJsonContent : HttpContent + { + private readonly byte[] _bytes; + private readonly TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public GatedJsonContent(string json) + { + _bytes = Encoding.UTF8.GetBytes(json); + Headers.ContentType = new MediaTypeHeaderValue("application/json"); + } + + public void Release() + { + _release.TrySetResult(); + } + + protected override bool TryComputeLength(out long length) + { + length = _bytes.Length; + return true; + } + + protected override Task SerializeToStreamAsync(Stream stream, TransportContext? context) + { + return SerializeToStreamAsync(stream, context, CancellationToken.None); + } + + protected override async Task SerializeToStreamAsync(Stream stream, TransportContext? context, CancellationToken cancellationToken) + { + await stream.WriteAsync(_bytes.AsMemory(0, 1), cancellationToken); + await stream.FlushAsync(cancellationToken); + await _release.Task.WaitAsync(cancellationToken); + await stream.WriteAsync(_bytes.AsMemory(1), cancellationToken); + } + } + + [Test] + public async Task QueryExport_RequiresControlAuthorizationAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(config => config.RequireBearerToken("export-key")); + using var http = host.GetTestClient(); + using var rejected = await http.GetAsync("/admin-api/requests/export"); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.Unauthorized); + http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", "export-key"); + var exported = await new LlmTestServerClient(http, "export-key").ExportRequestsAsync(); + await Assert.That(exported.Evidence.Requests).IsEmpty(); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Hosting/ServiceHealthIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Hosting/ServiceHealthIntegrationTests.cs new file mode 100644 index 0000000..cf90b26 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Hosting/ServiceHealthIntegrationTests.cs @@ -0,0 +1,30 @@ +using System.Net; +using ManagedCode.LlmTestServer.Hosting; +using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.Hosting; + +namespace ManagedCode.LlmTestServer.Tests.Hosting; + +public sealed class ServiceHealthIntegrationTests +{ + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task ServiceDefaults_ServeHealthAndLivenessOverHttpAsync(bool healthChecksOnly) + { + var builder = WebApplication.CreateBuilder(new WebApplicationOptions { EnvironmentName = Environments.Development }); + builder.WebHost.UseTestServer(); + if (healthChecksOnly) { builder.AddDefaultHealthChecks(); } + else { builder.AddServiceDefaults(); } + await using var app = builder.Build(); + app.MapDefaultEndpoints(); + await app.StartAsync(); + using var http = app.GetTestClient(); + foreach (var path in new[] { "/health", "/alive" }) + { + using var response = await http.GetAsync(path); + await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.OK); + await Assert.That(await response.Content.ReadAsStringAsync()).IsEqualTo("Healthy"); + } + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Hosting/StoredResponseAdmissionIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Hosting/StoredResponseAdmissionIntegrationTests.cs new file mode 100644 index 0000000..b206dfc --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Hosting/StoredResponseAdmissionIntegrationTests.cs @@ -0,0 +1,149 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.Hosting; + +public sealed class StoredResponseAdmissionIntegrationTests +{ + [Test] + [Arguments(0, 1048576L)] + [Arguments(8, 1L)] + public async Task FailedStorageAdmission_DoesNotWarmPromptCacheOrConsumeFixtureAsync(int maximumResponses, long maximumBytes) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(config => config.UseGpt41Mini() + .UseChatResponseCapacity(maximumResponses, maximumBytes) + .UseChatScenario("admission", scenario => scenario.Responds("first queued answer").Responds("second queued answer"))); + var prompt = CacheablePrefix("Admission reference"); + + using var rejected = await http.PostAsJsonAsync("/openai/v1/responses", Request(prompt, store: true)); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + var error = await rejected.Content.ReadFromJsonAsync(); + await Assert.That(error.GetProperty("error").GetProperty("code").GetString()).IsEqualTo("llm_test_server_response_capacity_exceeded"); + + var admitted = await CompleteAsync(http, prompt); + await Assert.That(Text(admitted)).IsEqualTo("first queued answer"); + await Assert.That(CachedTokens(admitted)).IsEqualTo(0); + await Assert.That(admitted.GetProperty("usage").GetProperty("input_tokens").GetInt32()).IsGreaterThanOrEqualTo(1024); + var repeated = await CompleteAsync(http, prompt); + await Assert.That(Text(repeated)).IsEqualTo("second queued answer"); + await Assert.That(CachedTokens(repeated)).IsGreaterThanOrEqualTo(1024); + + var evidence = await control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending }); + await Assert.That(evidence.Requests.Count).IsEqualTo(3); + await Assert.That(evidence.Requests[0].StatusCode).IsEqualTo(409); + await Assert.That(evidence.Requests[1].StatusCode).IsEqualTo(200); + await Assert.That(evidence.Requests[2].StatusCode).IsEqualTo(200); + await Assert.That(evidence.Requests[1].Usage!.CachedInputTokens).IsEqualTo(0); + await Assert.That(evidence.Requests[1].Usage!.CacheCreationInputTokens).IsGreaterThanOrEqualTo(1024); + await Assert.That(evidence.Requests[2].Usage!.CachedInputTokens).IsEqualTo(CachedTokens(repeated)); + } + + [Test] + [Arguments(0, 1048576L)] + [Arguments(8, 1L)] + public async Task FailedStorageAdmission_DoesNotEvictPreviouslyWarmedPrefixAsync(int maximumResponses, long maximumBytes) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + var configuration = new LlmTestServerClientConfigurationBuilder().UseGpt41Mini() + .UseChatResponseCapacity(maximumResponses, maximumBytes) + .UseChatScenario("admission-eviction", scenario => scenario.Responds("warm prefix A").Responds("preserved queued answer")) + .Build() with + { MaxPromptCacheEntries = 1 }; + await control.ConfigureAsync(configuration); + var firstPrefix = CacheablePrefix("Alpha retained reference"); + var rejectedPrefix = CacheablePrefix("Zulu rejected reference"); + + var warm = await CompleteAsync(http, firstPrefix); + await Assert.That(Text(warm)).IsEqualTo("warm prefix A"); + await Assert.That(CachedTokens(warm)).IsEqualTo(0); + await Assert.That(warm.GetProperty("usage").GetProperty("input_tokens").GetInt32()).IsGreaterThanOrEqualTo(1024); + using var rejected = await http.PostAsJsonAsync("/openai/v1/responses", Request(rejectedPrefix, store: true)); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + var error = await rejected.Content.ReadFromJsonAsync(); + await Assert.That(error.GetProperty("error").GetProperty("code").GetString()).IsEqualTo("llm_test_server_response_capacity_exceeded"); + + var preserved = await CompleteAsync(http, firstPrefix); + await Assert.That(Text(preserved)).IsEqualTo("preserved queued answer"); + await Assert.That(CachedTokens(preserved)).IsGreaterThanOrEqualTo(1024); + var evidence = await control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending }); + await Assert.That(evidence.Requests.Count).IsEqualTo(3); + await Assert.That(evidence.Requests[0].Usage!.CacheCreationInputTokens).IsGreaterThanOrEqualTo(1024); + await Assert.That(evidence.Requests[2].Usage!.CachedInputTokens).IsEqualTo(CachedTokens(preserved)); + await Assert.That(evidence.Requests[2].Usage!.CacheCreationInputTokens).IsEqualTo(0); + } + + [Test] + [Arguments(0, 1048576L)] + [Arguments(8, 1L)] + public async Task FailedStorageAdmission_RollsBackConversationWithoutWarmingCacheAsync(int maximumResponses, long maximumBytes) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(config => config.UseGpt41Mini() + .UseChatResponseCapacity(maximumResponses, maximumBytes) + .UseConversation("admission-graph", graph => graph + .Step("answer", step => step.WhenTurn(1).Responds("unconsumed graph answer").Complete()))); + var prefix = CacheablePrefix("Conversation admission reference"); + using var rejected = await http.PostAsJsonAsync("/openai/v1/responses", Request(prefix, store: true)); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + var error = await rejected.Content.ReadFromJsonAsync(); + await Assert.That(error.GetProperty("error").GetProperty("code").GetString()).IsEqualTo("llm_test_server_response_capacity_exceeded"); + await Assert.That((await control.GetConversationsAsync()).Count).IsEqualTo(0); + + var admitted = await CompleteAsync(http, prefix); + await Assert.That(Text(admitted)).IsEqualTo("unconsumed graph answer"); + await Assert.That(CachedTokens(admitted)).IsEqualTo(0); + var state = (await control.GetConversationsAsync()).Single(); + await Assert.That(state.ExecutedSteps).IsEqualTo(1); + var evidence = await control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending }); + await Assert.That(evidence.Requests.Count).IsEqualTo(2); + await Assert.That(evidence.Requests[1].Usage!.CacheCreationInputTokens).IsGreaterThanOrEqualTo(1024); + await Assert.That(evidence.Requests[1].StepId).IsEqualTo("answer"); + } + + private static string CacheablePrefix(string reference) + { + return string.Join(' ', Enumerable.Repeat(reference, 1200)); + } + + private static object Request(string prefix, bool store) + { + return new + { + model = LlmTestServerKnownModelIds.Gpt41Mini, + instructions = prefix, + input = "Return the queued answer.", + store, + }; + } + + private static async Task CompleteAsync(HttpClient http, string prefix) + { + using var response = await http.PostAsJsonAsync("/openai/v1/responses", Request(prefix, store: false)); + response.EnsureSuccessStatusCode(); + return await response.Content.ReadFromJsonAsync(); + } + + private static int CachedTokens(JsonElement response) + { + return response.GetProperty("usage") + .GetProperty("input_tokens_details").GetProperty("cached_tokens").GetInt32(); + } + + private static string? Text(JsonElement response) + { + return response.GetProperty("output")[0] + .GetProperty("content")[0].GetProperty("text").GetString(); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/ManagedCode.LlmTestServer.Tests.csproj b/tests/ManagedCode.LlmTestServer.Tests/ManagedCode.LlmTestServer.Tests.csproj new file mode 100644 index 0000000..7bf94e6 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/ManagedCode.LlmTestServer.Tests.csproj @@ -0,0 +1,64 @@ + + + + true + true + DnxPinned + Exe + false + false + true + false + true + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/ManagedCode.LlmTestServer.Tests/MicrosoftExtensionsAI/MicrosoftExtensionsAiClientTests.cs b/tests/ManagedCode.LlmTestServer.Tests/MicrosoftExtensionsAI/MicrosoftExtensionsAiClientTests.cs new file mode 100644 index 0000000..bc61b3a --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/MicrosoftExtensionsAI/MicrosoftExtensionsAiClientTests.cs @@ -0,0 +1,54 @@ +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Tests.TestSupport; +using Microsoft.Extensions.AI; + +namespace ManagedCode.LlmTestServer.Tests.MicrosoftExtensionsAI; + +public sealed class MicrosoftExtensionsAiClientTests +{ + [Test] + public async Task MicrosoftExtensionsAiClients_InvokeLlmTestServerThroughOpenAiCompatibilityEndpointsAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(options => options + .WithDefaultEmbeddingVector(0.125f, 0.25f, 0.5f) + .AddChatScenario( + "extensions-ai-blue-whale", + scenario => scenario + .ForModel(LlmTestServerKnownModelIds.Gpt41Mini) + .WhenUserContains("largest animal") + .Responds("blue whale", "blue ", "whale") + .Responds("blue whale", "blue ", "whale") + )); + using var httpClient = host.GetTestClient(); + using IChatClient chatClient = new LlmTestServerChatClient(httpClient); + using IEmbeddingGenerator> embeddingGenerator = + new LlmTestServerEmbeddingGenerator(httpClient); + using IImageGenerator imageGenerator = new LlmTestServerImageGenerator(httpClient); + + var chatResponse = await chatClient.GetResponseAsync( + [new ChatMessage(ChatRole.User, "What is the largest animal?")] + ); + var streamUpdates = new List(); + await foreach ( + var update in chatClient.GetStreamingResponseAsync( + [new ChatMessage(ChatRole.User, "What is the largest animal?")] + ) + ) + { + streamUpdates.Add(update.Text); + } + + var embeddings = await embeddingGenerator.GenerateAsync(["alpha", "beta"]); + var image = await imageGenerator.GenerateAsync( + new ImageGenerationRequest { Prompt = "blue compatibility marker" } + ); + + await Assert.That(chatResponse.Text).IsEqualTo("blue whale"); + await Assert.That(string.Concat(streamUpdates)).IsEqualTo("blue whale"); + await Assert.That(embeddings).Count().IsEqualTo(2); + await Assert.That(embeddings[0].Vector.Length).IsEqualTo(3); + await Assert.That(image.Contents).Count().IsEqualTo(1); + await Assert.That(image.Contents[0]).IsTypeOf(); + } +} diff --git a/tests/ManagedCode.LlmTck.Tests/Ollama/OllamaEndpointTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Ollama/OllamaEndpointTests.cs similarity index 88% rename from tests/ManagedCode.LlmTck.Tests/Ollama/OllamaEndpointTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/Ollama/OllamaEndpointTests.cs index 00866e6..c6f676f 100644 --- a/tests/ManagedCode.LlmTck.Tests/Ollama/OllamaEndpointTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Ollama/OllamaEndpointTests.cs @@ -1,9 +1,9 @@ using System.Net.Http.Json; using System.Text.Json; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.Tests.TestSupport; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Tests.TestSupport; -namespace ManagedCode.LlmTck.Tests.Ollama; +namespace ManagedCode.LlmTestServer.Tests.Ollama; public sealed class OllamaEndpointTests { @@ -12,8 +12,8 @@ public sealed class OllamaEndpointTests [Test] public async Task ChatEndpoint_WithStreamFalse_ReturnsOllamaChatShapeAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("ollama-chat", LlmTckModelKind.Chat) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("ollama-chat", LlmTestServerModelKind.Chat) .AddChatScenario( "ollama-blue-whale", scenario => scenario @@ -51,8 +51,8 @@ await Assert.That(payload.GetProperty("message").GetProperty("content").GetStrin [Test] public async Task ChatEndpoint_DefaultStreamsOllamaJsonLinesAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("ollama-chat", LlmTckModelKind.Chat) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("ollama-chat", LlmTestServerModelKind.Chat) .AddChatScenario( "ollama-stream", scenario => scenario @@ -103,8 +103,8 @@ await Assert.That(JsonDocument.Parse(lines[^1]).RootElement.GetProperty("eval_co [Test] public async Task EmbedEndpoint_ReturnsOllamaEmbeddingShapeAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("ollama-embed", LlmTckModelKind.Embedding) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("ollama-embed", LlmTestServerModelKind.Embedding) .WithDefaultEmbeddingVector(0.25f, 0.5f)); using var client = host.GetTestClient(); diff --git a/tests/ManagedCode.LlmTestServer.Tests/OpenAI/ModalityFailureIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/ModalityFailureIntegrationTests.cs new file mode 100644 index 0000000..92830ba --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/ModalityFailureIntegrationTests.cs @@ -0,0 +1,82 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.OpenAI; + +public sealed class ModalityFailureIntegrationTests +{ + [Test] + [Arguments("images/generations", "gpt-image-1")] + [Arguments("audio/speech", "gpt-4o-mini-tts")] + [Arguments("audio/transcriptions", "gpt-4o-mini-tts")] + [Arguments("audio/translations", "gpt-4o-mini-tts")] + [Arguments("videos", "sora-2")] + [Arguments("embeddings", "text-embedding-3-small")] + public async Task Modality_RejectsCredentialsAndWrongModelWithoutConsumingSuccessAsync(string operation, string model) + { + using var host = await LlmTestServerTestHost.StartAsync(config => config.RequireBearerToken("fixture-key")); + using var http = host.GetTestClient(); + using var rejected = await SendAsync(http, operation, model, "hello"); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.Unauthorized); + await Assert.That(await rejected.Content.ReadAsStringAsync()).Contains("invalid_api_key"); + http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", "fixture-key"); + foreach (var invalidModel in new[] { "missing-model", "gpt-4.1-mini" }) + { + using var unknown = await SendAsync(http, operation, invalidModel, "hello"); + await Assert.That(unknown.StatusCode).IsEqualTo(HttpStatusCode.NotFound); + await Assert.That(await unknown.Content.ReadAsStringAsync()).Contains("llm_test_server_unknown_model"); + } + using var success = await SendAsync(http, operation, model, "hello"); + success.EnsureSuccessStatusCode(); + var summary = (await http.GetFromJsonAsync("/admin-api/assertions"))!; + await Assert.That(summary.AuthFailed).IsEqualTo(1); + await Assert.That(summary.ModelNotFound).IsEqualTo(2); + await Assert.That(summary.Matched).IsEqualTo(1); + } + + [Test] + [Arguments("images/generations", "gpt-image-1")] + [Arguments("audio/speech", "gpt-4o-mini-tts")] + [Arguments("audio/transcriptions", "gpt-4o-mini-tts")] + [Arguments("audio/translations", "gpt-4o-mini-tts")] + [Arguments("videos", "sora-2")] + [Arguments("embeddings", "text-embedding-3-small")] + public async Task Modality_AppliesContentFilterAndRateLimitAndRestoresBudgetOnResetAsync(string operation, string model) + { + using var host = await LlmTestServerTestHost.StartAsync(config => config + .SimulateContentFilter("blocked-fixture").SimulateRateLimitAfter(2)); + using var http = host.GetTestClient(); + using var filtered = await SendAsync(http, operation, model, "blocked-fixture"); + await Assert.That(filtered.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + await Assert.That(await filtered.Content.ReadAsStringAsync()).Contains("content_filter"); + using var success = await SendAsync(http, operation, model, "allowed"); + success.EnsureSuccessStatusCode(); + using var limited = await SendAsync(http, operation, model, "allowed"); + await Assert.That(limited.StatusCode).IsEqualTo(HttpStatusCode.TooManyRequests); + await Assert.That(await limited.Content.ReadAsStringAsync()).Contains("too_many_requests"); + using var reset = await http.PostAsync("/admin-api/reset", null); + reset.EnsureSuccessStatusCode(); + using var afterReset = await SendAsync(http, operation, model, "allowed"); + afterReset.EnsureSuccessStatusCode(); + var summary = (await http.GetFromJsonAsync("/admin-api/assertions"))!; + await Assert.That(summary.Matched).IsEqualTo(1); + await Assert.That(summary.ErrorsReturned).IsEqualTo(0); + } + + private static async Task SendAsync(HttpClient http, string operation, string model, string text) + { + if (operation is "audio/transcriptions" or "audio/translations") + { + using var form = new MultipartFormDataContent(); + form.Add(new StringContent(model), "model"); + form.Add(new StringContent(text), "prompt"); + form.Add(new ByteArrayContent([82, 73, 70, 70]), "file", "fixture.wav"); + return await http.PostAsync("/openai/v1/" + operation, form); + } + return await http.PostAsJsonAsync("/openai/v1/" + operation, + new { model, prompt = text, input = text, voice = "alloy" }); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeFileCodeCacheIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeFileCodeCacheIntegrationTests.cs new file mode 100644 index 0000000..5242984 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeFileCodeCacheIntegrationTests.cs @@ -0,0 +1,85 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Scenarios; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.OpenAI; + +public sealed partial class NativeFileCodeIntegrationTests +{ + [Test] + [Arguments("filter")] + [Arguments("store")] + [Arguments("ranker")] + [Arguments("score")] + [Arguments("container")] + [Arguments("files")] + public async Task NativeNestedOptions_PartitionPromptCacheAndSurvivePublicExportAsync(string option) + { + var code = option is "container" or "files"; + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + var configuration = Configuration(code, 3); + if (option is "store" or "container" or "files") + { + for (var index = 1; index < 3; index++) + { + var items = Items(code); + if (option == "store") { items[0] = ((LlmTestServerResponseFileSearchItem)items[0]) with { VectorStoreIds = ["vs_changed"] }; } + if (option == "files") { items[0] = ((LlmTestServerResponseCodeInterpreterItem)items[0]) with { InputFileIds = ["file_changed"] }; } + if (option == "container") + { + items[0] = ((LlmTestServerResponseCodeInterpreterItem)items[0]) with { ContainerId = "cntr_changed" }; + items[1] = ((LlmTestServerResponseMessageItem)items[1]) with + { Citations = [new LlmTestServerContainerFileCitation { ContainerId = "cntr_changed", FileId = _generatedFile, Filename = "result.txt", StartIndex = 0, EndIndex = 0 }] }; + } + configuration.ChatScenarios[0].Responses[index] = configuration.ChatScenarios[0].Responses[index] with { OutputItems = [.. items] }; + } + } + await control.ConfigureAsync(configuration); + var body = Request(code, auto: option == "files", input: string.Join(' ', Enumerable.Repeat("stable native option cache reference", 1000))); + var tool = body["tools"]![0]!; + if (option is "ranker" or "score") { tool["ranking_options"] = new JsonObject { ["ranker"] = "auto", ["score_threshold"] = 0.5 }; } + var first = await CompleteAsync(http, code, body); + await Assert.That(first.GetProperty("usage").GetProperty("input_tokens").GetInt32()).IsGreaterThanOrEqualTo(1024); + await Assert.That(first.GetProperty("usage").GetProperty("input_tokens_details").GetProperty("cached_tokens").GetInt32()).IsEqualTo(0); + switch (option) + { + case "filter": tool["filters"] = Comparison(); break; + case "store": tool["vector_store_ids"] = new JsonArray("vs_changed"); break; + case "ranker": tool["ranking_options"]!["ranker"] = "default-2024-11-15"; break; + case "score": tool["ranking_options"]!["score_threshold"] = 0.75; break; + case "container": tool["container"] = "cntr_changed"; break; + case "files": tool["container"]!["file_ids"] = new JsonArray("file_changed"); break; + } + var changed = await CompleteAsync(http, code, body); + await Assert.That(changed.GetProperty("usage").GetProperty("input_tokens_details").GetProperty("cached_tokens").GetInt32()).IsEqualTo(0); + var repeated = await CompleteAsync(http, code, body); + var cached = repeated.GetProperty("usage").GetProperty("input_tokens_details").GetProperty("cached_tokens").GetInt32(); + await Assert.That(cached).IsGreaterThanOrEqualTo(1024); + var exported = await control.ExportRequestsAsync(); + var trace = exported.Evidence.Requests[0]; + await Assert.That(trace.Usage!.CachedInputTokens).IsEqualTo(cached); + await Assert.That(trace.ResponseToolCalls.Count).IsEqualTo(0); + await Assert.That(trace.NativeTools.Count).IsEqualTo(1); + await Assert.That(trace.ResponseOutputItems.Count).IsEqualTo(2); + var normalized = JsonSerializer.SerializeToElement(trace.NativeTools.Single(), _configurationJson); + var options = normalized.GetProperty(code ? "codeInterpreterOptions" : "fileSearchOptions"); + switch (option) + { + case "filter": await Assert.That(options.GetProperty("filters").GetProperty("key").GetString()).IsEqualTo("category"); break; + case "store": await Assert.That(options.GetProperty("vectorStoreIds")[0].GetString()).IsEqualTo("vs_changed"); break; + case "ranker": await Assert.That(options.GetProperty("ranker").GetString()).IsEqualTo("default-2024-11-15"); break; + case "score": await Assert.That(options.GetProperty("scoreThreshold").GetDouble()).IsEqualTo(0.75); break; + case "container": await Assert.That(options.GetProperty("containerId").GetString()).IsEqualTo("cntr_changed"); break; + case "files": await Assert.That(options.GetProperty("fileIds")[0].GetString()).IsEqualTo("file_changed"); break; + } + await Assert.That(((LlmTestServerResponseMessageItem)trace.ResponseOutputItems[1]).Citations.Count).IsEqualTo(1); + var lines = (await control.ExportRequestsNdjsonAsync()).Split('\n', StringSplitOptions.RemoveEmptyEntries); + using var record = JsonDocument.Parse(lines[1]); + await Assert.That(record.RootElement.GetProperty("request").GetProperty("responseOutputItems")[0].GetProperty("kind").GetString()) + .IsEqualTo(code ? "code_interpreter" : "file_search"); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeFileCodeConfigurationIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeFileCodeConfigurationIntegrationTests.cs new file mode 100644 index 0000000..5f4f519 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeFileCodeConfigurationIntegrationTests.cs @@ -0,0 +1,105 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using System.Text.Json.Nodes; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.OpenAI; + +public sealed partial class NativeFileCodeIntegrationTests +{ + [Test] + [Arguments(false, "nullResults")] + [Arguments(false, "nullResult")] + [Arguments(false, "negativeScore")] + [Arguments(false, "largeScore")] + [Arguments(false, "objectAttribute")] + [Arguments(false, "tooManyAttributes")] + [Arguments(false, "longAttributeKey")] + [Arguments(false, "longAttributeValue")] + [Arguments(false, "foreignFileCitation")] + [Arguments(false, "wrongFilename")] + [Arguments(false, "negativeIndex")] + [Arguments(false, "oversizedIndex")] + [Arguments(false, "tooManyResults")] + [Arguments(false, "noStores")] + [Arguments(true, "blankContainer")] + [Arguments(true, "invalidChunks")] + [Arguments(true, "unknownOutput")] + [Arguments(true, "foreignFileCitation")] + [Arguments(true, "foreignContainerCitation")] + [Arguments(true, "reversedRange")] + [Arguments(true, "oversizedRange")] + [Arguments(true, "tooManyOutputs")] + [Arguments(true, "tooManyInputFiles")] + public async Task InvalidFileCodeConfiguration_Returns400AndPreservesOldFixturesAsync(bool code, string defect) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + await new LlmTestServerClient(http).ConfigureAsync(Configuration(code)); + var configuration = JsonSerializer.SerializeToNode(Configuration(code), _configurationJson)!; + var items = configuration["chatScenarios"]![0]!["responses"]![0]!["outputItems"]!; + var native = items[0]!; var message = items[1]!; + switch (defect) + { + case "nullResults": native["results"] = null; break; + case "nullResult": native["results"] = new JsonArray((JsonNode?)null); break; + case "negativeScore": native["results"]![0]!["score"] = -0.1; break; + case "largeScore": native["results"]![0]!["score"] = 1.1; break; + case "objectAttribute": native["results"]![0]!["attributes"]!["invalid"] = new JsonObject(); break; + case "tooManyAttributes": + var attributes = new JsonObject(); for (var index = 0; index < 17; index++) { attributes[$"key{index}"] = "value"; } + native["results"]![0]!["attributes"] = attributes; break; + case "longAttributeKey": native["results"]![0]!["attributes"] = new JsonObject { [new string('x', 65)] = "value" }; break; + case "longAttributeValue": native["results"]![0]!["attributes"]!["category"] = new string('x', 513); break; + case "foreignFileCitation": message["citations"]![0]!["fileId"] = "file_foreign"; break; + case "wrongFilename": message["citations"]![0]!["filename"] = "wrong.txt"; break; + case "negativeIndex": message["citations"]![0]!["index"] = -1; break; + case "oversizedIndex": message["citations"]![0]!["index"] = 1000; break; + case "tooManyResults": native["results"] = new JsonArray(Enumerable.Range(0, 129).Select(_ => native["results"]![0]!.DeepClone()).ToArray()); break; + case "noStores": native["vectorStoreIds"] = new JsonArray(); break; + case "blankContainer": native["containerId"] = ""; break; + case "invalidChunks": native["codeChunks"] = new JsonArray("other code"); break; + case "unknownOutput": native["outputs"]![0]!["kind"] = "other"; break; + case "foreignContainerCitation": message["citations"]![0]!["containerId"] = "cntr_foreign"; break; + case "reversedRange": message["citations"]![0]!["startIndex"] = 2; message["citations"]![0]!["endIndex"] = 1; break; + case "oversizedRange": message["citations"]![0]!["endIndex"] = 1000; break; + case "tooManyOutputs": native["outputs"] = new JsonArray(Enumerable.Range(0, 129).Select(_ => native["outputs"]![0]!.DeepClone()).ToArray()); break; + case "tooManyInputFiles": native["inputFileIds"] = new JsonArray(Enumerable.Range(0, 129).Select(index => JsonValue.Create($"file_{index}")).ToArray()); break; + } + using var rejected = await http.PostAsJsonAsync("/admin-api/configure", configuration); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + var result = await CompleteAsync(http, code); + await Assert.That(result.GetProperty("output")[0].GetProperty("id").GetString()).IsEqualTo(ItemId(code)); + } + + [Test] + [Arguments(false, false, false)] + [Arguments(false, false, true)] + [Arguments(false, true, false)] + [Arguments(false, true, true)] + [Arguments(true, false, false)] + [Arguments(true, false, true)] + [Arguments(true, true, false)] + [Arguments(true, true, true)] + public async Task DisabledOrAbsentTool_Returns409WithoutConsumingQueueOrGraphAsync(bool code, bool graph, bool absent) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + if (graph) + { + await control.ConfigureAsync(config => config.UseChatModel(Model(code)).UseConversation("native-mismatch", flow => flow.ForModel(Model(code)) + .Step("native", step => step.RespondsWithItems(Items(code)).Complete()))); + } + else { await control.ConfigureAsync(Configuration(code)); } + var body = Request(code); body["tool_choice"] = absent ? "auto" : "none"; + if (absent) { body["tools"] = new JsonArray(); } + using var rejected = await http.PostAsJsonAsync("/openai/v1/responses", body); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + await Assert.That((await control.GetConversationsAsync()).Count).IsEqualTo(0); + await CompleteAsync(http, code); + if (graph) { await Assert.That((await control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); } + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeFileCodeFilterIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeFileCodeFilterIntegrationTests.cs new file mode 100644 index 0000000..cb96f78 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeFileCodeFilterIntegrationTests.cs @@ -0,0 +1,118 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json.Nodes; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Scenarios; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.OpenAI; + +public sealed partial class NativeFileCodeIntegrationTests +{ + [Test] + [Arguments("""{"type":"eq","key":"category","value":"fixture"}""")] + [Arguments("""{"type":"ne","key":"category","value":"other"}""")] + [Arguments("""{"type":"gt","key":"year","value":2025}""")] + [Arguments("""{"type":"gte","key":"year","value":2026}""")] + [Arguments("""{"type":"lt","key":"year","value":2027}""")] + [Arguments("""{"type":"lte","key":"year","value":2026}""")] + [Arguments("""{"type":"in","key":"category","value":["fixture","other"]}""")] + [Arguments("""{"type":"nin","key":"category","value":["other"]}""")] + [Arguments("""{"type":"in","key":"year","value":[2026,2027]}""")] + [Arguments("""{"type":"eq","key":"approved","value":true}""")] + [Arguments("""{"type":"and","filters":[{"type":"eq","key":"category","value":"fixture"},{"type":"gte","key":"year","value":2026}]}""")] + [Arguments("""{"type":"or","filters":[{"type":"eq","key":"absent","value":1},{"type":"eq","key":"approved","value":true}]}""")] + public async Task FileFilters_AdmitOnlyMatchingConfiguredResultsAsync(string filter) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + await new LlmTestServerClient(http).ConfigureAsync(Configuration(false)); + var body = Request(false); body["tools"]![0]!["filters"] = JsonNode.Parse(filter); + var result = await CompleteAsync(http, false, body); + await Assert.That(result.GetProperty("output")[0].GetProperty("results")[0].GetProperty("file_id").GetString()).IsEqualTo(_fileId); + } + + [Test] + [Arguments("""{"type":"eq","key":"category","value":"other"}""")] + [Arguments("""{"type":"gt","key":"year","value":2026}""")] + [Arguments("""{"type":"eq","key":"year","value":"2026"}""")] + [Arguments("""{"type":"ne","key":"absent","value":1}""")] + [Arguments("""{"type":"nin","key":"absent","value":[1]}""")] + [Arguments("""{"type":"eq","key":"absent","value":1}""")] + [Arguments("""{"type":"gt","key":"absent","value":1}""")] + [Arguments("""{"type":"and","filters":[{"type":"eq","key":"approved","value":true},{"type":"eq","key":"category","value":"other"}]}""")] + public async Task NonmatchingFileFilters_Return409WithoutConsumingFixtureAsync(string filter) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + await new LlmTestServerClient(http).ConfigureAsync(Configuration(false)); + var body = Request(false); body["tools"]![0]!["filters"] = JsonNode.Parse(filter); + using var rejected = await http.PostAsJsonAsync("/openai/v1/responses", body); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + var result = await CompleteAsync(http, false); + await Assert.That(result.GetProperty("output")[0].GetProperty("id").GetString()).IsEqualTo(ItemId(false)); + } + + [Test] + [Arguments(1, "auto", 0d)] + [Arguments(50, "default-2024-11-15", 0.9d)] + public async Task FileRankingAndResultCount_RespectAcceptedBoundariesAsync(int maximum, string ranker, double threshold) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + await new LlmTestServerClient(http).ConfigureAsync(Configuration(false)); + var body = Request(false); + body["tools"]![0]!["max_num_results"] = maximum; + body["tools"]![0]!["ranking_options"] = new JsonObject { ["ranker"] = ranker, ["score_threshold"] = threshold }; + await CompleteAsync(http, false, body); + } + + [Test] + [Arguments("score")] + [Arguments("maximum")] + [Arguments("store")] + [Arguments("emptyStore")] + public async Task FileFixtureConstraints_Return409BeforeQueueConsumptionAsync(string mismatch) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var configuration = Configuration(false); + if (mismatch == "maximum") + { + var item = (LlmTestServerResponseFileSearchItem)Items(false)[0]; + configuration.ChatScenarios[0].Responses[0] = configuration.ChatScenarios[0].Responses[0] with + { OutputItems = [item with { Results = [item.Results![0], item.Results[0] with { FileId = "file_second" }] }, Items(false)[1]] }; + } + await new LlmTestServerClient(http).ConfigureAsync(configuration); + var body = Request(false); + if (mismatch == "score") { body["tools"]![0]!["ranking_options"] = new JsonObject { ["score_threshold"] = 1d }; } + if (mismatch == "maximum") { body["tools"]![0]!["max_num_results"] = 1; } + if (mismatch == "store") { body["tools"]![0]!["vector_store_ids"] = new JsonArray("vs_foreign"); } + if (mismatch == "emptyStore") { body["tools"]![0]!["vector_store_ids"] = new JsonArray(); } + using var rejected = await http.PostAsJsonAsync("/openai/v1/responses", body); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + await CompleteAsync(http, false); + } + + [Test] + [Arguments(8, 128)] + public async Task FileFilterTree_AdmitsDepthAndNodeBoundaryAsync(int depth, int nodes) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + await new LlmTestServerClient(http).ConfigureAsync(Configuration(false, 2)); + JsonNode nested = Comparison(); + for (var index = 1; index < depth; index++) { nested = new JsonObject { ["type"] = "and", ["filters"] = new JsonArray(nested) }; } + var body = Request(false); body["tools"]![0]!["filters"] = nested; + await CompleteAsync(http, false, body); + var leaves = new JsonArray(); + for (var index = 1; index < nodes; index++) { leaves.Add(Comparison()); } + body = Request(false); body["tools"]![0]!["filters"] = new JsonObject { ["type"] = "and", ["filters"] = leaves }; + await CompleteAsync(http, false, body); + } + + private static JsonObject Comparison() + { + return new() { ["type"] = "eq", ["key"] = "category", ["value"] = "fixture" }; + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeFileCodeHistoryIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeFileCodeHistoryIntegrationTests.cs new file mode 100644 index 0000000..f4d9dd2 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeFileCodeHistoryIntegrationTests.cs @@ -0,0 +1,117 @@ +#pragma warning disable OPENAI001 // Exercise official Responses SDK continuation through HTTP. +using System.Net; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Text.Json; +using System.Text.Json.Nodes; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Scenarios; +using ManagedCode.LlmTestServer.Tests.TestSupport; +using OpenAI.Responses; + +namespace ManagedCode.LlmTestServer.Tests.OpenAI; + +public sealed partial class NativeFileCodeIntegrationTests +{ + [Test] + [Arguments(false, false)] + [Arguments(false, true)] + [Arguments(true, false)] + [Arguments(true, true)] + public async Task OfficialSdk_MixedFunctionRoundPreservesNativeHistoryAndResourceIdsAsync(bool code, bool stored) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + var call = new LlmTestServerToolCall { Id = "call_file_code", Name = "client_fixture", ArgumentsJson = "{}" }; + await control.ConfigureAsync(config => config.UseChatModel(Model(code)).UseConversation("file-code-history", graph => graph.ForModel(Model(code)) + .Step("native", step => step.WhenLatestUserEquals("start").WhenTurn(1).RespondsWithItems( + new LlmTestServerResponseFunctionCallItem { Id = "fc_file_code", Call = call }, Items(code)[0], Items(code)[1]).Then("result")) + .Step("result", step => step.WhenToolResult(call.Name, call.Id).WhenTurn(2).WhenLatestTextContains("scripted client result") + .Responds("complete native client round").Complete()))); + using var conversation = control.ForConversation("file-code-history"); + var sdk = Sdk(conversation.HttpClient); + var initial = Options(code); initial.ToolChoice = ResponseToolChoice.CreateAutoChoice(); initial.StoredOutputEnabled = stored; + initial.Tools.Add(ResponseTool.CreateFunctionTool(call.Name, BinaryData.FromString("{\"type\":\"object\"}"), false, "Client fixture")); + var first = (await sdk.CreateResponseAsync(initial)).Value; + await Assert.That(((FunctionCallResponseItem)first.OutputItems[0]).Id).IsEqualTo("fc_file_code"); + await Assert.That(first.OutputItems[1].Id).IsEqualTo(ItemId(code)); + var next = Options(code); next.ToolChoice = ResponseToolChoice.CreateAutoChoice(); next.InputItems.Clear(); + next.Tools.Add(ResponseTool.CreateFunctionTool(call.Name, BinaryData.FromString("{\"type\":\"object\"}"), false, "Client fixture")); + if (stored) { next.PreviousResponseId = first.Id; } + else + { + next.InputItems.Add(ResponseItem.CreateUserMessageItem("start")); + foreach (var item in first.OutputItems) { next.InputItems.Add(item); } + } + next.InputItems.Add(ResponseItem.CreateFunctionCallOutputItem(call.Id, "scripted client result")); + var final = (await sdk.CreateResponseAsync(next)).Value; + await Assert.That(((MessageResponseItem)final.OutputItems.Single()).Content.Single().Text).IsEqualTo("complete native client round"); + await Assert.That((await conversation.Control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(2); + var trace = (await conversation.Control.GetRequestsAsync()).Requests[0]; + var historical = trace.Messages.Where(message => message.Role == "assistant").SelectMany(message => message.OutputItems).ToArray(); + await Assert.That(historical.Length).IsEqualTo(3); + await Assert.That(((LlmTestServerResponseFunctionCallItem)historical[0]).Id).IsEqualTo("fc_file_code"); + await Assert.That(historical[1] is LlmTestServerResponseCodeInterpreterItem).IsEqualTo(code); + await Assert.That(trace.Messages.Count(message => message.Role == "user")).IsEqualTo(1); + await Assert.That(trace.Messages.Single(message => message.Role == "tool").ToolCallId).IsEqualTo(call.Id); + } + + [Test] + [Arguments(false, "credential")] + [Arguments(false, "provider")] + [Arguments(false, "session")] + [Arguments(false, "reset")] + [Arguments(true, "credential")] + [Arguments(true, "provider")] + [Arguments(true, "session")] + [Arguments(true, "reset")] + public async Task NativeHistory_RespectsCredentialProviderSessionAndResetBoundariesAsync(bool code, string boundary) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(Configuration(code, 2)); + http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", "first-key"); + var first = await CompleteAsync(http, code); + var follow = new JsonObject { ["model"] = Model(code), ["input"] = "follow", ["previous_response_id"] = first.GetProperty("id").GetString() }; + if (boundary == "credential") { http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", "second-key"); } + if (boundary == "reset") { await control.ResetAsync(); } + var path = boundary == "provider" ? "/azure-openai/openai/v1/responses" : "/openai/v1/responses"; + if (boundary == "session") + { + await using var session = await control.CreateSessionAsync(config => config.UseChatModel(Model(code)) + .UseChatScenario("session-native-file-code", scenario => scenario.ForModel(Model(code)).RespondsWithItems(Items(code)))); + using var rejected = await session.HttpClient.PostAsJsonAsync(path, follow); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.NotFound); + await CompleteAsync(session.HttpClient, code); + } + else + { + using var rejected = await http.PostAsJsonAsync(path, follow); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.NotFound); + await Assert.That((await rejected.Content.ReadFromJsonAsync()).GetProperty("error").GetProperty("code").GetString()).IsEqualTo("response_not_found"); + http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", "first-key"); + await CompleteAsync(http, code); + } + } + + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task NativeGraphReplay_PreservesResourceAndCitationIdentityAsync(bool code) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(config => config.UseChatModel(Model(code)).UseConversation("file-code-replay", graph => graph.ForModel(Model(code)) + .Step("native", step => step.RespondsWithItems(Items(code)).Complete()))); + var first = await CompleteAsync(http, code); + var second = await CompleteAsync(http, code); + await Assert.That(first.GetProperty("output").GetRawText()).IsEqualTo(second.GetProperty("output").GetRawText()); + await Assert.That(first.GetProperty("id").GetString()).IsEqualTo(second.GetProperty("id").GetString()); + await Assert.That((await control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); + await Assert.That((await control.GetRequestsAsync()).Requests.Count(trace => trace.IsReplay)).IsEqualTo(1); + } +} +#pragma warning restore OPENAI001 diff --git a/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeFileCodeIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeFileCodeIntegrationTests.cs new file mode 100644 index 0000000..3a629d0 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeFileCodeIntegrationTests.cs @@ -0,0 +1,200 @@ +#pragma warning disable OPENAI001 // Exercise official Responses SDK contracts through the running server. +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Net.Http.Json; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Configuration; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Scenarios; +using OpenAI.Responses; + +namespace ManagedCode.LlmTestServer.Tests.OpenAI; + +public sealed partial class NativeFileCodeIntegrationTests +{ + private const string _storeId = "vs_fixture"; + private const string _fileId = "file_fixture"; + private const string _fileName = "fixture.txt"; + private const string _containerId = "cntr_fixture"; + private const string _inputFile = "file_input"; + private const string _generatedFile = "file_generated"; + private const string _imageUrl = "https://example.com/fixture.png"; + private const string _code = "print(42)\n"; + private const string _answer = "Paris fixture."; + private static readonly JsonSerializerOptions _configurationJson = new(JsonSerializerDefaults.Web) + { Converters = { new JsonStringEnumConverter(JsonNamingPolicy.CamelCase) } }; + + private static string Model(bool code) + { + return code ? LlmTestServerKnownModelIds.Gpt41 : LlmTestServerKnownModelIds.Gpt4OMini; + } + + private static string ItemId(bool code) + { + return code ? "ci_fixture" : "fs_fixture"; + } + + private static Dictionary Attributes() + { + return new() + { + ["category"] = JsonSerializer.SerializeToElement("fixture"), + ["year"] = JsonSerializer.SerializeToElement(2026), + ["approved"] = JsonSerializer.SerializeToElement(true), + }; + } + + private static LlmTestServerResponseItem[] Items(bool code) + { + if (code) + { + return [new LlmTestServerResponseCodeInterpreterItem + { + Id = ItemId(true), ContainerId = _containerId, InputFileIds = [_inputFile], GeneratedFileIds = [_generatedFile], + Code = _code, CodeChunks = ["print(", "42)", "\n"], + Outputs = [new LlmTestServerCodeInterpreterLogsOutput { Logs = "42\n" }, new LlmTestServerCodeInterpreterImageOutput { Url = _imageUrl }], + }, new LlmTestServerResponseMessageItem + { + Id = "msg_code_fixture", Content = _answer, StreamChunks = ["Paris ", "fixture."], + Citations = [new LlmTestServerContainerFileCitation + { ContainerId = _containerId, FileId = _generatedFile, Filename = "result.txt", StartIndex = 0, EndIndex = 0 }], + }]; + } + return [new LlmTestServerResponseFileSearchItem + { + Id = ItemId(false), VectorStoreIds = [_storeId], Queries = ["Paris fixture facts"], + Results = [new LlmTestServerFileSearchResult + { FileId = _fileId, Filename = _fileName, Text = "Scripted Paris source", Score = 0.9, Attributes = Attributes() }], + }, new LlmTestServerResponseMessageItem + { + Id = "msg_file_fixture", Content = _answer, StreamChunks = ["Paris ", "fixture."], + Citations = [new LlmTestServerFileCitation { FileId = _fileId, Filename = _fileName, Index = 5 }], + }]; + } + + private static LlmTestServerConfiguration Configuration(bool code, int responses = 1) + { + return new LlmTestServerClientConfigurationBuilder().UseChatModel(Model(code)).UseChatScenario("native-file-code", scenario => + { + scenario.ForModel(Model(code)); + for (var index = 0; index < responses; index++) { scenario.RespondsWithItems(Items(code)); } + }).Build(); + } + + private static JsonObject Request(bool code, bool include = true, bool auto = false, bool stream = false, string input = "start") + { + JsonNode container = auto ? new JsonObject { ["type"] = "auto", ["file_ids"] = new JsonArray(_inputFile) } : JsonValue.Create(_containerId)!; + var tool = code + ? new JsonObject { ["type"] = "code_interpreter", ["container"] = container } + : new JsonObject { ["type"] = "file_search", ["vector_store_ids"] = new JsonArray(_storeId) }; + var body = new JsonObject + { + ["model"] = Model(code), + ["input"] = input, + ["tools"] = new JsonArray(tool), + ["stream"] = stream, + ["tool_choice"] = new JsonObject { ["type"] = code ? "code_interpreter" : "file_search" }, + }; + if (include) { body["include"] = new JsonArray(code ? "code_interpreter_call.outputs" : "file_search_call.results"); } + return body; + } + + private static CreateResponseOptions Options(bool code, bool include = true, bool auto = false, string input = "start") + { + var options = new CreateResponseOptions + { + Model = Model(code), + InputItems = { ResponseItem.CreateUserMessageItem(input) }, + ToolChoice = code ? ResponseToolChoice.CreateRequiredChoice() : ResponseToolChoice.CreateFileSearchChoice(), + }; + if (code) + { + var container = auto + ? new CodeInterpreterToolContainer(CodeInterpreterToolContainerConfiguration.CreateAutomaticContainerConfiguration([_inputFile])) + : new CodeInterpreterToolContainer(_containerId); + options.Tools.Add(ResponseTool.CreateCodeInterpreterTool(container)); + } + else + { + options.Tools.Add(ResponseTool.CreateFileSearchTool([_storeId], 2, + new FileSearchToolRankingOptions { Ranker = FileSearchToolRanker.Auto, ScoreThreshold = 0.5f }, + BinaryData.FromString("""{"type":"eq","key":"category","value":"fixture"}"""))); + } + if (include) { options.IncludedProperties.Add(code ? IncludedResponseProperty.CodeInterpreterCallOutputs : IncludedResponseProperty.FileSearchCallResults); } + return options; + } + + private static ResponsesClient Sdk(HttpClient http) + { + return new(new ApiKeyCredential("provider-key"), new ResponsesClientOptions + { Endpoint = new Uri(http.BaseAddress!, "/openai/v1/"), Transport = new HttpClientPipelineTransport(http) }); + } + + private static async Task CompleteAsync(HttpClient http, bool code, JsonObject? body = null) + { + using var response = await http.PostAsJsonAsync("/openai/v1/responses", body ?? Request(code)); + response.EnsureSuccessStatusCode(); + return await response.Content.ReadFromJsonAsync(); + } + + private static async Task AssertSdkAsync(ResponseResult response, bool code, bool include = true) + { + await Assert.That(response.OutputItems.Count).IsEqualTo(2); + await Assert.That(response.OutputItems.OfType().Count()).IsEqualTo(0); + await Assert.That(response.OutputItems[0].Id).IsEqualTo(ItemId(code)); + if (code) + { + var call = (CodeInterpreterCallResponseItem)response.OutputItems[0]; + await Assert.That(call.ContainerId).IsEqualTo(_containerId); + await Assert.That(call.Status).IsEqualTo(CodeInterpreterCallStatus.Completed); + await Assert.That(call.Code).IsEqualTo(_code); + await Assert.That(call.Outputs?.Count ?? 0).IsEqualTo(include ? 2 : 0); + if (include) + { + await Assert.That(((CodeInterpreterCallLogsOutput)call.Outputs![0]).Logs).IsEqualTo("42\n"); + await Assert.That(((CodeInterpreterCallImageOutput)call.Outputs![1]).ImageUri.AbsoluteUri).IsEqualTo(_imageUrl); + } + } + else + { + var call = (FileSearchCallResponseItem)response.OutputItems[0]; + await Assert.That(call.Status).IsEqualTo(FileSearchCallStatus.Completed); + await Assert.That(call.Queries.Single()).IsEqualTo("Paris fixture facts"); + await Assert.That(call.Results?.Count ?? 0).IsEqualTo(include ? 1 : 0); + if (include) + { + var result = call.Results!.Single(); + await Assert.That(result.FileId).IsEqualTo(_fileId); + await Assert.That(result.Filename).IsEqualTo(_fileName); + await Assert.That(result.Text).IsEqualTo("Scripted Paris source"); + await Assert.That(result.Score).IsEqualTo(0.9f); + await Assert.That(JsonSerializer.Deserialize(result.Attributes["category"].ToString())).IsEqualTo("fixture"); + await Assert.That(JsonSerializer.Deserialize(result.Attributes["year"].ToString())).IsEqualTo(2026); + await Assert.That(JsonSerializer.Deserialize(result.Attributes["approved"].ToString())).IsTrue(); + } + } + var message = (MessageResponseItem)response.OutputItems[1]; + await Assert.That(message.Content.Single().Text).IsEqualTo(_answer); + var annotation = message.Content.Single().OutputTextAnnotations.Single(); + if (code) + { + var citation = (ContainerFileCitationMessageAnnotation)annotation; + await Assert.That(citation.ContainerId).IsEqualTo(_containerId); + await Assert.That(citation.FileId).IsEqualTo(_generatedFile); + await Assert.That(citation.Filename).IsEqualTo("result.txt"); + await Assert.That(citation.StartIndex).IsEqualTo(0); + await Assert.That(citation.EndIndex).IsEqualTo(0); + } + else + { + var citation = (FileCitationMessageAnnotation)annotation; + await Assert.That(citation.FileId).IsEqualTo(_fileId); + await Assert.That(citation.Filename).IsEqualTo(_fileName); + await Assert.That(citation.Index).IsEqualTo(5); + } + } +} +#pragma warning restore OPENAI001 diff --git a/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeFileCodeLifecycleIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeFileCodeLifecycleIntegrationTests.cs new file mode 100644 index 0000000..ff4eb93 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeFileCodeLifecycleIntegrationTests.cs @@ -0,0 +1,132 @@ +#pragma warning disable OPENAI001 // Verify official SDK nullable contracts through HTTP. +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Scenarios; +using ManagedCode.LlmTestServer.Tests.TestSupport; +using OpenAI.Responses; + +namespace ManagedCode.LlmTestServer.Tests.OpenAI; + +public sealed partial class NativeFileCodeIntegrationTests +{ + [Test] + public async Task FileResultOptionalFields_RemainNullWithoutInventedResourceValuesAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var configuration = Configuration(false, 2); + var native = (LlmTestServerResponseFileSearchItem)Items(false)[0]; + var message = (LlmTestServerResponseMessageItem)Items(false)[1]; + for (var index = 0; index < 2; index++) + { + configuration.ChatScenarios[0].Responses[index] = configuration.ChatScenarios[0].Responses[index] with + { OutputItems = [native with { Results = [new LlmTestServerFileSearchResult()] }, message with { Citations = [] }] }; + } + await new LlmTestServerClient(http).ConfigureAsync(configuration); + var wire = await CompleteAsync(http, false); + var result = wire.GetProperty("output")[0].GetProperty("results")[0]; + foreach (var member in new[] { "file_id", "filename", "text", "score", "attributes" }) + { await Assert.That(!result.TryGetProperty(member, out var value) || value.ValueKind == JsonValueKind.Null).IsTrue(); } + var options = Options(false); + options.Tools.Clear(); options.Tools.Add(ResponseTool.CreateFileSearchTool([_storeId], null, null, null)); + var parsed = ((FileSearchCallResponseItem)(await Sdk(http).CreateResponseAsync(options)).Value.OutputItems[0]).Results.Single(); + await Assert.That(parsed.FileId).IsNull(); + await Assert.That(parsed.Filename).IsNull(); + await Assert.That(parsed.Text).IsNull(); + await Assert.That(parsed.Score).IsNull(); + await Assert.That(parsed.Attributes.Count).IsEqualTo(0); + } + + [Test] + public async Task CodeNullableBodyAndOutputs_RemainNullThroughHttpAndSdkAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var configuration = Configuration(true, 2); + var item = (LlmTestServerResponseCodeInterpreterItem)Items(true)[0]; + for (var index = 0; index < 2; index++) + { + configuration.ChatScenarios[0].Responses[index] = configuration.ChatScenarios[0].Responses[index] with + { OutputItems = [item with { Code = null, CodeChunks = [], Outputs = null }, Items(true)[1]] }; + } + await new LlmTestServerClient(http).ConfigureAsync(configuration); + var wire = (await CompleteAsync(http, true)).GetProperty("output")[0]; + await Assert.That(wire.GetProperty("code").ValueKind).IsEqualTo(JsonValueKind.Null); + await Assert.That(wire.GetProperty("outputs").ValueKind).IsEqualTo(JsonValueKind.Null); + var parsed = (CodeInterpreterCallResponseItem)(await Sdk(http).CreateResponseAsync(Options(true))).Value.OutputItems[0]; + await Assert.That(parsed.Code).IsNull(); + await Assert.That(parsed.Outputs?.Count ?? 0).IsEqualTo(0); + } + + [Test] + [Arguments(false, false)] + [Arguments(false, true)] + [Arguments(true, false)] + [Arguments(true, true)] + public async Task CancellationOrReset_LeavesNativeGraphAvailableForRetryAsync(bool code, bool reset) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(config => config.UseChatModel(Model(code)).UseConversation("file-code-cancel", graph => graph.ForModel(Model(code)) + .Step("native", step => step.RespondsWithItems(Items(code)).DelaysBy(800).Complete()))); + using var cancellation = new CancellationTokenSource(); + var pending = http.PostAsJsonAsync("/openai/v1/responses", Request(code), cancellation.Token); + var busy = false; + for (var attempt = 0; attempt < 200; attempt++) + { + if ((await control.GetConversationsAsync()).Any(state => state.IsBusy)) { busy = true; break; } + if (pending.IsCompleted) { break; } + await Task.Delay(2); + } + await Assert.That(busy).IsTrue(); + if (reset) + { + await control.ResetAsync(); + using var superseded = await pending; + await Assert.That(superseded.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + } + else + { + cancellation.Cancel(); var cancelled = false; + try { using var response = await pending; } + catch (OperationCanceledException) when (cancellation.IsCancellationRequested) { cancelled = true; } + await Assert.That(cancelled).IsTrue(); + } + await Assert.That((await control.GetConversationsAsync()).Count).IsEqualTo(0); + await CompleteAsync(http, code); + await Assert.That((await control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); + } + + [Test] + [Arguments(false, false)] + [Arguments(false, true)] + [Arguments(true, false)] + [Arguments(true, true)] + public async Task NativePayloads_AreIncludedInHistoryAndGraphCapacityAccountingAsync(bool code, bool graph) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var items = Items(code); + items[0] = code + ? ((LlmTestServerResponseCodeInterpreterItem)items[0]) with { Outputs = [new LlmTestServerCodeInterpreterLogsOutput { Logs = new string('x', 10000) }] } + : ((LlmTestServerResponseFileSearchItem)items[0]) with + { + Results = [new LlmTestServerFileSearchResult + { FileId = _fileId, Filename = _fileName, Text = new string('x', 10000) }] + }; + var builder = new LlmTestServerClientConfigurationBuilder().UseChatModel(Model(code)); + if (graph) { builder.UseConversationCapacity(4, 3000).UseConversation("file-code-budget", flow => flow.ForModel(Model(code)).Step("native", step => step.RespondsWithItems(items).Complete())); } + else { builder.UseChatResponseCapacity(8, 3000).UseChatScenario("file-code-budget", scenario => scenario.ForModel(Model(code)).RespondsWithItems(items)); } + var control = new LlmTestServerClient(http); await control.ConfigureAsync(builder.Build()); + using var rejected = await http.PostAsJsonAsync("/openai/v1/responses", Request(code)); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + await Assert.That((await rejected.Content.ReadFromJsonAsync()).GetProperty("error").GetProperty("code").GetString()) + .IsEqualTo(graph ? "conversation_capacity_exceeded" : "llm_test_server_response_capacity_exceeded"); + await Assert.That((await control.GetConversationsAsync()).Count).IsEqualTo(0); + if (!graph) { var transient = Request(code); transient["store"] = false; await CompleteAsync(http, code, transient); } + } +} +#pragma warning restore OPENAI001 diff --git a/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeFileCodeNormalizationIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeFileCodeNormalizationIntegrationTests.cs new file mode 100644 index 0000000..f4de420 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeFileCodeNormalizationIntegrationTests.cs @@ -0,0 +1,98 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json.Nodes; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.OpenAI; + +public sealed partial class NativeFileCodeIntegrationTests +{ + [Test] + [Arguments(false, "\"ordinary part\"", "ordinary part")] + [Arguments(false, "null", "")] + [Arguments(false, "123", "123")] + [Arguments(true, "\"ordinary part\"", "ordinary part")] + [Arguments(true, "null", "")] + [Arguments(true, "123", "123")] + public async Task NativeHistory_OrdinaryAssistantScalarPartsPreserveTextAndOneLogicalTurnAsync(bool code, string part, string expected) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(config => config.UseChatModel(Model(code)).UseConversation("native-normalization", graph => graph.ForModel(Model(code)) + .Step("native", step => step.WhenLatestUserEquals("start").WhenTurn(1).RespondsWithItems(Items(code)).Then("follow")) + .Step("follow", step => step.WhenLatestUserEquals("follow").WhenTurn(2).Responds("normalized follow-up").Complete()))); + var first = await CompleteAsync(http, code); + var input = new JsonArray(new JsonObject { ["role"] = "user", ["content"] = "start" }); + foreach (var item in first.GetProperty("output").EnumerateArray()) { input.Add(JsonNode.Parse(item.GetRawText())); } + input.Add(new JsonObject { ["role"] = "assistant", ["content"] = new JsonArray(JsonNode.Parse(part)) }); + input.Add(new JsonObject { ["role"] = "user", ["content"] = "follow" }); + var body = Request(code); body["input"] = input; body["tool_choice"] = "auto"; + var final = await CompleteAsync(http, code, body); + await Assert.That(final.GetProperty("output")[0].GetProperty("content")[0].GetProperty("text").GetString()).IsEqualTo("normalized follow-up"); + await Assert.That((await control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(2); + var trace = (await control.GetRequestsAsync()).Requests[0]; + await Assert.That(trace.Messages.Single(message => message.Role == "assistant").Content).IsEqualTo(_answer + expected); + } + + [Test] + [Arguments(false, "123")] + [Arguments(false, "{}")] + [Arguments(false, "true")] + [Arguments(false, "[]")] + [Arguments(true, "123")] + [Arguments(true, "{}")] + [Arguments(true, "true")] + [Arguments(true, "[]")] + public async Task OrdinaryAssistantOrFunctionNonStringIds_Return400WithoutConsumingFixtureAsync(bool function, string id) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + await new LlmTestServerClient(http).ConfigureAsync(Configuration(false)); + var item = function ? OrdinaryFunction(JsonNode.Parse(id)) : new JsonObject + { ["role"] = "assistant", ["id"] = JsonNode.Parse(id), ["content"] = "ordinary assistant" }; + var body = Request(false); body["input"] = new JsonArray(item, new JsonObject { ["role"] = "user", ["content"] = "follow" }); + using var rejected = await http.PostAsJsonAsync("/openai/v1/responses", body); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + await CompleteAsync(http, false); + } + + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task OrdinaryLongStringIds_StayOutsideNativeGroupSizeRestrictionsAsync(bool function) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(Configuration(false)); + var id = new string('x', 300); + var item = function ? OrdinaryFunction(JsonValue.Create(id)) : new JsonObject + { ["role"] = "assistant", ["id"] = id, ["content"] = "ordinary assistant" }; + var body = Request(false); body["input"] = new JsonArray(item, new JsonObject { ["role"] = "user", ["content"] = "follow" }); + await CompleteAsync(http, false, body); + var trace = (await control.GetRequestsAsync()).Requests.Single(); + if (function) { await Assert.That(trace.Messages.Single(message => message.Role == "assistant").ToolCalls.Single().Id).IsEqualTo("call_ordinary"); } + else { await Assert.That(trace.Messages.Single(message => message.Role == "assistant").Content).IsEqualTo("ordinary assistant"); } + } + + [Test] + public async Task OrdinaryFunctionNullId_UsesExistingFunctionCorrelationWithoutNativeGroupRestrictionAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(Configuration(false)); + var body = Request(false); body["input"] = new JsonArray(OrdinaryFunction(null), new JsonObject { ["role"] = "user", ["content"] = "follow" }); + await CompleteAsync(http, false, body); + var trace = (await control.GetRequestsAsync()).Requests.Single(); + await Assert.That(trace.Messages.Single(message => message.Role == "assistant").ToolCalls.Single().Id).IsEqualTo("call_ordinary"); + } + + private static JsonObject OrdinaryFunction(JsonNode? id) + { + return new() + { ["type"] = "function_call", ["id"] = id, ["call_id"] = "call_ordinary", ["name"] = "client_fixture", ["arguments"] = "{}" }; + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeFileCodeOptionsIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeFileCodeOptionsIntegrationTests.cs new file mode 100644 index 0000000..a1b80f7 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeFileCodeOptionsIntegrationTests.cs @@ -0,0 +1,154 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json.Nodes; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Scenarios; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.OpenAI; + +public sealed partial class NativeFileCodeIntegrationTests +{ + [Test] + [Arguments(false, """{"type":"file_search"}""")] + [Arguments(false, """{"type":"file_search","vector_store_ids":null}""")] + [Arguments(false, """{"type":"file_search","vector_store_ids":[""]}""")] + [Arguments(false, """{"type":"file_search","vector_store_ids":["vs_fixture"],"max_num_results":0}""")] + [Arguments(false, """{"type":"file_search","vector_store_ids":["vs_fixture"],"max_num_results":51}""")] + [Arguments(false, """{"type":"file_search","vector_store_ids":["vs_fixture"],"max_num_results":1.5}""")] + [Arguments(false, """{"type":"file_search","vector_store_ids":["vs_fixture"],"max_num_results":null}""")] + [Arguments(false, """{"type":"file_search","vector_store_ids":["vs_fixture"],"ranking_options":null}""")] + [Arguments(false, """{"type":"file_search","vector_store_ids":["vs_fixture"],"ranking_options":{"ranker":"unknown"}}""")] + [Arguments(false, """{"type":"file_search","vector_store_ids":["vs_fixture"],"ranking_options":{"ranker":null}}""")] + [Arguments(false, """{"type":"file_search","vector_store_ids":["vs_fixture"],"ranking_options":{"score_threshold":-0.1}}""")] + [Arguments(false, """{"type":"file_search","vector_store_ids":["vs_fixture"],"ranking_options":{"score_threshold":1.1}}""")] + [Arguments(false, """{"type":"file_search","vector_store_ids":["vs_fixture"],"ranking_options":{"score_threshold":null}}""")] + [Arguments(false, """{"type":"file_search","vector_store_ids":["vs_fixture"],"ranking_options":{"hybrid_search":{"embedding_weight":1}}}""")] + [Arguments(true, """{"type":"code_interpreter"}""")] + [Arguments(true, """{"type":"code_interpreter","container":null}""")] + [Arguments(true, """{"type":"code_interpreter","container":""}""")] + [Arguments(true, """{"type":"code_interpreter","container":{"type":"other"}}""")] + [Arguments(true, """{"type":"code_interpreter","container":{"type":"auto","file_ids":null}}""")] + [Arguments(true, """{"type":"code_interpreter","container":{"type":"auto","file_ids":[""]}}""")] + [Arguments(true, """{"type":"code_interpreter","container":{"type":"auto","memory_limit":"1g"}}""")] + [Arguments(true, """{"type":"code_interpreter","container":{"type":"auto","network_policy":{"allow":[]}}}""")] + [Arguments(true, """{"type":"code_interpreter","container":"cntr_fixture","allowed_callers":[]}""")] + public async Task InvalidNativeDeclaration_Returns400BeforeFixtureConsumptionAsync(bool code, string declaration) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + await new LlmTestServerClient(http).ConfigureAsync(Configuration(code)); + var body = Request(code); body["tools"] = new JsonArray(JsonNode.Parse(declaration)); + using var rejected = await http.PostAsJsonAsync("/openai/v1/responses", body); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + await CompleteAsync(http, code); + } + + [Test] + [Arguments("""{"type":"unknown","key":"category","value":"fixture"}""")] + [Arguments("""{"type":"eq","key":"category","value":null}""")] + [Arguments("""{"type":"eq","key":"category","value":{}}""")] + [Arguments("""{"type":"eq","key":"category","value":["fixture"]}""")] + [Arguments("""{"type":"eq","value":"fixture"}""")] + [Arguments("""{"type":"in","key":"category","value":[]}""")] + [Arguments("""{"type":"in","key":"category","value":["fixture",1]}""")] + [Arguments("""{"type":"nin","key":"approved","value":[true]}""")] + [Arguments("""{"type":"and","filters":[]}""")] + [Arguments("""{"type":"or","filters":null}""")] + [Arguments("""{"type":"eq","key":"category","value":"fixture","ignored":true}""")] + public async Task InvalidFileFilters_Return400AndPreserveQueuedResponseAsync(string filter) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + await new LlmTestServerClient(http).ConfigureAsync(Configuration(false)); + var body = Request(false); body["tools"]![0]!["filters"] = JsonNode.Parse(filter); + using var rejected = await http.PostAsJsonAsync("/openai/v1/responses", body); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + await CompleteAsync(http, false); + } + + [Test] + [Arguments(true)] + [Arguments(false)] + public async Task OversizedFilterTree_Return400WithoutConsumptionAsync(bool depth) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + await new LlmTestServerClient(http).ConfigureAsync(Configuration(false)); + JsonNode filter = Comparison(); + if (depth) + { + for (var index = 1; index < 9; index++) { filter = new JsonObject { ["type"] = "and", ["filters"] = new JsonArray(filter) }; } + } + else + { + var children = new JsonArray(); for (var index = 0; index < 128; index++) { children.Add(Comparison()); } + filter = new JsonObject { ["type"] = "or", ["filters"] = children }; + } + var body = Request(false); body["tools"]![0]!["filters"] = filter; + using var rejected = await http.PostAsJsonAsync("/openai/v1/responses", body); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + await CompleteAsync(http, false); + } + + [Test] + [Arguments("container")] + [Arguments("foreignFile")] + [Arguments("missingFile")] + public async Task CodeResourceMismatch_Returns409WithoutConsumptionAsync(string mismatch) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + await new LlmTestServerClient(http).ConfigureAsync(Configuration(true)); + var body = Request(true, auto: mismatch != "container"); + body["tools"]![0]!["container"] = mismatch switch + { + "container" => JsonValue.Create("cntr_foreign"), + "foreignFile" => new JsonObject { ["type"] = "auto", ["file_ids"] = new JsonArray("file_foreign") }, + _ => new JsonObject { ["type"] = "auto" }, + }; + using var rejected = await http.PostAsJsonAsync("/openai/v1/responses", body); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + await CompleteAsync(http, true); + } + + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task AutomaticContainer_InputFilesAreAnExactSetIncludingEmptyAsync(bool empty) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var configuration = Configuration(true); + var code = (LlmTestServerResponseCodeInterpreterItem)Items(true)[0]; + configuration.ChatScenarios[0].Responses[0] = configuration.ChatScenarios[0].Responses[0] with + { OutputItems = [code with { InputFileIds = empty ? [] : [_inputFile, "file_second"] }, Items(true)[1]] }; + await new LlmTestServerClient(http).ConfigureAsync(configuration); + var body = Request(true, auto: true); + body["tools"]![0]!["container"]!["file_ids"] = empty ? new JsonArray() : new JsonArray("file_second", _inputFile); + await CompleteAsync(http, true, body); + } + + [Test] + [Arguments(false, "/azure-openai/openai/v1/responses")] + [Arguments(false, "/groq/openai/v1/responses")] + [Arguments(true, "/microsoft-foundry/openai/v1/responses")] + [Arguments(true, "/openrouter/api/v1/responses")] + [Arguments(false, "/openai/v1/chat/completions")] + [Arguments(true, "/openai/v1/chat/completions")] + public async Task NativeFileAndCode_DoNotLeakIntoOtherNamespacesAsync(bool code, string path) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + await new LlmTestServerClient(http).ConfigureAsync(Configuration(code)); + var body = Request(code); + if (path.EndsWith("chat/completions", StringComparison.Ordinal)) + { + body.Remove("input"); body.Remove("include"); + body["messages"] = new JsonArray(new JsonObject { ["role"] = "user", ["content"] = "start" }); + } + using var rejected = await http.PostAsJsonAsync(path, body); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + await CompleteAsync(http, code); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeFileCodePrecisionIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeFileCodePrecisionIntegrationTests.cs new file mode 100644 index 0000000..ec9e8e4 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeFileCodePrecisionIntegrationTests.cs @@ -0,0 +1,88 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using System.Text.Json.Nodes; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Scenarios; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.OpenAI; + +public sealed partial class NativeFileCodeIntegrationTests +{ + [Test] + [Arguments("1e-29", "2e-29")] + [Arguments("9007199254740993", "9007199254740992")] + public async Task FileNumericFilters_DoNotRoundDistinctJsonNumbersIntoFalseMatchesAsync(string configured, string different) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var configuration = Configuration(false); + var native = (LlmTestServerResponseFileSearchItem)Items(false)[0]; + var result = native.Results[0] with + { Attributes = new() { ["precision"] = JsonSerializer.Deserialize(configured) } }; + configuration.ChatScenarios[0].Responses[0] = configuration.ChatScenarios[0].Responses[0] with + { OutputItems = [native with { Results = [result] }, Items(false)[1]] }; + await new LlmTestServerClient(http).ConfigureAsync(configuration); + var body = Request(false); + body["tools"]![0]!["filters"] = new JsonObject + { ["type"] = "eq", ["key"] = "precision", ["value"] = JsonNode.Parse(different) }; + using var rejected = await http.PostAsJsonAsync("/openai/v1/responses", body); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + body["tools"]![0]!["filters"]!["value"] = JsonNode.Parse(configured); + var accepted = await CompleteAsync(http, false, body); + await Assert.That(accepted.GetProperty("output")[0].GetProperty("results")[0].GetProperty("file_id").GetString()).IsEqualTo(_fileId); + } + [Test] + [Arguments("1", "1.0", "eq", true)] + [Arguments("1", "1e0", "eq", true)] + [Arguments("0", "-0", "eq", true)] + [Arguments("-0", "0.000e200", "eq", true)] + [Arguments("1e200", "10e199", "eq", true)] + [Arguments("1e-200", "10e-201", "eq", true)] + [Arguments("1e-1000000", "2e-1000000", "eq", false)] + [Arguments("1e-1000000", "10e-1000001", "eq", true)] + [Arguments("12345678901234567890123456789012345678901234567890", "12345678901234567890123456789012345678901234567891", "eq", false)] + [Arguments("1e-29", "2e-29", "ne", true)] + [Arguments("2e-29", "1e-29", "gt", true)] + [Arguments("1e-29", "1e-29", "gte", true)] + [Arguments("-2e-29", "-1e-29", "lt", true)] + [Arguments("-2e-29", "-2e-29", "lte", true)] + [Arguments("-2e-29", "-1e-29", "gt", false)] + [Arguments("-0", "0", "gt", false)] + [Arguments("1e-29", "[2e-29,3e-29]", "in", false)] + [Arguments("1e-29", "[2e-29,3e-29]", "nin", true)] + [Arguments("1e-29", "[0.00000000000000000000000000001]", "in", true)] + [Arguments("1e-29", "[1e-29]", "nin", false)] + [Arguments("1", "[1.0]", "nin", false)] + [Arguments("1", "[1e0]", "in", true)] + public async Task FileNumericFilterOperators_PreserveExactJsonValueOrderingAsync(string configured, string compared, string operation, bool matches) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var configuration = Configuration(false); + var native = (LlmTestServerResponseFileSearchItem)Items(false)[0]; + var result = native.Results[0] with + { Attributes = new() { ["precision"] = JsonSerializer.Deserialize(configured) } }; + configuration.ChatScenarios[0].Responses[0] = configuration.ChatScenarios[0].Responses[0] with + { OutputItems = [native with { Results = [result] }, Items(false)[1]] }; + await new LlmTestServerClient(http).ConfigureAsync(configuration); + var body = Request(false); + body["tools"]![0]!["filters"] = new JsonObject + { ["type"] = operation, ["key"] = "precision", ["value"] = JsonNode.Parse(compared) }; + using var response = await http.PostAsJsonAsync("/openai/v1/responses", body); + await Assert.That(response.StatusCode).IsEqualTo(matches ? HttpStatusCode.OK : HttpStatusCode.Conflict); + if (!matches) + { + body["tools"]![0]!["filters"]!["type"] = "eq"; + body["tools"]![0]!["filters"]!["value"] = JsonNode.Parse(configured); + var preserved = await CompleteAsync(http, false, body); + await Assert.That(preserved.GetProperty("output")[0].GetProperty("results")[0].GetProperty("file_id").GetString()).IsEqualTo(_fileId); + } + else + { + var accepted = await response.Content.ReadFromJsonAsync(); + await Assert.That(accepted.GetProperty("output")[0].GetProperty("results")[0].GetProperty("file_id").GetString()).IsEqualTo(_fileId); + } + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeFileCodeResourceIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeFileCodeResourceIntegrationTests.cs new file mode 100644 index 0000000..d248ae8 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeFileCodeResourceIntegrationTests.cs @@ -0,0 +1,80 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using System.Text.Json.Nodes; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Scenarios; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.OpenAI; + +public sealed partial class NativeFileCodeIntegrationTests +{ + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task DuplicateDeclaredResources_ArePreservedAndMatchedAsSetsAsync(bool code) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(Configuration(code)); + var body = Request(code, auto: code); + if (code) { body["tools"]![0]!["container"]!["file_ids"] = new JsonArray(_inputFile, _inputFile); } + else { body["tools"]![0]!["vector_store_ids"] = new JsonArray(_storeId, _storeId, "vs_extra"); } + await CompleteAsync(http, code, body); + var trace = (await control.GetRequestsAsync()).Requests.Single(); + await Assert.That(trace.NativeTools.Count).IsEqualTo(1); + var count = code ? trace.NativeTools[0].CodeInterpreterOptions!.FileIds.Count + : trace.NativeTools[0].FileSearchOptions!.VectorStoreIds.Count; + await Assert.That(count).IsEqualTo(code ? 2 : 3); + } + + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task EveryFileResult_MustRespectTheDeclaredFilterAndScoreAsync(bool score) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var configuration = Configuration(false); + var item = (LlmTestServerResponseFileSearchItem)Items(false)[0]; + var other = item.Results[0] with + { + FileId = "file_second", + Score = score ? 0.1 : 0.9, + Attributes = new() { ["category"] = JsonSerializer.SerializeToElement("other") }, + }; + configuration.ChatScenarios[0].Responses[0] = configuration.ChatScenarios[0].Responses[0] with + { OutputItems = [item with { Results = [item.Results[0], other] }, Items(false)[1]] }; + await new LlmTestServerClient(http).ConfigureAsync(configuration); + var body = Request(false); + if (score) { body["tools"]![0]!["ranking_options"] = new JsonObject { ["score_threshold"] = 0.5 }; } + else { body["tools"]![0]!["filters"] = Comparison(); } + using var rejected = await http.PostAsJsonAsync("/openai/v1/responses", body); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + var accepted = await CompleteAsync(http, false); + await Assert.That(accepted.GetProperty("output")[0].GetProperty("results").GetArrayLength()).IsEqualTo(2); + } + + [Test] + [Arguments(0d)] + [Arguments(1d)] + public async Task FileAttributeAndScoreBoundaries_ArePreservedAsync(double score) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var configuration = Configuration(false); + var item = (LlmTestServerResponseFileSearchItem)Items(false)[0]; + var attributes = new Dictionary { [new string('k', 64)] = JsonSerializer.SerializeToElement(new string('v', 512)) }; + for (var index = 1; index < 16; index++) { attributes[$"key{index}"] = JsonSerializer.SerializeToElement(index); } + configuration.ChatScenarios[0].Responses[0] = configuration.ChatScenarios[0].Responses[0] with + { OutputItems = [item with { Results = [item.Results[0] with { Score = score, Attributes = attributes }] }, Items(false)[1]] }; + await new LlmTestServerClient(http).ConfigureAsync(configuration); + var body = Request(false); body["tools"]![0]!["ranking_options"] = new JsonObject { ["score_threshold"] = score }; + var result = (await CompleteAsync(http, false, body)).GetProperty("output")[0].GetProperty("results")[0]; + await Assert.That(result.GetProperty("score").GetDouble()).IsEqualTo(score); + await Assert.That(result.GetProperty("attributes").EnumerateObject().Count()).IsEqualTo(16); + await Assert.That(result.GetProperty("attributes").GetProperty(new string('k', 64)).GetString()).IsEqualTo(new string('v', 512)); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeFileCodeSdkIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeFileCodeSdkIntegrationTests.cs new file mode 100644 index 0000000..9cf34fa --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeFileCodeSdkIntegrationTests.cs @@ -0,0 +1,105 @@ +#pragma warning disable OPENAI001 // Exercise official Responses SDK contracts through HTTP. +using System.Net.Http.Json; +using System.Text.Json; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Scenarios; +using ManagedCode.LlmTestServer.Tests.TestSupport; +using OpenAI.Responses; + +namespace ManagedCode.LlmTestServer.Tests.OpenAI; + +public sealed partial class NativeFileCodeIntegrationTests +{ + [Test] + [Arguments(false, false, false)] + [Arguments(false, true, false)] + [Arguments(true, false, false)] + [Arguments(true, true, false)] + [Arguments(true, true, true)] + public async Task JsonAndSdk_PreserveNativeItemsNullableIncludesAndTypedCitationsAsync(bool code, bool include, bool auto) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(Configuration(code, 2)); + var wire = await CompleteAsync(http, code, Request(code, include, auto)); + var item = wire.GetProperty("output")[0]; + await Assert.That(item.GetProperty("type").GetString()).IsEqualTo(code ? "code_interpreter_call" : "file_search_call"); + await Assert.That(item.GetProperty("id").GetString()).IsEqualTo(ItemId(code)); + var member = code ? "outputs" : "results"; + if (include) { await Assert.That(item.GetProperty(member).GetArrayLength()).IsEqualTo(code ? 2 : 1); } + else { await Assert.That(!item.TryGetProperty(member, out var omitted) || omitted.ValueKind == JsonValueKind.Null).IsTrue(); } + await Assert.That(item.TryGetProperty("search_results", out _)).IsFalse(); + await AssertSdkAsync((await Sdk(http).CreateResponseAsync(Options(code, include, auto))).Value, code, include); + var trace = (await control.GetRequestsAsync()).Requests[0]; + await Assert.That(trace.NativeTools.Count).IsEqualTo(1); + await Assert.That(trace.ResponseOutputItems.Count).IsEqualTo(2); + await Assert.That(trace.ResponseToolCalls.Count).IsEqualTo(0); + var exported = await control.ExportRequestsAsync(); + await Assert.That(exported.Evidence.Requests[0].ResponseOutputItems[0] is LlmTestServerResponseCodeInterpreterItem).IsEqualTo(code); + await Assert.That(exported.Evidence.Requests[0].ResponseOutputItems[0] is LlmTestServerResponseFileSearchItem).IsEqualTo(!code); + } + + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task OfficialSdkStream_ParsesLifecycleNativeCodeChunksAndAnnotationsAsync(bool code) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(Configuration(code)); + var options = Options(code); options.StreamingEnabled = true; + var updates = new List(); + await foreach (var update in Sdk(http).CreateResponseStreamingAsync(options)) { updates.Add(update); } + await Assert.That(updates.Select(update => update.SequenceNumber).SequenceEqual(Enumerable.Range(0, updates.Count))).IsTrue(); + if (code) + { + await Assert.That(updates.OfType().Single().ItemId).IsEqualTo(ItemId(true)); + await Assert.That(updates.OfType().Single().OutputIndex).IsEqualTo(0); + await Assert.That(updates.OfType().Single().ItemId).IsEqualTo(ItemId(true)); + await Assert.That(string.Concat(updates.OfType().Select(update => update.Delta))).IsEqualTo(_code); + await Assert.That(updates.OfType().Single().Code).IsEqualTo(_code); + } + else + { + await Assert.That(updates.OfType().Single().ItemId).IsEqualTo(ItemId(false)); + await Assert.That(updates.OfType().Single().OutputIndex).IsEqualTo(0); + await Assert.That(updates.OfType().Single().ItemId).IsEqualTo(ItemId(false)); + } + var annotation = updates.OfType().Single(); + await Assert.That(annotation.OutputIndex).IsEqualTo(1); + await Assert.That(annotation.ContentIndex).IsEqualTo(0); + await Assert.That(annotation.AnnotationIndex).IsEqualTo(0); + await Assert.That(annotation.Annotation is ContainerFileCitationMessageAnnotation).IsEqualTo(code); + await Assert.That(annotation.Annotation is FileCitationMessageAnnotation).IsEqualTo(!code); + await Assert.That(updates.OfType().Count()).IsEqualTo(0); + await AssertSdkAsync(updates.OfType().Single().Response, code); + var trace = (await control.GetRequestsAsync()).Requests.Single(); + await Assert.That(trace.TerminalMarkerObserved).IsTrue(); + } + + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task RawSse_FinalOrderedOutputMatchesJsonAsync(bool code) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + await new LlmTestServerClient(http).ConfigureAsync(Configuration(code, 2)); + var json = await CompleteAsync(http, code); + using var response = await http.PostAsJsonAsync("/openai/v1/responses", Request(code, stream: true)); + response.EnsureSuccessStatusCode(); + var frames = (await response.Content.ReadAsStringAsync()).Split('\n').Where(line => line.StartsWith("data: ", StringComparison.Ordinal) && line != "data: [DONE]") + .Select(line => JsonDocument.Parse(line[6..]).RootElement.Clone()).ToArray(); + await Assert.That(frames.Select(frame => frame.GetProperty("sequence_number").GetInt32()).SequenceEqual(Enumerable.Range(0, frames.Length))).IsTrue(); + var prefix = code ? "response.code_interpreter_call." : "response.file_search_call."; + var lifecycle = frames.Where(frame => frame.GetProperty("type").GetString()!.StartsWith(prefix, StringComparison.Ordinal)).ToArray(); + await Assert.That(string.Join(',', lifecycle.Select(frame => frame.GetProperty("type").GetString()))) + .IsEqualTo(prefix + "in_progress," + prefix + (code ? "interpreting," : "searching,") + prefix + "completed"); + await Assert.That(lifecycle.All(frame => frame.GetProperty("item_id").GetString() == ItemId(code) && frame.GetProperty("output_index").GetInt32() == 0)).IsTrue(); + var final = frames.Single(frame => frame.GetProperty("type").GetString() == "response.completed").GetProperty("response"); + await Assert.That(final.GetProperty("output").GetRawText()).IsEqualTo(json.GetProperty("output").GetRawText()); + } +} +#pragma warning restore OPENAI001 diff --git a/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeWebSearchCacheIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeWebSearchCacheIntegrationTests.cs new file mode 100644 index 0000000..d290fb5 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeWebSearchCacheIntegrationTests.cs @@ -0,0 +1,82 @@ +using System.Text.Json.Nodes; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.OpenAI; + +public sealed partial class NativeWebSearchIntegrationTests +{ + [Test] + [Arguments("context")] + [Arguments("domains")] + [Arguments("location")] + public async Task NativeDeclarationOptions_PartitionActualPromptCachePrefixesAsync(string option) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(Configuration(3)); + var body = Request(input: string.Join(' ', Enumerable.Repeat("stable native search reference", 1200))); + var initial = await CompleteAsync(http, body); + await Assert.That(initial.GetProperty("usage").GetProperty("input_tokens").GetInt32()).IsGreaterThanOrEqualTo(1024); + await Assert.That(initial.GetProperty("usage").GetProperty("input_tokens_details").GetProperty("cached_tokens").GetInt32()).IsEqualTo(0); + var tool = body["tools"]![0]!; + switch (option) + { + case "context": tool["search_context_size"] = "high"; break; + case "domains": tool["filters"] = new JsonObject { ["allowed_domains"] = new JsonArray("example.com") }; break; + case "location": tool["user_location"] = new JsonObject { ["type"] = "approximate", ["country"] = "FR" }; break; + } + var changed = await CompleteAsync(http, body); + await Assert.That(changed.GetProperty("usage").GetProperty("input_tokens_details").GetProperty("cached_tokens").GetInt32()).IsEqualTo(0); + var repeated = await CompleteAsync(http, body); + var cached = repeated.GetProperty("usage").GetProperty("input_tokens_details").GetProperty("cached_tokens").GetInt32(); + await Assert.That(cached).IsGreaterThanOrEqualTo(1024); + var evidence = await control.GetRequestsAsync(); + await Assert.That(evidence.Requests.Count).IsEqualTo(3); + await Assert.That(evidence.Requests[0].Usage!.CachedInputTokens).IsEqualTo(cached); + } + [Test] + public async Task MixedNativeDefinitions_CacheAccountingStaysWithinRealInputAtTokenBoundaryAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(Configuration(130)); + var prefix = string.Join(' ', Enumerable.Repeat("native cache boundary reference", 400)); + JsonObject? boundaryRequest = null; + var boundaryInput = 0; + var observed = 0; + for (var padding = 0; padding < 128; padding++) + { + var body = Request(input: prefix + string.Concat(Enumerable.Repeat(" x", padding))); + body["tools"]!.AsArray().Add(new JsonObject + { + ["type"] = "function", + ["name"] = "cache_boundary", + ["description"] = "Scripted boundary fixture", + ["parameters"] = new JsonObject { ["type"] = "object" }, + }); + body["tool_choice"] = "auto"; + var result = await CompleteAsync(http, body); + observed++; + var usage = result.GetProperty("usage"); + var input = usage.GetProperty("input_tokens").GetInt32(); + var cached = usage.GetProperty("input_tokens_details").GetProperty("cached_tokens").GetInt32(); + await Assert.That(input).IsGreaterThanOrEqualTo(1024); + await Assert.That(cached).IsLessThanOrEqualTo(input); + if (input % 128 == 127) { boundaryRequest = body; boundaryInput = input; break; } + } + await Assert.That(boundaryRequest).IsNotNull(); + var repeated = await CompleteAsync(http, boundaryRequest!); + var repeatedInput = repeated.GetProperty("usage").GetProperty("input_tokens").GetInt32(); + var repeatedCached = repeated.GetProperty("usage").GetProperty("input_tokens_details").GetProperty("cached_tokens").GetInt32(); + await Assert.That(repeatedInput).IsEqualTo(boundaryInput); + await Assert.That(repeatedCached).IsEqualTo(boundaryInput / 128 * 128); + var evidence = await control.GetRequestsAsync(); + await Assert.That(evidence.Requests.Count).IsEqualTo(observed + 1); + await Assert.That(evidence.Requests.All(trace => trace.Usage is { } usage + && usage.CachedInputTokens + usage.CacheCreationInputTokens <= usage.InputTokens)).IsTrue(); + await Assert.That((double)evidence.Requests[0].Usage!.CachedInputTokens / evidence.Requests[0].Usage!.InputTokens).IsLessThanOrEqualTo(1d); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeWebSearchConfigurationIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeWebSearchConfigurationIntegrationTests.cs new file mode 100644 index 0000000..d0e844f --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeWebSearchConfigurationIntegrationTests.cs @@ -0,0 +1,131 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using System.Text.Json.Nodes; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Scenarios; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.OpenAI; + +public sealed partial class NativeWebSearchIntegrationTests +{ + [Test] + [Arguments("duplicateFunctionOutputId")] + [Arguments("missingFunctionOutputId")] + [Arguments("zeroLengthCitation")] + [Arguments("reversedCitation")] + [Arguments("unexpectedStatus")] + [Arguments("unexpectedAction")] + [Arguments("nullCitation")] + [Arguments("nullItem")] + [Arguments("unknownKind")] + [Arguments("nullQueries")] + [Arguments("emptyQueries")] + [Arguments("blankQuery")] + [Arguments("nullSource")] + [Arguments("invalidSourceUrl")] + [Arguments("foreignContract")] + [Arguments("duplicateIds")] + [Arguments("negativeCitation")] + [Arguments("oversizedCitation")] + [Arguments("invalidCitationUrl")] + [Arguments("nonAsciiCitationText")] + [Arguments("inconsistentChunks")] + [Arguments("tooManyQueries")] + [Arguments("tooManySources")] + [Arguments("tooManyItems")] + public async Task InvalidNativeConfiguration_IsAtomicAndLeavesOldFixtureUsableAsync(string defect) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + await new LlmTestServerClient(http).ConfigureAsync(Configuration()); + var configuration = JsonSerializer.SerializeToNode(Configuration(), _configurationJson)!; + var output = configuration["chatScenarios"]![0]!["responses"]![0]!["outputItems"]!.AsArray(); + var search = output[0]!; + var message = output[1]!; + switch (defect) + { + case "duplicateFunctionOutputId": + case "missingFunctionOutputId": + output.Add(new JsonObject + { + ["kind"] = "function_call", + ["id"] = defect == "duplicateFunctionOutputId" ? _searchId : "", + ["call"] = new JsonObject { ["id"] = "call_collision_fixture", ["name"] = "weather", ["argumentsJson"] = "{}" }, + }); + break; + case "zeroLengthCitation": message["citations"]![0]!["endIndex"] = 0; break; + case "reversedCitation": message["citations"]![0]!["startIndex"] = 4; message["citations"]![0]!["endIndex"] = 3; break; + case "unexpectedStatus": search["status"] = "failed"; break; + case "unexpectedAction": search["action"] = new JsonObject { ["type"] = "open_page" }; break; + case "nullCitation": message["citations"] = new JsonArray((JsonNode?)null); break; + case "nullItem": output[0] = null; break; + case "unknownKind": search["kind"] = "file_search"; break; + case "nullQueries": search["queries"] = null; break; + case "emptyQueries": search["queries"] = new JsonArray(); break; + case "blankQuery": search["queries"] = new JsonArray(" "); break; + case "nullSource": search["sources"] = new JsonArray((JsonNode?)null); break; + case "invalidSourceUrl": search["sources"]![0]!["url"] = "file:///fixture"; break; + case "foreignContract": search["contract"] = "unavailableContract"; break; + case "duplicateIds": message["id"] = _searchId; break; + case "negativeCitation": message["citations"]![0]!["startIndex"] = -1; break; + case "oversizedCitation": message["citations"]![0]!["endIndex"] = 999; break; + case "invalidCitationUrl": message["citations"]![0]!["url"] = "relative-source"; break; + case "nonAsciiCitationText": message["content"] = "Paris café."; message["streamChunks"] = new JsonArray(); break; + case "inconsistentChunks": message["streamChunks"] = new JsonArray("different answer"); break; + case "tooManyQueries": search["queries"] = new JsonArray(Enumerable.Range(0, 33).Select(index => JsonValue.Create($"query {index}")).ToArray()); break; + case "tooManySources": search["sources"] = new JsonArray(Enumerable.Range(0, 129).Select(_ => (JsonNode)new JsonObject { ["url"] = _sourceUrl }).ToArray()); break; + case "tooManyItems": + while (output.Count < 129) { output.Add(new JsonObject { ["kind"] = "message", ["content"] = "extra" }); } + break; + } + using var rejected = await http.PostAsJsonAsync("/admin-api/configure", configuration); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + await AssertWireAsync(await CompleteAsync(http)); + } + + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task NativePayload_UsesExistingHistoryAndConversationByteBudgetsAsync(bool graph) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + var search = (LlmTestServerResponseWebSearchItem)Items()[0]; + search = search with + { + Sources = Enumerable.Range(0, 100).Select(index => new LlmTestServerWebSearchSource + { Url = $"https://example.com/{index}/" + new string('x', 100) }).ToList() + }; + var builder = new LlmTestServerClientConfigurationBuilder().UseGpt41Mini(); + if (graph) + { + builder.UseConversationCapacity(4, 3000).UseConversation("bounded-native", flow => flow + .Step("search", step => step.RespondsWithItems(search, Items()[1]).Complete())); + } + else + { + builder.UseChatResponseCapacity(8, 3000).UseChatScenario("bounded-native", scenario => scenario.RespondsWithItems(search, Items()[1])); + } + await control.ConfigureAsync(builder.Build()); + using var rejected = await http.PostAsJsonAsync("/openai/v1/responses", Request()); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + var error = await rejected.Content.ReadFromJsonAsync(); + await Assert.That(error.GetProperty("error").GetProperty("code").GetString()) + .IsEqualTo(graph ? "conversation_capacity_exceeded" : "llm_test_server_response_capacity_exceeded"); + await Assert.That((await control.GetConversationsAsync()).Count).IsEqualTo(0); + if (!graph) + { + var transient = Request(); transient["store"] = false; + var accepted = await CompleteAsync(http, transient); + await Assert.That(accepted.GetProperty("output")[0].GetProperty("action").GetProperty("sources").GetArrayLength()).IsEqualTo(100); + } + else + { + await control.ConfigureAsync(Configuration()); + await AssertWireAsync(await CompleteAsync(http)); + } + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeWebSearchHistoryValidationIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeWebSearchHistoryValidationIntegrationTests.cs new file mode 100644 index 0000000..fac6b7e --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeWebSearchHistoryValidationIntegrationTests.cs @@ -0,0 +1,53 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json.Nodes; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.OpenAI; + +public sealed partial class NativeWebSearchIntegrationTests +{ + [Test] + [Arguments("oversizedGroup")] + [Arguments("duplicateOutputId")] + [Arguments("duplicateUncitedOutputId")] + [Arguments("oversizedCallId")] + [Arguments("tooManyQueries")] + [Arguments("tooManySources")] + [Arguments("tooManyCitations")] + [Arguments("zeroLengthCitation")] + public async Task InvalidExplicitNativeHistory_Returns400AndPreservesNextFixtureAsync(string defect) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + await new LlmTestServerClient(http).ConfigureAsync(Configuration(2)); + var first = await CompleteAsync(http); + var output = JsonNode.Parse(first.GetProperty("output").GetRawText())!.AsArray(); + var input = new JsonArray(new JsonObject { ["role"] = "user", ["content"] = "start" }, output[0]!.DeepClone(), output[1]!.DeepClone()); + var search = input[1]!; + var message = input[2]!; + var content = message["content"]![0]!; + switch (defect) + { + case "oversizedGroup": + for (var index = 2; index < 129; index++) + { + var extra = message.DeepClone(); extra["id"] = $"msg_bounded_{index}"; input.Add(extra); + } + break; + case "duplicateOutputId": message["id"] = search["id"]!.GetValue(); break; + case "duplicateUncitedOutputId": message["id"] = search["id"]!.GetValue(); content["annotations"] = new JsonArray(); break; + case "oversizedCallId": search["id"] = new string('x', 129); break; + case "tooManyQueries": search["action"]!["queries"] = new JsonArray(Enumerable.Range(0, 33).Select(index => JsonValue.Create($"query {index}")).ToArray()); break; + case "tooManySources": search["action"]!["sources"] = new JsonArray(Enumerable.Range(0, 129).Select(_ => (JsonNode)new JsonObject { ["type"] = "url", ["url"] = _sourceUrl }).ToArray()); break; + case "tooManyCitations": content["annotations"] = new JsonArray(Enumerable.Range(0, 129).Select(_ => content["annotations"]![0]!.DeepClone()).ToArray()); break; + case "zeroLengthCitation": content["annotations"]![0]!["end_index"] = 0; break; + } + input.Add(new JsonObject { ["role"] = "user", ["content"] = "follow" }); + var body = Request(); body["input"] = input; + using var rejected = await http.PostAsJsonAsync("/openai/v1/responses", body); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + await AssertWireAsync(await CompleteAsync(http)); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeWebSearchIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeWebSearchIntegrationTests.cs new file mode 100644 index 0000000..ca52ee9 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeWebSearchIntegrationTests.cs @@ -0,0 +1,150 @@ +#pragma warning disable OPENAI001 // Exercise the official experimental Responses SDK through HTTP. +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Net.Http.Json; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Configuration; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Scenarios; +using OpenAI.Responses; + +namespace ManagedCode.LlmTestServer.Tests.OpenAI; + +public sealed partial class NativeWebSearchIntegrationTests +{ + private const string _sourceUrl = "https://example.com/source"; + private const string _searchId = "ws_native_fixture"; + private const string _messageId = "msg_native_fixture"; + private const string _answer = "Paris fixture."; + private const string _query = "Paris fixture facts"; + private static readonly JsonSerializerOptions _configurationJson = new(JsonSerializerDefaults.Web) + { + Converters = { new JsonStringEnumConverter(JsonNamingPolicy.CamelCase) }, + }; + + private static LlmTestServerResponseItem[] Items() + { + return [ + new LlmTestServerResponseWebSearchItem + { + Id = _searchId, Queries = [_query], Sources = [new() { Url = _sourceUrl }], + }, + new LlmTestServerResponseMessageItem + { + Id = _messageId, Content = _answer, StreamChunks = ["Paris ", "fixture."], + Citations = [new LlmTestServerUrlCitation() { StartIndex = 0, EndIndex = 5, Url = _sourceUrl, Title = "Fixture source" }], + }, + ]; + } + + private static LlmTestServerConfiguration Configuration(int responses = 1) + { + return new LlmTestServerClientConfigurationBuilder().UseGpt41Mini().UseChatScenario("native-web", scenario => + { + for (var index = 0; index < responses; index++) { scenario.RespondsWithItems(Items()); } + }).Build(); + } + + private static JsonObject Request(bool includeSources = true, bool stream = false, string input = "start") + { + var body = new JsonObject + { + ["model"] = LlmTestServerKnownModelIds.Gpt41Mini, + ["input"] = input, + ["tools"] = new JsonArray(new JsonObject { ["type"] = "web_search" }), + ["tool_choice"] = new JsonObject { ["type"] = "web_search" }, + ["stream"] = stream, + }; + if (includeSources) { body["include"] = new JsonArray("web_search_call.action.sources"); } + return body; + } + + private static ResponsesClient Sdk(HttpClient http) + { + return new(new ApiKeyCredential("provider-key"), new ResponsesClientOptions + { + Endpoint = new Uri(http.BaseAddress!, "/openai/v1/"), + Transport = new HttpClientPipelineTransport(http), + }); + } + + private static CreateResponseOptions Options(bool includeSources = true, string context = "low", string input = "start") + { + var options = new CreateResponseOptions + { + Model = LlmTestServerKnownModelIds.Gpt41Mini, + InputItems = { ResponseItem.CreateUserMessageItem(input) }, + Tools = { ResponseTool.CreateWebSearchTool(new WebSearchToolApproximateLocation + { Country = "FR", Region = "Ile-de-France", City = "Paris", Timezone = "Europe/Paris" }, + new WebSearchToolContextSize(context), new WebSearchToolFilters { AllowedDomains = { "example.com" } }) }, + ToolChoice = ResponseToolChoice.CreateRequiredChoice(), + }; + if (includeSources) { options.IncludedProperties.Add(IncludedResponseProperty.WebSearchCallActionSources); } + return options; + } + + private static async Task CompleteAsync(HttpClient http, JsonObject? request = null) + { + using var response = await http.PostAsJsonAsync("/openai/v1/responses", request ?? Request()); + response.EnsureSuccessStatusCode(); + return await response.Content.ReadFromJsonAsync(); + } + + private static async Task AssertWireAsync(JsonElement result, bool sources = true) + { + var output = result.GetProperty("output"); + await Assert.That(output.GetArrayLength()).IsEqualTo(2); + var search = output[0]; + await Assert.That(search.GetProperty("type").GetString()).IsEqualTo("web_search_call"); + await Assert.That(search.GetProperty("id").GetString()).IsEqualTo(_searchId); + await Assert.That(search.GetProperty("status").GetString()).IsEqualTo("completed"); + var action = search.GetProperty("action"); + await Assert.That(action.GetProperty("type").GetString()).IsEqualTo("search"); + await Assert.That(action.GetProperty("queries")[0].GetString()).IsEqualTo(_query); + if (sources) + { + await Assert.That(action.GetProperty("sources").GetArrayLength()).IsEqualTo(1); + await Assert.That(action.GetProperty("sources")[0].GetProperty("type").GetString()).IsEqualTo("url"); + await Assert.That(action.GetProperty("sources")[0].GetProperty("url").GetString()).IsEqualTo(_sourceUrl); + } + else { await Assert.That(!action.TryGetProperty("sources", out var omitted) || omitted.ValueKind == JsonValueKind.Null).IsTrue(); } + var message = output[1]; + await Assert.That(message.GetProperty("type").GetString()).IsEqualTo("message"); + await Assert.That(message.GetProperty("id").GetString()).IsEqualTo(_messageId); + var text = message.GetProperty("content")[0]; + await Assert.That(text.GetProperty("text").GetString()).IsEqualTo(_answer); + var citation = text.GetProperty("annotations")[0]; + await Assert.That(text.GetProperty("annotations").GetArrayLength()).IsEqualTo(1); + await Assert.That(citation.GetProperty("type").GetString()).IsEqualTo("url_citation"); + await Assert.That(citation.GetProperty("start_index").GetInt32()).IsEqualTo(0); + await Assert.That(citation.GetProperty("end_index").GetInt32()).IsEqualTo(5); + await Assert.That(citation.GetProperty("url").GetString()).IsEqualTo(_sourceUrl); + await Assert.That(citation.GetProperty("title").GetString()).IsEqualTo("Fixture source"); + } + + private static async Task AssertSdkAsync(ResponseResult result, bool sources = true) + { + await Assert.That(result.OutputItems.Count).IsEqualTo(2); + await Assert.That(result.OutputItems.OfType().Count()).IsEqualTo(0); + var search = (WebSearchCallResponseItem)result.OutputItems[0]; + await Assert.That(search.Id).IsEqualTo(_searchId); + await Assert.That(search.Status).IsEqualTo(WebSearchCallStatus.Completed); + var action = (WebSearchSearchAction)search.Action!; + await Assert.That(action.Queries.Single()).IsEqualTo(_query); + await Assert.That(action.Sources.Count).IsEqualTo(sources ? 1 : 0); + if (sources) { await Assert.That(((WebSearchActionUriSource)action.Sources.Single()).Uri.AbsoluteUri).IsEqualTo(_sourceUrl); } + var message = (MessageResponseItem)result.OutputItems[1]; + await Assert.That(message.Id).IsEqualTo(_messageId); + await Assert.That(message.Content.Single().Text).IsEqualTo(_answer); + var citation = (UriCitationMessageAnnotation)message.Content.Single().OutputTextAnnotations.Single(); + await Assert.That(citation.Uri.AbsoluteUri).IsEqualTo(_sourceUrl); + await Assert.That(citation.StartIndex).IsEqualTo(0); + await Assert.That(citation.EndIndex).IsEqualTo(5); + await Assert.That(citation.Title).IsEqualTo("Fixture source"); + } +} + +#pragma warning restore OPENAI001 diff --git a/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeWebSearchJsonIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeWebSearchJsonIntegrationTests.cs new file mode 100644 index 0000000..311e711 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeWebSearchJsonIntegrationTests.cs @@ -0,0 +1,205 @@ +#pragma warning disable OPENAI001 // Exercise the official experimental Responses SDK through HTTP. +using System.Text.Json; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Scenarios; +using ManagedCode.LlmTestServer.Tests.TestSupport; +using OpenAI.Responses; + +namespace ManagedCode.LlmTestServer.Tests.OpenAI; + +public sealed partial class NativeWebSearchIntegrationTests +{ + [Test] + [Arguments(false, "low")] + [Arguments(true, "low")] + [Arguments(true, "medium")] + [Arguments(true, "high")] + public async Task JsonAndOfficialSdk_PreserveNativeSearchSourcesAndTypedCitationAsync(bool sources, string context) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(Configuration(2)); + var request = Request(sources); + request["tools"]![0]!["search_context_size"] = context; + await AssertWireAsync(await CompleteAsync(http, request), sources); + await AssertSdkAsync((await Sdk(http).CreateResponseAsync(Options(sources, context))).Value, sources); + var evidence = await control.GetRequestsAsync(); + await Assert.That(evidence.Requests.Count).IsEqualTo(2); + await Assert.That(evidence.Requests.All(trace => trace.IsSuccess == true)).IsTrue(); + await Assert.That(evidence.Requests.All(trace => trace.ResponseToolCalls.Count == 0)).IsTrue(); + await Assert.That(evidence.Requests.All(trace => trace.NativeTools.Count == 1 && trace.ResponseOutputItems.Count == 2)).IsTrue(); + var exported = await control.ExportRequestsAsync(); + var exportedSearch = (LlmTestServerResponseWebSearchItem)exported.Evidence.Requests[0].ResponseOutputItems[0]; + await Assert.That(exportedSearch.Id).IsEqualTo(_searchId); + await Assert.That(exportedSearch.Sources.Single().Url).IsEqualTo(_sourceUrl); + var lines = (await control.ExportRequestsNdjsonAsync()).Split('\n', StringSplitOptions.RemoveEmptyEntries); + using var record = JsonDocument.Parse(lines[1]); + await Assert.That(record.RootElement.GetProperty("request").GetProperty("responseOutputItems")[0].GetProperty("kind").GetString()).IsEqualTo("web_search"); + } + + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task OfficialSdk_PreservesMixedFunctionNativeAndMessageOrderAsync(bool streaming) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + var call = new LlmTestServerToolCall { Id = "call_mixed_fixture", Name = "weather", ArgumentsJson = "{}" }; + await control.ConfigureAsync(config => config.UseGpt41Mini().UseChatScenario("mixed-native", scenario => scenario.RespondsWithItems( + new LlmTestServerResponseFunctionCallItem { Id = "fc_mixed_fixture", Call = call }, Items()[0], Items()[1]))); + var options = Options(); + options.ToolChoice = ResponseToolChoice.CreateAutoChoice(); + options.Tools.Add(ResponseTool.CreateFunctionTool("weather", BinaryData.FromString("{\"type\":\"object\"}"), false, "Scripted weather")); + ResponseResult result; + if (streaming) + { + options.StreamingEnabled = true; + var updates = new List(); + await foreach (var update in Sdk(http).CreateResponseStreamingAsync(options)) { updates.Add(update); } + result = updates.OfType().Single().Response; + await Assert.That(updates.OfType().Single().OutputIndex).IsEqualTo(1); + await Assert.That(updates.OfType().Count()).IsEqualTo(1); + } + else { result = (await Sdk(http).CreateResponseAsync(options)).Value; } + await Assert.That(result.OutputItems.Count).IsEqualTo(3); + var function = (FunctionCallResponseItem)result.OutputItems[0]; + await Assert.That(function.Id).IsEqualTo("fc_mixed_fixture"); + await Assert.That(function.FunctionName).IsEqualTo(call.Name); + await Assert.That(function.CallId).IsEqualTo(call.Id); + await Assert.That(function.FunctionArguments.ToString()).IsEqualTo(call.ArgumentsJson); + await Assert.That(((WebSearchCallResponseItem)result.OutputItems[1]).Id).IsEqualTo(_searchId); + await Assert.That(((MessageResponseItem)result.OutputItems[2]).Content.Single().Text).IsEqualTo(_answer); + var trace = (await control.GetRequestsAsync()).Requests.Single(); + await Assert.That(trace.ResponseToolCalls.Single().Id).IsEqualTo(call.Id); + } + + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task OfficialSdk_ExplicitAndStoredNativeHistoryAdvanceOneLogicalTurnAsync(bool stored) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(config => config.UseGpt41Mini().UseConversation("native-continuation", graph => graph + .Step("search", step => step.WhenLatestUserEquals("start").WhenTurn(1).RespondsWithItems(Items()).Then("follow")) + .Step("follow", step => step.WhenLatestUserEquals("follow").WhenTurn(2).Responds("continued native history").Complete()))); + using var conversation = control.ForConversation("native-continuation"); + var sdk = Sdk(conversation.HttpClient); + var firstOptions = Options(); + firstOptions.StoredOutputEnabled = stored; + var first = (await sdk.CreateResponseAsync(firstOptions)).Value; + await AssertSdkAsync(first); + var next = Options(input: "follow"); + if (stored) { next.PreviousResponseId = first.Id; } + else + { + next.InputItems.Clear(); + next.InputItems.Add(ResponseItem.CreateUserMessageItem("start")); + foreach (var item in first.OutputItems) { next.InputItems.Add(item); } + next.InputItems.Add(ResponseItem.CreateUserMessageItem("follow")); + } + next.ToolChoice = ResponseToolChoice.CreateAutoChoice(); + var continued = (await sdk.CreateResponseAsync(next)).Value; + await Assert.That(((MessageResponseItem)continued.OutputItems.Single()).Content.Single().Text).IsEqualTo("continued native history"); + var state = (await conversation.Control.GetConversationsAsync()).Single(); + await Assert.That(state.ExecutedSteps).IsEqualTo(2); + var trace = (await conversation.Control.GetRequestsAsync()).Requests[0]; + await Assert.That(trace.StepId).IsEqualTo("follow"); + await Assert.That(trace.Messages.Count(message => message.Role == "user")).IsEqualTo(2); + var priorSearch = trace.Messages.Where(message => message.Role == "assistant").SelectMany(message => message.OutputItems) + .OfType().Single(); + await Assert.That(priorSearch.Id).IsEqualTo(_searchId); + await Assert.That(priorSearch.Queries.Single()).IsEqualTo(_query); + await Assert.That(trace.Messages.Any(message => message.Role == "user" && message.Content.Contains("web_search_call", StringComparison.Ordinal))).IsFalse(); + } + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task OfficialSdk_MixedNativeFunctionRoundRetainsOutputIdsAcrossContinuationAsync(bool stored) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + var call = new LlmTestServerToolCall { Id = "call_native_weather", Name = "weather", ArgumentsJson = "{}" }; + await control.ConfigureAsync(config => config.UseGpt41Mini().UseConversation("mixed-native-history", graph => graph + .Step("search", step => step.WhenLatestUserEquals("start").WhenTurn(1).RespondsWithItems( + new LlmTestServerResponseFunctionCallItem { Id = "fc_native_weather", Call = call }, Items()[0], Items()[1]).Then("result")) + .Step("result", step => step.WhenToolResult(call.Name, call.Id).WhenTurn(2) + .WhenLatestTextContains("18 degrees").Responds("native search and weather complete").Complete()))); + using var conversation = control.ForConversation("mixed-native-history"); + var sdk = Sdk(conversation.HttpClient); + var firstOptions = Options(); + firstOptions.StoredOutputEnabled = stored; + firstOptions.ToolChoice = ResponseToolChoice.CreateAutoChoice(); + firstOptions.Tools.Add(ResponseTool.CreateFunctionTool("weather", BinaryData.FromString("{\"type\":\"object\"}"), false, "Scripted weather")); + var first = (await sdk.CreateResponseAsync(firstOptions)).Value; + await Assert.That(first.OutputItems.Count).IsEqualTo(3); + await Assert.That(((FunctionCallResponseItem)first.OutputItems[0]).Id).IsEqualTo("fc_native_weather"); + await Assert.That(((WebSearchCallResponseItem)first.OutputItems[1]).Id).IsEqualTo(_searchId); + var next = Options(); + next.ToolChoice = ResponseToolChoice.CreateAutoChoice(); + next.Tools.Add(ResponseTool.CreateFunctionTool("weather", BinaryData.FromString("{\"type\":\"object\"}"), false, "Scripted weather")); + next.InputItems.Clear(); + if (stored) { next.PreviousResponseId = first.Id; } + else + { + next.InputItems.Add(ResponseItem.CreateUserMessageItem("start")); + foreach (var item in first.OutputItems) { next.InputItems.Add(item); } + } + next.InputItems.Add(ResponseItem.CreateFunctionCallOutputItem(call.Id, "18 degrees")); + var final = (await sdk.CreateResponseAsync(next)).Value; + await Assert.That(((MessageResponseItem)final.OutputItems.Single()).Content.Single().Text).IsEqualTo("native search and weather complete"); + var state = (await conversation.Control.GetConversationsAsync()).Single(); + await Assert.That(state.ExecutedSteps).IsEqualTo(2); + var trace = (await conversation.Control.GetRequestsAsync()).Requests[0]; + var historical = trace.Messages.Where(message => message.Role == "assistant").SelectMany(message => message.OutputItems).ToArray(); + await Assert.That(historical.Length).IsEqualTo(3); + await Assert.That(((LlmTestServerResponseFunctionCallItem)historical[0]).Id).IsEqualTo("fc_native_weather"); + await Assert.That(((LlmTestServerResponseFunctionCallItem)historical[0]).Call.Id).IsEqualTo(call.Id); + await Assert.That(((LlmTestServerResponseWebSearchItem)historical[1]).Id).IsEqualTo(_searchId); + await Assert.That(((LlmTestServerResponseMessageItem)historical[2]).Id).IsEqualTo(_messageId); + await Assert.That(trace.Messages.Single(message => message.Role == "tool").ToolCallId).IsEqualTo(call.Id); + } + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task OfficialSdk_ExplicitNativeHistoryPreservesUncitedAndEmptyMessageIdentityAsync(bool empty) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + var text = empty ? "" : "Uncited fixture answer."; + await control.ConfigureAsync(config => config.UseGpt41Mini().UseConversation("uncited-native-history", graph => graph + .Step("search", step => step.WhenLatestUserEquals("start").WhenTurn(1).RespondsWithItems(Items()[0], + new LlmTestServerResponseMessageItem { Id = _messageId, Content = text }).Then("follow")) + .Step("follow", step => step.WhenLatestUserEquals("follow").WhenTurn(2).Responds("continued uncited history").Complete()))); + using var conversation = control.ForConversation("uncited-native-history"); + var sdk = Sdk(conversation.HttpClient); + var first = (await sdk.CreateResponseAsync(Options())).Value; + var firstMessage = (MessageResponseItem)first.OutputItems[1]; + await Assert.That(firstMessage.Id).IsEqualTo(_messageId); + await Assert.That(firstMessage.Content.Single().Text).IsEqualTo(text); + await Assert.That(firstMessage.Content.Single().OutputTextAnnotations.Count).IsEqualTo(0); + var next = Options(); + next.ToolChoice = ResponseToolChoice.CreateAutoChoice(); + next.InputItems.Clear(); + next.InputItems.Add(ResponseItem.CreateUserMessageItem("start")); + foreach (var item in first.OutputItems) { next.InputItems.Add(item); } + next.InputItems.Add(ResponseItem.CreateUserMessageItem("follow")); + var final = (await sdk.CreateResponseAsync(next)).Value; + await Assert.That(((MessageResponseItem)final.OutputItems.Single()).Content.Single().Text).IsEqualTo("continued uncited history"); + await Assert.That((await conversation.Control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(2); + var trace = (await conversation.Control.GetRequestsAsync()).Requests[0]; + await Assert.That(trace.Messages.Count(message => message.Role == "user")).IsEqualTo(2); + var historical = trace.Messages.Where(message => message.Role == "assistant").SelectMany(message => message.OutputItems) + .OfType().Single(); + await Assert.That(historical.Id).IsEqualTo(_messageId); + await Assert.That(historical.Content).IsEqualTo(text); + await Assert.That(historical.Citations.Count).IsEqualTo(0); + } +} + +#pragma warning restore OPENAI001 diff --git a/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeWebSearchLegacyOutputIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeWebSearchLegacyOutputIntegrationTests.cs new file mode 100644 index 0000000..71413b6 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeWebSearchLegacyOutputIntegrationTests.cs @@ -0,0 +1,96 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using System.Text.Json.Nodes; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Scenarios; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.OpenAI; + +public sealed class NativeWebSearchLegacyOutputIntegrationTests +{ + [Test] + [Arguments(false, 128, true)] + [Arguments(true, 128, true)] + [Arguments(false, 129, false)] + [Arguments(true, 129, false)] + public async Task LegacyFunctionCalls_PreserveLargeResponseAndQueueOrderAsync(bool responses, int callCount, bool withContent) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + var calls = Enumerable.Range(0, callCount).Select(index => new LlmTestServerToolCall + { + Id = $"call_legacy_{index}", + Name = "legacy_function", + ArgumentsJson = JsonSerializer.Serialize(new { index }), + }).ToArray(); + var configuration = new LlmTestServerClientConfigurationBuilder().UseGpt41Mini() + .UseChatScenario("legacy-many-calls", scenario => scenario.CallsTools(calls).Responds("queued answer after large fixture")) + .Build(); + var content = withContent ? "Legacy text accompanying function calls." : string.Empty; + var scenario = configuration.ChatScenarios.Single(); + scenario.Responses[0] = scenario.Responses[0] with { Content = content }; + await control.ConfigureAsync(configuration); + var body = new JsonObject + { + ["model"] = LlmTestServerKnownModelIds.Gpt41Mini, + ["tool_choice"] = "auto", + ["parallel_tool_calls"] = true, + }; + if (responses) + { + body["input"] = "legacy calls"; + body["tools"] = new JsonArray(new JsonObject + { ["type"] = "function", ["name"] = "legacy_function", ["parameters"] = new JsonObject { ["type"] = "object" } }); + } + else + { + body["messages"] = new JsonArray(new JsonObject { ["role"] = "user", ["content"] = "legacy calls" }); + body["tools"] = new JsonArray(new JsonObject + { ["type"] = "function", ["function"] = new JsonObject { ["name"] = "legacy_function", ["parameters"] = new JsonObject { ["type"] = "object" } } }); + } + var path = responses ? "/openai/v1/responses" : "/openai/v1/chat/completions"; + using var first = await http.PostAsJsonAsync(path, body); + await Assert.That(first.StatusCode).IsEqualTo(HttpStatusCode.OK); + var firstPayload = await first.Content.ReadFromJsonAsync(); + var returned = responses + ? firstPayload.GetProperty("output").EnumerateArray().Where(item => item.GetProperty("type").GetString() == "function_call").ToArray() + : firstPayload.GetProperty("choices")[0].GetProperty("message").GetProperty("tool_calls").EnumerateArray().ToArray(); + await Assert.That(returned.Length).IsEqualTo(callCount); + for (var index = 0; index < callCount; index++) + { + await Assert.That(returned[index].GetProperty(responses ? "call_id" : "id").GetString()).IsEqualTo(calls[index].Id); + var function = responses ? returned[index] : returned[index].GetProperty("function"); + await Assert.That(function.GetProperty("name").GetString()).IsEqualTo(calls[index].Name); + await Assert.That(function.GetProperty("arguments").GetString()).IsEqualTo(calls[index].ArgumentsJson); + } + if (responses) + { + var messages = firstPayload.GetProperty("output").EnumerateArray().Where(item => item.GetProperty("type").GetString() == "message").ToArray(); + await Assert.That(messages.Length).IsEqualTo(withContent ? 1 : 0); + if (withContent) { await Assert.That(messages.Single().GetProperty("content")[0].GetProperty("text").GetString()).IsEqualTo(content); } + } + else + { + await Assert.That(firstPayload.GetProperty("choices")[0].GetProperty("message").GetProperty("content").GetString()).IsEqualTo(content); + } + using var second = await http.PostAsJsonAsync(path, body); + await Assert.That(second.StatusCode).IsEqualTo(HttpStatusCode.OK); + var secondPayload = await second.Content.ReadFromJsonAsync(); + var nextText = responses + ? secondPayload.GetProperty("output").EnumerateArray().Single().GetProperty("content")[0].GetProperty("text").GetString() + : secondPayload.GetProperty("choices")[0].GetProperty("message").GetProperty("content").GetString(); + await Assert.That(nextText).IsEqualTo("queued answer after large fixture"); + using var exhausted = await http.PostAsJsonAsync(path, body); + await Assert.That(exhausted.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + await Assert.That((await exhausted.Content.ReadFromJsonAsync()).GetProperty("error").GetProperty("code").GetString()).IsEqualTo("llm_test_server_scenario_exhausted"); + var evidence = await control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending }); + await Assert.That(evidence.Requests.Count).IsEqualTo(3); + await Assert.That(evidence.Requests[0].ResponseToolCalls.Count).IsEqualTo(callCount); + await Assert.That(evidence.Requests[1].ResponseToolCalls.Count).IsEqualTo(0); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeWebSearchLifecycleIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeWebSearchLifecycleIntegrationTests.cs new file mode 100644 index 0000000..9885b35 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeWebSearchLifecycleIntegrationTests.cs @@ -0,0 +1,132 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Text.Json; +using System.Text.Json.Nodes; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.OpenAI; + +public sealed partial class NativeWebSearchIntegrationTests +{ + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task InFlightNativeResponse_CancellationAndResetReleaseGraphReservationAsync(bool reset) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(config => config.UseGpt41Mini().UseConversation("native-inflight", graph => graph + .Step("search", step => step.RespondsWithItems(Items()).DelaysBy(800).Complete()))); + using var cancellation = new CancellationTokenSource(); + var pending = http.PostAsJsonAsync("/openai/v1/responses", Request(), cancellation.Token); + var busy = false; + for (var attempt = 0; attempt < 200; attempt++) + { + if ((await control.GetConversationsAsync()).Any(state => state.IsBusy)) { busy = true; break; } + if (pending.IsCompleted) { break; } + await Task.Delay(2); + } + await Assert.That(busy).IsTrue(); + if (reset) + { + await control.ResetAsync(); + using var superseded = await pending; + await Assert.That(superseded.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + } + else + { + cancellation.Cancel(); + var cancelled = false; + try { using var response = await pending; } + catch (OperationCanceledException) when (cancellation.IsCancellationRequested) { cancelled = true; } + await Assert.That(cancelled).IsTrue(); + } + await Assert.That((await control.GetConversationsAsync()).Count).IsEqualTo(0); + await AssertWireAsync(await CompleteAsync(http)); + await Assert.That((await control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); + } + + [Test] + public async Task GraphReplay_PreservesOrderedNativeItemsAndDoesNotAdvanceTwiceAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(config => config.UseGpt41Mini().UseDeterminism("native-replay", new DateTimeOffset(2026, 1, 2, 3, 4, 5, TimeSpan.Zero)) + .UseConversation("native-replay", graph => graph.Step("search", step => step.RespondsWithItems(Items()).Complete()))); + var first = await CompleteAsync(http); + var replay = await CompleteAsync(http); + await AssertWireAsync(replay); + await Assert.That(replay.GetProperty("id").GetString()).IsEqualTo(first.GetProperty("id").GetString()); + await Assert.That(replay.GetProperty("output").GetRawText()).IsEqualTo(first.GetProperty("output").GetRawText()); + await Assert.That((await control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); + var evidence = await control.GetRequestsAsync(); + await Assert.That(evidence.Requests.Count(trace => trace.IsReplay)).IsEqualTo(1); + await control.ResetAsync(); + var reset = await CompleteAsync(http); + await Assert.That(reset.GetProperty("output").GetRawText()).IsEqualTo(first.GetProperty("output").GetRawText()); + await Assert.That(reset.GetProperty("created_at").GetInt64()).IsEqualTo(first.GetProperty("created_at").GetInt64()); + } + + [Test] + [Arguments("credential")] + [Arguments("provider")] + [Arguments("session")] + [Arguments("reset")] + [Arguments("storeFalse")] + public async Task StoredNativeHistory_PreservesExistingOwnershipBoundariesAsync(string boundary) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(Configuration(2)); + http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", "first-key"); + var initial = Request(); + if (boundary == "storeFalse") { initial["store"] = false; } + var first = await CompleteAsync(http, initial); + await AssertWireAsync(first); + var follow = new JsonObject + { + ["model"] = "gpt-4.1-mini", + ["input"] = "follow", + ["previous_response_id"] = first.GetProperty("id").GetString(), + }; + if (boundary == "credential") { http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", "second-key"); } + if (boundary == "reset") { await control.ResetAsync(); } + var path = boundary == "provider" ? "/azure-openai/openai/v1/responses" : "/openai/v1/responses"; + if (boundary == "session") + { + await using var session = await control.CreateSessionAsync(config => config.UseGpt41Mini() + .UseChatScenario("session-native", scenario => scenario.RespondsWithItems(Items()))); + using var rejected = await session.HttpClient.PostAsJsonAsync(path, follow); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.NotFound); + await Assert.That((await rejected.Content.ReadFromJsonAsync()).GetProperty("error").GetProperty("code").GetString()).IsEqualTo("response_not_found"); + await AssertWireAsync(await CompleteAsync(session.HttpClient)); + } + else + { + using var rejected = await http.PostAsJsonAsync(path, follow); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.NotFound); + await Assert.That((await rejected.Content.ReadFromJsonAsync()).GetProperty("error").GetProperty("code").GetString()).IsEqualTo("response_not_found"); + http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", "first-key"); + await AssertWireAsync(await CompleteAsync(http)); + } + } + + [Test] + public async Task NativeFixtureSourcesAndCitations_RespectAllowedDomainsWithoutConsumingResponseAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + await new LlmTestServerClient(http).ConfigureAsync(Configuration()); + var body = Request(); + body["tools"]![0]!["filters"] = new JsonObject { ["allowed_domains"] = new JsonArray("unrelated.example.org") }; + using var rejected = await http.PostAsJsonAsync("/openai/v1/responses", body); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + await Assert.That((await rejected.Content.ReadFromJsonAsync()).GetProperty("error").GetProperty("code").GetString()).IsEqualTo("llm_test_server_fixture_mismatch"); + await AssertWireAsync(await CompleteAsync(http)); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeWebSearchStreamIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeWebSearchStreamIntegrationTests.cs new file mode 100644 index 0000000..046a9e4 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeWebSearchStreamIntegrationTests.cs @@ -0,0 +1,76 @@ +#pragma warning disable OPENAI001 // Exercise the official experimental Responses SDK through HTTP. +using System.Net.Http.Json; +using System.Text; +using System.Text.Json; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Tests.TestSupport; +using OpenAI.Responses; + +namespace ManagedCode.LlmTestServer.Tests.OpenAI; + +public sealed partial class NativeWebSearchIntegrationTests +{ + [Test] + public async Task OfficialSdkStream_ParsesNativeProgressCitationAndCompletedOutputAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(Configuration()); + var updates = new List(); + var text = new StringBuilder(); + var options = Options(); + options.StreamingEnabled = true; + await foreach (var update in Sdk(http).CreateResponseStreamingAsync(options)) + { + updates.Add(update); + if (update is StreamingResponseOutputTextDeltaUpdate delta) { text.Append(delta.Delta); } + } + await Assert.That(text.ToString()).IsEqualTo(_answer); + await Assert.That(updates.Select(update => update.SequenceNumber).SequenceEqual(Enumerable.Range(0, updates.Count))).IsTrue(); + await Assert.That(updates.OfType().Single().ItemId).IsEqualTo(_searchId); + await Assert.That(updates.OfType().Single().OutputIndex).IsEqualTo(0); + await Assert.That(updates.OfType().Single().ItemId).IsEqualTo(_searchId); + var citation = updates.OfType().Single(); + await Assert.That(citation.ItemId).IsEqualTo(_messageId); + await Assert.That(citation.OutputIndex).IsEqualTo(1); + await Assert.That(citation.ContentIndex).IsEqualTo(0); + await Assert.That(citation.AnnotationIndex).IsEqualTo(0); + await Assert.That(((UriCitationMessageAnnotation)citation.Annotation).Uri.AbsoluteUri).IsEqualTo(_sourceUrl); + await Assert.That(updates.OfType().Count()).IsEqualTo(0); + await AssertSdkAsync(updates.OfType().Single().Response); + var trace = (await control.GetRequestsAsync()).Requests.Single(); + await Assert.That(trace.IsStreaming).IsTrue(); + await Assert.That(trace.TerminalMarkerObserved).IsTrue(); + } + + [Test] + public async Task RawStream_FinalOutputMatchesJsonAndNativeLifecycleIsOrderedAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(Configuration(2)); + var json = await CompleteAsync(http); + using var response = await http.PostAsJsonAsync("/openai/v1/responses", Request(stream: true)); + response.EnsureSuccessStatusCode(); + await Assert.That(response.Content.Headers.ContentType!.MediaType).IsEqualTo("text/event-stream"); + var frames = (await response.Content.ReadAsStringAsync()).Split('\n') + .Where(line => line.StartsWith("data: ", StringComparison.Ordinal) && line != "data: [DONE]") + .Select(line => JsonDocument.Parse(line[6..]).RootElement.Clone()).ToArray(); + await Assert.That(frames.Select(frame => frame.GetProperty("sequence_number").GetInt32()).SequenceEqual(Enumerable.Range(0, frames.Length))).IsTrue(); + var native = frames.Where(frame => frame.GetProperty("type").GetString()!.StartsWith("response.web_search_call.", StringComparison.Ordinal)).ToArray(); + await Assert.That(string.Join(',', native.Select(frame => frame.GetProperty("type").GetString()))) + .IsEqualTo("response.web_search_call.in_progress,response.web_search_call.searching,response.web_search_call.completed"); + await Assert.That(native.All(frame => frame.GetProperty("item_id").GetString() == _searchId && frame.GetProperty("output_index").GetInt32() == 0)).IsTrue(); + var names = frames.Select(frame => frame.GetProperty("type").GetString()).ToArray(); + await Assert.That(Array.IndexOf(names, "response.web_search_call.completed") < Array.IndexOf(names, "response.output_text.delta")).IsTrue(); + await Assert.That(names.Count(name => name == "response.output_text.annotation.added")).IsEqualTo(1); + await Assert.That(names.Any(name => name!.Contains("function_call_arguments", StringComparison.Ordinal))).IsFalse(); + var completed = frames.Single(frame => frame.GetProperty("type").GetString() == "response.completed").GetProperty("response"); + await AssertWireAsync(completed); + await Assert.That(completed.GetProperty("output").GetRawText()).IsEqualTo(json.GetProperty("output").GetRawText()); + } +} + +#pragma warning restore OPENAI001 diff --git a/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeWebSearchValidationIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeWebSearchValidationIntegrationTests.cs new file mode 100644 index 0000000..ba28c0a --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/NativeWebSearchValidationIntegrationTests.cs @@ -0,0 +1,189 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.OpenAI; + +public sealed partial class NativeWebSearchIntegrationTests +{ + [Test] + [Arguments(false, false)] + [Arguments(false, true)] + [Arguments(true, false)] + [Arguments(true, true)] + public async Task MissingOrDisabledNativeDeclaration_DoesNotConsumeQueueOrGraphAsync(bool graph, bool none) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + if (graph) + { + await control.ConfigureAsync(config => config.UseGpt41Mini().UseConversation("native-mismatch", flow => flow + .Step("search", step => step.RespondsWithItems(Items()).Complete()))); + } + else { await control.ConfigureAsync(Configuration()); } + var invalid = Request(); + invalid["tool_choice"] = none ? "none" : "auto"; + if (!none) { invalid["tools"] = new JsonArray(); } + using var rejected = await http.PostAsJsonAsync("/openai/v1/responses", invalid); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + var failure = await rejected.Content.ReadFromJsonAsync(); + await Assert.That(failure.GetProperty("error").GetProperty("code").GetString()).IsEqualTo("llm_test_server_fixture_mismatch"); + await Assert.That((await control.GetConversationsAsync()).Count).IsEqualTo(0); + await AssertWireAsync(await CompleteAsync(http)); + if (graph) { await Assert.That((await control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); } + } + + [Test] + [Arguments("/azure-openai/openai/v1/responses")] + [Arguments("/microsoft-foundry/openai/v1/responses")] + [Arguments("/groq/openai/v1/responses")] + [Arguments("/openrouter/api/v1/responses")] + [Arguments("/openai/v1/chat/completions")] + public async Task NativeDeclaration_StaysWithinOpenAiResponsesNamespaceAsync(string path) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + await new LlmTestServerClient(http).ConfigureAsync(Configuration()); + var body = Request(); + if (path.EndsWith("chat/completions", StringComparison.Ordinal)) + { + body.Remove("input"); body.Remove("include"); + body["messages"] = new JsonArray(new JsonObject { ["role"] = "user", ["content"] = "start" }); + } + using var rejected = await http.PostAsJsonAsync(path, body); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + await AssertWireAsync(await CompleteAsync(http)); + } + + [Test] + [Arguments("{\"type\":\"web_search\",\"search_context_size\":\"huge\"}")] + [Arguments("{\"type\":\"web_search\",\"search_context_size\":null}")] + [Arguments("{\"type\":\"web_search\",\"external_web_access\":true}")] + [Arguments("{\"type\":\"web_search\",\"return_token_budget\":\"unlimited\"}")] + [Arguments("{\"type\":\"web_search\",\"user_location\":{\"type\":\"exact\",\"city\":\"Paris\"}}")] + [Arguments("{\"type\":\"web_search\",\"user_location\":{\"type\":\"approximate\",\"country\":\"ZZ\"}}")] + [Arguments("{\"type\":\"web_search\",\"user_location\":{\"type\":\"approximate\",\"timezone\":\"Invalid/Fixture\"}}")] + [Arguments("{\"type\":\"web_search\",\"filters\":{\"allowed_domains\":[\"https://example.com\"]}}")] + [Arguments("{\"type\":\"web_search\",\"filters\":{\"allowed_domains\":[\"*.example.com\"]}}")] + [Arguments("{\"type\":\"web_search\",\"filters\":{\"allowed_domains\":[\"127.0.0.1\"]}}")] + [Arguments("{\"type\":\"web_search\",\"filters\":{\"allowed_domains\":[\"example.com/path\"]}}")] + [Arguments("{\"type\":\"web_search\",\"filters\":{\"blocked_domains\":[\"example.com\"]}}")] + public async Task InvalidNativeOptions_Return400AndLeaveFixtureAvailableAsync(string declaration) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + await new LlmTestServerClient(http).ConfigureAsync(Configuration()); + var body = Request(); body["tools"] = new JsonArray(JsonNode.Parse(declaration)); + using var rejected = await http.PostAsJsonAsync("/openai/v1/responses", body); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + await AssertWireAsync(await CompleteAsync(http)); + } + + [Test] + [Arguments("foreignInclude")] + [Arguments("missingNamedDeclaration")] + [Arguments("unsupportedModel")] + [Arguments("tooManyDomains")] + public async Task InvalidNativeRequestBoundaries_DoNotConsumeFixtureAsync(string defect) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + await new LlmTestServerClient(http).ConfigureAsync(Configuration()); + var body = Request(); + switch (defect) + { + case "foreignInclude": body["include"] = new JsonArray("file_search_call.unsupported"); break; + case "missingNamedDeclaration": body["tools"] = new JsonArray(); break; + case "unsupportedModel": body["model"] = "gpt-5-mini"; break; + case "tooManyDomains": + body["tools"]![0]!["filters"] = new JsonObject + { ["allowed_domains"] = new JsonArray(Enumerable.Range(0, 101).Select(index => JsonValue.Create($"d{index}.example.com")).ToArray()) }; break; + } + using var rejected = await http.PostAsJsonAsync("/openai/v1/responses", body); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + await AssertWireAsync(await CompleteAsync(http)); + } + + [Test] + [Arguments("file_search_call.results")] + [Arguments("code_interpreter_call.outputs")] + public async Task SupportedUnusedIncludes_DoNotInventOtherNativeOutputAsync(string include) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + await new LlmTestServerClient(http).ConfigureAsync(Configuration()); + var body = Request(); + body["include"] = new JsonArray(include); + var payload = await CompleteAsync(http, body); + await AssertWireAsync(payload, sources: false); + await Assert.That(payload.GetProperty("output").GetArrayLength()).IsEqualTo(2); + } + + [Test] + public async Task RepeatedNativeDeclaration_RejectsUnsupportedFixtureWithoutConsumptionAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + await new LlmTestServerClient(http).ConfigureAsync(Configuration()); + var body = Request(); + body["tools"]!.AsArray().Add(new JsonObject { ["type"] = "web_search" }); + using var rejected = await http.PostAsJsonAsync("/openai/v1/responses", body); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + await Assert.That((await rejected.Content.ReadFromJsonAsync()).GetProperty("error").GetProperty("code").GetString()).IsEqualTo("llm_test_server_fixture_mismatch"); + await AssertWireAsync(await CompleteAsync(http)); + } + + [Test] + public async Task DuplicateNativeField_IsRejectedBeforeDeserializationAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + await new LlmTestServerClient(http).ConfigureAsync(Configuration()); + using var body = new StringContent(""" + {"model":"gpt-4.1-mini","input":"start","tools":[{"type":"web_search","search_context_size":"low","search_context_size":"high"}]} + """, Encoding.UTF8, "application/json"); + using var rejected = await http.PostAsync("/openai/v1/responses", body); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + await AssertWireAsync(await CompleteAsync(http)); + } + + [Test] + [Arguments("{\"type\":\"web_search\",\"user_location\":null}")] + [Arguments("{\"type\":\"web_search\",\"user_location\":{}}")] + [Arguments("{\"type\":\"web_search\",\"user_location\":{\"type\":\"approximate\"}}")] + [Arguments("{\"type\":\"web_search\",\"user_location\":{\"country\":\"fr\",\"city\":\"\",\"region\":\"\"}}")] + [Arguments("{\"type\":\"web_search\",\"user_location\":{\"country\":null,\"city\":null,\"region\":null,\"timezone\":null}}")] + [Arguments("{\"type\":\"web_search\",\"filters\":null}")] + [Arguments("{\"type\":\"web_search\",\"filters\":{}}")] + [Arguments("{\"type\":\"web_search\",\"filters\":{\"allowed_domains\":null}}")] + [Arguments("{\"type\":\"web_search\",\"filters\":{\"allowed_domains\":[]}}")] + [Arguments("{\"type\":\"web_search\",\"filters\":{\"allowed_domains\":[\"example.com\",\"EXAMPLE.COM\"]}}")] + public async Task OptionalNativeValues_FollowProviderSchemaAsync(string declaration) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + await new LlmTestServerClient(http).ConfigureAsync(Configuration()); + var body = Request(); + body["tools"] = new JsonArray(JsonNode.Parse(declaration)); + body["include"]!.AsArray().Add("web_search_call.action.sources"); + await AssertWireAsync(await CompleteAsync(http, body)); + } + + [Test] + public async Task AllowedDomainUpperBoundary_IsAcceptedAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + await new LlmTestServerClient(http).ConfigureAsync(Configuration()); + var body = Request(); + var domains = new JsonArray("example.com"); + for (var index = 1; index < 100; index++) { domains.Add($"d{index}.example.com"); } + body["tools"]![0]!["filters"] = new JsonObject { ["allowed_domains"] = domains }; + await AssertWireAsync(await CompleteAsync(http, body)); + } +} diff --git a/tests/ManagedCode.LlmTck.Tests/OpenAI/OpenAiEndpointTests.cs b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/OpenAiEndpointTests.cs similarity index 89% rename from tests/ManagedCode.LlmTck.Tests/OpenAI/OpenAiEndpointTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/OpenAI/OpenAiEndpointTests.cs index c8a9851..2c75f1a 100644 --- a/tests/ManagedCode.LlmTck.Tests/OpenAI/OpenAiEndpointTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/OpenAiEndpointTests.cs @@ -3,14 +3,14 @@ using System.Net.Http.Json; using System.Text; using System.Text.Json; -using ManagedCode.LlmTck.Client; -using ManagedCode.LlmTck.Configuration; -using ManagedCode.LlmTck.Control; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Tests.TestSupport; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Configuration; +using ManagedCode.LlmTestServer.Control; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Tests.TestSupport; -namespace ManagedCode.LlmTck.Tests.OpenAI; +namespace ManagedCode.LlmTestServer.Tests.OpenAI; public sealed class OpenAiEndpointTests { @@ -19,13 +19,13 @@ public sealed class OpenAiEndpointTests [Test] public async Task OpenAiEndpoints_ExposeModelsAndAllModalitiesAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options + using var host = await LlmTestServerTestHost.StartAsync(options => options .RequireBearerToken("test-key") - .AddModel("chat-model", LlmTckModelKind.Chat) - .AddModel("embedding-model", LlmTckModelKind.Embedding) - .AddModel("image-model", LlmTckModelKind.Image) - .AddModel("audio-model", LlmTckModelKind.Audio) - .AddModel("video-model", LlmTckModelKind.Video) + .AddModel("chat-model", LlmTestServerModelKind.Chat) + .AddModel("embedding-model", LlmTestServerModelKind.Embedding) + .AddModel("image-model", LlmTestServerModelKind.Image) + .AddModel("audio-model", LlmTestServerModelKind.Audio) + .AddModel("video-model", LlmTestServerModelKind.Video) .WithDefaultEmbeddingVector(0.25f, 0.5f) .WithDefaultTranscriptionText("audio transcript") .AddChatScenario( @@ -118,7 +118,7 @@ await Assert.That(videoJson.GetProperty("prompt").GetString()) .IsEqualTo("generate a blue compatibility marker"); var assertions = await client.GetFromJsonAsync( - LlmTckControlRoutes.Assertions, + LlmTestServerControlRoutes.Assertions, _jsonOptions ); await Assert.That(assertions.GetProperty("matched").GetInt32()).IsEqualTo(6); @@ -127,10 +127,10 @@ await Assert.That(videoJson.GetProperty("prompt").GetString()) [Test] public async Task StreamingChatCompletion_ReturnsServerSentChunksAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options.AddChatScenario( + using var host = await LlmTestServerTestHost.StartAsync(options => options.AddChatScenario( "streaming-blue-whale", scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) + .ForModel(LlmTestServerKnownModelIds.Gpt41Mini) .WhenUserContains("stream") .Responds("blue whale", "blue ", "whale") )); @@ -140,7 +140,7 @@ public async Task StreamingChatCompletion_ReturnsServerSentChunksAsync() "/openai/v1/chat/completions", new { - model = LlmTckKnownModelIds.Gpt41Mini, + model = LlmTestServerKnownModelIds.Gpt41Mini, stream = true, messages = new[] { new { role = "user", content = "stream response" } }, }, @@ -169,7 +169,7 @@ public async Task OpenAiUsage_IncludesReasoningTokenDetailsForReasoningModelsAsy { const string reasoningModel = "gpt-5-nano"; const int reasoningTokens = 19; - using var host = await LlmTckTestHost.StartAsync(options => options + using var host = await LlmTestServerTestHost.StartAsync(options => options .AddReasoningChatModel(reasoningModel, reasoningTokens) .AddChatScenario( "reasoning-usage", @@ -240,8 +240,8 @@ public async Task OpenAiUsage_IncludesPromptCacheDetailsForChatAndResponsesAsync { var chatPrompt = CreatePromptWithAtLeastTokens(1100); var responsesPrompt = CreatePromptWithAtLeastTokens(1100) + " responses"; - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("cache-chat", LlmTckModelKind.Chat) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("cache-chat", LlmTestServerModelKind.Chat) .AddChatScenario( "openai-chat-cache", scenario => scenario @@ -369,12 +369,12 @@ await Assert.That(secondResponseUsage.GetProperty("input_tokens").GetInt32()) [Test] public async Task BearerTokenRequirement_ReturnsOpenAiErrorShapeAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options.RequireBearerToken("test-key")); + using var host = await LlmTestServerTestHost.StartAsync(options => options.RequireBearerToken("test-key")); using var client = host.GetTestClient(); var response = await client.PostAsJsonAsync( "/openai/v1/embeddings", - new { model = LlmTckKnownModelIds.TextEmbedding3Small, input = "hello" }, + new { model = LlmTestServerKnownModelIds.TextEmbedding3Small, input = "hello" }, _jsonOptions ); @@ -387,13 +387,13 @@ await Assert.That(payload.GetProperty("error").GetProperty("code").GetString()) [Test] public async Task ControlEndpoints_RequireConfiguredBearerTokenAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options.RequireBearerToken("test-key")); + using var host = await LlmTestServerTestHost.StartAsync(options => options.RequireBearerToken("test-key")); using var client = host.GetTestClient(); - var assertions = await client.GetAsync(LlmTckControlRoutes.Assertions); - var reset = await client.PostAsync(LlmTckControlRoutes.Reset, content: null); + var assertions = await client.GetAsync(LlmTestServerControlRoutes.Assertions); + var reset = await client.PostAsync(LlmTestServerControlRoutes.Reset, content: null); var configure = await client.PostAsJsonAsync( - LlmTckControlRoutes.Configure, + LlmTestServerControlRoutes.Configure, new { models = Array.Empty(), chatScenarios = Array.Empty() }, _jsonOptions ); @@ -403,7 +403,7 @@ public async Task ControlEndpoints_RequireConfiguredBearerTokenAsync() await Assert.That(configure.StatusCode).IsEqualTo(HttpStatusCode.Unauthorized); client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", "test-key"); - var authorizedAssertions = await client.GetAsync(LlmTckControlRoutes.Assertions); + var authorizedAssertions = await client.GetAsync(LlmTestServerControlRoutes.Assertions); authorizedAssertions.EnsureSuccessStatusCode(); } @@ -411,7 +411,7 @@ public async Task ControlEndpoints_RequireConfiguredBearerTokenAsync() [Test] public async Task ProviderEndpoints_RejectUnknownModelsAsync() { - using var host = await LlmTckTestHost.StartAsync(); + using var host = await LlmTestServerTestHost.StartAsync(); using var client = host.GetTestClient(); var response = await client.PostAsJsonAsync( @@ -423,13 +423,13 @@ public async Task ProviderEndpoints_RejectUnknownModelsAsync() await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.NotFound); var payload = await response.Content.ReadFromJsonAsync(_jsonOptions); await Assert.That(payload.GetProperty("error").GetProperty("code").GetString()) - .IsEqualTo("llm_tck_unknown_model"); + .IsEqualTo("llm_test_server_unknown_model"); } [Test] public async Task MalformedChatCompletion_ReturnsOpenAiBadRequestAsync() { - using var host = await LlmTckTestHost.StartAsync(); + using var host = await LlmTestServerTestHost.StartAsync(); using var client = host.GetTestClient(); var malformed = await client.PostAsync( @@ -438,7 +438,7 @@ public async Task MalformedChatCompletion_ReturnsOpenAiBadRequestAsync() ); var missingMessages = await client.PostAsJsonAsync( "/openai/v1/chat/completions", - new { model = LlmTckKnownModelIds.Gpt41Mini, messages = (object?)null }, + new { model = LlmTestServerKnownModelIds.Gpt41Mini, messages = (object?)null }, _jsonOptions ); @@ -453,11 +453,11 @@ await Assert.That(payload.GetProperty("error").GetProperty("code").GetString()) [Test] public async Task ConfigureEndpoint_AcceptsReadableJsonEnumsAsync() { - using var host = await LlmTckTestHost.StartAsync(); + using var host = await LlmTestServerTestHost.StartAsync(); using var client = host.GetTestClient(); var configure = await client.PostAsJsonAsync( - LlmTckControlRoutes.Configure, + LlmTestServerControlRoutes.Configure, new { models = new[] @@ -520,16 +520,16 @@ await Assert.That( [Test] public async Task ControlClient_ConfiguresReadsAssertionsAndResetsRuntimeAsync() { - using var host = await LlmTckTestHost.StartAsync(); + using var host = await LlmTestServerTestHost.StartAsync(); using var client = host.GetTestClient(); - var controlClient = new LlmTckClient(client); + var controlClient = new LlmTestServerClient(client); await controlClient.ConfigureAsync( - new LlmTckConfigurationBuilder() + new LlmTestServerConfigurationBuilder() .AddChatScenario( "control-client-blue-whale", scenario => scenario - .ForModel(LlmTckKnownModelIds.Gpt41Mini) + .ForModel(LlmTestServerKnownModelIds.Gpt41Mini) .WhenUserContains("largest animal") .Responds("blue whale") ) @@ -539,7 +539,7 @@ await controlClient.ConfigureAsync( "/openai/v1/chat/completions", new { - model = LlmTckKnownModelIds.Gpt41Mini, + model = LlmTestServerKnownModelIds.Gpt41Mini, messages = new[] { new { role = "user", content = "largest animal" } }, }, _jsonOptions @@ -559,7 +559,7 @@ await controlClient.ConfigureAsync( [Test] public async Task ProviderEndpoints_ReturnBadRequestForMissingRequiredFieldsAsync() { - using var host = await LlmTckTestHost.StartAsync(); + using var host = await LlmTestServerTestHost.StartAsync(); using var client = host.GetTestClient(); var missingEmbeddingModel = await client.PostAsJsonAsync( @@ -569,7 +569,7 @@ public async Task ProviderEndpoints_ReturnBadRequestForMissingRequiredFieldsAsyn ); var missingEmbeddingInput = await client.PostAsJsonAsync( "/openai/v1/embeddings", - new { model = LlmTckKnownModelIds.TextEmbedding3Small, input = (object?)null }, + new { model = LlmTestServerKnownModelIds.TextEmbedding3Small, input = (object?)null }, _jsonOptions ); var missingImageModel = await client.PostAsJsonAsync( @@ -579,7 +579,7 @@ public async Task ProviderEndpoints_ReturnBadRequestForMissingRequiredFieldsAsyn ); var missingImagePrompt = await client.PostAsJsonAsync( "/openai/v1/images/generations", - new { model = LlmTckKnownModelIds.GptImage1, prompt = "" }, + new { model = LlmTestServerKnownModelIds.GptImage1, prompt = "" }, _jsonOptions ); var missingAudioModel = await client.PostAsJsonAsync( @@ -589,15 +589,15 @@ public async Task ProviderEndpoints_ReturnBadRequestForMissingRequiredFieldsAsyn ); var missingAudioInput = await client.PostAsJsonAsync( "/openai/v1/audio/speech", - new { model = LlmTckKnownModelIds.Gpt4OMiniTts, input = "", voice = "alloy" }, + new { model = LlmTestServerKnownModelIds.Gpt4OMiniTts, input = "", voice = "alloy" }, _jsonOptions ); var missingAudioVoice = await client.PostAsJsonAsync( "/openai/v1/audio/speech", - new { model = LlmTckKnownModelIds.Gpt4OMiniTts, input = "speak this", voice = "" }, + new { model = LlmTestServerKnownModelIds.Gpt4OMiniTts, input = "speak this", voice = "" }, _jsonOptions ); - var controlModels = await client.GetAsync(LlmTckControlRoutes.Models); + var controlModels = await client.GetAsync(LlmTestServerControlRoutes.Models); await Assert.That(missingEmbeddingModel.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); await Assert.That(missingEmbeddingInput.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); @@ -612,12 +612,12 @@ public async Task ProviderEndpoints_ReturnBadRequestForMissingRequiredFieldsAsyn [Test] public async Task AudioSpeech_ReturnsTruthfulWavFixtureAsync() { - using var host = await LlmTckTestHost.StartAsync(); + using var host = await LlmTestServerTestHost.StartAsync(); using var client = host.GetTestClient(); var response = await client.PostAsJsonAsync( "/openai/v1/audio/speech", - new { model = LlmTckKnownModelIds.Gpt4OMiniTts, input = "audio fixture", voice = "alloy" }, + new { model = LlmTestServerKnownModelIds.Gpt4OMiniTts, input = "audio fixture", voice = "alloy" }, _jsonOptions ); @@ -632,9 +632,9 @@ public async Task AudioSpeech_ReturnsTruthfulWavFixtureAsync() [Test] public async Task ImageRoutes_ReturnDocumentedEditVariationAndStreamingShapesAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("gpt-image-1.5", LlmTckModelKind.Image) - .AddModel("dall-e-2", LlmTckModelKind.Image)); + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("gpt-image-1.5", LlmTestServerModelKind.Image) + .AddModel("dall-e-2", LlmTestServerModelKind.Image)); using var client = host.GetTestClient(); var stream = await client.PostAsJsonAsync( @@ -725,19 +725,19 @@ await Assert.That(variationPayload.GetProperty("data")[0].GetProperty("b64_json" [Test] public async Task AudioTranscription_ReturnsJsonTextAndStreamingShapesAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options + using var host = await LlmTestServerTestHost.StartAsync(options => options .WithDefaultTranscriptionText("deterministic transcript")); using var client = host.GetTestClient(); - using var jsonContent = CreateTranscriptionContent(LlmTckKnownModelIds.Gpt4OMiniTts); + using var jsonContent = CreateTranscriptionContent(LlmTestServerKnownModelIds.Gpt4OMiniTts); var json = await client.PostAsync("/openai/v1/audio/transcriptions", jsonContent); using var textContent = CreateTranscriptionContent( - LlmTckKnownModelIds.Gpt4OMiniTts, + LlmTestServerKnownModelIds.Gpt4OMiniTts, responseFormat: "text" ); var text = await client.PostAsync("/openai/v1/audio/transcriptions", textContent); using var streamContent = CreateTranscriptionContent( - LlmTckKnownModelIds.Gpt4OMiniTts, + LlmTestServerKnownModelIds.Gpt4OMiniTts, stream: true ); var stream = await client.PostAsync("/openai/v1/audio/transcriptions", streamContent); @@ -765,24 +765,24 @@ await Assert.That(stream.Content.Headers.ContentType?.MediaType) [Test] public async Task AudioTranslation_ReturnsJsonTextAndRejectsStreamingAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options + using var host = await LlmTestServerTestHost.StartAsync(options => options .WithDefaultTranslationText("deterministic translation")); using var client = host.GetTestClient(); - using var jsonContent = CreateTranscriptionContent(LlmTckKnownModelIds.Gpt4OMiniTts); + using var jsonContent = CreateTranscriptionContent(LlmTestServerKnownModelIds.Gpt4OMiniTts); var json = await client.PostAsync("/openai/v1/audio/translations", jsonContent); using var textContent = CreateTranscriptionContent( - LlmTckKnownModelIds.Gpt4OMiniTts, + LlmTestServerKnownModelIds.Gpt4OMiniTts, responseFormat: "text" ); var text = await client.PostAsync("/openai/v1/audio/translations", textContent); using var streamContent = CreateTranscriptionContent( - LlmTckKnownModelIds.Gpt4OMiniTts, + LlmTestServerKnownModelIds.Gpt4OMiniTts, stream: true ); var stream = await client.PostAsync("/openai/v1/audio/translations", streamContent); using var diarizedContent = CreateTranscriptionContent( - LlmTckKnownModelIds.Gpt4OMiniTts, + LlmTestServerKnownModelIds.Gpt4OMiniTts, responseFormat: "diarized_json" ); var diarized = await client.PostAsync("/openai/v1/audio/translations", diarizedContent); @@ -810,12 +810,12 @@ await Assert.That(diarizedPayload.GetProperty("error").GetProperty("message").Ge [Test] public async Task VideoRoutes_ReturnDocumentedOpenAiShapesAndValidateEnumsAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel(LlmTckKnownModelIds.Sora2, LlmTckModelKind.Video)); + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel(LlmTestServerKnownModelIds.Sora2, LlmTestServerModelKind.Video)); using var client = host.GetTestClient(); using var createContent = CreateVideoContent( - LlmTckKnownModelIds.Sora2, + LlmTestServerKnownModelIds.Sora2, "generate a deterministic clip", seconds: "8", size: "1280x720" @@ -849,13 +849,13 @@ public async Task VideoRoutes_ReturnDocumentedOpenAiShapesAndValidateEnumsAsync( var delete = await client.DeleteAsync($"/openai/v1/videos/{createdId}"); using var invalidSecondsContent = CreateVideoContent( - LlmTckKnownModelIds.Sora2, + LlmTestServerKnownModelIds.Sora2, "generate a deterministic clip", seconds: "16" ); var invalidSeconds = await client.PostAsync("/openai/v1/videos", invalidSecondsContent); using var invalidSizeContent = CreateVideoContent( - LlmTckKnownModelIds.Sora2, + LlmTestServerKnownModelIds.Sora2, "generate a deterministic clip", size: "640x480" ); @@ -885,7 +885,7 @@ public async Task VideoRoutes_ReturnDocumentedOpenAiShapesAndValidateEnumsAsync( var deletePayload = await delete.Content.ReadFromJsonAsync(_jsonOptions); await Assert.That(createPayload.GetProperty("object").GetString()).IsEqualTo("video"); - await Assert.That(createPayload.GetProperty("model").GetString()).IsEqualTo(LlmTckKnownModelIds.Sora2); + await Assert.That(createPayload.GetProperty("model").GetString()).IsEqualTo(LlmTestServerKnownModelIds.Sora2); await Assert.That(createPayload.GetProperty("seconds").GetString()).IsEqualTo("8"); await Assert.That(createPayload.GetProperty("size").GetString()).IsEqualTo("1280x720"); await Assert.That(listPayload.GetProperty("object").GetString()).IsEqualTo("list"); @@ -956,7 +956,7 @@ private static MultipartFormDataContent CreateVideoContent( private static string CreatePromptWithAtLeastTokens(int minimumTokens) { var builder = new StringBuilder("cacheable fixture"); - while (LlmTckTokenCounter.CountTextTokens(builder.ToString()) < minimumTokens) + while (LlmTestServerTokenCounter.CountTextTokens(builder.ToString()) < minimumTokens) { builder.Append(" stable-prefix"); } diff --git a/tests/ManagedCode.LlmTck.Tests/OpenAI/OpenAiSdkCompatibilityTests.cs b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/OpenAiSdkCompatibilityTests.cs similarity index 83% rename from tests/ManagedCode.LlmTck.Tests/OpenAI/OpenAiSdkCompatibilityTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/OpenAI/OpenAiSdkCompatibilityTests.cs index c030c28..b66b948 100644 --- a/tests/ManagedCode.LlmTck.Tests/OpenAI/OpenAiSdkCompatibilityTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/OpenAiSdkCompatibilityTests.cs @@ -1,22 +1,22 @@ using System.ClientModel; using System.ClientModel.Primitives; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.Tests.TestSupport; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Tests.TestSupport; using OpenAI; using OpenAiChatClient = OpenAI.Chat.ChatClient; using OpenAiUserChatMessage = OpenAI.Chat.UserChatMessage; -using ProviderRoutes = ManagedCode.LlmTck.Providers.LlmTckProviderRouteNamespaces; +using ProviderRoutes = ManagedCode.LlmTestServer.Providers.LlmTestServerProviderRouteNamespaces; -namespace ManagedCode.LlmTck.Tests.OpenAI; +namespace ManagedCode.LlmTestServer.Tests.OpenAI; public sealed class OpenAiSdkCompatibilityTests { [Test] public async Task OpenAiChatClient_CanUseNamespacedChatStreamingAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options + using var host = await LlmTestServerTestHost.StartAsync(options => options .RequireBearerToken("test-key") - .AddModel("openai-chat", LlmTckModelKind.Chat) + .AddModel("openai-chat", LlmTestServerModelKind.Chat) .AddChatScenario( "openai-sdk-streaming-blue-whale", scenario => scenario diff --git a/tests/ManagedCode.LlmTestServer.Tests/OpenAI/ResponseHistoryIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/ResponseHistoryIntegrationTests.cs new file mode 100644 index 0000000..47d69ed --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/ResponseHistoryIntegrationTests.cs @@ -0,0 +1,66 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Text.Json; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Configuration; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.OpenAI; + +public sealed class ResponseHistoryIntegrationTests +{ + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task StoredResponse_IsUnavailableAcrossCredentialsProvidersAndResetAsync(bool reconfigure) + { + using var host = await LlmTestServerTestHost.StartAsync(config => config + .AddChatScenario("history", scenario => scenario.Responds("first").Responds("second"))); + using var http = host.GetTestClient(); + http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", "first-key"); + using var first = await http.PostAsJsonAsync("/openai/v1/responses", new { model = "gpt-4.1-mini", input = "hello" }); + first.EnsureSuccessStatusCode(); + var id = (await first.Content.ReadFromJsonAsync()).GetProperty("id").GetString(); + http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", "second-key"); + using var otherCredential = await http.PostAsJsonAsync("/openai/v1/responses", new { model = "gpt-4.1-mini", input = "continue", previous_response_id = id }); + await Assert.That(otherCredential.StatusCode).IsEqualTo(HttpStatusCode.NotFound); + await Assert.That(await otherCredential.Content.ReadAsStringAsync()).Contains("response_not_found"); + http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", "first-key"); + using var otherProvider = await http.PostAsJsonAsync("/azure-openai/openai/v1/responses", new { model = "gpt-4.1-mini", input = "continue", previous_response_id = id }); + await Assert.That(otherProvider.StatusCode).IsEqualTo(HttpStatusCode.NotFound); + using var cleared = reconfigure + ? await http.PostAsJsonAsync("/admin-api/configure", LlmTestServerConfiguration.CreateDefault()) + : await http.PostAsync("/admin-api/reset", null); + cleared.EnsureSuccessStatusCode(); + using var after = await http.PostAsJsonAsync("/openai/v1/responses", new { model = "gpt-4.1-mini", input = "continue", previous_response_id = id }); + await Assert.That(after.StatusCode).IsEqualTo(HttpStatusCode.NotFound); + await Assert.That(await after.Content.ReadAsStringAsync()).Contains("response_not_found"); + } + + [Test] + [Arguments(0, 1048576)] + [Arguments(5, 1)] + [Arguments(1, 1048576)] + public async Task ResponseCapacity_RejectsOverflowAndStoreFalseRemainsAvailableAsync(int maxResponses, long maxBytes) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + await new LlmTestServerClient(http).ConfigureAsync(config => config.UseGpt41Mini() + .UseChatResponseCapacity(maxResponses, maxBytes) + .UseChatScenario("capacity", scenario => scenario.Responds("one").Responds("two").Responds("three"))); + if (maxResponses == 1) + { + using var first = await http.PostAsJsonAsync("/openai/v1/responses", new { model = "gpt-4.1-mini", input = "hello" }); + first.EnsureSuccessStatusCode(); + } + using var overflow = await http.PostAsJsonAsync("/openai/v1/responses", new { model = "gpt-4.1-mini", input = "hello" }); + await Assert.That(overflow.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + await Assert.That(await overflow.Content.ReadAsStringAsync()).Contains("llm_test_server_response_capacity_exceeded"); + using var transient = await http.PostAsJsonAsync("/openai/v1/responses", new { model = "gpt-4.1-mini", input = "hello", store = false }); + transient.EnsureSuccessStatusCode(); + var id = (await transient.Content.ReadFromJsonAsync()).GetProperty("id").GetString(); + using var continuation = await http.PostAsJsonAsync("/openai/v1/responses", new { model = "gpt-4.1-mini", input = "hello", previous_response_id = id }); + await Assert.That(continuation.StatusCode).IsEqualTo(HttpStatusCode.NotFound); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/OpenAI/ScenarioFlowIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/ScenarioFlowIntegrationTests.cs new file mode 100644 index 0000000..a0d6683 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/OpenAI/ScenarioFlowIntegrationTests.cs @@ -0,0 +1,71 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Scenarios; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.OpenAI; + +public sealed class ScenarioFlowIntegrationTests +{ + [Test] + public async Task ExactConversation_RequiresScenarioCredentialAndOrderedHistoryAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(config => config.AddChatScenario("exact", scenario => scenario + .RequireBearerToken("scenario-key").WithExactMatch( + new LlmTestServerMessage { Role = "system", Content = "Follow the script" }, + new LlmTestServerMessage { Role = "user", Content = "hello" }) + .Responds("first").DelaysBy(10).Responds("second"))); + using var http = host.GetTestClient(); + var body = new + { + model = "gpt-4.1-mini", + messages = new[] { + new { role = "system", content = "Follow the script" }, new { role = "user", content = "hello" } } + }; + using var unauthenticated = await http.PostAsJsonAsync("/openai/v1/chat/completions", body); + await Assert.That(unauthenticated.StatusCode).IsEqualTo(HttpStatusCode.Unauthorized); + http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", "scenario-key"); + using var reordered = await http.PostAsJsonAsync("/openai/v1/chat/completions", new { body.model, messages = body.messages.Reverse().ToArray() }); + await Assert.That(reordered.StatusCode).IsEqualTo(HttpStatusCode.NotFound); + await Assert.That(await reordered.Content.ReadAsStringAsync()).Contains("llm_test_server_unmatched_request"); + foreach (var expected in new[] { "first", "second" }) + { + using var response = await http.PostAsJsonAsync("/openai/v1/chat/completions", body); + response.EnsureSuccessStatusCode(); + await Assert.That(await response.Content.ReadAsStringAsync()).Contains(expected); + } + using var exhausted = await http.PostAsJsonAsync("/openai/v1/chat/completions", body); + await Assert.That(exhausted.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + var summary = (await http.GetFromJsonAsync("/admin-api/assertions"))!; + await Assert.That(summary.AuthFailed).IsEqualTo(1); + await Assert.That(summary.Unmatched).IsEqualTo(1); + await Assert.That(summary.ScenarioExhausted).IsEqualTo(1); + await Assert.That(summary.Matched).IsEqualTo(2); + } + + [Test] + public async Task CancelledDelayedResponse_DoesNotConsumeScenarioAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(config => config.AddChatScenario("cancel", scenario => scenario.Responds("retry succeeds").DelaysBy(1000))); + using var http = host.GetTestClient(); + var body = new { model = "gpt-4.1-mini", messages = new[] { new { role = "user", content = "hello" } } }; + using var cancellation = new CancellationTokenSource(TimeSpan.FromMilliseconds(150)); + var cancelled = false; + try + { + using var unexpected = await http.PostAsJsonAsync("/openai/v1/chat/completions", body, cancellation.Token); + } + catch (OperationCanceledException) when (cancellation.IsCancellationRequested) + { + cancelled = true; + } + await Assert.That(cancelled).IsTrue(); + using var retry = await http.PostAsJsonAsync("/openai/v1/chat/completions", body); + retry.EnsureSuccessStatusCode(); + await Assert.That(await retry.Content.ReadAsStringAsync()).Contains("retry succeeds"); + var summary = (await http.GetFromJsonAsync("/admin-api/assertions"))!; + await Assert.That(summary.Matched).IsEqualTo(1); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Policy/AzureNativeOutputPolicyIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Policy/AzureNativeOutputPolicyIntegrationTests.cs new file mode 100644 index 0000000..c23f705 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Policy/AzureNativeOutputPolicyIntegrationTests.cs @@ -0,0 +1,461 @@ +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Collections.Concurrent; +using System.Net; +using System.Net.Http.Json; +using System.Text; +using System.Text.Json; +using global::Azure.AI.Inference; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Scenarios; +using OpenAI.Chat; + +namespace ManagedCode.LlmTestServer.Tests.Policy; + +public sealed partial class OutputDispositionIntegrationTests +{ + private static ChatClient AzurePolicySdk(HttpClient http) + { + var client = new global::Azure.AI.OpenAI.AzureOpenAIClient( + new Uri(http.BaseAddress!, "/azure-openai/"), new ApiKeyCredential(_key), + new global::Azure.AI.OpenAI.AzureOpenAIClientOptions + { + Transport = new HttpClientPipelineTransport(http), + RetryPolicy = new ClientRetryPolicy(0), + }); + return client.GetChatClient(LlmTestServerKnownModelIds.Gpt41Mini); + } + + private static ChatCompletionsClient FoundryPolicySdk(HttpClient http, string endpoint) + { + var options = new AzureAIInferenceClientOptions + { + Transport = new global::Azure.Core.Pipeline.HttpClientTransport(http), + }; + options.Retry.MaxRetries = 0; + return new(new Uri(http.BaseAddress!, endpoint), new global::Azure.AzureKeyCredential(_key), options); + } + + // Pass-through observation of the original SDK request; no version, headers, or content are rewritten. + private sealed class AzurePolicyObserver : DelegatingHandler + { + public ConcurrentQueue<(Uri Uri, HttpStatusCode Status)> Observed { get; } = new(); + + public AzurePolicyObserver() : base(new HttpClientHandler()) { } + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + var response = await base.SendAsync(request, cancellationToken); + Observed.Enqueue((request.RequestUri!, response.StatusCode)); + return response; + } + } + + [Test] + [Arguments(LlmTestServerOutputDispositionKind.Filtered, false, false)] + [Arguments(LlmTestServerOutputDispositionKind.Filtered, true, false)] + [Arguments(LlmTestServerOutputDispositionKind.Filtered, false, true)] + [Arguments(LlmTestServerOutputDispositionKind.Filtered, true, true)] + [Arguments(LlmTestServerOutputDispositionKind.Refused, false, false)] + [Arguments(LlmTestServerOutputDispositionKind.Refused, true, false)] + [Arguments(LlmTestServerOutputDispositionKind.Refused, false, true)] + [Arguments(LlmTestServerOutputDispositionKind.Refused, true, true)] + public async Task AzureDeployment_OfficialSdkDefaultVersionReturnsNativePolicyJsonAndSseThenRecoversAsync( + LlmTestServerOutputDispositionKind kind, bool streaming, bool empty) + { + await using var host = await StartAsync(config => + { + config.RequireBearerToken(_key); + ConfigureQueue(config, LlmTestServerKnownModelIds.Gpt41Mini, kind, empty); + }); + using var http = host.Client(); var control = new LlmTestServerClient(http); + using var observer = new AzurePolicyObserver(); + using var sdkHttp = new HttpClient(observer) { BaseAddress = http.BaseAddress }; + var sdk = AzurePolicySdk(sdkHttp); + var text = new StringBuilder(); var refusal = new StringBuilder(); + ChatFinishReason? finish = null; ChatTokenUsage? usage = null; var hasRefusal = false; + using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + if (streaming) + { + await foreach (var update in sdk.CompleteChatStreamingAsync([new UserChatMessage("policy")], cancellationToken: deadline.Token)) + { + foreach (var part in update.ContentUpdate) { text.Append(part.Text); } + refusal.Append(update.RefusalUpdate); hasRefusal |= update.RefusalUpdate is not null; + finish = update.FinishReason ?? finish; usage = update.Usage ?? usage; + } + } + else + { + var result = (await sdk.CompleteChatAsync([new UserChatMessage("policy")], cancellationToken: deadline.Token)).Value; + foreach (var part in result.Content) { text.Append(part.Text); } + refusal.Append(result.Refusal); hasRefusal = result.Refusal is not null; + finish = result.FinishReason; usage = result.Usage; + } + var expected = empty ? string.Empty : _text; + await Assert.That(text.ToString()).IsEqualTo(kind == LlmTestServerOutputDispositionKind.Filtered ? expected : string.Empty); + await Assert.That(refusal.ToString()).IsEqualTo(kind == LlmTestServerOutputDispositionKind.Refused ? expected : string.Empty); + await Assert.That(hasRefusal).IsEqualTo(kind == LlmTestServerOutputDispositionKind.Refused); + await Assert.That(finish).IsEqualTo(kind == LlmTestServerOutputDispositionKind.Filtered ? ChatFinishReason.ContentFilter : ChatFinishReason.Stop); + await EvidenceAsync(control, kind, streaming); + var trace = await CompletedAsync(control, 1); + await Assert.That(trace.Path).IsEqualTo("/azure-openai/openai/deployments/" + LlmTestServerKnownModelIds.Gpt41Mini + "/chat/completions"); + await Assert.That(usage).IsNotNull(); + await Assert.That(usage!.InputTokenCount).IsEqualTo(trace.Usage!.InputTokens); + await Assert.That(usage.OutputTokenCount).IsEqualTo(trace.Usage.OutputTokens); + await Assert.That(usage.TotalTokenCount).IsEqualTo(trace.Usage.TotalTokens); + var observed = observer.Observed.Single(); + await Assert.That(observed.Status).IsEqualTo(HttpStatusCode.OK); + await Assert.That(observed.Uri.AbsolutePath).IsEqualTo(trace.Path); + // This value is observed from the pinned official SDK default, never selected in options. + await Assert.That(observed.Uri.Query).IsEqualTo("?api-version=2025-04-01-preview"); + var next = (await sdk.CompleteChatAsync([new UserChatMessage("next")], cancellationToken: deadline.Token)).Value; + await Assert.That(next.Content.Single().Text).IsEqualTo(_recovery); + await RecoveredAsync(control); + } + + [Test] + [Arguments("/microsoft-foundry", false, false)] + [Arguments("/microsoft-foundry", true, false)] + [Arguments("/microsoft-foundry", false, true)] + [Arguments("/microsoft-foundry", true, true)] + [Arguments("/microsoft-foundry/models", false, false)] + [Arguments("/microsoft-foundry/models", true, false)] + [Arguments("/microsoft-foundry/models", false, true)] + [Arguments("/microsoft-foundry/models", true, true)] + public async Task FoundryInference_OfficialSdkDefaultVersionReturnsContentFilteredThenRecoversAsync(string endpoint, bool streaming, bool empty) + { + await using var host = await StartAsync(config => + { + config.RequireBearerToken(_key); + ConfigureQueue(config, LlmTestServerKnownModelIds.Gpt41Mini, LlmTestServerOutputDispositionKind.Filtered, empty); + }); + using var http = host.Client(); var control = new LlmTestServerClient(http); + using var observer = new AzurePolicyObserver(); + using var sdkHttp = new HttpClient(observer) { BaseAddress = http.BaseAddress }; + var sdk = FoundryPolicySdk(sdkHttp, endpoint); + var options = new ChatCompletionsOptions([new ChatRequestUserMessage("policy")]) { Model = LlmTestServerKnownModelIds.Gpt41Mini }; + using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + var text = new StringBuilder(); CompletionsFinishReason? finish = null; CompletionsUsage? usage = null; var usageUpdates = 0; + if (streaming) + { + using var response = await sdk.CompleteStreamingAsync(options, deadline.Token); + await foreach (var update in response) + { + text.Append(update.ContentUpdate); + finish = update.FinishReason ?? finish; usage = update.Usage ?? usage; + if (update.Usage is not null) { usageUpdates++; } + } + } + else + { + var response = (await sdk.CompleteAsync(options, deadline.Token)).Value; + text.Append(response.Content); finish = response.FinishReason; usage = response.Usage; + } + await Assert.That(text.ToString()).IsEqualTo(empty ? string.Empty : _text); + await Assert.That(finish).IsEqualTo(CompletionsFinishReason.ContentFiltered); + await EvidenceAsync(control, LlmTestServerOutputDispositionKind.Filtered, streaming); + var trace = await CompletedAsync(control, 1); + await Assert.That(trace.Path).IsEqualTo(endpoint + "/chat/completions"); + await Assert.That(usage).IsNotNull(); + await Assert.That(usage!.PromptTokens).IsEqualTo(trace.Usage!.InputTokens); + await Assert.That(usage.CompletionTokens).IsEqualTo(trace.Usage.OutputTokens); + await Assert.That(usage.TotalTokens).IsEqualTo(trace.Usage.TotalTokens); + if (streaming) { await Assert.That(usageUpdates).IsEqualTo(1); } + var observed = observer.Observed.Single(); + await Assert.That(observed.Status).IsEqualTo(HttpStatusCode.OK); + await Assert.That(observed.Uri.AbsolutePath).IsEqualTo(trace.Path); + await Assert.That(observed.Uri.Query).IsEqualTo("?api-version=2024-05-01-preview"); + var next = (await sdk.CompleteAsync(new ChatCompletionsOptions([new ChatRequestUserMessage("next")]) + { Model = LlmTestServerKnownModelIds.Gpt41Mini }, deadline.Token)).Value; + await Assert.That(next.Content).IsEqualTo(_recovery); + await RecoveredAsync(control); + } + + [Test] + [Arguments("/microsoft-foundry", false)] + [Arguments("/microsoft-foundry", true)] + [Arguments("/microsoft-foundry/models", false)] + [Arguments("/microsoft-foundry/models", true)] + public async Task FoundryInference_RefusedIs409BeforeConsumptionAndAzureDeploymentCanUseTheOriginalFixtureAsync(string endpoint, bool streaming) + { + await using var host = await StartAsync(config => ConfigureQueue(config, LlmTestServerKnownModelIds.Gpt41Mini, LlmTestServerOutputDispositionKind.Refused)); + using var http = host.Client(); var control = new LlmTestServerClient(http); + var sdk = FoundryPolicySdk(http, endpoint); + var options = new ChatCompletionsOptions([new ChatRequestUserMessage("policy")]) { Model = LlmTestServerKnownModelIds.Gpt41Mini }; + global::Azure.RequestFailedException? failure = null; + try + { + if (streaming) + { + using var response = await sdk.CompleteStreamingAsync(options); + await foreach (var update in response) { _ = update; } + } + else { await sdk.CompleteAsync(options); } + } + catch (global::Azure.RequestFailedException exception) { failure = exception; } + await Assert.That(failure?.Status).IsEqualTo(409); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(0); + var qualified = (await AzurePolicySdk(http).CompleteChatAsync([new UserChatMessage("policy")])).Value; + await Assert.That(qualified.Refusal).IsEqualTo(_text); + var next = (await sdk.CompleteAsync(new ChatCompletionsOptions([new ChatRequestUserMessage("next")]) + { Model = LlmTestServerKnownModelIds.Gpt41Mini })).Value; + await Assert.That(next.Content).IsEqualTo(_recovery); + await Assert.That((await CompletedAsync(control, 3)).OutputDisposition).IsNull(); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(2); + } + + [Test] + [Arguments("deployment")] + [Arguments("/microsoft-foundry")] + [Arguments("/microsoft-foundry/models")] + public async Task AzureNativeSdk_InputFilterIs400WhileOutputFilterIs200AndDoesNotConsumeQueueAsync(string surface) + { + await using var host = await StartAsync(config => + { + config.SimulateContentFilter("blocked-input"); + ConfigureQueue(config, LlmTestServerKnownModelIds.Gpt41Mini, LlmTestServerOutputDispositionKind.Filtered); + }); + using var http = host.Client(); var control = new LlmTestServerClient(http); + var status = 0; + if (surface == "deployment") + { + try { await AzurePolicySdk(http).CompleteChatAsync([new UserChatMessage("blocked-input")]); } + catch (ClientResultException exception) { status = exception.Status; } + } + else + { + try { await FoundryPolicySdk(http, surface).CompleteAsync(new ChatCompletionsOptions([new ChatRequestUserMessage("blocked-input")]) + { Model = LlmTestServerKnownModelIds.Gpt41Mini }); } + catch (global::Azure.RequestFailedException exception) { status = exception.Status; } + } + await Assert.That(status).IsEqualTo(400); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(0); + await Assert.That((await CompletedAsync(control, 1)).OutputDisposition).IsNull(); + if (surface == "deployment") + { + var sdk = AzurePolicySdk(http); + await Assert.That((await sdk.CompleteChatAsync([new UserChatMessage("policy")])).Value.FinishReason).IsEqualTo(ChatFinishReason.ContentFilter); + await Assert.That((await sdk.CompleteChatAsync([new UserChatMessage("next")])).Value.Content.Single().Text).IsEqualTo(_recovery); + } + else + { + var sdk = FoundryPolicySdk(http, surface); + await Assert.That((await sdk.CompleteAsync(new ChatCompletionsOptions([new ChatRequestUserMessage("policy")]) + { Model = LlmTestServerKnownModelIds.Gpt41Mini })).Value.FinishReason).IsEqualTo(CompletionsFinishReason.ContentFiltered); + await Assert.That((await sdk.CompleteAsync(new ChatCompletionsOptions([new ChatRequestUserMessage("next")]) + { Model = LlmTestServerKnownModelIds.Gpt41Mini })).Value.Content).IsEqualTo(_recovery); + } + await Assert.That((await CompletedAsync(control, 2)).OutputDisposition!.Kind).IsEqualTo(LlmTestServerOutputDispositionKind.Filtered); + await Assert.That((await CompletedAsync(control, 3)).OutputDisposition).IsNull(); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(2); + } + + [Test] + [Arguments("deployment", false)] + [Arguments("deployment", true)] + [Arguments("/microsoft-foundry", false)] + [Arguments("/microsoft-foundry", true)] + [Arguments("/microsoft-foundry/models", false)] + [Arguments("/microsoft-foundry/models", true)] + public async Task AzureNativeSdk_ToolsAndStructuredRequestsAre409BeforeConsumingFilteredFixtureAsync(string surface, bool schema) + { + await using var host = await StartAsync(config => ConfigureQueue(config, LlmTestServerKnownModelIds.Gpt41Mini, LlmTestServerOutputDispositionKind.Filtered)); + using var http = host.Client(); var control = new LlmTestServerClient(http); + var status = 0; + if (surface == "deployment") + { + var options = new ChatCompletionOptions(); + if (schema) { options.ResponseFormat = ChatResponseFormat.CreateJsonObjectFormat(); } + else { options.Tools.Add(ChatTool.CreateFunctionTool("weather", functionParameters: BinaryData.FromString("{\"type\":\"object\",\"properties\":{}}"))); } + try { await AzurePolicySdk(http).CompleteChatAsync([new UserChatMessage("policy")], options); } + catch (ClientResultException exception) { status = exception.Status; } + } + else + { + var options = new ChatCompletionsOptions([new ChatRequestUserMessage("policy")]) { Model = LlmTestServerKnownModelIds.Gpt41Mini }; + if (schema) { options.ResponseFormat = ChatCompletionsResponseFormat.CreateJsonFormat(); } + else { options.Tools.Add(new ChatCompletionsToolDefinition(new FunctionDefinition("weather") + { Parameters = BinaryData.FromString("{\"type\":\"object\",\"properties\":{}}") })); } + try { await FoundryPolicySdk(http, surface).CompleteAsync(options); } + catch (global::Azure.RequestFailedException exception) { status = exception.Status; } + } + await Assert.That(status).IsEqualTo(409); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(0); + var first = (await AzurePolicySdk(http).CompleteChatAsync([new UserChatMessage("policy")])).Value; + await Assert.That(first.FinishReason).IsEqualTo(ChatFinishReason.ContentFilter); + await Assert.That(first.Content.Single().Text).IsEqualTo(_text); + var next = (await AzurePolicySdk(http).CompleteChatAsync([new UserChatMessage("next")])).Value; + await Assert.That(next.Content.Single().Text).IsEqualTo(_recovery); + await Assert.That((await CompletedAsync(control, 3)).OutputDisposition).IsNull(); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(2); + } + + [Test] + [Arguments(LlmTestServerOutputDispositionKind.Filtered)] + [Arguments(LlmTestServerOutputDispositionKind.Refused)] + public async Task AzureDeployment_PolicyGraphReplaysAcrossJsonSseAndCanonicalSdkHistoryWhileSessionsStayIsolatedAsync(LlmTestServerOutputDispositionKind kind) + { + await using var host = await StartAsync(_ => { }); + using var http = host.Client(); var control = new LlmTestServerClient(http); + void Configure(LlmTestServerClientConfigurationBuilder config) + { + config.UseGpt41Mini().UseConversation("azure-policy-graph", graph => graph.ForModel(LlmTestServerKnownModelIds.Gpt41Mini) + .Step("policy", step => step.WhenLatestUserEquals("go").Responds(_text, "Visible ", "fixture text") + .WithOutputDisposition(new() { Kind = kind }).Then("next")) + .Step("next", step => step.WhenLatestUserEquals("next").Responds(_recovery).Complete())); + } + await using var firstSession = await control.CreateSessionAsync(Configure); + await using var secondSession = await control.CreateSessionAsync(Configure); + using var firstConversation = firstSession.Control.ForConversation("same-azure-label"); + using var secondConversation = secondSession.Control.ForConversation("same-azure-label"); + global::OpenAI.Chat.ChatMessage[] input = [new SystemChatMessage(string.Join(" ", Enumerable.Repeat("stable policy context", 1400))), new UserChatMessage("go")]; + var firstSdk = AzurePolicySdk(firstConversation.HttpClient); + var first = (await firstSdk.CompleteChatAsync(input)).Value; + var expectedReason = kind == LlmTestServerOutputDispositionKind.Filtered ? ChatFinishReason.ContentFilter : ChatFinishReason.Stop; + await Assert.That(first.FinishReason).IsEqualTo(expectedReason); + await Assert.That(first.Usage.InputTokenDetails.CachedTokenCount).IsEqualTo(0); + var replayText = new StringBuilder(); ChatFinishReason? replayFinish = null; + await foreach (var update in firstSdk.CompleteChatStreamingAsync(input)) + { + foreach (var part in update.ContentUpdate) { replayText.Append(part.Text); } + replayText.Append(update.RefusalUpdate); replayFinish = update.FinishReason ?? replayFinish; + } + await Assert.That(replayText.ToString()).IsEqualTo(_text); + await Assert.That(replayFinish).IsEqualTo(expectedReason); + var replay = await CompletedAsync(firstConversation.Control, 2); + await Assert.That(replay.IsReplay).IsTrue(); + await Assert.That(replay.TerminalMarkerObserved).IsTrue(); + await Assert.That((await firstConversation.Control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); + await Assert.That((await secondSession.Control.GetConversationsAsync()).Count).IsEqualTo(0); + var isolated = (await AzurePolicySdk(secondConversation.HttpClient).CompleteChatAsync(input)).Value; + await Assert.That(isolated.FinishReason).IsEqualTo(expectedReason); + await Assert.That(isolated.Usage.InputTokenDetails.CachedTokenCount).IsEqualTo(0); + var next = (await firstSdk.CompleteChatAsync([.. input, new AssistantChatMessage(first), new UserChatMessage("next")])).Value; + await Assert.That(next.Content.Single().Text).IsEqualTo(_recovery); + await Assert.That(next.Usage.InputTokenDetails.CachedTokenCount).IsGreaterThan(0); + var trace = await CompletedAsync(firstConversation.Control, 3); + var assistant = trace.Messages.Single(message => message.Role == "assistant"); + await Assert.That(assistant.Content).IsEqualTo(_text); + await Assert.That(assistant.OutputDisposition?.Kind).IsEqualTo(kind == LlmTestServerOutputDispositionKind.Refused ? kind : (LlmTestServerOutputDispositionKind?)null); + await Assert.That((await firstConversation.Control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(2); + await Assert.That((await secondConversation.Control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); + } + + [Test] + [Arguments("/microsoft-foundry", false)] + [Arguments("/microsoft-foundry", true)] + [Arguments("/microsoft-foundry/models", false)] + [Arguments("/microsoft-foundry/models", true)] + public async Task FoundryInference_NativeSseCarriesFinalUsageWithOneChoiceWithoutUsageOnlyFramesAsync(string endpoint, bool filtered) + { + await using var host = await StartAsync(config => config.AddModel(LlmTestServerKnownModelIds.Gpt41Mini, LlmTestServerModelKind.Chat) + .AddChatScenario("foundry-native-stream", scenario => + { + scenario.ForModel(LlmTestServerKnownModelIds.Gpt41Mini).Responds(_text, "Visible ", "fixture text"); + if (filtered) { scenario.WithOutputDisposition(new() { Kind = LlmTestServerOutputDispositionKind.Filtered }); } + scenario.Responds(_recovery); + })); + using var http = host.Client(); var control = new LlmTestServerClient(http); + // Raw protocol evidence supplements the original SDK tests. No stream_options extension is sent. + using var response = await http.PostAsJsonAsync(endpoint + "/chat/completions?api-version=2024-05-01-preview", new + { + model = LlmTestServerKnownModelIds.Gpt41Mini, + stream = true, + messages = new[] { new { role = "user", content = "policy" } }, + }); + await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.OK); + var records = (await response.Content.ReadAsStringAsync()).Split('\n', StringSplitOptions.RemoveEmptyEntries) + .Where(line => line.StartsWith("data:", StringComparison.Ordinal)).Select(line => line[5..].Trim()).ToArray(); + await Assert.That(records[^1]).IsEqualTo("[DONE]"); + var frames = records[..^1].Select(record => JsonSerializer.Deserialize(record)).ToArray(); + await Assert.That(frames.Length).IsGreaterThan(1); + await Assert.That(frames.All(frame => frame.GetProperty("choices").GetArrayLength() == 1)).IsTrue(); + foreach (var frame in frames[..^1]) + { + await Assert.That(frame.GetProperty("choices")[0].GetProperty("finish_reason").ValueKind).IsEqualTo(JsonValueKind.Null); + await Assert.That(!frame.TryGetProperty("usage", out var intermediateUsage) || intermediateUsage.ValueKind == JsonValueKind.Null).IsTrue(); + } + var final = frames[^1]; + await Assert.That(final.GetProperty("choices")[0].GetProperty("finish_reason").GetString()).IsEqualTo(filtered ? "content_filter" : "stop"); + var usage = final.GetProperty("usage"); + await Assert.That(usage.ValueKind).IsEqualTo(JsonValueKind.Object); + await Assert.That(frames.Count(frame => frame.TryGetProperty("usage", out var value) && value.ValueKind == JsonValueKind.Object)).IsEqualTo(1); + var trace = await CompletedAsync(control, 1); + await Assert.That(trace.TerminalMarkerObserved).IsTrue(); + await Assert.That(trace.OutputDisposition?.Kind == LlmTestServerOutputDispositionKind.Filtered).IsEqualTo(filtered); + await Assert.That(usage.GetProperty("prompt_tokens").GetInt32()).IsEqualTo(trace.Usage!.InputTokens); + await Assert.That(usage.GetProperty("completion_tokens").GetInt32()).IsEqualTo(trace.Usage.OutputTokens); + await Assert.That(usage.GetProperty("total_tokens").GetInt32()).IsEqualTo(trace.Usage.TotalTokens); + await Assert.That((await control.VerifyAsync(new() + { Requests = [new() { RequireStreamingTerminalMarker = true }] })).Passed).IsTrue(); + var next = (await FoundryPolicySdk(http, endpoint).CompleteAsync(new ChatCompletionsOptions([new ChatRequestUserMessage("next")]) + { Model = LlmTestServerKnownModelIds.Gpt41Mini })).Value; + await Assert.That(next.Content).IsEqualTo(_recovery); + await RecoveredAsync(control); + } + + [Test] + [Arguments("2024-10-21", LlmTestServerOutputDispositionKind.Filtered, false)] + [Arguments("2024-10-21", LlmTestServerOutputDispositionKind.Filtered, true)] + [Arguments("2024-10-21", LlmTestServerOutputDispositionKind.Refused, false)] + [Arguments("2024-10-21", LlmTestServerOutputDispositionKind.Refused, true)] + [Arguments("2025-04-01-preview", LlmTestServerOutputDispositionKind.Filtered, false)] + [Arguments("2025-04-01-preview", LlmTestServerOutputDispositionKind.Filtered, true)] + [Arguments("2025-04-01-preview", LlmTestServerOutputDispositionKind.Refused, false)] + [Arguments("2025-04-01-preview", LlmTestServerOutputDispositionKind.Refused, true)] + public async Task AzureDeployment_BothDocumentedVersionsReturnNativePolicyJsonAndSseAsync( + string apiVersion, LlmTestServerOutputDispositionKind kind, bool streaming) + { + await using var host = await StartAsync(config => + { + config.RequireBearerToken(_key); + ConfigureQueue(config, LlmTestServerKnownModelIds.Gpt41Mini, kind); + }); + using var http = host.Client(); var control = new LlmTestServerClient(http); + using var request = new HttpRequestMessage(HttpMethod.Post, + "/azure-openai/openai/deployments/" + LlmTestServerKnownModelIds.Gpt41Mini + "/chat/completions?api-version=" + apiVersion) + { + Content = JsonContent.Create(new + { + stream = streaming, + messages = new[] { new { role = "user", content = "policy" } }, + }), + }; + request.Headers.Add("api-key", _key); + using var response = await http.SendAsync(request); + await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.OK); + var text = new StringBuilder(); var refusal = new StringBuilder(); string? finish; + if (streaming) + { + var records = (await response.Content.ReadAsStringAsync()).Split('\n', StringSplitOptions.RemoveEmptyEntries) + .Where(line => line.StartsWith("data:", StringComparison.Ordinal)).Select(line => line[5..].Trim()).ToArray(); + await Assert.That(records[^1]).IsEqualTo("[DONE]"); + var choices = records[..^1].Select(record => JsonSerializer.Deserialize(record)) + .SelectMany(frame => frame.GetProperty("choices").EnumerateArray()).ToArray(); + foreach (var choice in choices) + { + var delta = choice.GetProperty("delta"); + if (delta.TryGetProperty("content", out var content)) { text.Append(content.GetString()); } + if (delta.TryGetProperty("refusal", out var refusalPart)) { refusal.Append(refusalPart.GetString()); } + } + finish = choices.Last().GetProperty("finish_reason").GetString(); + } + else + { + var body = await response.Content.ReadFromJsonAsync(); + var choice = body.GetProperty("choices")[0]; var message = choice.GetProperty("message"); + if (message.TryGetProperty("content", out var content)) { text.Append(content.GetString()); } + if (message.TryGetProperty("refusal", out var refusalPart)) { refusal.Append(refusalPart.GetString()); } + finish = choice.GetProperty("finish_reason").GetString(); + } + var filtered = kind == LlmTestServerOutputDispositionKind.Filtered; + await Assert.That(finish).IsEqualTo(filtered ? "content_filter" : "stop"); + await Assert.That(text.ToString()).IsEqualTo(filtered ? _text : string.Empty); + await Assert.That(refusal.ToString()).IsEqualTo(filtered ? string.Empty : _text); + await EvidenceAsync(control, kind, streaming); + var recovered = (await AzurePolicySdk(http).CompleteChatAsync([new UserChatMessage("next")])).Value; + await Assert.That(recovered.Content.Single().Text).IsEqualTo(_recovery); + await RecoveredAsync(control); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Policy/AzureV1OutputPolicyIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Policy/AzureV1OutputPolicyIntegrationTests.cs new file mode 100644 index 0000000..0090926 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Policy/AzureV1OutputPolicyIntegrationTests.cs @@ -0,0 +1,438 @@ +#pragma warning disable OPENAI001 // Qualify the pinned official Responses SDK against native policy outcomes. +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Net; +using System.Net.Http.Json; +using System.Text; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Providers; +using ManagedCode.LlmTestServer.Scenarios; +using OpenAI; +using OpenAI.Chat; +using OpenAI.Responses; + +namespace ManagedCode.LlmTestServer.Tests.Policy; + +public sealed partial class OutputDispositionIntegrationTests +{ + [Test] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI, LlmTestServerOutputDispositionKind.Filtered, false, false)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry, LlmTestServerOutputDispositionKind.Filtered, false, false)] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI, LlmTestServerOutputDispositionKind.Filtered, true, false)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry, LlmTestServerOutputDispositionKind.Filtered, true, false)] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI, LlmTestServerOutputDispositionKind.Filtered, false, true)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry, LlmTestServerOutputDispositionKind.Filtered, false, true)] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI, LlmTestServerOutputDispositionKind.Filtered, true, true)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry, LlmTestServerOutputDispositionKind.Filtered, true, true)] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI, LlmTestServerOutputDispositionKind.Refused, false, false)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry, LlmTestServerOutputDispositionKind.Refused, false, false)] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI, LlmTestServerOutputDispositionKind.Refused, true, false)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry, LlmTestServerOutputDispositionKind.Refused, true, false)] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI, LlmTestServerOutputDispositionKind.Refused, false, true)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry, LlmTestServerOutputDispositionKind.Refused, false, true)] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI, LlmTestServerOutputDispositionKind.Refused, true, true)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry, LlmTestServerOutputDispositionKind.Refused, true, true)] + public async Task AzureV1Chat_OfficialSdkPolicyOutcomesAndRecoveryAsync( + string provider, LlmTestServerOutputDispositionKind kind, bool streaming, bool empty) + { + await using var host = await StartAsync(config => ConfigureQueue(config, LlmTestServerKnownModelIds.Gpt41Mini, kind, empty)); + using var http = host.Client(); + var control = new LlmTestServerClient(http); + var sdk = AzurePolicyChatSdk(http, provider); + var content = new StringBuilder(); var refusal = new StringBuilder(); ChatFinishReason? finish = null; var sawRefusal = false; ChatTokenUsage? sdkUsage = null; + using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + if (streaming) + { + await foreach (var update in sdk.CompleteChatStreamingAsync([new UserChatMessage("policy")], cancellationToken: deadline.Token)) + { + foreach (var part in update.ContentUpdate) { content.Append(part.Text); } + refusal.Append(update.RefusalUpdate); + sawRefusal |= update.RefusalUpdate is not null; + finish = update.FinishReason ?? finish; + sdkUsage = update.Usage ?? sdkUsage; + } + } + else + { + var response = (await sdk.CompleteChatAsync([new UserChatMessage("policy")], cancellationToken: deadline.Token)).Value; + foreach (var part in response.Content) { content.Append(part.Text); } + refusal.Append(response.Refusal); + sawRefusal = response.Refusal is not null; + finish = response.FinishReason; + sdkUsage = response.Usage; + } + var expected = empty ? string.Empty : _text; + await Assert.That(sawRefusal).IsEqualTo(kind == LlmTestServerOutputDispositionKind.Refused); + await Assert.That(finish).IsEqualTo(kind == LlmTestServerOutputDispositionKind.Filtered ? ChatFinishReason.ContentFilter : ChatFinishReason.Stop); + await Assert.That(content.ToString()).IsEqualTo(kind == LlmTestServerOutputDispositionKind.Filtered ? expected : string.Empty); + await Assert.That(refusal.ToString()).IsEqualTo(kind == LlmTestServerOutputDispositionKind.Refused ? expected : string.Empty); + await EvidenceAsync(control, kind, streaming); + var trace = await CompletedAsync(control, 1); + await Assert.That(trace.Path).IsEqualTo(provider + "/openai/v1/chat/completions"); + await Assert.That(trace.Usage!.InputTokens).IsGreaterThan(0); + await Assert.That(sdkUsage!.TotalTokenCount).IsEqualTo(trace.Usage.TotalTokens); + var recovered = (await sdk.CompleteChatAsync([new UserChatMessage("next")], cancellationToken: deadline.Token)).Value; + await Assert.That(recovered.Content.Single().Text).IsEqualTo(_recovery); + await Assert.That(recovered.FinishReason).IsEqualTo(ChatFinishReason.Stop); + await RecoveredAsync(control); + } + + [Test] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI, LlmTestServerOutputDispositionKind.Filtered, false, false)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry, LlmTestServerOutputDispositionKind.Filtered, false, false)] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI, LlmTestServerOutputDispositionKind.Filtered, true, false)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry, LlmTestServerOutputDispositionKind.Filtered, true, false)] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI, LlmTestServerOutputDispositionKind.Filtered, false, true)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry, LlmTestServerOutputDispositionKind.Filtered, false, true)] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI, LlmTestServerOutputDispositionKind.Filtered, true, true)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry, LlmTestServerOutputDispositionKind.Filtered, true, true)] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI, LlmTestServerOutputDispositionKind.Refused, false, false)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry, LlmTestServerOutputDispositionKind.Refused, false, false)] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI, LlmTestServerOutputDispositionKind.Refused, true, false)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry, LlmTestServerOutputDispositionKind.Refused, true, false)] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI, LlmTestServerOutputDispositionKind.Refused, false, true)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry, LlmTestServerOutputDispositionKind.Refused, false, true)] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI, LlmTestServerOutputDispositionKind.Refused, true, true)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry, LlmTestServerOutputDispositionKind.Refused, true, true)] + public async Task AzureV1Responses_OfficialSdkPolicyOutcomesAndRecoveryAsync( + string provider, LlmTestServerOutputDispositionKind kind, bool streaming, bool empty) + { + await using var host = await StartAsync(config => ConfigureQueue(config, LlmTestServerKnownModelIds.Gpt41Mini, kind, empty)); + using var http = host.Client(); + var control = new LlmTestServerClient(http); + var sdk = AzurePolicyResponsesSdk(http, provider); + var options = new CreateResponseOptions + { Model = LlmTestServerKnownModelIds.Gpt41Mini, StoredOutputEnabled = false, StreamingEnabled = streaming, InputItems = { ResponseItem.CreateUserMessageItem("policy") } }; + ResponseResult? result = null; var deltas = new StringBuilder(); var refusals = new StringBuilder(); var incomplete = false; var completed = false; + using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + if (streaming) + { + await foreach (var update in sdk.CreateResponseStreamingAsync(options, deadline.Token)) + { + if (update is StreamingResponseOutputTextDeltaUpdate text) { deltas.Append(text.Delta); } + if (update is StreamingResponseRefusalDeltaUpdate refusal) { refusals.Append(refusal.Delta); } + if (update is StreamingResponseIncompleteUpdate incompleteEnding) { incomplete = true; result = incompleteEnding.Response; } + if (update is StreamingResponseCompletedUpdate completedEnding) { completed = true; result = completedEnding.Response; } + await Assert.That(update is StreamingResponseFailedUpdate or StreamingResponseErrorUpdate).IsFalse(); + } + await Assert.That(incomplete).IsEqualTo(kind == LlmTestServerOutputDispositionKind.Filtered); + await Assert.That(completed).IsEqualTo(kind == LlmTestServerOutputDispositionKind.Refused); + await Assert.That(deltas.ToString()).IsEqualTo(kind == LlmTestServerOutputDispositionKind.Filtered && !empty ? _text : string.Empty); + await Assert.That(refusals.ToString()).IsEqualTo(kind == LlmTestServerOutputDispositionKind.Refused && !empty ? _text : string.Empty); + } + else { result = (await sdk.CreateResponseAsync(options, deadline.Token)).Value; } + await Assert.That(result).IsNotNull(); + await Assert.That(result!.Status).IsEqualTo(kind == LlmTestServerOutputDispositionKind.Filtered ? ResponseStatus.Incomplete : ResponseStatus.Completed); + if (kind == LlmTestServerOutputDispositionKind.Filtered) + { + await Assert.That(result.IncompleteStatusDetails.Reason).IsEqualTo(ResponseIncompleteStatusReason.ContentFilter); + await Assert.That(result.GetOutputText()).IsEqualTo(empty ? string.Empty : _text); + } + else + { + var message = (MessageResponseItem)result.OutputItems.Single(); + await Assert.That(message.Content.Single().Kind).IsEqualTo(ResponseContentPartKind.Refusal); + await Assert.That(message.Content.Single().Refusal).IsEqualTo(empty ? string.Empty : _text); + await Assert.That(result.IncompleteStatusDetails).IsNull(); + } + await EvidenceAsync(control, kind, streaming); + var trace = await CompletedAsync(control, 1); + await Assert.That(trace.Path).IsEqualTo(provider + "/openai/v1/responses"); + await Assert.That(trace.Usage!.InputTokens).IsGreaterThan(0); + await Assert.That(result.Usage.TotalTokenCount).IsEqualTo(trace.Usage.TotalTokens); + var next = (await sdk.CreateResponseAsync(new CreateResponseOptions + { Model = LlmTestServerKnownModelIds.Gpt41Mini, StoredOutputEnabled = false, InputItems = { ResponseItem.CreateUserMessageItem("next") } }, deadline.Token)).Value; + await Assert.That(next.Status).IsEqualTo(ResponseStatus.Completed); + await Assert.That(next.GetOutputText()).IsEqualTo(_recovery); + await RecoveredAsync(control); + } + + [Test] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI, LlmTestServerOutputDispositionKind.Filtered)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry, LlmTestServerOutputDispositionKind.Filtered)] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI, LlmTestServerOutputDispositionKind.Refused)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry, LlmTestServerOutputDispositionKind.Refused)] + public async Task AzureV1Chat_PolicyHistoryReplayAndSessionsRemainIsolatedAsync(string provider, LlmTestServerOutputDispositionKind kind) + { + await using var host = await StartAsync(_ => { }); + using var http = host.Client(); var control = new LlmTestServerClient(http); + void Configure(LlmTestServerClientConfigurationBuilder config) + { + config.UseGpt41Mini().UseConversation("policy-graph", graph => graph.ForModel(LlmTestServerKnownModelIds.Gpt41Mini) + .Step("policy", step => step.WhenLatestUserEquals("go").Responds(_text, "Visible ", "fixture text") + .WithOutputDisposition(new() { Kind = kind }).Then("next")) + .Step("next", step => step.WhenLatestUserEquals("next").Responds(_recovery).Complete())); + } + await using var firstSession = await control.CreateSessionAsync(Configure); + await using var secondSession = await control.CreateSessionAsync(Configure); + using var firstConversation = firstSession.Control.ForConversation("same-label"); + using var secondConversation = secondSession.Control.ForConversation("same-label"); + var prompt = new SystemChatMessage(string.Join(" ", Enumerable.Repeat("stable policy context", 1400))); + global::OpenAI.Chat.ChatMessage[] messages = [prompt, new UserChatMessage("go")]; + using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + var firstSdk = AzurePolicyChatSdk(firstConversation.HttpClient, provider); + var first = (await firstSdk.CompleteChatAsync(messages, cancellationToken: deadline.Token)).Value; + await Assert.That(first.FinishReason).IsEqualTo(kind == LlmTestServerOutputDispositionKind.Filtered ? ChatFinishReason.ContentFilter : ChatFinishReason.Stop); + await Assert.That(first.Usage.InputTokenDetails.CachedTokenCount).IsEqualTo(0); + var replayText = new StringBuilder(); var replayRefusal = new StringBuilder(); ChatFinishReason? replayReason = null; + await foreach (var update in firstSdk.CompleteChatStreamingAsync(messages, cancellationToken: deadline.Token)) + { + foreach (var part in update.ContentUpdate) { replayText.Append(part.Text); } + replayRefusal.Append(update.RefusalUpdate); + replayReason = update.FinishReason ?? replayReason; + } + await Assert.That(replayReason).IsEqualTo(first.FinishReason); + await Assert.That(kind == LlmTestServerOutputDispositionKind.Filtered ? replayText.ToString() : replayRefusal.ToString()).IsEqualTo(_text); + await Assert.That((await firstConversation.Control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); + await Assert.That((await secondSession.Control.GetConversationsAsync()).Count).IsEqualTo(0); + var replay = await CompletedAsync(firstConversation.Control, 2); + await Assert.That(replay.IsReplay).IsTrue(); + await Assert.That(replay.OutputDisposition!.Kind).IsEqualTo(kind); + await Assert.That(replay.TerminalMarkerObserved).IsTrue(); + var second = (await AzurePolicyChatSdk(secondConversation.HttpClient, provider).CompleteChatAsync(messages, cancellationToken: deadline.Token)).Value; + await Assert.That(second.Usage.InputTokenDetails.CachedTokenCount).IsEqualTo(0); + await Assert.That((await secondConversation.Control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); + var recovered = (await firstSdk.CompleteChatAsync([.. messages, kind == LlmTestServerOutputDispositionKind.Refused ? new AssistantChatMessage(ChatMessageContentPart.CreateRefusalPart(first.Refusal)) : new AssistantChatMessage(first), new UserChatMessage("next")], cancellationToken: deadline.Token)).Value; + await Assert.That(recovered.Content.Single().Text).IsEqualTo(_recovery); + await Assert.That(recovered.Usage.InputTokenDetails.CachedTokenCount).IsGreaterThan(0); + await Assert.That((await firstConversation.Control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(2); + await Assert.That((await firstConversation.Control.GetConversationsAsync()).Single().IsTerminal).IsTrue(); + await Assert.That((await secondConversation.Control.GetConversationsAsync()).Single().IsTerminal).IsFalse(); + var followTrace = await CompletedAsync(firstConversation.Control, 3); + var assistant = followTrace.Messages.Single(message => message.Role == "assistant"); + await Assert.That(assistant.Content).IsEqualTo(_text); + if (kind == LlmTestServerOutputDispositionKind.Refused) + { await Assert.That(assistant.OutputDisposition!.Kind).IsEqualTo(kind); } + } + + [Test] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI, LlmTestServerOutputDispositionKind.Refused)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry, LlmTestServerOutputDispositionKind.Refused)] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI, LlmTestServerOutputDispositionKind.Filtered)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry, LlmTestServerOutputDispositionKind.Filtered)] + public async Task AzureV1Responses_StoredAndManualHistoryReplaySameTurnWithCommittedCacheAsync(string provider, LlmTestServerOutputDispositionKind kind) + { + await using var host = await StartAsync(config => config.AddConversation("policy-history-projection", graph => graph + .ForModel(LlmTestServerKnownModelIds.Gpt41Mini) + .Step("policy", step => step.WhenLatestUserEquals("go").Responds(_text) + .WithOutputDisposition(new() + { + Kind = kind, + Category = kind == LlmTestServerOutputDispositionKind.Refused ? "cyber" : null, + Explanation = kind == LlmTestServerOutputDispositionKind.Refused ? "Full diagnostic metadata" : null, + }).Then("next")) + .Step("next", step => step.WhenLatestUserEquals("next").Responds(_recovery).Then("tail")) + .Step("tail", step => step.WhenLatestUserEquals("tail").Responds("Remaining graph step").Complete()))); + using var http = host.Client(); var control = new LlmTestServerClient(http); + using var conversation = control.ForConversation("canonical-policy-history"); + var sdk = AzurePolicyResponsesSdk(conversation.HttpClient, provider); + using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + var prefix = string.Join(" ", Enumerable.Repeat("stable Azure v1 policy context", 1400)); + var first = (await sdk.CreateResponseAsync(new CreateResponseOptions + { + Model = LlmTestServerKnownModelIds.Gpt41Mini, + Instructions = prefix, + StoredOutputEnabled = true, + InputItems = { ResponseItem.CreateUserMessageItem("go") }, + }, deadline.Token)).Value; + await Assert.That(first.Status).IsEqualTo(kind == LlmTestServerOutputDispositionKind.Filtered ? ResponseStatus.Incomplete : ResponseStatus.Completed); + var automatic = (await sdk.CreateResponseAsync(new CreateResponseOptions + { + Model = LlmTestServerKnownModelIds.Gpt41Mini, + Instructions = prefix, + StoredOutputEnabled = false, + PreviousResponseId = first.Id, + InputItems = { ResponseItem.CreateUserMessageItem("next") }, + }, deadline.Token)).Value; + await Assert.That(automatic.GetOutputText()).IsEqualTo(_recovery); + await Assert.That((await conversation.Control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(2); + var assistant = kind == LlmTestServerOutputDispositionKind.Refused + ? ResponseItem.CreateAssistantMessageItem([ResponseContentPart.CreateRefusalPart(_text)]) + : ResponseItem.CreateAssistantMessageItem(_text); + var canonical = (await sdk.CreateResponseAsync(new CreateResponseOptions + { + Model = LlmTestServerKnownModelIds.Gpt41Mini, + Instructions = prefix, + StoredOutputEnabled = false, + InputItems = { ResponseItem.CreateUserMessageItem("go"), assistant, ResponseItem.CreateUserMessageItem("next") }, + }, deadline.Token)).Value; + await Assert.That(canonical.Id).IsEqualTo(automatic.Id); + await Assert.That(canonical.GetOutputText()).IsEqualTo(_recovery); + await Assert.That(canonical.Usage.InputTokenCount).IsEqualTo(automatic.Usage.InputTokenCount); + await Assert.That(canonical.Usage.OutputTokenCount).IsEqualTo(automatic.Usage.OutputTokenCount); + var state = (await conversation.Control.GetConversationsAsync()).Single(); + await Assert.That(state.ExecutedSteps).IsEqualTo(2); + await Assert.That(state.NextSteps.Single()).IsEqualTo("tail"); + await Assert.That(state.IsTerminal).IsFalse(); + var replay = await CompletedAsync(conversation.Control, 3); + await Assert.That(replay.IsReplay).IsTrue(); + var history = replay.Messages.Single(message => message.Role == "assistant"); + await Assert.That(history.Content).IsEqualTo(_text); + if (kind == LlmTestServerOutputDispositionKind.Refused) + { + await Assert.That(history.OutputDisposition!.Kind).IsEqualTo(kind); + await Assert.That(history.OutputDisposition.Category).IsNull(); + await Assert.That(history.OutputDisposition.Explanation).IsNull(); + } + else { await Assert.That(history.OutputDisposition).IsNull(); } + var journal = await conversation.Control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending }); + await Assert.That(journal.Requests[0].OutputDisposition!.Kind).IsEqualTo(kind); + await Assert.That(journal.Requests[0].OutputDisposition!.Explanation).IsEqualTo(kind == LlmTestServerOutputDispositionKind.Refused ? "Full diagnostic metadata" : null); + await Assert.That(journal.Requests[1].IsReplay).IsFalse(); + await Assert.That(journal.Requests[0].Usage!.CachedInputTokens).IsEqualTo(0); + await Assert.That(journal.Requests[1].Usage!.CachedInputTokens).IsGreaterThan(0); + await Assert.That(replay.Usage!.TotalTokens).IsEqualTo(journal.Requests[1].Usage!.TotalTokens); + } + + [Test] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI, false, LlmTestServerOutputDispositionKind.Filtered, false)] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI, false, LlmTestServerOutputDispositionKind.Filtered, true)] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI, false, LlmTestServerOutputDispositionKind.Refused, false)] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI, false, LlmTestServerOutputDispositionKind.Refused, true)] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI, true, LlmTestServerOutputDispositionKind.Filtered, false)] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI, true, LlmTestServerOutputDispositionKind.Filtered, true)] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI, true, LlmTestServerOutputDispositionKind.Refused, false)] + [Arguments(LlmTestServerProviderRouteNamespaces.AzureOpenAI, true, LlmTestServerOutputDispositionKind.Refused, true)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry, false, LlmTestServerOutputDispositionKind.Filtered, false)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry, false, LlmTestServerOutputDispositionKind.Filtered, true)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry, false, LlmTestServerOutputDispositionKind.Refused, false)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry, false, LlmTestServerOutputDispositionKind.Refused, true)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry, true, LlmTestServerOutputDispositionKind.Filtered, false)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry, true, LlmTestServerOutputDispositionKind.Filtered, true)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry, true, LlmTestServerOutputDispositionKind.Refused, false)] + [Arguments(LlmTestServerProviderRouteNamespaces.MicrosoftFoundry, true, LlmTestServerOutputDispositionKind.Refused, true)] + public async Task AzureV1_PolicyRejectsToolsAndStructuredOutputBeforeFixtureConsumptionAsync( + string provider, bool responses, LlmTestServerOutputDispositionKind kind, bool schema) + { + await using var host = await StartAsync(config => ConfigureQueue(config, LlmTestServerKnownModelIds.Gpt41Mini, kind)); + using var http = host.Client(); var control = new LlmTestServerClient(http); + if (responses && schema) + { + // Malformed/unsupported wire configuration supplements genuine SDK admission below. + using var rejected = await http.PostAsJsonAsync(provider + "/openai/v1/responses", new + { + model = LlmTestServerKnownModelIds.Gpt41Mini, + input = "policy", + text = new { format = new { type = "json_object" } }, + }); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + } + else + { + ClientResultException? rejected = null; + try + { + if (responses) + { + var options = new CreateResponseOptions + { Model = LlmTestServerKnownModelIds.Gpt41Mini, StoredOutputEnabled = false, InputItems = { ResponseItem.CreateUserMessageItem("policy") } }; + options.Tools.Add(ResponseTool.CreateFunctionTool("weather", BinaryData.FromString("{\"type\":\"object\"}"), false, "Scripted weather")); + await AzurePolicyResponsesSdk(http, provider).CreateResponseAsync(options); + } + else + { + var options = new ChatCompletionOptions(); + if (schema) { options.ResponseFormat = ChatResponseFormat.CreateJsonObjectFormat(); } + else { options.Tools.Add(ChatTool.CreateFunctionTool("weather")); } + await AzurePolicyChatSdk(http, provider).CompleteChatAsync([new UserChatMessage("policy")], options); + } + } + catch (ClientResultException exception) { rejected = exception; } + await Assert.That(rejected?.Status).IsEqualTo(409); + } + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(0); + if (responses) + { + var sdk = AzurePolicyResponsesSdk(http, provider); + var first = (await sdk.CreateResponseAsync(new CreateResponseOptions + { Model = LlmTestServerKnownModelIds.Gpt41Mini, StoredOutputEnabled = false, InputItems = { ResponseItem.CreateUserMessageItem("policy") } })).Value; + await Assert.That(first.Status).IsEqualTo(kind == LlmTestServerOutputDispositionKind.Filtered ? ResponseStatus.Incomplete : ResponseStatus.Completed); + if (kind == LlmTestServerOutputDispositionKind.Refused) + { await Assert.That(((MessageResponseItem)first.OutputItems.Single()).Content.Single().Refusal).IsEqualTo(_text); } + else { await Assert.That(first.GetOutputText()).IsEqualTo(_text); } + var next = (await sdk.CreateResponseAsync(new CreateResponseOptions + { Model = LlmTestServerKnownModelIds.Gpt41Mini, StoredOutputEnabled = false, InputItems = { ResponseItem.CreateUserMessageItem("next") } })).Value; + await Assert.That(next.GetOutputText()).IsEqualTo(_recovery); + } + else + { + var sdk = AzurePolicyChatSdk(http, provider); + var first = (await sdk.CompleteChatAsync([new UserChatMessage("policy")])).Value; + await Assert.That(first.FinishReason).IsEqualTo(kind == LlmTestServerOutputDispositionKind.Filtered ? ChatFinishReason.ContentFilter : ChatFinishReason.Stop); + await Assert.That(kind == LlmTestServerOutputDispositionKind.Filtered ? first.Content.Single().Text : first.Refusal).IsEqualTo(_text); + var next = (await sdk.CompleteChatAsync([new UserChatMessage("next")])).Value; + await Assert.That(next.Content.Single().Text).IsEqualTo(_recovery); + } + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(2); + var evidence = await control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending }); + await Assert.That(evidence.TotalObserved).IsEqualTo(3); + await Assert.That(evidence.Requests[0].StatusCode).IsEqualTo(409); + await Assert.That(evidence.Requests[1].OutputDisposition!.Kind).IsEqualTo(kind); + await Assert.That(evidence.Requests[2].OutputDisposition).IsNull(); + } + + [Test] + [Arguments(LlmTestServerOutputDispositionKind.Filtered)] + [Arguments(LlmTestServerOutputDispositionKind.Refused)] + public async Task AzureV1Responses_StoredPolicyAndGraphStateAreIsolatedByProviderAndSessionAsync(LlmTestServerOutputDispositionKind kind) + { + await using var host = await StartAsync(_ => { }); + using var http = host.Client(); var control = new LlmTestServerClient(http); + void Configure(LlmTestServerClientConfigurationBuilder config) => config.UseGpt41Mini() + .UseConversation("isolated-policy", graph => graph.ForModel(LlmTestServerKnownModelIds.Gpt41Mini) + .Step("policy", step => step.WhenLatestUserEquals("go").Responds(_text) + .WithOutputDisposition(new() { Kind = kind }).Then("next")) + .Step("next", step => step.WhenLatestUserEquals("next").Responds(_recovery).Complete())); + await using var firstSession = await control.CreateSessionAsync(Configure); + await using var secondSession = await control.CreateSessionAsync(Configure); + using var firstConversation = firstSession.Control.ForConversation("shared-label"); + using var otherSessionConversation = secondSession.Control.ForConversation("shared-label"); + var azure = AzurePolicyResponsesSdk(firstConversation.HttpClient, LlmTestServerProviderRouteNamespaces.AzureOpenAI); + var foundry = AzurePolicyResponsesSdk(firstConversation.HttpClient, LlmTestServerProviderRouteNamespaces.MicrosoftFoundry); + var options = new CreateResponseOptions + { Model = LlmTestServerKnownModelIds.Gpt41Mini, StoredOutputEnabled = true, InputItems = { ResponseItem.CreateUserMessageItem("go") } }; + var first = (await azure.CreateResponseAsync(options)).Value; + await Assert.That(string.IsNullOrWhiteSpace(first.Id)).IsFalse(); + var continued = new CreateResponseOptions + { Model = LlmTestServerKnownModelIds.Gpt41Mini, StoredOutputEnabled = false, PreviousResponseId = first.Id, InputItems = { ResponseItem.CreateUserMessageItem("next") } }; + foreach (var unauthorized in new[] + { + foundry, + AzurePolicyResponsesSdk(otherSessionConversation.HttpClient, LlmTestServerProviderRouteNamespaces.AzureOpenAI), + }) + { + ClientResultException? rejected = null; + try { await unauthorized.CreateResponseAsync(continued); } + catch (ClientResultException exception) { rejected = exception; } + await Assert.That(rejected?.Status).IsEqualTo(404); + } + await Assert.That((await secondSession.Control.GetConversationsAsync()).Count).IsEqualTo(0); + await Assert.That((await firstSession.Control.GetAssertionsAsync()).Matched).IsEqualTo(1); + var independent = (await foundry.CreateResponseAsync(options)).Value; + await Assert.That(independent.Status).IsEqualTo(first.Status); + var states = await firstSession.Control.GetConversationsAsync(); + await Assert.That(states.Count).IsEqualTo(2); + await Assert.That(states.All(state => state.ExecutedSteps == 1 && !state.IsTerminal)).IsTrue(); + var recovered = (await azure.CreateResponseAsync(continued)).Value; + await Assert.That(recovered.GetOutputText()).IsEqualTo(_recovery); + states = await firstSession.Control.GetConversationsAsync(); + await Assert.That(states.Count(state => state.ExecutedSteps == 2 && state.IsTerminal)).IsEqualTo(1); + await Assert.That(states.Count(state => state.ExecutedSteps == 1 && !state.IsTerminal)).IsEqualTo(1); + var other = (await AzurePolicyResponsesSdk(otherSessionConversation.HttpClient, + LlmTestServerProviderRouteNamespaces.AzureOpenAI).CreateResponseAsync(options)).Value; + await Assert.That(other.Status).IsEqualTo(first.Status); + await Assert.That((await secondSession.Control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); + await Assert.That((await firstSession.Control.GetAssertionsAsync()).Matched).IsEqualTo(3); + } + + private static Uri AzurePolicyEndpoint(HttpClient http, string provider) => new(http.BaseAddress!, provider + "/openai/v1/"); + + private static ChatClient AzurePolicyChatSdk(HttpClient http, string provider) => new( + LlmTestServerKnownModelIds.Gpt41Mini, new ApiKeyCredential(_key), new OpenAIClientOptions + { Endpoint = AzurePolicyEndpoint(http, provider), Transport = new HttpClientPipelineTransport(http), RetryPolicy = new ClientRetryPolicy(0) }); + + private static ResponsesClient AzurePolicyResponsesSdk(HttpClient http, string provider) => new(new ApiKeyCredential(_key), new ResponsesClientOptions + { Endpoint = AzurePolicyEndpoint(http, provider), Transport = new HttpClientPipelineTransport(http), RetryPolicy = new ClientRetryPolicy(0) }); +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Policy/DeepSeekOutputPolicyIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Policy/DeepSeekOutputPolicyIntegrationTests.cs new file mode 100644 index 0000000..1e18608 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Policy/DeepSeekOutputPolicyIntegrationTests.cs @@ -0,0 +1,227 @@ +using System.ClientModel; +using System.Net; +using System.Net.Http.Json; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Scenarios; +using OpenAI.Chat; + +namespace ManagedCode.LlmTestServer.Tests.Policy; + +public sealed partial class OutputDispositionIntegrationTests +{ + // Current primary contract: https://api-docs.deepseek.com/api/create-chat-completion/ + private const string _deepSeekModel = "deepseek-flash"; + private const string _deepSeekPath = "/deepseek/v1/chat/completions"; + + [Test] + [Arguments(false, false)] + [Arguments(true, false)] + [Arguments(false, true)] + [Arguments(true, true)] + public async Task DeepSeek_OfficialOpenAiSdkParsesFilteredJsonAndSseWithUsageThenRecoversAsync(bool streaming, bool empty) + { + await using var host = await StartAsync(config => ConfigureQueue(config, _deepSeekModel, LlmTestServerOutputDispositionKind.Filtered, empty)); + using var http = host.Client(); var control = new LlmTestServerClient(http); + var sdk = ChatSdk(http, "deepseek", _deepSeekModel); + using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + var text = new StringBuilder(); ChatFinishReason? finish = null; ChatTokenUsage? usage = null; var usageUpdates = 0; + if (streaming) + { + await foreach (var update in sdk.CompleteChatStreamingAsync([new UserChatMessage("policy")], cancellationToken: deadline.Token)) + { + foreach (var part in update.ContentUpdate) { text.Append(part.Text); } + finish = update.FinishReason ?? finish; + await Assert.That(update.RefusalUpdate).IsNull(); + if (update.Usage is not null) { usage = update.Usage; usageUpdates++; } + } + await Assert.That(usageUpdates).IsEqualTo(1); + } + else + { + var result = (await sdk.CompleteChatAsync([new UserChatMessage("policy")], cancellationToken: deadline.Token)).Value; + foreach (var part in result.Content) { text.Append(part.Text); } + finish = result.FinishReason; usage = result.Usage; + await Assert.That(result.Refusal).IsNull(); + } + await Assert.That(text.ToString()).IsEqualTo(empty ? string.Empty : _text); + await Assert.That(finish).IsEqualTo(ChatFinishReason.ContentFilter); + await Assert.That(usage).IsNotNull(); + await EvidenceAsync(control, LlmTestServerOutputDispositionKind.Filtered, streaming); + var first = await CompletedAsync(control, 1); + await Assert.That(first.Path).IsEqualTo(_deepSeekPath); + await Assert.That(first.ModelId).IsEqualTo(_deepSeekModel); + await Assert.That(usage!.InputTokenCount).IsEqualTo(first.Usage!.InputTokens); + await Assert.That(usage.OutputTokenCount).IsEqualTo(first.Usage.OutputTokens); + await Assert.That(usage.TotalTokenCount).IsEqualTo(first.Usage.TotalTokens); + var next = (await sdk.CompleteChatAsync([new UserChatMessage("next")], cancellationToken: deadline.Token)).Value; + await Assert.That(next.Content.Single().Text).IsEqualTo(_recovery); + await Assert.That(next.FinishReason).IsEqualTo(ChatFinishReason.Stop); + await RecoveredAsync(control); + } + + [Test] + [Arguments(false, false)] + [Arguments(false, true)] + [Arguments(false, null)] + [Arguments(true, false)] + [Arguments(true, true)] + [Arguments(true, null)] + public async Task DeepSeek_NativeSseFinalChoiceCarriesUsageForAllOptionsAndPolicyOutcomesAsync(bool filtered, bool? includeUsage) + { + await using var host = await StartAsync(config => config.AddModel(_deepSeekModel, LlmTestServerModelKind.Chat) + .AddChatScenario("deepseek-native-stream", scenario => + { + scenario.ForModel(_deepSeekModel).Responds(_text, "Visible ", "fixture text"); + if (filtered) { scenario.WithOutputDisposition(new() { Kind = LlmTestServerOutputDispositionKind.Filtered }); } + scenario.Responds(_recovery); + })); + using var http = host.Client(); var control = new LlmTestServerClient(http); + // Exact native frame placement supplements the original SDK success coverage above. + var body = new JsonObject + { + ["model"] = _deepSeekModel, + ["stream"] = true, + ["messages"] = JsonSerializer.SerializeToNode(new[] { new { role = "user", content = "policy" } }), + }; + if (includeUsage is { } requestedUsage) { body["stream_options"] = new JsonObject { ["include_usage"] = requestedUsage }; } + using var response = await http.PostAsJsonAsync(_deepSeekPath, body); + await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.OK); + await Assert.That(response.Content.Headers.ContentType!.MediaType).IsEqualTo("text/event-stream"); + var records = (await response.Content.ReadAsStringAsync()).Split('\n', StringSplitOptions.RemoveEmptyEntries) + .Where(line => line.StartsWith("data:", StringComparison.Ordinal)).Select(line => line[5..].Trim()).ToArray(); + await Assert.That(records[^1]).IsEqualTo("[DONE]"); + var frames = records[..^1].Select(record => JsonSerializer.Deserialize(record)).ToArray(); + await Assert.That(frames.Length).IsEqualTo(3); + await Assert.That(frames.All(frame => frame.GetProperty("choices").GetArrayLength() == 1)).IsTrue(); + foreach (var frame in frames[..^1]) + { + await Assert.That(frame.GetProperty("choices")[0].GetProperty("finish_reason").ValueKind).IsEqualTo(JsonValueKind.Null); + await Assert.That(frame.TryGetProperty("usage", out var interimUsage)).IsEqualTo(includeUsage == true); + if (includeUsage == true) { await Assert.That(interimUsage.ValueKind).IsEqualTo(JsonValueKind.Null); } + } + var final = frames[^1]; var finalChoice = final.GetProperty("choices")[0]; + await Assert.That(finalChoice.GetProperty("finish_reason").GetString()).IsEqualTo(filtered ? "content_filter" : "stop"); + await Assert.That(finalChoice.GetProperty("delta").GetProperty("content").GetString()).IsEqualTo(string.Empty); + var usage = final.GetProperty("usage"); + await Assert.That(usage.ValueKind).IsEqualTo(JsonValueKind.Object); + await Assert.That(frames.Count(frame => frame.TryGetProperty("usage", out var value) && value.ValueKind == JsonValueKind.Object)).IsEqualTo(1); + var trace = await CompletedAsync(control, 1); + await Assert.That(trace.TerminalMarkerObserved).IsTrue(); + await Assert.That(trace.OutputDisposition?.Kind == LlmTestServerOutputDispositionKind.Filtered).IsEqualTo(filtered); + await Assert.That(usage.GetProperty("prompt_tokens").GetInt32()).IsEqualTo(trace.Usage!.InputTokens); + await Assert.That(usage.GetProperty("completion_tokens").GetInt32()).IsEqualTo(trace.Usage.OutputTokens); + await Assert.That(usage.GetProperty("total_tokens").GetInt32()).IsEqualTo(trace.Usage.TotalTokens); + await Assert.That(usage.GetProperty("prompt_cache_hit_tokens").GetInt32()).IsEqualTo(trace.Usage.CachedInputTokens); + await Assert.That(usage.GetProperty("prompt_cache_miss_tokens").GetInt32()).IsEqualTo(trace.Usage.InputTokens - trace.Usage.CachedInputTokens); + await Assert.That((await control.VerifyAsync(new() + { Requests = [new() { RequireStreamingTerminalMarker = true }] })).Passed).IsTrue(); + var recovered = (await ChatSdk(http, "deepseek", _deepSeekModel).CompleteChatAsync([new UserChatMessage("next")])).Value; + await Assert.That(recovered.Content.Single().Text).IsEqualTo(_recovery); + await RecoveredAsync(control); + } + + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task DeepSeek_RefusedIsRejectedBeforeConsumptionThenQualifiedOpenAiRouteUsesSameFixtureAsync(bool streaming) + { + await using var host = await StartAsync(config => ConfigureQueue(config, _deepSeekModel, LlmTestServerOutputDispositionKind.Refused)); + using var http = host.Client(); var control = new LlmTestServerClient(http); + var sdk = ChatSdk(http, "deepseek", _deepSeekModel); + ClientResultException? failure = null; + try + { + if (streaming) { await foreach (var update in sdk.CompleteChatStreamingAsync([new UserChatMessage("policy")])) { _ = update; } } + else { await sdk.CompleteChatAsync([new UserChatMessage("policy")]); } + } + catch (ClientResultException exception) { failure = exception; } + await Assert.That(failure?.Status).IsEqualTo(409); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(0); + var qualified = (await ChatSdk(http, model: _deepSeekModel).CompleteChatAsync([new UserChatMessage("policy")])).Value; + await Assert.That(qualified.Refusal).IsEqualTo(_text); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(1); + var recovered = (await sdk.CompleteChatAsync([new UserChatMessage("next")])).Value; + await Assert.That(recovered.Content.Single().Text).IsEqualTo(_recovery); + await Assert.That((await CompletedAsync(control, 3)).OutputDisposition).IsNull(); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(2); + } + + [Test] + [Arguments(false, "true")] + [Arguments(false, "false")] + [Arguments(false, "empty")] + [Arguments(null, "true")] + [Arguments(null, "false")] + [Arguments(null, "empty")] + public async Task DeepSeek_NonStreamingUsageOptionsAre400AndLeaveOriginalPolicyFixtureAvailableAsync(bool? streaming, string usageOption) + { + await using var host = await StartAsync(config => ConfigureQueue(config, _deepSeekModel, LlmTestServerOutputDispositionKind.Filtered)); + using var http = host.Client(); var control = new LlmTestServerClient(http); + var options = new JsonObject(); + if (usageOption != "empty") { options["include_usage"] = usageOption == "true"; } + var body = new JsonObject + { + ["model"] = _deepSeekModel, + ["stream_options"] = options, + ["messages"] = JsonSerializer.SerializeToNode(new[] { new { role = "user", content = "policy" } }), + }; + if (streaming is { } stream) { body["stream"] = stream; } + using var rejected = await http.PostAsJsonAsync(_deepSeekPath, body); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(0); + var sdk = ChatSdk(http, "deepseek", _deepSeekModel); + var first = (await sdk.CompleteChatAsync([new UserChatMessage("policy")])).Value; + await Assert.That(first.FinishReason).IsEqualTo(ChatFinishReason.ContentFilter); + await Assert.That(first.Content.Single().Text).IsEqualTo(_text); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(1); + var generated = await CompletedAsync(control, 2); + await Assert.That(generated.OutputDisposition!.Kind).IsEqualTo(LlmTestServerOutputDispositionKind.Filtered); + var next = (await sdk.CompleteChatAsync([new UserChatMessage("next")])).Value; + await Assert.That(next.Content.Single().Text).IsEqualTo(_recovery); + await Assert.That((await CompletedAsync(control, 3)).OutputDisposition).IsNull(); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(2); + await Assert.That((await control.GetRequestsAsync()).TotalObserved).IsEqualTo(3); + } + + [Test] + public async Task DeepSeek_FilteredGraphReplaysAcrossJsonAndSseAndCommittedCacheIsRetainedAsync() + { + await using var host = await StartAsync(config => config.AddModel(_deepSeekModel, LlmTestServerModelKind.Chat) + .AddConversation("deepseek-policy-replay", graph => graph.ForModel(_deepSeekModel) + .Step("filtered", step => step.WhenLatestUserEquals("go").Responds(_text, "Visible ", "fixture text") + .WithOutputDisposition(new() { Kind = LlmTestServerOutputDispositionKind.Filtered }).Then("next")) + .Step("next", step => step.WhenLatestUserEquals("next").Responds(_recovery).Complete()))); + using var http = host.Client(); var control = new LlmTestServerClient(http); + using var conversation = control.ForConversation("deepseek-policy-replay"); + var sdk = ChatSdk(conversation.HttpClient, "deepseek", _deepSeekModel); + global::OpenAI.Chat.ChatMessage[] prompt = + [new SystemChatMessage(string.Join(" ", Enumerable.Repeat("DeepSeek cache fixture", 1200))), new UserChatMessage("go")]; + var first = (await sdk.CompleteChatAsync(prompt)).Value; + await Assert.That(first.FinishReason).IsEqualTo(ChatFinishReason.ContentFilter); + var replay = new StringBuilder(); ChatFinishReason? reason = null; ChatTokenUsage? replayUsage = null; + await foreach (var update in sdk.CompleteChatStreamingAsync(prompt)) + { + foreach (var part in update.ContentUpdate) { replay.Append(part.Text); } + reason = update.FinishReason ?? reason; + replayUsage = update.Usage ?? replayUsage; + } + await Assert.That(replay.ToString()).IsEqualTo(_text); + await Assert.That(reason).IsEqualTo(ChatFinishReason.ContentFilter); + await Assert.That(replayUsage!.TotalTokenCount).IsEqualTo(first.Usage.TotalTokenCount); + await Assert.That((await conversation.Control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); + var replayTrace = await CompletedAsync(conversation.Control, 2); + await Assert.That(replayTrace.IsReplay).IsTrue(); + await Assert.That(replayTrace.TerminalMarkerObserved).IsTrue(); + await Assert.That(replayTrace.OutputDisposition!.Kind).IsEqualTo(LlmTestServerOutputDispositionKind.Filtered); + var recovered = (await sdk.CompleteChatAsync([.. prompt, new AssistantChatMessage(first), new UserChatMessage("next")])).Value; + await Assert.That(recovered.Content.Single().Text).IsEqualTo(_recovery); + var recoveryTrace = await CompletedAsync(conversation.Control, 3); + await Assert.That(recoveryTrace.Usage!.CachedInputTokens).IsGreaterThan(0); + await Assert.That((await conversation.Control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(2); + await Assert.That((await conversation.Control.GetConversationsAsync()).Single().IsTerminal).IsTrue(); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Policy/OutputDispositionAdmissionIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Policy/OutputDispositionAdmissionIntegrationTests.cs new file mode 100644 index 0000000..b20c581 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Policy/OutputDispositionAdmissionIntegrationTests.cs @@ -0,0 +1,153 @@ +using System.ClientModel; +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using System.Text.Json.Nodes; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Scenarios; +using OpenAI.Chat; + +namespace ManagedCode.LlmTestServer.Tests.Policy; + +public sealed partial class OutputDispositionIntegrationTests +{ + [Test] + [Arguments("kind-zero")] + [Arguments("kind-unknown")] + [Arguments("category")] + [Arguments("explanation")] + [Arguments("error")] + [Arguments("delivery")] + [Arguments("tools")] + [Arguments("items")] + [Arguments("chunks")] + public async Task InvalidPolicyConfigurationIsAtomicAndPriorFixtureStillRunsThroughOfficialSdkAsync(string defect) + { + await using var host = await StartAsync(config => config.AddChatScenario("preserved", scenario => scenario.Responds(_recovery))); + using var http = host.Client(); + var baseline = new + { + models = new[] { new { id = LlmTestServerKnownModelIds.Gpt41Mini, kind = "chat" } }, + chatScenarios = new[] { new { id = "preserved", responses = new[] { new { content = _recovery, outputDisposition = new { kind = "Filtered" } } } } }, + }; + using (var configured = await http.PostAsJsonAsync("/admin-api/configure", baseline)) + { await Assert.That(configured.StatusCode).IsEqualTo(HttpStatusCode.OK); } + var response = JsonNode.Parse("""{"content":"visible","outputDisposition":{"kind":"Filtered"}}""")!.AsObject(); + var disposition = response["outputDisposition"]!.AsObject(); + switch (defect) + { + case "kind-zero": disposition["kind"] = 0; break; + case "kind-unknown": disposition["kind"] = 999; break; + case "category": disposition["category"] = new string('x', 257); break; + case "explanation": disposition["explanation"] = new string('x', 4097); break; + case "error": response["error"] = JsonNode.Parse("""{"statusCode":500,"code":"fixture-error","message":"explicit"}"""); break; + case "delivery": response["streamDelivery"] = JsonNode.Parse("""{"mode":1}"""); break; + case "tools": response["toolCalls"] = JsonNode.Parse("""[{"id":"call_policy","name":"weather","argumentsJson":"{}"}]"""); break; + case "items": response["content"] = ""; response["outputItems"] = JsonNode.Parse("""[{"kind":"message","id":"msg_policy","content":"visible"}]"""); break; + case "chunks": response["streamChunks"] = JsonNode.Parse("""["different"]"""); break; + } + var body = new JsonObject + { + ["models"] = JsonNode.Parse("""[{"id":"gpt-4.1-mini","kind":"chat"}]"""), + ["chatScenarios"] = new JsonArray(new JsonObject { ["id"] = "invalid", ["responses"] = new JsonArray(response) }), + }; + using var rejected = await http.PostAsJsonAsync("/admin-api/configure", body); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + var error = await rejected.Content.ReadFromJsonAsync(); + await Assert.That(error.GetProperty("error").GetProperty("message").GetString()).IsEqualTo("Invalid runtime configuration."); + var recovered = (await ChatSdk(http).CompleteChatAsync([new UserChatMessage("preserved")])).Value; + await Assert.That(recovered.Content.Single().Text).IsEqualTo(_recovery); + await Assert.That(recovered.FinishReason).IsEqualTo(ChatFinishReason.ContentFilter); + await Assert.That((await new LlmTestServerClient(http).GetAssertionsAsync()).Matched).IsEqualTo(1); + } + + [Test] + [Arguments("openai", "Filtered")] + [Arguments("openai", "Refused")] + [Arguments("deepseek", "Filtered")] + public async Task RawConfigurationAcceptsDocumentedPolicyKindNamesAndOriginalSdkObservesOutcomeAsync(string provider, string kind) + { + var model = provider == "deepseek" ? _deepSeekModel : LlmTestServerKnownModelIds.Gpt41Mini; + await using var host = await StartAsync(_ => { }); + using var http = host.Client(); var control = new LlmTestServerClient(http); + var body = new JsonObject + { + ["models"] = new JsonArray(new JsonObject { ["id"] = model, ["kind"] = "chat" }), + ["chatScenarios"] = new JsonArray(new JsonObject + { + ["id"] = "raw-policy-name", + ["modelId"] = model, + ["responses"] = new JsonArray( + new JsonObject { ["content"] = _text, ["outputDisposition"] = new JsonObject { ["kind"] = kind } }, + new JsonObject { ["content"] = _recovery }), + }), + }; + using var configured = await http.PostAsJsonAsync("/admin-api/configure", body); + await Assert.That(configured.StatusCode).IsEqualTo(HttpStatusCode.OK); + var sdk = ChatSdk(http, provider, model); + var first = (await sdk.CompleteChatAsync([new UserChatMessage("policy")])).Value; + var filtered = kind == "Filtered"; + await Assert.That(first.FinishReason).IsEqualTo(filtered ? ChatFinishReason.ContentFilter : ChatFinishReason.Stop); + if (filtered) { await Assert.That(first.Content.Single().Text).IsEqualTo(_text); } + else { await Assert.That(first.Refusal).IsEqualTo(_text); } + await EvidenceAsync(control, filtered ? LlmTestServerOutputDispositionKind.Filtered : LlmTestServerOutputDispositionKind.Refused, false); + var trace = await CompletedAsync(control, 1); + await Assert.That(trace.Path).IsEqualTo(provider == "deepseek" ? "/deepseek/v1/chat/completions" : "/openai/v1/chat/completions"); + var next = (await sdk.CompleteChatAsync([new UserChatMessage("next")])).Value; + await Assert.That(next.Content.Single().Text).IsEqualTo(_recovery); + await RecoveredAsync(control); + } + + [Test] + [Arguments("provider")] + [Arguments("tools")] + [Arguments("schema")] + public async Task UnsupportedPolicyRequestFailsBeforeConsumptionAndNativePlainRequestRecoversAsync(string defect) + { + await using var host = await StartAsync(config => ConfigureQueue(config, LlmTestServerKnownModelIds.Gpt41Mini, LlmTestServerOutputDispositionKind.Filtered)); + using var http = host.Client(); var control = new LlmTestServerClient(http); + var options = new ChatCompletionOptions(); + if (defect == "tools") { options.Tools.Add(ChatTool.CreateFunctionTool("weather")); } + if (defect == "schema") { options.ResponseFormat = ChatResponseFormat.CreateJsonObjectFormat(); } + ClientResultException? rejected = null; + try { await ChatSdk(http, defect == "provider" ? "groq/openai" : "openai").CompleteChatAsync([new UserChatMessage("policy")], options); } + catch (ClientResultException exception) { rejected = exception; } + await Assert.That(rejected?.Status).IsEqualTo(409); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(0); + var first = (await ChatSdk(http).CompleteChatAsync([new UserChatMessage("policy")])).Value; + await Assert.That(first.FinishReason).IsEqualTo(ChatFinishReason.ContentFilter); + await Assert.That(first.Content.Single().Text).IsEqualTo(_text); + var next = (await ChatSdk(http).CompleteChatAsync([new UserChatMessage("next")])).Value; + await Assert.That(next.Content.Single().Text).IsEqualTo(_recovery); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(2); + } + + [Test] + [Arguments("anthropic")] + [Arguments("gemini")] + public async Task UnsupportedNativePolicyPairIs409AndDoesNotConsumeSelectedFixtureAsync(string provider) + { + var model = provider == "anthropic" + ? JsonSerializer.SerializeToElement(global::Anthropic.Models.Messages.Model.ClaudeSonnet5_5).GetString()! : _geminiModel; + var kind = provider == "anthropic" ? LlmTestServerOutputDispositionKind.Filtered : LlmTestServerOutputDispositionKind.Refused; + await using var host = await StartAsync(config => ConfigureQueue(config, model, kind)); + using var http = host.Client(); var control = new LlmTestServerClient(http); + // This malformed pairing is checked over HTTP; the supported recovery uses the original official SDK. + var path = provider == "anthropic" ? "/anthropic/v1/messages" : "/gemini/v1beta/models/" + model + ":generateContent"; + object body = provider == "anthropic" + ? new { model, max_tokens = 256, messages = new[] { new { role = "user", content = "policy" } } } + : new { contents = new[] { new { role = "user", parts = new[] { new { text = "policy" } } } } }; + using var request = new HttpRequestMessage(HttpMethod.Post, path) { Content = JsonContent.Create(body) }; + if (provider == "anthropic") + { request.Headers.Add(ManagedCode.LlmTestServer.Anthropic.AnthropicWireMapper.VersionHeaderName, ManagedCode.LlmTestServer.Anthropic.AnthropicWireMapper.SupportedVersion); } + using var rejected = await http.SendAsync(request); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(0); + var supported = (await ChatSdk(http, model: model).CompleteChatAsync([new UserChatMessage("policy")])).Value; + await Assert.That(supported.FinishReason).IsEqualTo(kind == LlmTestServerOutputDispositionKind.Filtered ? ChatFinishReason.ContentFilter : ChatFinishReason.Stop); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(1); + var recovery = (await ChatSdk(http, model: model).CompleteChatAsync([new UserChatMessage("next")])).Value; + await Assert.That(recovery.Content.Single().Text).IsEqualTo(_recovery); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Policy/OutputDispositionAnthropicIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Policy/OutputDispositionAnthropicIntegrationTests.cs new file mode 100644 index 0000000..8abb9c1 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Policy/OutputDispositionAnthropicIntegrationTests.cs @@ -0,0 +1,96 @@ +using System.Text; +using System.Text.Json; +using global::Anthropic; +using global::Anthropic.Core; +using global::Anthropic.Models.Messages; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Scenarios; + +namespace ManagedCode.LlmTestServer.Tests.Policy; + +public sealed partial class OutputDispositionIntegrationTests +{ + [Test] + [Arguments(false, false, false)] + [Arguments(true, false, false)] + [Arguments(false, true, false)] + [Arguments(true, true, false)] + [Arguments(false, false, true)] + [Arguments(true, false, true)] + public async Task OfficialAnthropic_StrictRefusalHasNativeStopDetailsAndRecoveryAsync(bool streaming, bool empty, bool metadata) + { + var model = JsonSerializer.SerializeToElement(Model.ClaudeSonnet5_5).GetString()!; + await using var host = await StartAsync(config => ConfigureQueue(config, model, LlmTestServerOutputDispositionKind.Refused, empty, metadata)); + using var http = host.Client(); + var control = new LlmTestServerClient(http); + using var sdk = AnthropicSdk(http); + using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + var text = new StringBuilder(); RefusalStopDetails? details = null; string? reason = null; var stopped = false; + if (streaming) + { + await foreach (var update in sdk.Messages.CreateStreaming(AnthropicOptions(), deadline.Token)) + { + if (update.Value is RawMessageStartEvent start) + { await Assert.That(start.Message.StopReason).IsNull(); await Assert.That(start.Message.StopDetails).IsNull(); } + if (update.Value is RawContentBlockDeltaEvent block && block.Delta.Value is TextDelta delta) { text.Append(delta.Text); } + if (update.Value is RawMessageDeltaEvent ending) { reason = ending.Delta.StopReason?.Json.GetString(); details = ending.Delta.StopDetails; } + if (update.Value is RawMessageStopEvent) { stopped = true; } + } + await Assert.That(stopped).IsTrue(); + } + else + { + var response = await sdk.Messages.Create(AnthropicOptions(), deadline.Token); + foreach (var block in response.Content) { text.Append(((TextBlock)block.Value!).Text); } + reason = response.StopReason?.Json.GetString(); details = response.StopDetails; + } + await Assert.That(text.ToString()).IsEqualTo(empty ? string.Empty : _text); + await Assert.That(reason).IsEqualTo("refusal"); + await Assert.That(details).IsNotNull(); + await Assert.That(details!.Type.GetString()).IsEqualTo("refusal"); + await Assert.That(details.Category?.Json.GetString()).IsEqualTo(metadata ? "cyber" : null); + await Assert.That(details.Explanation).IsEqualTo(metadata ? "Explicit fixture policy" : null); + await EvidenceAsync(control, LlmTestServerOutputDispositionKind.Refused, streaming, metadata); + var recovered = await sdk.Messages.Create(AnthropicOptions(), deadline.Token); + await Assert.That(((TextBlock)recovered.Content.Single().Value!).Text).IsEqualTo(_recovery); + await Assert.That(recovered.StopReason?.Json.GetString()).IsEqualTo("end_turn"); + await Assert.That(recovered.StopDetails).IsNull(); + await RecoveredAsync(control); + } + + [Test] + public async Task OfficialAnthropic_PrefillLeavesPolicyFixtureUnconsumedAndWarmsCacheAsync() + { + var model = JsonSerializer.SerializeToElement(Model.ClaudeSonnet5_5).GetString()!; + await using var host = await StartAsync(config => ConfigureQueue(config, model, LlmTestServerOutputDispositionKind.Refused)); + using var http = host.Client(); var control = new LlmTestServerClient(http); + using var sdk = AnthropicSdk(http); + using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + var options = AnthropicOptions() with + { MaxTokens = 0, CacheControl = new(), System = string.Join(" ", Enumerable.Repeat("shared reference", 1800)) }; + var prefill = await sdk.Messages.Create(options, deadline.Token); + await Assert.That(prefill.Content).IsEmpty(); + await Assert.That(prefill.StopReason?.Json.GetString()).IsEqualTo("max_tokens"); + await Assert.That(prefill.Usage.CacheCreationInputTokens > 0).IsTrue(); + var trace = await CompletedAsync(control, 1); + await Assert.That(trace.OutputDisposition).IsNull(); + await Assert.That((await control.GetConversationsAsync()).Count).IsEqualTo(0); + var refused = await sdk.Messages.Create(options with { MaxTokens = 256 }, deadline.Token); + await Assert.That(refused.StopReason?.Json.GetString()).IsEqualTo("refusal"); + await Assert.That(refused.Usage.CacheReadInputTokens > 0).IsTrue(); + await Assert.That((await CompletedAsync(control, 2)).OutputDisposition!.Kind).IsEqualTo(LlmTestServerOutputDispositionKind.Refused); + var next = await sdk.Messages.Create(options with { MaxTokens = 256 }, deadline.Token); + await Assert.That(((TextBlock)next.Content.Single().Value!).Text).IsEqualTo(_recovery); + } + + private static AnthropicClient AnthropicSdk(HttpClient http) + { + return new(new ClientOptions + { HttpClient = http, BaseUrl = new Uri(http.BaseAddress!, "/anthropic").ToString(), ApiKey = _key, MaxRetries = 0, ResponseValidation = true }); + } + + private static MessageCreateParams AnthropicOptions() + { + return new() { Model = Model.ClaudeSonnet5_5, MaxTokens = 256, Messages = [new() { Role = Role.User, Content = "policy" }] }; + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Policy/OutputDispositionChatAbstractionIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Policy/OutputDispositionChatAbstractionIntegrationTests.cs new file mode 100644 index 0000000..0be5ec4 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Policy/OutputDispositionChatAbstractionIntegrationTests.cs @@ -0,0 +1,37 @@ +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Scenarios; +using Microsoft.Extensions.AI; + +namespace ManagedCode.LlmTestServer.Tests.Policy; + +public sealed partial class OutputDispositionIntegrationTests +{ + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task MicrosoftChatAbstraction_RefusalIsErrorContentAndAssistantHistoryRoundTripsAsync(bool streaming) + { + await using var host = await StartAsync(config => ConfigureQueue(config, LlmTestServerKnownModelIds.Gpt41Mini, LlmTestServerOutputDispositionKind.Refused)); + using var http = host.Client(); var control = new LlmTestServerClient(http); + using var chat = control.CreateChatClient(); + using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + Microsoft.Extensions.AI.ChatMessage[] prompt = [new(ChatRole.User, "policy")]; + var response = streaming + ? await chat.GetStreamingResponseAsync(prompt, cancellationToken: deadline.Token).ToChatResponseAsync(deadline.Token) + : await chat.GetResponseAsync(prompt, cancellationToken: deadline.Token); + var refusal = response.Messages.SelectMany(message => message.Contents).OfType().Single(); + await Assert.That(refusal.ErrorCode).IsEqualTo("Refusal"); + await Assert.That(refusal.Message).IsEqualTo(_text); + await Assert.That(response.Messages.SelectMany(message => message.Contents).OfType()).IsEmpty(); + await Assert.That(response.Text).IsEqualTo(string.Empty); + await EvidenceAsync(control, LlmTestServerOutputDispositionKind.Refused, streaming); + var recovered = await chat.GetResponseAsync([.. prompt, .. response.Messages, new(ChatRole.User, "next")], cancellationToken: deadline.Token); + await Assert.That(recovered.Text).IsEqualTo(_recovery); + var trace = await CompletedAsync(control, 2); + var assistant = trace.Messages.Single(message => message.Role == "assistant"); + await Assert.That(assistant.Content).IsEqualTo(_text); + await Assert.That(assistant.OutputDisposition!.Kind).IsEqualTo(LlmTestServerOutputDispositionKind.Refused); + await RecoveredAsync(control); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Policy/OutputDispositionConversationIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Policy/OutputDispositionConversationIntegrationTests.cs new file mode 100644 index 0000000..2190121 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Policy/OutputDispositionConversationIntegrationTests.cs @@ -0,0 +1,211 @@ +#pragma warning disable OPENAI001 // Qualify refusal history through the pinned official Responses SDK. +using System.Text; +using System.Text.Json; +using global::Anthropic.Models.Messages; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Scenarios; +using OpenAI.Chat; +using OpenAI.Responses; + +namespace ManagedCode.LlmTestServer.Tests.Policy; + +public sealed partial class OutputDispositionIntegrationTests +{ + [Test] + [Arguments(LlmTestServerOutputDispositionKind.Filtered)] + [Arguments(LlmTestServerOutputDispositionKind.Refused)] + public async Task OfficialChat_PolicyGraphReplaysAndSessionsKeepIndependentStateAndCacheAsync(LlmTestServerOutputDispositionKind kind) + { + await using var host = await StartAsync(_ => { }); + using var http = host.Client(); var control = new LlmTestServerClient(http); + void Configure(LlmTestServerClientConfigurationBuilder config) + { + config.UseGpt41Mini().UseConversation("policy-graph", graph => graph.ForModel(LlmTestServerKnownModelIds.Gpt41Mini) + .Step("policy", step => step.WhenLatestUserEquals("go").Responds(_text, "Visible ", "fixture text") + .WithOutputDisposition(new() { Kind = kind }).Then("next")) + .Step("next", step => step.WhenLatestUserEquals("next").Responds(_recovery).Complete())); + } + await using var firstSession = await control.CreateSessionAsync(Configure); + await using var secondSession = await control.CreateSessionAsync(Configure); + using var firstConversation = firstSession.Control.ForConversation("same-label"); + using var secondConversation = secondSession.Control.ForConversation("same-label"); + var prompt = new SystemChatMessage(string.Join(" ", Enumerable.Repeat("stable policy context", 1400))); + global::OpenAI.Chat.ChatMessage[] messages = [prompt, new UserChatMessage("go")]; + using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + var firstSdk = ChatSdk(firstConversation.HttpClient); + var first = (await firstSdk.CompleteChatAsync(messages, cancellationToken: deadline.Token)).Value; + await Assert.That(first.FinishReason).IsEqualTo(kind == LlmTestServerOutputDispositionKind.Filtered ? ChatFinishReason.ContentFilter : ChatFinishReason.Stop); + await Assert.That(first.Usage.InputTokenDetails.CachedTokenCount).IsEqualTo(0); + var replayText = new StringBuilder(); var replayRefusal = new StringBuilder(); ChatFinishReason? replayReason = null; + await foreach (var update in firstSdk.CompleteChatStreamingAsync(messages, cancellationToken: deadline.Token)) + { + foreach (var part in update.ContentUpdate) { replayText.Append(part.Text); } + replayRefusal.Append(update.RefusalUpdate); + replayReason = update.FinishReason ?? replayReason; + } + await Assert.That(replayReason).IsEqualTo(first.FinishReason); + await Assert.That(kind == LlmTestServerOutputDispositionKind.Filtered ? replayText.ToString() : replayRefusal.ToString()).IsEqualTo(_text); + await Assert.That((await firstConversation.Control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); + await Assert.That((await secondSession.Control.GetConversationsAsync()).Count).IsEqualTo(0); + var replay = await CompletedAsync(firstConversation.Control, 2); + await Assert.That(replay.IsReplay).IsTrue(); + await Assert.That(replay.OutputDisposition!.Kind).IsEqualTo(kind); + await Assert.That(replay.TerminalMarkerObserved).IsTrue(); + var second = (await ChatSdk(secondConversation.HttpClient).CompleteChatAsync(messages, cancellationToken: deadline.Token)).Value; + await Assert.That(second.Usage.InputTokenDetails.CachedTokenCount).IsEqualTo(0); + await Assert.That((await secondConversation.Control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); + var recovered = (await firstSdk.CompleteChatAsync([.. messages, new AssistantChatMessage(first), new UserChatMessage("next")], cancellationToken: deadline.Token)).Value; + await Assert.That(recovered.Content.Single().Text).IsEqualTo(_recovery); + await Assert.That(recovered.Usage.InputTokenDetails.CachedTokenCount).IsGreaterThan(0); + await Assert.That((await firstConversation.Control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(2); + await Assert.That((await firstConversation.Control.GetConversationsAsync()).Single().IsTerminal).IsTrue(); + await Assert.That((await secondConversation.Control.GetConversationsAsync()).Single().IsTerminal).IsFalse(); + var followTrace = await CompletedAsync(firstConversation.Control, 3); + var assistant = followTrace.Messages.Single(message => message.Role == "assistant"); + await Assert.That(assistant.Content).IsEqualTo(_text); + if (kind == LlmTestServerOutputDispositionKind.Refused) + { await Assert.That(assistant.OutputDisposition!.Kind).IsEqualTo(kind); } + } + + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task OfficialChat_PureRefusalContentPartsRoundTripIntoNormalizedAssistantHistoryAsync(bool multipleParts) + { + await using var host = await StartAsync(config => ConfigureQueue(config, LlmTestServerKnownModelIds.Gpt41Mini, LlmTestServerOutputDispositionKind.Refused)); + using var http = host.Client(); var control = new LlmTestServerClient(http); + var sdk = ChatSdk(http); + var initial = (await sdk.CompleteChatAsync([new UserChatMessage("policy")])).Value; + await Assert.That(initial.Refusal).IsEqualTo(_text); + var assistant = multipleParts + ? new AssistantChatMessage(ChatMessageContentPart.CreateRefusalPart("Visible "), ChatMessageContentPart.CreateRefusalPart("fixture text")) + : new AssistantChatMessage(ChatMessageContentPart.CreateRefusalPart(initial.Refusal)); + var follow = (await sdk.CompleteChatAsync([new UserChatMessage("policy"), assistant, new UserChatMessage("next")])).Value; + await Assert.That(follow.Content.Single().Text).IsEqualTo(_recovery); + var trace = await CompletedAsync(control, 2); + var history = trace.Messages.Single(message => message.Role == "assistant"); + await Assert.That(history.Content).IsEqualTo(_text); + await Assert.That(history.OutputDisposition!.Kind).IsEqualTo(LlmTestServerOutputDispositionKind.Refused); + await Assert.That(trace.OutputDisposition).IsNull(); + } + + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task OfficialResponses_RefusalContinuationPreservesNativeHistoryForExplicitAndStoredInputAsync(bool stored) + { + await using var host = await StartAsync(config => config.AddChatScenario("refusal-history", scenario => scenario + .Responds(_text).WithOutputDisposition(new() { Kind = LlmTestServerOutputDispositionKind.Refused }).Responds(_recovery))); + using var http = host.Client(); var control = new LlmTestServerClient(http); + var sdk = ResponsesSdk(http); + var options = new CreateResponseOptions + { Model = LlmTestServerKnownModelIds.Gpt41Mini, StoredOutputEnabled = stored, InputItems = { ResponseItem.CreateUserMessageItem("policy") } }; + var first = (await sdk.CreateResponseAsync(options)).Value; + await Assert.That(((MessageResponseItem)first.OutputItems.Single()).Content.Single().Refusal).IsEqualTo(_text); + var next = new CreateResponseOptions { Model = LlmTestServerKnownModelIds.Gpt41Mini, StoredOutputEnabled = false }; + if (stored) { next.PreviousResponseId = first.Id; } + else + { + foreach (var item in options.InputItems) { next.InputItems.Add(item); } + foreach (var item in first.OutputItems) { next.InputItems.Add(item); } + } + next.InputItems.Add(ResponseItem.CreateUserMessageItem("next")); + var continued = (await sdk.CreateResponseAsync(next)).Value; + await Assert.That(continued.GetOutputText()).IsEqualTo(_recovery); + var trace = await CompletedAsync(control, 2); + var history = trace.Messages.Single(message => message.Role == "assistant"); + await Assert.That(history.Content).IsEqualTo(_text); + await Assert.That(history.OutputDisposition!.Kind).IsEqualTo(LlmTestServerOutputDispositionKind.Refused); + await Assert.That(trace.OutputDisposition).IsNull(); + } + + [Test] + [Arguments(LlmTestServerOutputDispositionKind.Refused)] + [Arguments(LlmTestServerOutputDispositionKind.Filtered)] + public async Task OfficialResponses_StoredPolicyHistoryAndCanonicalManualHistoryReplayTheSameGraphTurnAsync(LlmTestServerOutputDispositionKind kind) + { + await using var host = await StartAsync(config => config.AddConversation("policy-history-projection", graph => graph + .ForModel(LlmTestServerKnownModelIds.Gpt41Mini) + .Step("policy", step => step.WhenLatestUserEquals("go").Responds(_text) + .WithOutputDisposition(new() + { + Kind = kind, + Category = kind == LlmTestServerOutputDispositionKind.Refused ? "cyber" : null, + Explanation = kind == LlmTestServerOutputDispositionKind.Refused ? "Full diagnostic metadata" : null, + }).Then("next")) + .Step("next", step => step.WhenLatestUserEquals("next").Responds(_recovery).Then("tail")) + .Step("tail", step => step.WhenLatestUserEquals("tail").Responds("Remaining graph step").Complete()))); + using var http = host.Client(); var control = new LlmTestServerClient(http); + using var conversation = control.ForConversation("canonical-policy-history"); + var sdk = ResponsesSdk(conversation.HttpClient); + using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + var first = (await sdk.CreateResponseAsync(new CreateResponseOptions + { + Model = LlmTestServerKnownModelIds.Gpt41Mini, + StoredOutputEnabled = true, + InputItems = { ResponseItem.CreateUserMessageItem("go") }, + }, deadline.Token)).Value; + await Assert.That(first.Status).IsEqualTo(kind == LlmTestServerOutputDispositionKind.Filtered ? ResponseStatus.Incomplete : ResponseStatus.Completed); + var automatic = (await sdk.CreateResponseAsync(new CreateResponseOptions + { + Model = LlmTestServerKnownModelIds.Gpt41Mini, + StoredOutputEnabled = false, + PreviousResponseId = first.Id, + InputItems = { ResponseItem.CreateUserMessageItem("next") }, + }, deadline.Token)).Value; + await Assert.That(automatic.GetOutputText()).IsEqualTo(_recovery); + await Assert.That((await conversation.Control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(2); + var assistant = kind == LlmTestServerOutputDispositionKind.Refused + ? ResponseItem.CreateAssistantMessageItem([ResponseContentPart.CreateRefusalPart(_text)]) + : ResponseItem.CreateAssistantMessageItem(_text); + var canonical = (await sdk.CreateResponseAsync(new CreateResponseOptions + { + Model = LlmTestServerKnownModelIds.Gpt41Mini, + StoredOutputEnabled = false, + InputItems = { ResponseItem.CreateUserMessageItem("go"), assistant, ResponseItem.CreateUserMessageItem("next") }, + }, deadline.Token)).Value; + await Assert.That(canonical.Id).IsEqualTo(automatic.Id); + await Assert.That(canonical.GetOutputText()).IsEqualTo(_recovery); + await Assert.That(canonical.Usage.InputTokenCount).IsEqualTo(automatic.Usage.InputTokenCount); + await Assert.That(canonical.Usage.OutputTokenCount).IsEqualTo(automatic.Usage.OutputTokenCount); + var state = (await conversation.Control.GetConversationsAsync()).Single(); + await Assert.That(state.ExecutedSteps).IsEqualTo(2); + await Assert.That(state.NextSteps.Single()).IsEqualTo("tail"); + await Assert.That(state.IsTerminal).IsFalse(); + var replay = await CompletedAsync(conversation.Control, 3); + await Assert.That(replay.IsReplay).IsTrue(); + var history = replay.Messages.Single(message => message.Role == "assistant"); + await Assert.That(history.Content).IsEqualTo(_text); + if (kind == LlmTestServerOutputDispositionKind.Refused) + { + await Assert.That(history.OutputDisposition!.Kind).IsEqualTo(kind); + await Assert.That(history.OutputDisposition.Category).IsNull(); + await Assert.That(history.OutputDisposition.Explanation).IsNull(); + } + else { await Assert.That(history.OutputDisposition).IsNull(); } + var journal = await conversation.Control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending }); + await Assert.That(journal.Requests[0].OutputDisposition!.Kind).IsEqualTo(kind); + await Assert.That(journal.Requests[0].OutputDisposition!.Explanation).IsEqualTo(kind == LlmTestServerOutputDispositionKind.Refused ? "Full diagnostic metadata" : null); + await Assert.That(journal.Requests[1].IsReplay).IsFalse(); + } + + [Test] + public async Task OfficialAnthropic_StrictRefusalAssistantContentCanBeReturnedInNextTurnAsync() + { + var model = JsonSerializer.SerializeToElement(Model.ClaudeSonnet5_5).GetString()!; + await using var host = await StartAsync(config => ConfigureQueue(config, model, LlmTestServerOutputDispositionKind.Refused)); + using var http = host.Client(); var control = new LlmTestServerClient(http); + using var sdk = AnthropicSdk(http); + var options = AnthropicOptions(); + var refused = await sdk.Messages.Create(options); + var history = JsonSerializer.Deserialize>(JsonSerializer.Serialize(refused.Content))!; + var continued = await sdk.Messages.Create(options with + { Messages = [.. options.Messages, new() { Role = Role.Assistant, Content = history }, new() { Role = Role.User, Content = "next" }] }); + await Assert.That(((TextBlock)continued.Content.Single().Value!).Text).IsEqualTo(_recovery); + var trace = await CompletedAsync(control, 2); + await Assert.That(trace.Messages.Single(message => message.Role == "assistant").Content).IsEqualTo(_text); + await Assert.That(trace.OutputDisposition).IsNull(); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Policy/OutputDispositionGeminiIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Policy/OutputDispositionGeminiIntegrationTests.cs new file mode 100644 index 0000000..fbf288b --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Policy/OutputDispositionGeminiIntegrationTests.cs @@ -0,0 +1,96 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text; +using System.Text.Json; +using Google.GenAI.Types; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Scenarios; + +namespace ManagedCode.LlmTestServer.Tests.Policy; + +public sealed partial class OutputDispositionIntegrationTests +{ + private const string _geminiModel = "gemini-3.8-flash"; + + [Test] + [Arguments(false, false)] + [Arguments(true, false)] + [Arguments(false, true)] + [Arguments(true, true)] + public async Task OfficialGoogle_OutputSafetyReturnsCandidateRatherThanPromptBlockAndRecoversAsync(bool streaming, bool empty) + { + await using var host = await StartAsync(config => ConfigureQueue(config, _geminiModel, LlmTestServerOutputDispositionKind.Filtered, empty)); + using var http = host.Client(); var control = new LlmTestServerClient(http); + using var sdk = GoogleSdk(http); + using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + var text = new StringBuilder(); GenerateContentResponse? final = null; + var options = new GenerateContentConfig { AutomaticContinuation = false }; + if (streaming) + { + await foreach (var chunk in sdk.Models.GenerateContentStreamAsync(_geminiModel, "policy", options, cancellationToken: deadline.Token)) + { + text.Append(chunk.Text); + await Assert.That(chunk.PromptFeedback).IsNull(); + if (chunk.Candidates?.Any(candidate => candidate.FinishReason is not null) == true) { final = chunk; } + } + } + else + { + final = await sdk.Models.GenerateContentAsync(_geminiModel, "policy", options, cancellationToken: deadline.Token); + text.Append(final.Text); + } + await Assert.That(final).IsNotNull(); + await Assert.That(final!.Candidates!.Single().FinishReason == FinishReason.Safety).IsTrue(); + await Assert.That(final.PromptFeedback).IsNull(); + await Assert.That(text.ToString()).IsEqualTo(empty || !streaming ? string.Empty : _text); + if (streaming) { await Assert.That(final.Text ?? string.Empty).IsEqualTo(string.Empty); } + await EvidenceAsync(control, LlmTestServerOutputDispositionKind.Filtered, streaming); + var recovered = await sdk.Models.GenerateContentAsync(_geminiModel, "next", options, cancellationToken: deadline.Token); + await Assert.That(recovered.Text).IsEqualTo(_recovery); + await Assert.That(recovered.Candidates!.Single().FinishReason == FinishReason.Stop).IsTrue(); + await RecoveredAsync(control); + } + + [Test] + public async Task GeminiJsonArrayStreamCarriesPrefixAndSafetyTerminalThenOfficialSdkRecoveryAsync() + { + await using var host = await StartAsync(config => ConfigureQueue(config, _geminiModel, LlmTestServerOutputDispositionKind.Filtered)); + using var http = host.Client(); var control = new LlmTestServerClient(http); + // The official SDK uses SSE; raw HTTP supplements this distinct native JSON-array transport. + using var response = await http.PostAsJsonAsync("/gemini/v1beta/models/" + _geminiModel + ":streamGenerateContent", new + { contents = new[] { new { role = "user", parts = new[] { new { text = "policy" } } } } }); + await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.OK); + await Assert.That(response.Content.Headers.ContentType!.MediaType).IsEqualTo("application/json"); + var frames = await response.Content.ReadFromJsonAsync(); + await Assert.That(frames.ValueKind).IsEqualTo(JsonValueKind.Array); + var text = new StringBuilder(); + foreach (var frame in frames.EnumerateArray()) + { + var candidate = frame.GetProperty("candidates")[0]; + if (candidate.TryGetProperty("content", out var content) && content.ValueKind == JsonValueKind.Object + && content.TryGetProperty("parts", out var parts)) + { + foreach (var part in parts.EnumerateArray()) + { if (part.TryGetProperty("text", out var value)) { text.Append(value.GetString()); } } + } + } + await Assert.That(text.ToString()).IsEqualTo(_text); + var final = frames[frames.GetArrayLength() - 1].GetProperty("candidates")[0]; + await Assert.That(final.GetProperty("finishReason").GetString()).IsEqualTo("SAFETY"); + await Assert.That(final.TryGetProperty("content", out var finalContent) + && finalContent.ValueKind == JsonValueKind.Object && finalContent.TryGetProperty("parts", out var finalParts) + && finalParts.EnumerateArray().Any(part => part.TryGetProperty("text", out var value) && !string.IsNullOrEmpty(value.GetString()))).IsFalse(); + await EvidenceAsync(control, LlmTestServerOutputDispositionKind.Filtered, true); + using var sdk = GoogleSdk(http); + var recovered = await sdk.Models.GenerateContentAsync(_geminiModel, "next", new GenerateContentConfig { AutomaticContinuation = false }); + await Assert.That(recovered.Text).IsEqualTo(_recovery); + await RecoveredAsync(control); + } + + private static Google.GenAI.Client GoogleSdk(HttpClient http) + { + return new(apiKey: _key, enterprise: false, + httpOptions: new() { BaseUrl = new Uri(http.BaseAddress!, "/gemini").ToString(), ApiVersion = "v1beta", RetryOptions = new() { Attempts = 1 } }, + clientOptions: new() { HttpClientFactory = () => http }); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Policy/OutputDispositionIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Policy/OutputDispositionIntegrationTests.cs new file mode 100644 index 0000000..f3fbbcb --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Policy/OutputDispositionIntegrationTests.cs @@ -0,0 +1,138 @@ +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Net; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Configuration; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Hosting; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Scenarios; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Hosting.Server; +using Microsoft.AspNetCore.Hosting.Server.Features; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using OpenAI; +using OpenAI.Chat; + +namespace ManagedCode.LlmTestServer.Tests.Policy; + +public sealed partial class OutputDispositionIntegrationTests +{ + private const string _key = "policy-sdk-key"; + private const string _text = "Visible fixture text"; + private const string _recovery = "Explicit recovery"; + + private static async Task StartAsync(Action configure, int? traceBudgetBytes = null) + { + var builder = WebApplication.CreateBuilder(new WebApplicationOptions { EnvironmentName = Environments.Development }); + builder.WebHost.ConfigureKestrel(options => options.Listen(IPAddress.Loopback, 0)); + builder.Services.AddLlmTestServer(configure); + if (traceBudgetBytes is { } budget) + { + builder.Services.Configure(options => + { options.MaxPreviewBytes = 128; options.MaxRetainedBytes = budget; }); + } + builder.AddDefaultHealthChecks(); + var app = builder.Build(); + app.MapLlmTestServer(); + app.MapDefaultEndpoints(); + var host = new PolicyHost(app); + try + { + using var startup = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + await app.StartAsync(startup.Token); + using var warmup = host.Client(); + using var health = await warmup.GetAsync("/health", startup.Token); + health.EnsureSuccessStatusCode(); + return host; + } + catch { await host.DisposeAsync(); throw; } + } + + private sealed class PolicyHost(WebApplication app) : IAsyncDisposable + { + public HttpClient Client() + { + var server = app.Services.GetRequiredService(); + var address = server.Features.Get()!.Addresses.Single(); + return new() { BaseAddress = new Uri(address), Timeout = TimeSpan.FromSeconds(60) }; + } + + public async ValueTask DisposeAsync() + { + using var stop = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + try { await app.StopAsync(stop.Token); } + finally { await app.DisposeAsync(); } + } + } + + private static ChatClient ChatSdk(HttpClient http, string provider = "openai", string model = LlmTestServerKnownModelIds.Gpt41Mini) + { + return new(model, new ApiKeyCredential(_key), new OpenAIClientOptions + { + Endpoint = new Uri(http.BaseAddress!, "/" + provider + "/v1/"), + Transport = new HttpClientPipelineTransport(http), + RetryPolicy = new ClientRetryPolicy(0), + }); + } + + private static void ConfigureQueue(LlmTestServerConfigurationBuilder config, string model, + LlmTestServerOutputDispositionKind kind, bool empty = false, bool metadata = false) + { + config.AddModel(model, LlmTestServerModelKind.Chat).AddChatScenario("policy", scenario => scenario.ForModel(model) + .Responds(empty ? string.Empty : _text, empty ? [string.Empty] : ["Visible ", "fixture text"]) + .WithOutputDisposition(new() { Kind = kind, Category = metadata ? "cyber" : null, Explanation = metadata ? "Explicit fixture policy" : null }) + .Responds(_recovery)); + } + + private static async Task CompletedAsync(LlmTestServerClient control, int count) + { + using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + while (true) + { + var snapshot = await control.GetRequestsAsync(cancellationToken: deadline.Token); + if (snapshot.Requests.Count >= count && snapshot.Requests.All(trace => trace.CompletedAt is not null)) + { + return snapshot.Requests.OrderBy(trace => trace.Sequence).ElementAt(count - 1); + } + + await Task.Delay(20, deadline.Token); + } + } + + private static async Task EvidenceAsync(LlmTestServerClient control, LlmTestServerOutputDispositionKind kind, bool streaming, + bool metadata = false) + { + var trace = await CompletedAsync(control, 1); + await Assert.That(trace.StatusCode).IsEqualTo(200); + await Assert.That(trace.TransportState).IsEqualTo(LlmTestServerProviderHttpTraceTransportState.Completed); + await Assert.That(trace.OutputDisposition!.Kind).IsEqualTo(kind); + await Assert.That(trace.OutputDisposition.Category).IsEqualTo(metadata ? "cyber" : null); + await Assert.That(trace.OutputDisposition.Explanation).IsEqualTo(metadata ? "Explicit fixture policy" : null); + await Assert.That(trace.Usage).IsNotNull(); + await Assert.That(trace.StreamDelivery).IsNull(); + if (streaming) + { + await Assert.That(trace.TerminalMarkerObserved).IsTrue(); + var verification = await control.VerifyAsync(new() + { Requests = [new() { Query = new() { RequestId = trace.RequestId }, RequireStreamingTerminalMarker = true }] }); + await Assert.That(verification.Passed).IsTrue(); + } + var exported = await control.ExportRequestsAsync(new() { RequestId = trace.RequestId }); + await Assert.That(exported.Evidence.Requests.Single().OutputDisposition!.Kind).IsEqualTo(kind); + var summary = await control.GetAssertionsAsync(); + await Assert.That(summary.Matched).IsEqualTo(1); + await Assert.That(summary.ErrorsReturned).IsEqualTo(0); + await Assert.That(summary.Events.Single().OutputDisposition!.Kind).IsEqualTo(kind); + } + + private static async Task RecoveredAsync(LlmTestServerClient control) + { + var trace = await CompletedAsync(control, 2); + await Assert.That(trace.StatusCode).IsEqualTo(200); + await Assert.That(trace.OutputDisposition).IsNull(); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(2); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Policy/OutputDispositionMalformedHistoryIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Policy/OutputDispositionMalformedHistoryIntegrationTests.cs new file mode 100644 index 0000000..7beaffa --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Policy/OutputDispositionMalformedHistoryIntegrationTests.cs @@ -0,0 +1,60 @@ +#pragma warning disable OPENAI001 // Validate native Responses admission with the original SDK recovery path. +using System.Net; +using System.Net.Http.Json; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Scenarios; +using OpenAI.Chat; +using OpenAI.Responses; + +namespace ManagedCode.LlmTestServer.Tests.Policy; + +public sealed partial class OutputDispositionIntegrationTests +{ + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task RefusalHistoryObjectIsRejectedBeforeFlatteningAndOriginalSdkStillReceivesFirstPolicyAsync(bool responses) + { + await using var host = await StartAsync(config => ConfigureQueue(config, LlmTestServerKnownModelIds.Gpt41Mini, LlmTestServerOutputDispositionKind.Filtered)); + using var http = host.Client(); var control = new LlmTestServerClient(http); + object[] history = + [ + new { role = "assistant", content = new { type = "refusal", refusal = "x" } }, + new { role = "user", content = "policy" }, + ]; + object body = responses + ? new { model = LlmTestServerKnownModelIds.Gpt41Mini, input = history } + : new { model = LlmTestServerKnownModelIds.Gpt41Mini, messages = history }; + using var rejected = await http.PostAsJsonAsync(responses ? "/openai/v1/responses" : "/openai/v1/chat/completions", body); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(0); + if (responses) + { + var sdk = ResponsesSdk(http); + var first = (await sdk.CreateResponseAsync(new CreateResponseOptions + { Model = LlmTestServerKnownModelIds.Gpt41Mini, StoredOutputEnabled = false, InputItems = { ResponseItem.CreateUserMessageItem("policy") } })).Value; + await Assert.That(first.Status).IsEqualTo(ResponseStatus.Incomplete); + await Assert.That(first.IncompleteStatusDetails.Reason).IsEqualTo(ResponseIncompleteStatusReason.ContentFilter); + await Assert.That(first.GetOutputText()).IsEqualTo(_text); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(1); + var next = (await sdk.CreateResponseAsync(new CreateResponseOptions + { Model = LlmTestServerKnownModelIds.Gpt41Mini, StoredOutputEnabled = false, InputItems = { ResponseItem.CreateUserMessageItem("next") } })).Value; + await Assert.That(next.GetOutputText()).IsEqualTo(_recovery); + } + else + { + var sdk = ChatSdk(http); + var first = (await sdk.CompleteChatAsync([new UserChatMessage("policy")])).Value; + await Assert.That(first.FinishReason).IsEqualTo(ChatFinishReason.ContentFilter); + await Assert.That(first.Content.Single().Text).IsEqualTo(_text); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(1); + var next = (await sdk.CompleteChatAsync([new UserChatMessage("next")])).Value; + await Assert.That(next.Content.Single().Text).IsEqualTo(_recovery); + } + var recovery = await CompletedAsync(control, 3); + await Assert.That(recovery.OutputDisposition).IsNull(); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(2); + await Assert.That((await control.GetRequestsAsync()).TotalObserved).IsEqualTo(3); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Policy/OutputDispositionOpenAiIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Policy/OutputDispositionOpenAiIntegrationTests.cs new file mode 100644 index 0000000..8305f12 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Policy/OutputDispositionOpenAiIntegrationTests.cs @@ -0,0 +1,126 @@ +#pragma warning disable OPENAI001 // Qualify the pinned official Responses SDK against native policy outcomes. +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Text; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Scenarios; +using OpenAI.Chat; +using OpenAI.Responses; + +namespace ManagedCode.LlmTestServer.Tests.Policy; + +public sealed partial class OutputDispositionIntegrationTests +{ + [Test] + [Arguments(LlmTestServerOutputDispositionKind.Filtered, false, false)] + [Arguments(LlmTestServerOutputDispositionKind.Filtered, true, false)] + [Arguments(LlmTestServerOutputDispositionKind.Filtered, false, true)] + [Arguments(LlmTestServerOutputDispositionKind.Filtered, true, true)] + [Arguments(LlmTestServerOutputDispositionKind.Refused, false, false)] + [Arguments(LlmTestServerOutputDispositionKind.Refused, true, false)] + [Arguments(LlmTestServerOutputDispositionKind.Refused, false, true)] + [Arguments(LlmTestServerOutputDispositionKind.Refused, true, true)] + public async Task OfficialOpenAiChat_ReturnsNativePolicyOutcomeThenQueuedRecoveryAsync( + LlmTestServerOutputDispositionKind kind, bool streaming, bool empty) + { + await using var host = await StartAsync(config => ConfigureQueue(config, LlmTestServerKnownModelIds.Gpt41Mini, kind, empty)); + using var http = host.Client(); + var control = new LlmTestServerClient(http); + var sdk = ChatSdk(http); + var content = new StringBuilder(); var refusal = new StringBuilder(); ChatFinishReason? finish = null; var sawRefusal = false; + using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + if (streaming) + { + await foreach (var update in sdk.CompleteChatStreamingAsync([new UserChatMessage("policy")], cancellationToken: deadline.Token)) + { + foreach (var part in update.ContentUpdate) { content.Append(part.Text); } + refusal.Append(update.RefusalUpdate); + sawRefusal |= update.RefusalUpdate is not null; + finish = update.FinishReason ?? finish; + } + } + else + { + var response = (await sdk.CompleteChatAsync([new UserChatMessage("policy")], cancellationToken: deadline.Token)).Value; + foreach (var part in response.Content) { content.Append(part.Text); } + refusal.Append(response.Refusal); + sawRefusal = response.Refusal is not null; + finish = response.FinishReason; + } + var expected = empty ? string.Empty : _text; + await Assert.That(sawRefusal).IsEqualTo(kind == LlmTestServerOutputDispositionKind.Refused); + await Assert.That(finish).IsEqualTo(kind == LlmTestServerOutputDispositionKind.Filtered ? ChatFinishReason.ContentFilter : ChatFinishReason.Stop); + await Assert.That(content.ToString()).IsEqualTo(kind == LlmTestServerOutputDispositionKind.Filtered ? expected : string.Empty); + await Assert.That(refusal.ToString()).IsEqualTo(kind == LlmTestServerOutputDispositionKind.Refused ? expected : string.Empty); + await EvidenceAsync(control, kind, streaming); + var recovered = (await sdk.CompleteChatAsync([new UserChatMessage("next")], cancellationToken: deadline.Token)).Value; + await Assert.That(recovered.Content.Single().Text).IsEqualTo(_recovery); + await Assert.That(recovered.FinishReason).IsEqualTo(ChatFinishReason.Stop); + await RecoveredAsync(control); + } + + [Test] + [Arguments(LlmTestServerOutputDispositionKind.Filtered, false, false)] + [Arguments(LlmTestServerOutputDispositionKind.Filtered, true, false)] + [Arguments(LlmTestServerOutputDispositionKind.Filtered, false, true)] + [Arguments(LlmTestServerOutputDispositionKind.Filtered, true, true)] + [Arguments(LlmTestServerOutputDispositionKind.Refused, false, false)] + [Arguments(LlmTestServerOutputDispositionKind.Refused, true, false)] + [Arguments(LlmTestServerOutputDispositionKind.Refused, false, true)] + [Arguments(LlmTestServerOutputDispositionKind.Refused, true, true)] + public async Task OfficialResponses_ReturnsTypedIncompleteOrRefusalThenQueuedRecoveryAsync( + LlmTestServerOutputDispositionKind kind, bool streaming, bool empty) + { + await using var host = await StartAsync(config => ConfigureQueue(config, LlmTestServerKnownModelIds.Gpt41Mini, kind, empty)); + using var http = host.Client(); + var control = new LlmTestServerClient(http); + var sdk = ResponsesSdk(http); + var options = new CreateResponseOptions + { Model = LlmTestServerKnownModelIds.Gpt41Mini, StoredOutputEnabled = false, StreamingEnabled = streaming, InputItems = { ResponseItem.CreateUserMessageItem("policy") } }; + ResponseResult? result = null; var deltas = new StringBuilder(); var refusals = new StringBuilder(); var incomplete = false; var completed = false; + using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + if (streaming) + { + await foreach (var update in sdk.CreateResponseStreamingAsync(options, deadline.Token)) + { + if (update is StreamingResponseOutputTextDeltaUpdate text) { deltas.Append(text.Delta); } + if (update is StreamingResponseRefusalDeltaUpdate refusal) { refusals.Append(refusal.Delta); } + if (update is StreamingResponseIncompleteUpdate incompleteEnding) { incomplete = true; result = incompleteEnding.Response; } + if (update is StreamingResponseCompletedUpdate completedEnding) { completed = true; result = completedEnding.Response; } + await Assert.That(update is StreamingResponseFailedUpdate or StreamingResponseErrorUpdate).IsFalse(); + } + await Assert.That(incomplete).IsEqualTo(kind == LlmTestServerOutputDispositionKind.Filtered); + await Assert.That(completed).IsEqualTo(kind == LlmTestServerOutputDispositionKind.Refused); + await Assert.That(deltas.ToString()).IsEqualTo(kind == LlmTestServerOutputDispositionKind.Filtered && !empty ? _text : string.Empty); + await Assert.That(refusals.ToString()).IsEqualTo(kind == LlmTestServerOutputDispositionKind.Refused && !empty ? _text : string.Empty); + } + else { result = (await sdk.CreateResponseAsync(options, deadline.Token)).Value; } + await Assert.That(result).IsNotNull(); + await Assert.That(result!.Status).IsEqualTo(kind == LlmTestServerOutputDispositionKind.Filtered ? ResponseStatus.Incomplete : ResponseStatus.Completed); + if (kind == LlmTestServerOutputDispositionKind.Filtered) + { + await Assert.That(result.IncompleteStatusDetails.Reason).IsEqualTo(ResponseIncompleteStatusReason.ContentFilter); + await Assert.That(result.GetOutputText()).IsEqualTo(empty ? string.Empty : _text); + } + else + { + var message = (MessageResponseItem)result.OutputItems.Single(); + await Assert.That(message.Content.Single().Kind).IsEqualTo(ResponseContentPartKind.Refusal); + await Assert.That(message.Content.Single().Refusal).IsEqualTo(empty ? string.Empty : _text); + await Assert.That(result.IncompleteStatusDetails).IsNull(); + } + await EvidenceAsync(control, kind, streaming); + var next = (await sdk.CreateResponseAsync(new CreateResponseOptions + { Model = LlmTestServerKnownModelIds.Gpt41Mini, StoredOutputEnabled = false, InputItems = { ResponseItem.CreateUserMessageItem("next") } }, deadline.Token)).Value; + await Assert.That(next.Status).IsEqualTo(ResponseStatus.Completed); + await Assert.That(next.GetOutputText()).IsEqualTo(_recovery); + await RecoveredAsync(control); + } + + private static ResponsesClient ResponsesSdk(HttpClient http) + { + return new(new ApiKeyCredential(_key), new ResponsesClientOptions + { Endpoint = new Uri(http.BaseAddress!, "/openai/v1/"), Transport = new HttpClientPipelineTransport(http), RetryPolicy = new ClientRetryPolicy(0) }); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Policy/OutputDispositionRetentionIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Policy/OutputDispositionRetentionIntegrationTests.cs new file mode 100644 index 0000000..db7b20a --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Policy/OutputDispositionRetentionIntegrationTests.cs @@ -0,0 +1,49 @@ +using System.Text.Json; +using global::Anthropic.Models.Messages; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Scenarios; + +namespace ManagedCode.LlmTestServer.Tests.Policy; + +public sealed partial class OutputDispositionIntegrationTests +{ + [Test] + [Arguments(2048)] + [Arguments(4096)] + public async Task NativePolicyMetadataRemainsValidWhilePublicDiagnosticBudgetMarksTruncationAsync(int retainedBytes) + { + var model = JsonSerializer.SerializeToElement(Model.ClaudeSonnet5_5).GetString()!; + var explanation = new string('x', 4096); + await using var host = await StartAsync(config => config.AddModel(model, LlmTestServerModelKind.Chat) + .AddChatScenario("policy-retention", scenario => scenario.ForModel(model).Responds(_text) + .WithOutputDisposition(new() { Kind = LlmTestServerOutputDispositionKind.Refused, Category = "cyber", Explanation = explanation }) + .Responds(_recovery)), traceBudgetBytes: retainedBytes); + using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + using var http = host.Client(); + var control = new LlmTestServerClient(http); + using var sdk = AnthropicSdk(http); + var native = await sdk.Messages.Create(AnthropicOptions(), deadline.Token); + await Assert.That(native.StopReason?.Json.GetString()).IsEqualTo("refusal"); + await Assert.That(native.StopDetails!.Explanation).IsEqualTo(explanation); + await Assert.That(((TextBlock)native.Content.Single().Value!).Text).IsEqualTo(_text); + var trace = await CompletedAsync(control, 1); + await Assert.That(trace.RetentionTruncated).IsTrue(); + await Assert.That(trace.RetainedByteCount <= retainedBytes).IsTrue(); + await Assert.That(trace.ResponsePreviewTruncated || trace.OutputDisposition?.Explanation != explanation).IsTrue(); + await Assert.That(trace.Usage!.InputTokens).IsEqualTo((int)native.Usage.InputTokens); + await Assert.That(trace.Usage.OutputTokens).IsEqualTo((int)native.Usage.OutputTokens); + await Assert.That(trace.OutputDisposition?.Explanation == explanation).IsFalse(); + var exported = await control.ExportRequestsAsync(cancellationToken: deadline.Token); + await Assert.That(exported.Evidence.RetentionTruncated).IsTrue(); + await Assert.That(exported.Evidence.RetainedByteCount <= retainedBytes).IsTrue(); + await Assert.That(exported.Evidence.Requests.Single().RetentionTruncated).IsTrue(); + // Runtime evidence has its own existing budget; a smaller wire journal does not mutate the inference result. + var events = (await control.GetAssertionsAsync(deadline.Token)).Events; + await Assert.That(events.Single().OutputDisposition!.Explanation).IsEqualTo(explanation); + var recovered = await sdk.Messages.Create(AnthropicOptions(), deadline.Token); + await Assert.That(recovered.StopReason?.Json.GetString()).IsEqualTo("end_turn"); + await Assert.That(((TextBlock)recovered.Content.Single().Value!).Text).IsEqualTo(_recovery); + await Assert.That((await control.GetRequestsAsync(cancellationToken: deadline.Token)).TotalObserved).IsEqualTo(2); + } +} diff --git a/tests/ManagedCode.LlmTck.Tests/Providers/OpenAiCompatibleProviderRouteTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Providers/OpenAiCompatibleProviderRouteTests.cs similarity index 90% rename from tests/ManagedCode.LlmTck.Tests/Providers/OpenAiCompatibleProviderRouteTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/Providers/OpenAiCompatibleProviderRouteTests.cs index 6f37e7f..a2b9336 100644 --- a/tests/ManagedCode.LlmTck.Tests/Providers/OpenAiCompatibleProviderRouteTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Providers/OpenAiCompatibleProviderRouteTests.cs @@ -3,11 +3,11 @@ using System.Net.Http.Json; using System.Text; using System.Text.Json; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Tests.TestSupport; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Tests.TestSupport; -namespace ManagedCode.LlmTck.Tests.Providers; +namespace ManagedCode.LlmTestServer.Tests.Providers; public sealed class OpenAiCompatibleProviderRouteTests { @@ -24,8 +24,8 @@ public sealed class OpenAiCompatibleProviderRouteTests [Arguments("/microsoft-foundry/models/chat/completions?api-version=2024-05-01-preview")] public async Task OpenAiCompatibleChatRoutes_ReturnChatCompletionShapeAsync(string path) { - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("compat-chat", LlmTckModelKind.Chat) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("compat-chat", LlmTestServerModelKind.Chat) .AddChatScenario( "compat-route-blue-whale", scenario => scenario @@ -79,8 +79,8 @@ bool usesDeepSeekUsageFields ) { var cacheablePrompt = CreatePromptWithAtLeastTokens(1100); - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("provider-cache-chat", LlmTckModelKind.Chat) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("provider-cache-chat", LlmTestServerModelKind.Chat) .AddChatScenario( "provider-cache", scenario => scenario @@ -144,8 +144,8 @@ await Assert.That(secondDetails.GetProperty("cache_write_tokens").GetInt32()) public async Task OpenAiCompatibleChatRoutes_WithoutProviderPromptCaching_DoNotReportCacheUsageAsync() { var cacheablePrompt = CreatePromptWithAtLeastTokens(1100); - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("perplexity-cache-chat", LlmTckModelKind.Chat) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("perplexity-cache-chat", LlmTestServerModelKind.Chat) .AddChatScenario( "perplexity-cache", scenario => scenario @@ -193,8 +193,8 @@ public async Task OpenAiCompatibleChatRoutes_WithoutProviderPromptCaching_DoNotR [Arguments("/microsoft-foundry/models/chat/completions?api-version=2024-05-01-preview")] public async Task OpenAiCompatibleChatRoutes_StreamServerSentChunksAsync(string path) { - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("compat-stream-chat", LlmTckModelKind.Chat) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("compat-stream-chat", LlmTestServerModelKind.Chat) .AddChatScenario( "compat-route-stream", scenario => scenario @@ -233,8 +233,8 @@ await Assert.That(response.Content.Headers.ContentType?.MediaType) [Arguments("/deepseek/models")] public async Task OpenAiCompatibleModelRoutes_ReturnModelListShapeAsync(string path) { - using var host = await LlmTckTestHost.StartAsync(options => - options.AddModel("compat-chat", LlmTckModelKind.Chat) + using var host = await LlmTestServerTestHost.StartAsync(options => + options.AddModel("compat-chat", LlmTestServerModelKind.Chat) ); using var client = host.GetTestClient(); @@ -259,8 +259,8 @@ public async Task OpenAiCompatibleModelRoutes_ReturnModelListShapeAsync(string p [Arguments("POST", "/openai/deployments/compat-chat/chat/completions")] public async Task GenericRootProviderRoutes_AreNotMappedAsync(string method, string path) { - using var host = await LlmTckTestHost.StartAsync(options => - options.AddModel("compat-chat", LlmTckModelKind.Chat) + using var host = await LlmTestServerTestHost.StartAsync(options => + options.AddModel("compat-chat", LlmTestServerModelKind.Chat) ); using var client = host.GetTestClient(); using var request = new HttpRequestMessage(new HttpMethod(method), path); @@ -276,8 +276,8 @@ public async Task GenericRootProviderRoutes_AreNotMappedAsync(string method, str [Arguments("/openrouter/api/v1/responses")] public async Task OpenAiCompatibleResponsesRoutes_ReturnResponseShapeAsync(string path) { - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("compat-responses", LlmTckModelKind.Chat) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("compat-responses", LlmTestServerModelKind.Chat) .AddChatScenario( "compat-responses", scenario => scenario @@ -341,8 +341,8 @@ await Assert.That(payload.GetProperty("usage").GetProperty("total_tokens").GetIn [Arguments("/openrouter/api/v1/responses")] public async Task OpenAiCompatibleResponsesRoutes_StreamResponseServerSentEventsAsync(string path) { - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("compat-responses", LlmTckModelKind.Chat) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("compat-responses", LlmTestServerModelKind.Chat) .AddChatScenario( "compat-responses-stream", scenario => scenario @@ -385,8 +385,8 @@ await Assert.That(response.Content.Headers.ContentType?.MediaType) [Arguments("/microsoft-foundry/models/embeddings?api-version=2024-05-01-preview")] public async Task FoundryEmbeddingRoutes_ReturnEmbeddingShapeAsync(string path) { - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("foundry-embedding", LlmTckModelKind.Embedding) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("foundry-embedding", LlmTestServerModelKind.Embedding) .WithDefaultEmbeddingVector(0.25f, 0.5f)); using var client = host.GetTestClient(); @@ -407,8 +407,8 @@ await Assert.That(payload.GetProperty("data")[0].GetProperty("embedding").GetArr [Test] public async Task MistralEmbeddingRoute_ReturnsEmbeddingShapeAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("mistral-embedding", LlmTckModelKind.Embedding) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("mistral-embedding", LlmTestServerModelKind.Embedding) .WithDefaultEmbeddingVector(0.25f, 0.5f)); using var client = host.GetTestClient(); @@ -429,8 +429,8 @@ await Assert.That(payload.GetProperty("data")[0].GetProperty("embedding").GetArr [Test] public async Task GroqAudioSpeechRoute_ReturnsAudioAndValidatesDocumentedRequestAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("playai-tts", LlmTckModelKind.Audio)); + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("playai-tts", LlmTestServerModelKind.Audio)); using var client = host.GetTestClient(); var response = await client.PostAsJsonAsync( @@ -481,8 +481,8 @@ await Assert.That(unsupportedFormatPayload.GetProperty("error").GetProperty("mes [Test] public async Task GroqAudioTranscriptionRoute_ReturnsGroqTranscriptionShapeAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("whisper-large-v3", LlmTckModelKind.Audio) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("whisper-large-v3", LlmTestServerModelKind.Audio) .WithDefaultTranscriptionText("groq transcript")); using var client = host.GetTestClient(); using var content = CreateTranscriptionContent("whisper-large-v3"); @@ -500,8 +500,8 @@ await Assert.That(payload.GetProperty("x_groq").GetProperty("id").GetString()) [Test] public async Task GroqAudioTranslationRoute_ReturnsGroqTranslationShapeAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("whisper-large-v3", LlmTckModelKind.Audio) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("whisper-large-v3", LlmTestServerModelKind.Audio) .WithDefaultTranslationText("groq translation")); using var client = host.GetTestClient(); using var content = CreateTranscriptionContent("whisper-large-v3"); @@ -519,9 +519,9 @@ await Assert.That(payload.GetProperty("x_groq").GetProperty("id").GetString()) [Test] public async Task AzureOpenAiAudioTranscriptionRoute_UsesDeploymentModelAndApiKeyAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options + using var host = await LlmTestServerTestHost.StartAsync(options => options .RequireBearerToken("azure-key") - .AddModel("whisper-deployment", LlmTckModelKind.Audio) + .AddModel("whisper-deployment", LlmTestServerModelKind.Audio) .WithDefaultTranscriptionText("azure transcript")); using var client = host.GetTestClient(); client.DefaultRequestHeaders.Add("api-key", "azure-key"); @@ -546,9 +546,9 @@ public async Task AzureOpenAiAudioTranscriptionRoute_UsesDeploymentModelAndApiKe [Test] public async Task AzureOpenAiAudioTranslationRoute_UsesDeploymentModelAndApiKeyAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options + using var host = await LlmTestServerTestHost.StartAsync(options => options .RequireBearerToken("azure-key") - .AddModel("whisper-deployment", LlmTckModelKind.Audio) + .AddModel("whisper-deployment", LlmTestServerModelKind.Audio) .WithDefaultTranslationText("azure translation")); using var client = host.GetTestClient(); client.DefaultRequestHeaders.Add("api-key", "azure-key"); @@ -573,9 +573,9 @@ public async Task AzureOpenAiAudioTranslationRoute_UsesDeploymentModelAndApiKeyA [Test] public async Task AzureOpenAiVideoRoutes_FollowPreviewJobAndContentShapesAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options + using var host = await LlmTestServerTestHost.StartAsync(options => options .RequireBearerToken("azure-key") - .AddModel("sora-deployment", LlmTckModelKind.Video)); + .AddModel("sora-deployment", LlmTestServerModelKind.Video)); using var client = host.GetTestClient(); client.DefaultRequestHeaders.Add("api-key", "azure-key"); @@ -657,7 +657,7 @@ await Assert.That(createPayload.GetProperty("object").GetString()) await Assert.That(listPayload.GetProperty("data").GetArrayLength()).IsEqualTo(1); await Assert.That(jobPayload.GetProperty("id").GetString()).IsEqualTo(jobId); await Assert.That(generationPayload.GetProperty("id").GetString()).IsEqualTo(generationId); - await Assert.That(generationPayload.GetProperty("job_id").GetString()).IsEqualTo("video_llm_tck"); + await Assert.That(generationPayload.GetProperty("job_id").GetString()).IsEqualTo("video_llm_test_server"); await Assert.That(thumbnail.Content.Headers.ContentType?.MediaType).IsEqualTo("image/jpg"); await Assert.That(video.Content.Headers.ContentType?.MediaType).IsEqualTo("video/mp4"); await Assert.That(Encoding.ASCII.GetString(videoBytes, 4, 4)).IsEqualTo("ftyp"); @@ -674,8 +674,8 @@ public async Task GroqAudioRoutes_RejectOpenAiOnlyResponseFormatsAsync( string operationName ) { - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("whisper-large-v3", LlmTckModelKind.Audio)); + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("whisper-large-v3", LlmTestServerModelKind.Audio)); using var client = host.GetTestClient(); using var content = CreateTranscriptionContent("whisper-large-v3", responseFormat: "vtt"); @@ -729,7 +729,7 @@ private static Task PostProviderCacheRequestAsync( private static string CreatePromptWithAtLeastTokens(int minimumTokens) { var builder = new StringBuilder("cacheable fixture"); - while (LlmTckTokenCounter.CountTextTokens(builder.ToString()) < minimumTokens) + while (LlmTestServerTokenCounter.CountTextTokens(builder.ToString()) < minimumTokens) { builder.Append(" stable-prefix"); } diff --git a/tests/ManagedCode.LlmTestServer.Tests/Providers/OpenAiSdkProviderCompatibilityTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Providers/OpenAiSdkProviderCompatibilityTests.cs new file mode 100644 index 0000000..941f7e6 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Providers/OpenAiSdkProviderCompatibilityTests.cs @@ -0,0 +1,103 @@ +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Text; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Tests.TestSupport; +using OpenAI; +using OpenAI.Chat; + +namespace ManagedCode.LlmTestServer.Tests.Providers; + +public sealed class OpenAiSdkProviderCompatibilityTests +{ + private const string _key = "sdk-provider-key"; + + [Test] + [Arguments("/groq/openai/v1", false)] + [Arguments("/groq/openai/v1", true)] + [Arguments("/openrouter/api/v1", false)] + [Arguments("/openrouter/api/v1", true)] + [Arguments("/mistral/v1", false)] + [Arguments("/mistral/v1", true)] + [Arguments("/deepseek/v1", false)] + [Arguments("/deepseek/v1", true)] + public async Task OfficialOpenAiClient_CompletesCorrelatedFunctionRoundThroughCompatibleProviderAsync(string prefix, bool stream) + { + using var host = await LlmTestServerTestHost.StartAsync(config => config.RequireBearerToken(_key) + .AddModel(LlmTestServerKnownModelIds.Gpt55, LlmTestServerModelKind.Chat) + .AddConversation("provider-sdk", graph => graph.ForModel(LlmTestServerKnownModelIds.Gpt55) + .Step("call", step => step.WhenLatestUserEquals("weather") + .CallsTool("weather", "{\"city\":\"Paris\"}", "call_provider_sdk").Then("result")) + .Step("result", step => step.WhenToolResult("weather", "call_provider_sdk") + .WhenLatestTextContains("18 degrees").Responds("Paris is 18 degrees.", "Paris is ", "18 degrees.").Complete()))); + using var http = host.GetTestClient(); + using var conversation = new LlmTestServerClient(http, _key).ForConversation("sdk-round"); + var sdk = CreateSdk(conversation.HttpClient, prefix); + var options = new ChatCompletionOptions + { + Tools = { ChatTool.CreateFunctionTool("weather", "Weather fixture", BinaryData.FromString("{\"type\":\"object\",\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}")) }, + Temperature = 0, + }; + List messages = [new UserChatMessage("weather")]; + var called = (await sdk.CompleteChatAsync(messages, options)).Value; + var call = called.ToolCalls.Single(); + await Assert.That(call.Id).IsEqualTo("call_provider_sdk"); + await Assert.That(call.FunctionName).IsEqualTo("weather"); + await Assert.That(call.FunctionArguments.ToString()).IsEqualTo("{\"city\":\"Paris\"}"); + messages.Add(new AssistantChatMessage(called)); + messages.Add(new ToolChatMessage(call.Id, "18 degrees")); + var output = new StringBuilder(); + if (stream) + { + await foreach (var update in sdk.CompleteChatStreamingAsync(messages, options)) + { + foreach (var part in update.ContentUpdate) { output.Append(part.Text); } + } + } + else { output.Append((await sdk.CompleteChatAsync(messages, options)).Value.Content.Single().Text); } + await Assert.That(output.ToString()).IsEqualTo("Paris is 18 degrees."); + var state = (await conversation.Control.GetConversationsAsync()).Single(); + await Assert.That(state.ExecutedSteps).IsEqualTo(2); + await Assert.That(state.IsTerminal).IsTrue(); + var evidence = (await conversation.Control.ExportRequestsAsync()).Evidence.Requests; + await Assert.That(evidence.Count).IsEqualTo(2); + await Assert.That(evidence.All(request => request.StatusCode == 200 && request.Path == prefix + "/chat/completions")).IsTrue(); + var result = evidence.Single(request => request.Messages.Any(message => message.Role == "tool")); + await Assert.That(result.Messages.Single(message => message.Role == "tool").ToolCallId).IsEqualTo(call.Id); + await Assert.That(result.Messages.Single(message => message.Role == "tool").Content).IsEqualTo("18 degrees"); + if (stream) { await Assert.That(result.TerminalMarkerObserved).IsTrue(); } + } + + [Test] + [Arguments("/groq/openai/v1", 429)] + [Arguments("/groq/openai/v1", 500)] + [Arguments("/openrouter/api/v1", 429)] + [Arguments("/openrouter/api/v1", 500)] + [Arguments("/mistral/v1", 429)] + [Arguments("/mistral/v1", 500)] + [Arguments("/deepseek/v1", 429)] + [Arguments("/deepseek/v1", 500)] + public async Task OfficialOpenAiClient_ObservesOneProviderErrorThenExplicitRecoveryAsync(string prefix, int status) + { + using var host = await LlmTestServerTestHost.StartAsync(config => config.RequireBearerToken(_key) + .AddModel(LlmTestServerKnownModelIds.Gpt55, LlmTestServerModelKind.Chat).AddChatScenario("sdk-error", scenario => scenario + .ForModel(LlmTestServerKnownModelIds.Gpt55).Fails(status, "fixture_unavailable", "Scripted provider failure").Responds("SDK recovered"))); + using var http = host.GetTestClient(); + var sdk = CreateSdk(http, prefix); + ClientResultException? failure = null; + try { await sdk.CompleteChatAsync([new UserChatMessage("recover")]); } + catch (ClientResultException exception) { failure = exception; } + await Assert.That(failure?.Status).IsEqualTo(status); + await Assert.That((await sdk.CompleteChatAsync([new UserChatMessage("recover")])).Value.Content.Single().Text).IsEqualTo("SDK recovered"); + var evidence = (await new LlmTestServerClient(http, _key).ExportRequestsAsync()).Evidence.Requests; + await Assert.That(evidence.Count).IsEqualTo(2); + await Assert.That(evidence.Select(request => request.StatusCode)).IsEquivalentTo(new int?[] { status, 200 }); + } + + private static ChatClient CreateSdk(HttpClient http, string prefix) + { + return new(LlmTestServerKnownModelIds.Gpt55, new ApiKeyCredential(_key), new OpenAIClientOptions + { Endpoint = new Uri(http.BaseAddress!, prefix), Transport = new HttpClientPipelineTransport(http), RetryPolicy = new ClientRetryPolicy(0) }); + } +} diff --git a/tests/ManagedCode.LlmTck.Tests/Providers/ProviderApiDriftTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Providers/ProviderApiDriftTests.cs similarity index 90% rename from tests/ManagedCode.LlmTck.Tests/Providers/ProviderApiDriftTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/Providers/ProviderApiDriftTests.cs index 82bc6fe..a7aa635 100644 --- a/tests/ManagedCode.LlmTck.Tests/Providers/ProviderApiDriftTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Providers/ProviderApiDriftTests.cs @@ -1,10 +1,10 @@ using System.Net; using System.Net.Http.Json; using System.Text.Json; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.Tests.TestSupport; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Tests.TestSupport; -namespace ManagedCode.LlmTck.Tests.Providers; +namespace ManagedCode.LlmTestServer.Tests.Providers; public sealed class ProviderApiDriftTests { @@ -14,7 +14,7 @@ public sealed class ProviderApiDriftTests [Arguments("?alt=sse")] public async Task GeminiStreaming_PreservesResponseIdAndReasoningUsageAsync(string query) { - using var host = await LlmTckTestHost.StartAsync(options => options + using var host = await LlmTestServerTestHost.StartAsync(options => options .AddReasoningChatModel("gemini-2.5-flash", 17) .AddChatScenario("gemini-reasoning", scenario => scenario.ForModel("gemini-2.5-flash") .WhenUserContains("whale").Responds("blue whale", "blue ", "whale"))); @@ -45,8 +45,8 @@ await Assert.That(string.Concat(chunks.Select(chunk => chunk.GetProperty("candid [Arguments("/openrouter/api/v1/responses")] public async Task ResponsesStreaming_UsesDocumentedEventsAndStableItemIdentityAsync(string path) { - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("gpt-4.1-mini", LlmTckModelKind.Chat) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("gpt-4.1-mini", LlmTestServerModelKind.Chat) .AddChatScenario("response-events", scenario => scenario.ForModel("gpt-4.1-mini") .WhenUserContains("whale").Responds("blue whale", "blue ", "whale"))); using var client = host.GetTestClient(); @@ -83,8 +83,8 @@ await Assert.That(events.Select(item => item.GetProperty("type").GetString()!).S [Arguments(null, "")] public async Task OpenRouterResponses_RejectsStateBeforeConsumingScenarioAsync(bool? store, string? previousResponseId) { - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("gpt-4.1-mini", LlmTckModelKind.Chat) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("gpt-4.1-mini", LlmTestServerModelKind.Chat) .AddChatScenario("stateless", scenario => scenario.ForModel("gpt-4.1-mini") .WhenUserContains("whale").Responds("blue whale"))); using var client = host.GetTestClient(); @@ -114,8 +114,8 @@ await Assert.That(payload.GetProperty("output")[0].GetProperty("content")[0].Get [Test] public async Task OllamaChat_ReportsCacheReadsInFinalStreamingChunkAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("llama3.2", LlmTckModelKind.Chat) + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel("llama3.2", LlmTestServerModelKind.Chat) .AddChatScenario("ollama-cache", scenario => scenario.ForModel("llama3.2") .WhenUserContains("whale").Responds("blue whale").Responds("blue whale", "blue ", "whale"))); using var client = host.GetTestClient(); @@ -142,9 +142,9 @@ await Assert.That(lines[^1].GetProperty("prompt_eval_cached_count").GetInt32()) [Arguments(true)] public async Task AnthropicZeroMaxTokens_WarmsCacheWithoutConsumingResponseAsync(bool stream) { - using var host = await LlmTckTestHost.StartAsync(options => options - .AddModel("claude-sonnet-4-5", LlmTckModelKind.Chat) - .AddChatScenario("cache-only", scenario => scenario.ForModel("claude-sonnet-4-5") + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddModel(LlmTestServerKnownModelIds.ClaudeSonnet55, LlmTestServerModelKind.Chat) + .AddChatScenario("cache-only", scenario => scenario.ForModel(LlmTestServerKnownModelIds.ClaudeSonnet55) .WhenUserContains("whale").Responds("blue whale"))); using var client = host.GetTestClient(); client.DefaultRequestHeaders.Add("anthropic-version", "2023-06-01"); @@ -152,7 +152,7 @@ public async Task AnthropicZeroMaxTokens_WarmsCacheWithoutConsumingResponseAsync var cacheControl = new { type = "ephemeral" }; using var warm = await client.PostAsJsonAsync("/anthropic/v1/messages", new { - model = "claude-sonnet-4-5", + model = LlmTestServerKnownModelIds.ClaudeSonnet55, messages, max_tokens = 0, cache_control = cacheControl, @@ -166,7 +166,7 @@ public async Task AnthropicZeroMaxTokens_WarmsCacheWithoutConsumingResponseAsync await Assert.That(warmBody).DoesNotContain("content_block_delta"); using var response = await client.PostAsJsonAsync("/anthropic/v1/messages", new { - model = "claude-sonnet-4-5", + model = LlmTestServerKnownModelIds.ClaudeSonnet55, messages, max_tokens = 64, cache_control = cacheControl, diff --git a/tests/ManagedCode.LlmTestServer.Tests/Providers/ProviderCapabilityEvidenceTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Providers/ProviderCapabilityEvidenceTests.cs new file mode 100644 index 0000000..50b69a0 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Providers/ProviderCapabilityEvidenceTests.cs @@ -0,0 +1,92 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text; +using ManagedCode.LlmTestServer.Providers; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Tests.Regression; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.Providers; + +public sealed class ProviderCapabilityEvidenceTests +{ + [Test] + [Arguments(LlmTestServerCompatibilityTags.SystemOne)] + [Arguments(LlmTestServerCompatibilityTags.Cloudflare)] + [Arguments(LlmTestServerCompatibilityTags.OpenAI)] + [Arguments(LlmTestServerCompatibilityTags.AzureOpenAI)] + [Arguments(LlmTestServerCompatibilityTags.MicrosoftFoundry)] + [Arguments(LlmTestServerCompatibilityTags.Anthropic)] + [Arguments(LlmTestServerCompatibilityTags.Gemini)] + [Arguments(LlmTestServerCompatibilityTags.Groq)] + [Arguments(LlmTestServerCompatibilityTags.Mistral)] + [Arguments(LlmTestServerCompatibilityTags.Ollama)] + [Arguments(LlmTestServerCompatibilityTags.Cohere)] + [Arguments(LlmTestServerCompatibilityTags.Bedrock)] + [Arguments(LlmTestServerCompatibilityTags.OpenRouter)] + [Arguments(LlmTestServerCompatibilityTags.DeepSeek)] + [Arguments(LlmTestServerCompatibilityTags.Perplexity)] + public async Task ClaimedToolAndSchemaCases_ExecutePositiveAndNegativeHttpEvidenceAsync(string provider) + { + var profile = LlmTestServerProviderFixtures.GetProfiles().Single(item => item.Id == provider); + foreach (var operation in profile.ApiContract.Operations.Where(operation => operation.ImplementedByHosting)) + { + foreach (var capability in operation.Capabilities.Where(capability => capability is LlmTestServerProviderCapability.Tools or LlmTestServerProviderCapability.StructuredOutput)) + { + var streamingOnly = operation.Id is LlmTestServerProviderOperationIds.Gemini.ModelsStreamGenerateContent or LlmTestServerProviderOperationIds.Bedrock.ConverseStream; + var modes = streamingOnly ? new[] { true } : operation.SupportsStreaming ? [false, true] : [false]; + foreach (var stream in modes) + { + await VerifyCaseAsync(provider, operation, capability, stream); + } + } + } + } + + private static async Task VerifyCaseAsync(string provider, LlmTestServerProviderOperation operation, LlmTestServerProviderCapability capability, bool stream) + { + var caseId = $"{provider}/{operation.Id}/{capability}/{(stream ? "stream" : "json")}"; + var tools = capability == LlmTestServerProviderCapability.Tools; + const string answer = "{\"city\":\"Paris\"}"; + using var host = await LlmTestServerTestHost.StartAsync(b => b.AddChatScenario(caseId, s => + { + if (tools) { s.CallsTool("weather", answer); } + else { s.RespondsJson(answer); } + })); + using var http = host.GetTestClient(); + http.DefaultRequestHeaders.Add("anthropic-version", "2023-06-01"); + var format = provider switch + { + LlmTestServerCompatibilityTags.Anthropic => 2, + LlmTestServerCompatibilityTags.Gemini => 3, + LlmTestServerCompatibilityTags.Ollama => 4, + LlmTestServerCompatibilityTags.Cohere => 5, + LlmTestServerCompatibilityTags.Bedrock => 6, + _ => operation.Id is LlmTestServerProviderOperationIds.OpenAI.ResponsesCreate or LlmTestServerProviderOperationIds.AzureOpenAI.V1ResponsesCreate ? 1 : 0, + }; + var body = ToolFixtureEndpointTests.Request(format, tools); + body["stream"] = stream; + var path = operation.Path.Replace("{model}", "gpt-4.1-mini", StringComparison.Ordinal) + .Replace("{modelId}", "gpt-4.1-mini", StringComparison.Ordinal) + .Replace("{deployment}", "gpt-4.1-mini", StringComparison.Ordinal); + if (operation.ApiVersion is { } version && version != "v1") { path += "?api-version=" + version; } + if (provider == LlmTestServerCompatibilityTags.Gemini && stream) { path += "?alt=sse"; } + var invalid = body.ToJsonString().Replace("Paris", "London", StringComparison.Ordinal); + using var rejected = await http.PostAsync(path, new StringContent(invalid, Encoding.UTF8, "application/json")); + await Assert.That((caseId + "/reject", rejected.StatusCode)).IsEqualTo((caseId + "/reject", HttpStatusCode.Conflict)); + using var accepted = await http.PostAsync(path, new StringContent(body.ToJsonString(), Encoding.UTF8, "application/json")); + await Assert.That((caseId + "/accept", accepted.StatusCode)).IsEqualTo((caseId + "/accept", HttpStatusCode.OK)); + var content = await accepted.Content.ReadAsStringAsync(); + await Assert.That(content).Contains(tools ? "weather" : "Paris"); + if (stream) + { + var marker = format switch { 1 => "response.completed", 2 => "message_stop", 3 => "data:", 4 => "\"done\":true", 5 => "message-end", 6 => "messageStop", _ => "[DONE]" }; + await Assert.That(content).Contains(marker); + } + var summary = (await http.GetFromJsonAsync("/admin-api/assertions"))!; + await Assert.That(summary.ErrorsReturned).IsEqualTo(1); + await Assert.That(summary.Matched).IsEqualTo(1); + await Assert.That(summary.Events.All(entry => entry.ScenarioId == caseId)).IsTrue(); + } + +} diff --git a/tests/ManagedCode.LlmTck.Tests/Providers/ProviderFaultSimulationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Providers/ProviderFaultSimulationTests.cs similarity index 94% rename from tests/ManagedCode.LlmTck.Tests/Providers/ProviderFaultSimulationTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/Providers/ProviderFaultSimulationTests.cs index 82e148b..4ffb8cd 100644 --- a/tests/ManagedCode.LlmTck.Tests/Providers/ProviderFaultSimulationTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Providers/ProviderFaultSimulationTests.cs @@ -1,9 +1,9 @@ using System.Net; using System.Net.Http.Json; using System.Text.Json; -using ManagedCode.LlmTck.Tests.TestSupport; +using ManagedCode.LlmTestServer.Tests.TestSupport; -namespace ManagedCode.LlmTck.Tests.Providers; +namespace ManagedCode.LlmTestServer.Tests.Providers; public sealed class ProviderFaultSimulationTests { @@ -14,7 +14,7 @@ public sealed class ProviderFaultSimulationTests [Test] public async Task ProviderFamilies_ReturnConfiguredContentFilterErrorsAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options + using var host = await LlmTestServerTestHost.StartAsync(options => options .SimulateContentFilter(_blockedTerm)); using var client = host.GetTestClient(); @@ -31,7 +31,7 @@ public async Task ProviderFamilies_ReturnConfiguredContentFilterErrorsAsync() [Test] public async Task ProviderFamilies_ReturnConfiguredRateLimitErrorsAsync() { - using var host = await LlmTckTestHost.StartAsync(options => options + using var host = await LlmTestServerTestHost.StartAsync(options => options .SimulateRateLimitAfter(0)); using var client = host.GetTestClient(); @@ -45,7 +45,7 @@ public async Task ProviderFamilies_ReturnConfiguredRateLimitErrorsAsync() } } - private static ProviderFaultCase[] GetProviderFaultCases() + internal static ProviderFaultCase[] GetProviderFaultCases() { return [ @@ -192,7 +192,7 @@ string content ); } - private sealed record ProviderFaultCase( + internal sealed record ProviderFaultCase( string ProviderName, Func> SendAsync ); diff --git a/tests/ManagedCode.LlmTck.Tests/Regression/FollowUpEndpointTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Regression/FollowUpEndpointTests.cs similarity index 82% rename from tests/ManagedCode.LlmTck.Tests/Regression/FollowUpEndpointTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/Regression/FollowUpEndpointTests.cs index 157fc6b..a4cd08a 100644 --- a/tests/ManagedCode.LlmTck.Tests/Regression/FollowUpEndpointTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Regression/FollowUpEndpointTests.cs @@ -3,12 +3,11 @@ using System.Text; using System.Text.Json; using System.Text.Json.Nodes; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Scenarios; -using ManagedCode.LlmTck.Tests.TestSupport; -using Microsoft.Extensions.DependencyInjection; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Scenarios; +using ManagedCode.LlmTestServer.Tests.TestSupport; -namespace ManagedCode.LlmTck.Tests.Regression; +namespace ManagedCode.LlmTestServer.Tests.Regression; public sealed class FollowUpEndpointTests { @@ -21,9 +20,9 @@ public sealed class FollowUpEndpointTests [Arguments("/anthropic/v1/messages", 2, true)] public async Task ParallelTools_RespectRequestLimitAndPreserveQueueAsync(string path, int format, bool stream) { - using var host = await LlmTckTestHost.StartAsync(b => b.AddChatScenario("parallel", s => s.CallsTools( - new LlmTckToolCall { Id = "first", Name = "weather", ArgumentsJson = "{\"city\":\"Paris\"}" }, - new LlmTckToolCall { Id = "second", Name = "weather", ArgumentsJson = "{\"city\":\"Paris\"}" }))); + using var host = await LlmTestServerTestHost.StartAsync(b => b.AddChatScenario("parallel", s => s.CallsTools( + new LlmTestServerToolCall { Id = "first", Name = "weather", ArgumentsJson = "{\"city\":\"Paris\"}" }, + new LlmTestServerToolCall { Id = "second", Name = "weather", ArgumentsJson = "{\"city\":\"Paris\"}" }))); using var http = host.GetTestClient(); http.DefaultRequestHeaders.Add("anthropic-version", "2023-06-01"); var body = ToolFixtureEndpointTests.Request(format, tools: true); @@ -40,7 +39,7 @@ public async Task ParallelTools_RespectRequestLimitAndPreserveQueueAsync(string using var rejected = await PostAsync(http, path, body); await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.Conflict); await Assert.That(await rejected.Content.ReadAsStringAsync()).Contains("parallel tool calls"); - var summary = host.Services.GetRequiredService().GetAssertionSummary(); + var summary = (await http.GetFromJsonAsync("/admin-api/assertions"))!; await Assert.That(summary.ErrorsReturned).IsEqualTo(1); body.Remove(format == 2 ? "tool_choice" : "parallel_tool_calls"); using var accepted = await PostAsync(http, path, body); @@ -57,7 +56,7 @@ public async Task ParallelTools_RespectRequestLimitAndPreserveQueueAsync(string [Arguments(true)] public async Task ChatModel_IsRequiredExceptForAzureDeploymentRoutesAsync(bool azureDeployment) { - using var host = await LlmTckTestHost.StartAsync(b => b.AddChatScenario("model", s => s.Responds("fixture"))); + using var host = await LlmTestServerTestHost.StartAsync(b => b.AddChatScenario("model", s => s.Responds("fixture"))); using var http = host.GetTestClient(); var body = JsonNode.Parse("""{"messages":[{"role":"user","content":"hello"}]}""")!; const string chat = "/openai/v1/chat/completions"; @@ -66,7 +65,7 @@ public async Task ChatModel_IsRequiredExceptForAzureDeploymentRoutesAsync(bool a await Assert.That(response.StatusCode).IsEqualTo(azureDeployment ? HttpStatusCode.OK : HttpStatusCode.BadRequest); if (!azureDeployment) { - await Assert.That(host.Services.GetRequiredService().GetAssertionSummary().TotalEvents).IsEqualTo(0); + await Assert.That((await http.GetFromJsonAsync("/admin-api/assertions"))!.TotalEvents).IsEqualTo(0); body["model"] = "gpt-4.1-mini"; using var valid = await PostAsync(http, path, body); await Assert.That(valid.StatusCode).IsEqualTo(HttpStatusCode.OK); @@ -80,7 +79,7 @@ public async Task ChatModel_IsRequiredExceptForAzureDeploymentRoutesAsync(bool a [Arguments("nested-anyof", "{\"city\":\"Paris\"}", "{\"type\":\"OBJECT\",\"minProperties\":\"1\",\"maxProperties\":\"1\",\"properties\":{\"city\":{\"anyOf\":[{\"type\":\"STRING\",\"enum\":[\"Paris\"]},{\"type\":\"INTEGER\",\"minimum\":1,\"maximum\":2}]}}}")] public async Task GeminiSchema_ValidatesNullableNestedAlternativesAndStringLimitsAsync(string caseId, string fixture, string schema) { - using var host = await LlmTckTestHost.StartAsync(b => b.AddChatScenario(caseId, s => s.RespondsJson(fixture))); + using var host = await LlmTestServerTestHost.StartAsync(b => b.AddChatScenario(caseId, s => s.RespondsJson(fixture))); using var http = host.GetTestClient(); var body = JsonNode.Parse("""{"contents":[{"role":"user","parts":[{"text":"answer"}]}],"generationConfig":{"responseMimeType":"application/json"}}""")!; body["generationConfig"]!["responseSchema"] = JsonNode.Parse("""{"type":"BOOLEAN"}"""); @@ -99,11 +98,11 @@ public async Task GeminiSchema_ValidatesNullableNestedAlternativesAndStringLimit [Arguments("/azure-openai/openai/v1/video/generations/jobs?api-version=preview")] public async Task VideoCapacity_ReturnsConflictThroughProviderEndpointAsync(string path) { - using var host = await LlmTckTestHost.StartAsync(b => b.WithVideoCapacity(0, 0)); + using var host = await LlmTestServerTestHost.StartAsync(b => b.WithVideoCapacity(0, 0)); using var http = host.GetTestClient(); using var response = await http.PostAsJsonAsync(path, new { model = "sora-2", prompt = "whale" }); await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.Conflict); - await Assert.That(await response.Content.ReadAsStringAsync()).Contains("llm_tck_video_capacity_exceeded"); + await Assert.That(await response.Content.ReadAsStringAsync()).Contains("llm_test_server_video_capacity_exceeded"); using var list = await http.GetAsync(path); await Assert.That((await list.Content.ReadFromJsonAsync()).GetProperty("data").GetArrayLength()).IsEqualTo(0); } diff --git a/tests/ManagedCode.LlmTck.Tests/Regression/ProviderValidationBoundaryTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Regression/ProviderValidationBoundaryTests.cs similarity index 77% rename from tests/ManagedCode.LlmTck.Tests/Regression/ProviderValidationBoundaryTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/Regression/ProviderValidationBoundaryTests.cs index b4cfaad..eb9146f 100644 --- a/tests/ManagedCode.LlmTck.Tests/Regression/ProviderValidationBoundaryTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Regression/ProviderValidationBoundaryTests.cs @@ -1,11 +1,11 @@ using System.Net; +using System.Net.Http.Json; using System.Text; using System.Text.Json.Nodes; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Tests.TestSupport; -using Microsoft.Extensions.DependencyInjection; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Tests.TestSupport; -namespace ManagedCode.LlmTck.Tests.Regression; +namespace ManagedCode.LlmTestServer.Tests.Regression; public sealed class ProviderValidationBoundaryTests { @@ -19,7 +19,7 @@ public sealed class ProviderValidationBoundaryTests [Arguments("/bedrock/model/gpt-4.1-mini/converse", 6)] public async Task ProviderValidator_RejectsMalformedToolsBeforeMappingOrRuntimeAsync(string path, int format) { - using var host = await LlmTckTestHost.StartAsync(b => b.AddChatScenario("tools", s => s.CallsTool("weather", "{\"city\":\"Paris\"}"))); + using var host = await LlmTestServerTestHost.StartAsync(b => b.AddChatScenario("tools", s => s.CallsTool("weather", "{\"city\":\"Paris\"}"))); using var http = host.GetTestClient(); http.DefaultRequestHeaders.Add("anthropic-version", "2023-06-01"); var valid = ToolFixtureEndpointTests.Request(format, tools: true); @@ -31,7 +31,7 @@ public async Task ProviderValidator_RejectsMalformedToolsBeforeMappingOrRuntimeA using var response = await PostAsync(http, path, invalid); await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); } - await Assert.That(host.Services.GetRequiredService().GetAssertionSummary().TotalEvents).IsEqualTo(0); + await Assert.That((await http.GetFromJsonAsync("/admin-api/assertions"))!.TotalEvents).IsEqualTo(0); using var accepted = await PostAsync(http, path, valid); await Assert.That(accepted.StatusCode).IsEqualTo(HttpStatusCode.OK); } @@ -42,7 +42,7 @@ public async Task ProviderValidator_RejectsMalformedToolsBeforeMappingOrRuntimeA [Arguments("/anthropic/v1/messages", 2)] public async Task DisabledParallelTools_AllowsZeroOrOneCallAsync(string path, int format) { - using var host = await LlmTckTestHost.StartAsync(b => b.AddChatScenario("single", s => s.Responds("plain").CallsTool("weather", "{\"city\":\"Paris\"}"))); + using var host = await LlmTestServerTestHost.StartAsync(b => b.AddChatScenario("single", s => s.Responds("plain").CallsTool("weather", "{\"city\":\"Paris\"}"))); using var http = host.GetTestClient(); http.DefaultRequestHeaders.Add("anthropic-version", "2023-06-01"); var body = ToolFixtureEndpointTests.Request(format, tools: true); @@ -54,7 +54,7 @@ public async Task DisabledParallelTools_AllowsZeroOrOneCallAsync(string path, in using var single = await PostAsync(http, path, body); await Assert.That(single.StatusCode).IsEqualTo(HttpStatusCode.OK); await Assert.That(await single.Content.ReadAsStringAsync()).Contains("weather"); - await Assert.That(host.Services.GetRequiredService().GetAssertionSummary().ErrorsReturned).IsEqualTo(0); + await Assert.That((await http.GetFromJsonAsync("/admin-api/assertions"))!.ErrorsReturned).IsEqualTo(0); } private static Task PostAsync(HttpClient http, string path, JsonNode body) diff --git a/tests/ManagedCode.LlmTck.Tests/Regression/ReviewCapabilityRegressionTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Regression/ReviewCapabilityRegressionTests.cs similarity index 67% rename from tests/ManagedCode.LlmTck.Tests/Regression/ReviewCapabilityRegressionTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/Regression/ReviewCapabilityRegressionTests.cs index bdded30..1fcf2cf 100644 --- a/tests/ManagedCode.LlmTck.Tests/Regression/ReviewCapabilityRegressionTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Regression/ReviewCapabilityRegressionTests.cs @@ -1,13 +1,11 @@ using System.Net; +using System.Net.Http.Json; using System.Text; -using System.Text.Json; using System.Text.Json.Nodes; -using ManagedCode.LlmTck.Hosting; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Tests.TestSupport; -using Microsoft.Extensions.DependencyInjection; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Tests.TestSupport; -namespace ManagedCode.LlmTck.Tests.Regression; +namespace ManagedCode.LlmTestServer.Tests.Regression; public sealed class ReviewCapabilityRegressionTests { @@ -31,7 +29,7 @@ public sealed class ReviewCapabilityRegressionTests [Arguments("/openai/v1/responses")] public async Task LegacyFunctions_ReturnExplicitErrorWithoutConsumingFixtureAsync(string path) { - using var host = await LlmTckTestHost.StartAsync(b => b.AddChatScenario("queue", s => s.Responds("blue whale"))); + using var host = await LlmTestServerTestHost.StartAsync(b => b.AddChatScenario("queue", s => s.Responds("blue whale"))); using var client = host.GetTestClient(); client.DefaultRequestHeaders.Add("anthropic-version", "2023-06-01"); var body = JsonNode.Parse("""{"model":"gpt-4.1-mini","max_tokens":10,"messages":[{"role":"user","content":"whale"}],"input":"whale","contents":[{"parts":[{"text":"whale"}]}]}""")!.AsObject(); @@ -39,8 +37,8 @@ public async Task LegacyFunctions_ReturnExplicitErrorWithoutConsumingFixtureAsyn using var content = new StringContent(body.ToJsonString(), Encoding.UTF8, "application/json"); using var response = await client.PostAsync(path, content); await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); - var runtime = host.Services.GetRequiredService(); - await Assert.That(runtime.GetAssertionSummary().TotalEvents).IsEqualTo(0); + var summary = (await client.GetFromJsonAsync("/admin-api/assertions"))!; + await Assert.That(summary.TotalEvents).IsEqualTo(0); } [Test] @@ -53,6 +51,15 @@ public async Task LegacyFunctions_ReturnExplicitErrorWithoutConsumingFixtureAsyn [Arguments("{\"generationConfig\":{\"responseJsonSchema\":{}}}", true)] public async Task CapabilityPolicy_DistinguishesTextDefaultsFromUnsupportedFeaturesAsync(string json, bool unsupported) { - await Assert.That(LlmTckRequestPolicy.HasUnsupportedChatFeatures(JsonSerializer.Deserialize(json))).IsEqualTo(unsupported); + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddChatScenario("policy", scenario => scenario.Responds("accepted"))); + using var http = host.GetTestClient(); + var body = JsonNode.Parse(json)!.AsObject(); + body["inputText"] = "policy"; + using var response = await http.PostAsJsonAsync("/bedrock/model/gpt-4.1-mini/invoke", body); + await Assert.That(response.StatusCode).IsEqualTo(unsupported ? HttpStatusCode.BadRequest : HttpStatusCode.OK); + if (!unsupported) + { + await Assert.That(await response.Content.ReadAsStringAsync()).Contains("accepted"); + } } } diff --git a/tests/ManagedCode.LlmTck.Tests/Regression/ReviewHttpRegressionTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Regression/ReviewHttpRegressionTests.cs similarity index 81% rename from tests/ManagedCode.LlmTck.Tests/Regression/ReviewHttpRegressionTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/Regression/ReviewHttpRegressionTests.cs index 374a8fa..83e3a83 100644 --- a/tests/ManagedCode.LlmTck.Tests/Regression/ReviewHttpRegressionTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Regression/ReviewHttpRegressionTests.cs @@ -2,15 +2,14 @@ using System.Net.Http.Json; using System.Text; using System.Text.Json; -using ManagedCode.LlmTck.Client; -using ManagedCode.LlmTck.Hosting; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Tests.TestSupport; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Tests.TestSupport; using Microsoft.Extensions.AI; -using Microsoft.Extensions.DependencyInjection; -namespace ManagedCode.LlmTck.Tests.Regression; +namespace ManagedCode.LlmTestServer.Tests.Regression; public sealed class ReviewHttpRegressionTests { @@ -23,10 +22,10 @@ private static StringContent Json(string body) [Test] public async Task OfficialOpenAiSdk_ReceivesFinalStreamingUsageAsync() { - using var host = await LlmTckTestHost.StartAsync(b => b.AddReasoningChatModel(LlmTckKnownModelIds.Gpt41Mini, 17) + using var host = await LlmTestServerTestHost.StartAsync(b => b.AddReasoningChatModel(LlmTestServerKnownModelIds.Gpt41Mini, 17) .AddChatScenario("queue", scenario => scenario.Responds("blue whale", "blue ", "whale"))); using var http = host.GetTestClient(); - var sdk = new global::OpenAI.Chat.ChatClient(LlmTckKnownModelIds.Gpt41Mini, + var sdk = new global::OpenAI.Chat.ChatClient(LlmTestServerKnownModelIds.Gpt41Mini, new System.ClientModel.ApiKeyCredential("test-key"), new global::OpenAI.OpenAIClientOptions { Endpoint = new Uri(http.BaseAddress!, "/openai/v1"), @@ -52,15 +51,14 @@ public async Task OfficialOpenAiSdk_ReceivesFinalStreamingUsageAsync() [Arguments("defaultAudioBytes")] public async Task InvalidConfigure_PreservesConfigurationAndTracesAsync(string property) { - using var host = await LlmTckTestHost.StartAsync(b => b.AddChatScenario("queue", s => s.Responds("first").Responds("second"))); + using var host = await LlmTestServerTestHost.StartAsync(b => b.AddChatScenario("queue", s => s.Responds("first").Responds("second"))); using var client = host.GetTestClient(); using var first = await client.PostAsync("/openai/v1/chat/completions", Json(_chat)); first.EnsureSuccessStatusCode(); - var traces = host.Services.GetRequiredService(); - var before = traces.GetSnapshot().Select(trace => trace.RequestId).ToArray(); + var before = (await client.GetFromJsonAsync>("/admin-api/requests"))!.Select(trace => trace.RequestId).ToArray(); using var invalid = await client.PostAsync("/admin-api/configure", Json($"{{\"{property}\":null}}")); await Assert.That(invalid.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); - await Assert.That(traces.GetSnapshot().Select(trace => trace.RequestId).ToArray()).IsEquivalentTo(before); + await Assert.That((await client.GetFromJsonAsync>("/admin-api/requests"))!.Select(trace => trace.RequestId).ToArray()).IsEquivalentTo(before); using var second = await client.PostAsync("/openai/v1/chat/completions", Json(_chat)); second.EnsureSuccessStatusCode(); await Assert.That(await second.Content.ReadAsStringAsync()).Contains("second"); @@ -77,12 +75,12 @@ public async Task InvalidConfigure_PreservesConfigurationAndTracesAsync(string p [Arguments("/openai/v1/chat/completions", "{\"model\":\"gpt-4.1-mini\",\"messages\":null}")] public async Task NestedNullPayloads_ReturnBadRequestAsync(string path, string body) { - using var host = await LlmTckTestHost.StartAsync(); + using var host = await LlmTestServerTestHost.StartAsync(); using var client = host.GetTestClient(); client.DefaultRequestHeaders.Add("anthropic-version", "2023-06-01"); using var response = await client.PostAsync(path, Json(body)); await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); - await Assert.That(host.Services.GetRequiredService().GetAssertionSummary().TotalEvents).IsEqualTo(0); + await Assert.That((await client.GetFromJsonAsync("/admin-api/assertions"))!.TotalEvents).IsEqualTo(0); } [Test] @@ -100,7 +98,7 @@ public async Task NestedNullPayloads_ReturnBadRequestAsync(string path, string b [Arguments("?api-version=2025-04-02-preview")] public async Task AzureLegacyVersion_IsValidatedBeforeQueueConsumptionAsync(string query) { - using var host = await LlmTckTestHost.StartAsync(b => b.AddChatScenario("queue", s => s.Responds("first"))); + using var host = await LlmTestServerTestHost.StartAsync(b => b.AddChatScenario("queue", s => s.Responds("first"))); using var client = host.GetTestClient(); const string path = "/azure-openai/openai/deployments/gpt-4.1-mini/chat/completions"; using var invalid = await client.PostAsync(path + query, Json(_chat)); @@ -115,7 +113,7 @@ public async Task AzureLegacyVersion_IsValidatedBeforeQueueConsumptionAsync(stri [Arguments("2025-04-01-preview")] public async Task AzureLegacyVersion_AcceptsDocumentedStableAndPreviewVersionsAsync(string version) { - using var host = await LlmTckTestHost.StartAsync(b => b.AddChatScenario("queue", s => s.Responds("first"))); + using var host = await LlmTestServerTestHost.StartAsync(b => b.AddChatScenario("queue", s => s.Responds("first"))); using var client = host.GetTestClient(); using var response = await client.PostAsync( "/azure-openai/openai/deployments/gpt-4.1-mini/chat/completions?api-version=" + version, Json(_chat)); @@ -127,10 +125,10 @@ public async Task AzureLegacyVersion_AcceptsDocumentedStableAndPreviewVersionsAs [Test] public async Task ChatClient_PreservesUsageAndTerminalUpdate_EmbeddingsPreserveUsageAsync() { - using var host = await LlmTckTestHost.StartAsync(b => b.AddReasoningChatModel(LlmTckKnownModelIds.Gpt41Mini, 17) + using var host = await LlmTestServerTestHost.StartAsync(b => b.AddReasoningChatModel(LlmTestServerKnownModelIds.Gpt41Mini, 17) .AddChatScenario("queue", s => s.Responds("blue whale").Responds("blue whale", "blue ", "whale"))); using var http = host.GetTestClient(); - using var client = new LlmTckChatClient(http); + using var client = new LlmTestServerChatClient(http); var response = await client.GetResponseAsync([new(ChatRole.User, "whale")]); await Assert.That(response.Usage!.ReasoningTokenCount).IsEqualTo(17); await Assert.That(response.Usage.InputTokenCount.GetValueOrDefault()).IsGreaterThan(0); @@ -146,7 +144,7 @@ public async Task ChatClient_PreservesUsageAndTerminalUpdate_EmbeddingsPreserveU var usage = updates.SelectMany(update => update.Contents).OfType().Single().Details; await Assert.That(usage.TotalTokenCount).IsEqualTo(response.Usage.TotalTokenCount); await Assert.That(usage.ReasoningTokenCount).IsEqualTo(17); - using var embeddings = new LlmTckEmbeddingGenerator(http); + using var embeddings = new LlmTestServerEmbeddingGenerator(http); var generated = await embeddings.GenerateAsync(["blue whale"]); await Assert.That(generated.Usage!.InputTokenCount.GetValueOrDefault()).IsGreaterThan(0); await Assert.That(generated.Usage.TotalTokenCount).IsEqualTo(generated.Usage.InputTokenCount); @@ -157,11 +155,11 @@ public async Task ChatClient_PreservesUsageAndTerminalUpdate_EmbeddingsPreserveU [Arguments(false)] public async Task StreamingUsage_IsOptInAndFollowsTerminalChoiceAsync(bool includeUsage) { - using var host = await LlmTckTestHost.StartAsync(b => b.AddChatScenario("queue", s => s.Responds("blue whale", "blue ", "whale"))); + using var host = await LlmTestServerTestHost.StartAsync(b => b.AddChatScenario("queue", s => s.Responds("blue whale", "blue ", "whale"))); using var http = host.GetTestClient(); using var response = await http.PostAsJsonAsync("/openai/v1/chat/completions", new { - model = LlmTckKnownModelIds.Gpt41Mini, + model = LlmTestServerKnownModelIds.Gpt41Mini, messages = new[] { new { role = "user", content = "whale" } }, stream = true, stream_options = new { include_usage = includeUsage }, @@ -185,15 +183,15 @@ public async Task StreamingUsage_IsOptInAndFollowsTerminalChoiceAsync(bool inclu [Arguments("/microsoft-foundry/models/embeddings")] public async Task FoundryInference_RequiresDocumentedVersionWithoutConsumingQueueAsync(string path) { - using var host = await LlmTckTestHost.StartAsync(b => b.AddChatScenario("queue", s => s.Responds("first"))); + using var host = await LlmTestServerTestHost.StartAsync(b => b.AddChatScenario("queue", s => s.Responds("first"))); using var http = host.GetTestClient(); foreach (var query in new[] { "", "?api-version=wrong", "?api-version=2024-05-01-preview&api-version=wrong" }) { using var response = await http.PostAsync(path + query, Json(_chat)); await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); } - var runtime = host.Services.GetRequiredService(); - await Assert.That(runtime.GetAssertionSummary().TotalEvents).IsEqualTo(0); + var summary = (await http.GetFromJsonAsync("/admin-api/assertions"))!; + await Assert.That(summary.TotalEvents).IsEqualTo(0); using var success = await http.PostAsync("/microsoft-foundry/chat/completions?api-version=2024-05-01-preview", Json(_chat)); await Assert.That(await success.Content.ReadAsStringAsync()).Contains("first"); } diff --git a/tests/ManagedCode.LlmTck.Tests/Regression/ReviewVideoRegressionTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Regression/ReviewVideoRegressionTests.cs similarity index 71% rename from tests/ManagedCode.LlmTck.Tests/Regression/ReviewVideoRegressionTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/Regression/ReviewVideoRegressionTests.cs index 4adcbcf..ad4ed41 100644 --- a/tests/ManagedCode.LlmTck.Tests/Regression/ReviewVideoRegressionTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Regression/ReviewVideoRegressionTests.cs @@ -1,13 +1,10 @@ using System.Net; using System.Net.Http.Json; using System.Text.Json; -using ManagedCode.LlmTck.Configuration; -using ManagedCode.LlmTck.Providers; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Tests.TestSupport; -using Microsoft.Extensions.DependencyInjection; +using ManagedCode.LlmTestServer.Configuration; +using ManagedCode.LlmTestServer.Tests.TestSupport; -namespace ManagedCode.LlmTck.Tests.Regression; +namespace ManagedCode.LlmTestServer.Tests.Regression; public sealed class ReviewVideoRegressionTests { @@ -16,7 +13,7 @@ public sealed class ReviewVideoRegressionTests [Arguments("/azure-openai/openai/v1/video/generations/jobs", "?api-version=preview", true)] public async Task VideoLifecycle_PersistsCompletedJobsAndDeletesContentAsync(string path, string query, bool azure) { - using var host = await LlmTckTestHost.StartAsync(); + using var host = await LlmTestServerTestHost.StartAsync(); using var client = host.GetTestClient(); using var unknown = await client.GetAsync(path + "/never-created" + query); await Assert.That(unknown.StatusCode).IsEqualTo(HttpStatusCode.NotFound); @@ -61,22 +58,25 @@ public async Task VideoLifecycle_PersistsCompletedJobsAndDeletesContentAsync(str await Assert.That(afterDelete.StatusCode).IsEqualTo(HttpStatusCode.NotFound); await Assert.That(deletedContent.StatusCode).IsEqualTo(HttpStatusCode.NotFound); await Assert.That(repeatedDelete.StatusCode).IsEqualTo(HttpStatusCode.NotFound); - await host.Services.GetRequiredService().ResetAsync(); + using var reset = await client.PostAsync("/admin-api/reset", null); + reset.EnsureSuccessStatusCode(); using var afterReset = await client.GetAsync($"{path}/{secondId}{query}"); await Assert.That(afterReset.StatusCode).IsEqualTo(HttpStatusCode.NotFound); } [Test] - public async Task VideoStore_IsolatesProvidersSnapshotsBytesAndClearsOnConfigureAsync() + public async Task VideoEndpoints_IsolateProviderJobsAndClearOnConfigureAsync() { - var runtime = new LlmTckRuntime(); - var generated = await runtime.GenerateVideoAsync("sora-2", "whale"); - var stored = runtime.StoreVideo(LlmTckCompatibilityTags.OpenAI, generated); - var firstByte = stored.Bytes[0]; - stored.Bytes[0] = 255; - await Assert.That(runtime.FindVideo(LlmTckCompatibilityTags.OpenAI, stored.VideoId)!.Bytes[0]).IsEqualTo(firstByte); - await Assert.That(runtime.FindVideo("azure", stored.VideoId)).IsNull(); - await runtime.ConfigureAsync(LlmTckConfiguration.CreateDefault()); - await Assert.That(runtime.ListVideos(LlmTckCompatibilityTags.OpenAI).Count).IsEqualTo(0); + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + using var created = await http.PostAsJsonAsync("/openai/v1/videos", new { model = "sora-2", prompt = "whale" }); + created.EnsureSuccessStatusCode(); + var videoId = (await created.Content.ReadFromJsonAsync()).GetProperty("id").GetString(); + using var foreign = await http.GetAsync($"/azure-openai/openai/v1/video/generations/jobs/{videoId}?api-version=preview"); + await Assert.That(foreign.StatusCode).IsEqualTo(HttpStatusCode.NotFound); + using var configured = await http.PostAsJsonAsync("/admin-api/configure", LlmTestServerConfiguration.CreateDefault()); + configured.EnsureSuccessStatusCode(); + using var after = await http.GetAsync($"/openai/v1/videos/{videoId}"); + await Assert.That(after.StatusCode).IsEqualTo(HttpStatusCode.NotFound); } } diff --git a/tests/ManagedCode.LlmTck.Tests/Regression/ToolFixtureClientTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Regression/ToolFixtureClientTests.cs similarity index 81% rename from tests/ManagedCode.LlmTck.Tests/Regression/ToolFixtureClientTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/Regression/ToolFixtureClientTests.cs index 2ba5f64..315e7d7 100644 --- a/tests/ManagedCode.LlmTck.Tests/Regression/ToolFixtureClientTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Regression/ToolFixtureClientTests.cs @@ -1,11 +1,11 @@ using System.Text.Json; -using ManagedCode.LlmTck.Client; -using ManagedCode.LlmTck.Models; -using ManagedCode.LlmTck.Scenarios; -using ManagedCode.LlmTck.Tests.TestSupport; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Scenarios; +using ManagedCode.LlmTestServer.Tests.TestSupport; using Microsoft.Extensions.AI; -namespace ManagedCode.LlmTck.Tests.Regression; +namespace ManagedCode.LlmTestServer.Tests.Regression; // Responses remains experimental in the official SDK. #pragma warning disable OPENAI001 @@ -16,10 +16,10 @@ public sealed class ToolFixtureClientTests [Arguments(true)] public async Task ExtensionsAi_RespectsSingleToolOptionWithoutConsumingFixtureAsync(bool stream) { - using var host = await LlmTckTestHost.StartAsync(b => b.AddChatScenario("parallel", s => s.CallsTools( - new LlmTckToolCall { Id = "first", Name = "weather" }, new LlmTckToolCall { Id = "second", Name = "weather" }))); + using var host = await LlmTestServerTestHost.StartAsync(b => b.AddChatScenario("parallel", s => s.CallsTools( + new LlmTestServerToolCall { Id = "first", Name = "weather" }, new LlmTestServerToolCall { Id = "second", Name = "weather" }))); using var http = host.GetTestClient(); - using var client = new LlmTckChatClient(http); + using var client = new LlmTestServerChatClient(http); var options = new ChatOptions { Tools = [AIFunctionFactory.Create(() => "sunny", "weather")], AllowMultipleToolCalls = false }; async Task SendAsync() { @@ -40,9 +40,9 @@ await Assert.That(accepted.Messages.SelectMany(message => message.Contents).OfTy [Arguments(true)] public async Task ExtensionsAi_InvokesToolAndReturnsValidatedJsonAsync(bool stream) { - using var host = await LlmTckTestHost.StartAsync(b => b.AddChatScenario("weather", s => s.CallsTool("weather", """{"city":"Paris"}""").RespondsJson("""{"forecast":"sunny"}"""))); + using var host = await LlmTestServerTestHost.StartAsync(b => b.AddChatScenario("weather", s => s.CallsTool("weather", """{"city":"Paris"}""").RespondsJson("""{"forecast":"sunny"}"""))); using var http = host.GetTestClient(); - using var client = new ChatClientBuilder(new LlmTckChatClient(http)).UseFunctionInvocation().Build(); + using var client = new ChatClientBuilder(new LlmTestServerChatClient(http)).UseFunctionInvocation().Build(); var calls = new List(); var tool = AIFunctionFactory.Create((string city) => { calls.Add(city); return "sunny"; }, "weather"); var options = new ChatOptions @@ -61,9 +61,9 @@ public async Task ExtensionsAi_InvokesToolAndReturnsValidatedJsonAsync(bool stre [Arguments(true)] public async Task OfficialOpenAiSdk_ParsesToolCallAndStreamingArgumentsAsync(bool stream) { - using var host = await LlmTckTestHost.StartAsync(b => b.AddChatScenario("weather", s => s.CallsTool("weather", """{"city":"Paris"}"""))); + using var host = await LlmTestServerTestHost.StartAsync(b => b.AddChatScenario("weather", s => s.CallsTool("weather", """{"city":"Paris"}"""))); using var http = host.GetTestClient(); - var client = new global::OpenAI.Chat.ChatClient(LlmTckKnownModelIds.Gpt41Mini, + var client = new global::OpenAI.Chat.ChatClient(LlmTestServerKnownModelIds.Gpt41Mini, new System.ClientModel.ApiKeyCredential("test-key"), new global::OpenAI.OpenAIClientOptions { Endpoint = new Uri(http.BaseAddress!, "/openai/v1"), @@ -108,13 +108,13 @@ public async Task OfficialOpenAiSdk_ParsesToolCallAndStreamingArgumentsAsync(boo [Arguments("/microsoft-foundry/openai/v1")] public async Task OfficialResponsesSdk_ParsesFunctionCallsAndStreamAsync(string endpoint) { - using var host = await LlmTckTestHost.StartAsync(b => b.AddChatScenario("weather", s => s.CallsTool("weather", "{}").CallsTool("weather", "{}"))); + using var host = await LlmTestServerTestHost.StartAsync(b => b.AddChatScenario("weather", s => s.CallsTool("weather", "{}").CallsTool("weather", "{}"))); using var http = host.GetTestClient(); var client = new global::OpenAI.Responses.ResponsesClient(new System.ClientModel.ApiKeyCredential("test-key"), new global::OpenAI.Responses.ResponsesClientOptions { Endpoint = new Uri(http.BaseAddress!, endpoint), Transport = new System.ClientModel.Primitives.HttpClientPipelineTransport(http) }); var options = new global::OpenAI.Responses.CreateResponseOptions { - Model = LlmTckKnownModelIds.Gpt41Mini, + Model = LlmTestServerKnownModelIds.Gpt41Mini, InputItems = { global::OpenAI.Responses.ResponseItem.CreateUserMessageItem("weather") }, Tools = { global::OpenAI.Responses.ResponseTool.CreateFunctionTool("weather", BinaryData.FromString("{\"type\":\"object\"}"), null, null) }, }; @@ -139,12 +139,12 @@ public async Task OfficialResponsesSdk_ParsesFunctionCallsAndStreamAsync(string [Test] public async Task OfficialAzureSdk_ParsesFunctionToolCallsAsync() { - using var host = await LlmTckTestHost.StartAsync(b => b.AddChatScenario("weather", s => s.CallsTool("weather", "{}"))); + using var host = await LlmTestServerTestHost.StartAsync(b => b.AddChatScenario("weather", s => s.CallsTool("weather", "{}"))); using var http = host.GetTestClient(); var azure = new global::Azure.AI.OpenAI.AzureOpenAIClient(new Uri(http.BaseAddress!, "/azure-openai"), new global::Azure.AzureKeyCredential("test-key"), new global::Azure.AI.OpenAI.AzureOpenAIClientOptions { Transport = new System.ClientModel.Primitives.HttpClientPipelineTransport(http) }); - var response = (await azure.GetChatClient(LlmTckKnownModelIds.Gpt41Mini).CompleteChatAsync([new global::OpenAI.Chat.UserChatMessage("weather")], new() + var response = (await azure.GetChatClient(LlmTestServerKnownModelIds.Gpt41Mini).CompleteChatAsync([new global::OpenAI.Chat.UserChatMessage("weather")], new() { Tools = { global::OpenAI.Chat.ChatTool.CreateFunctionTool("weather", functionParameters: BinaryData.FromString("{\"type\":\"object\"}")) } })).Value; await Assert.That(response.ToolCalls.Single().FunctionName).IsEqualTo("weather"); } diff --git a/tests/ManagedCode.LlmTck.Tests/Regression/ToolFixtureEndpointTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Regression/ToolFixtureEndpointTests.cs similarity index 93% rename from tests/ManagedCode.LlmTck.Tests/Regression/ToolFixtureEndpointTests.cs rename to tests/ManagedCode.LlmTestServer.Tests/Regression/ToolFixtureEndpointTests.cs index 8ef011d..5740650 100644 --- a/tests/ManagedCode.LlmTck.Tests/Regression/ToolFixtureEndpointTests.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/Regression/ToolFixtureEndpointTests.cs @@ -1,11 +1,11 @@ using System.Net; +using System.Net.Http.Json; using System.Text; using System.Text.Json.Nodes; -using ManagedCode.LlmTck.Runtime; -using ManagedCode.LlmTck.Tests.TestSupport; -using Microsoft.Extensions.DependencyInjection; +using ManagedCode.LlmTestServer.Runtime; +using ManagedCode.LlmTestServer.Tests.TestSupport; -namespace ManagedCode.LlmTck.Tests.Regression; +namespace ManagedCode.LlmTestServer.Tests.Regression; public sealed class ToolFixtureEndpointTests { @@ -35,7 +35,7 @@ public sealed class ToolFixtureEndpointTests [Arguments("/bedrock/model/gpt-4.1-mini/converse", 6)] public async Task ToolsAndSchema_RoundTripAndRejectMismatchedFixturesAsync(string path, int format) { - using var host = await LlmTckTestHost.StartAsync(b => b.AddChatScenario("tools", s => s.CallsTool("weather", _answer).RespondsJson(_answer))); + using var host = await LlmTestServerTestHost.StartAsync(b => b.AddChatScenario("tools", s => s.CallsTool("weather", _answer).RespondsJson(_answer))); using var http = host.GetTestClient(); http.DefaultRequestHeaders.Add("anthropic-version", "2023-06-01"); var request = Request(format, tools: true); @@ -67,7 +67,7 @@ public async Task ToolsAndSchema_RoundTripAndRejectMismatchedFixturesAsync(strin using var final = await PostAsync(http, path, second); await Assert.That(final.StatusCode).IsEqualTo(HttpStatusCode.OK); await Assert.That(await final.Content.ReadAsStringAsync()).Contains("Paris"); - var events = host.Services.GetRequiredService().GetAssertionSummary().Events; + var events = (await http.GetFromJsonAsync("/admin-api/assertions"))!.Events; await Assert.That(events.Any(entry => entry.Messages.Any(message => message.Role == "tool" && message.Content.Contains("sunny", StringComparison.Ordinal)))).IsTrue(); } @@ -81,7 +81,7 @@ public async Task ToolsAndSchema_RoundTripAndRejectMismatchedFixturesAsync(strin [Arguments("/bedrock/model/gpt-4.1-mini/converse-stream", 6, "contentBlockDelta")] public async Task StreamingTools_IncludeArgumentsAndProviderEventsAsync(string path, int format, string eventType) { - using var host = await LlmTckTestHost.StartAsync(b => b.AddChatScenario("tools", s => s.CallsTool("weather", _answer))); + using var host = await LlmTestServerTestHost.StartAsync(b => b.AddChatScenario("tools", s => s.CallsTool("weather", _answer))); using var http = host.GetTestClient(); http.DefaultRequestHeaders.Add("anthropic-version", "2023-06-01"); var request = Request(format, tools: true); @@ -97,7 +97,7 @@ public async Task StreamingTools_IncludeArgumentsAndProviderEventsAsync(string p [Test] public async Task Perplexity_ValidatesJsonSchemaAndRejectsUnclaimedFunctionToolsAsync() { - using var host = await LlmTckTestHost.StartAsync(b => b.AddChatScenario("json", s => s.RespondsJson(_answer))); + using var host = await LlmTestServerTestHost.StartAsync(b => b.AddChatScenario("json", s => s.RespondsJson(_answer))); using var http = host.GetTestClient(); using var rejected = await PostAsync(http, "/perplexity/v1/sonar", Request(0, tools: true)); await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); diff --git a/tests/ManagedCode.LlmTestServer.Tests/Regression/ToolFixtureValidationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Regression/ToolFixtureValidationTests.cs new file mode 100644 index 0000000..3a164de --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Regression/ToolFixtureValidationTests.cs @@ -0,0 +1,135 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.Regression; + +public sealed class ToolFixtureValidationTests +{ + [Test] + [Arguments("/openai/v1/chat/completions", "\"tools\":null")] + [Arguments("/openai/v1/chat/completions", "\"tools\":[null]")] + [Arguments("/openai/v1/chat/completions", "\"tools\":[{\"type\":\"function\",\"function\":null}]")] + [Arguments("/openai/v1/chat/completions", "\"tools\":[{\"type\":\"web_search\"}]")] + [Arguments("/openai/v1/chat/completions", "\"tool_choice\":\"maybe\"")] + [Arguments("/openai/v1/chat/completions", "\"tool_choice\":{\"type\":\"function\",\"function\":{}}")] + [Arguments("/openai/v1/chat/completions", "\"response_format\":{\"type\":\"json_schema\"}")] + [Arguments("/openai/v1/chat/completions", "\"response_format\":{\"type\":\"xml\"}")] + [Arguments("/openai/v1/responses", "\"text\":{\"format\":{\"type\":\"json_schema\"}}")] + [Arguments("/anthropic/v1/messages", "\"tools\":[{\"name\":\"weather\"}]")] + [Arguments("/anthropic/v1/messages", "\"tool_choice\":{\"type\":\"tool\"}")] + [Arguments("/gemini/v1beta/models/gpt-4.1-mini:generateContent", "\"tools\":[{}]")] + [Arguments("/gemini/v1beta/models/gpt-4.1-mini:generateContent", "\"toolConfig\":{\"functionCallingConfig\":{\"mode\":\"WRONG\"}}")] + [Arguments("/ollama/api/chat", "\"format\":\"xml\"")] + [Arguments("/cohere/v2/chat", "\"tool_choice\":\"MAYBE\"")] + [Arguments("/bedrock/model/gpt-4.1-mini/converse", "\"toolConfig\":{\"tools\":[null]}")] + [Arguments("/bedrock/model/gpt-4.1-mini/converse", "\"outputConfig\":{\"textFormat\":{\"type\":\"json_schema\"}}")] + [Arguments("/openai/v1/chat/completions", "\"messages\":[{\"tool_calls\":[null]}]")] + [Arguments("/openai/v1/chat/completions", "\"messages\":[{\"tool_calls\":[{\"id\":\"call\",\"function\":{\"name\":\"weather\",\"arguments\":null}}]}]")] + [Arguments("/openai/v1/responses", "\"input\":[{\"type\":\"function_call\",\"name\":\"weather\"}]")] + [Arguments("/anthropic/v1/messages", "\"messages\":[{\"content\":[{\"type\":\"tool_use\"}]}]")] + [Arguments("/gemini/v1beta/models/gpt-4.1-mini:generateContent", "\"contents\":[{\"parts\":[{\"functionCall\":{}}]}]")] + [Arguments("/bedrock/model/gpt-4.1-mini/converse", "\"messages\":[{\"content\":[{\"toolResult\":{\"toolUseId\":\"call\",\"content\":[null]}}]}]")] + public async Task InvalidToolOptions_Return400BeforeRuntimeAsync(string path, string fields) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + http.DefaultRequestHeaders.Add("anthropic-version", "2023-06-01"); + using var response = await http.PostAsync(path, new StringContent("{" + fields + "}", Encoding.UTF8, "application/json")); + await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + } + + [Test] + [Arguments("{\"$ref\":\"https://example.com/schema\"}", HttpStatusCode.Conflict)] + [Arguments("{\"type\":42}", HttpStatusCode.Conflict)] + [Arguments("\"not json\"", HttpStatusCode.BadRequest)] + public async Task InvalidSchema_RejectsWithoutConsumingResponseAsync(string schema, HttpStatusCode status) + { + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddChatScenario("json", scenario => scenario.RespondsJson("{}"))); + using var http = host.GetTestClient(); + var body = Request(); + body["response_format"] = JsonNode.Parse("{\"type\":\"json_schema\",\"json_schema\":{\"name\":\"fixture\",\"schema\":" + schema + "}}"); + using var rejected = await http.PostAsJsonAsync("/openai/v1/chat/completions", body); + await Assert.That(rejected.StatusCode).IsEqualTo(status); + body["response_format"]!["json_schema"]!["schema"] = new JsonObject { ["type"] = "object" }; + using var accepted = await http.PostAsJsonAsync("/openai/v1/chat/completions", body); + accepted.EnsureSuccessStatusCode(); + await Assert.That(await accepted.Content.ReadAsStringAsync()).Contains("{}"); + } + + [Test] + public async Task RequiredAndNoneChoices_DoNotConsumeMismatchedToolFixturesAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddChatScenario("tools", scenario => scenario.CallsTool("weather", "{}").Responds("done"))); + using var http = host.GetTestClient(); + var body = Request(); + body["tools"] = Tools(new JsonObject { ["type"] = "object" }); + body["tool_choice"] = "none"; + using var none = await http.PostAsJsonAsync("/openai/v1/chat/completions", body); + await Assert.That(none.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + body["tool_choice"] = new JsonObject { ["type"] = "function", ["function"] = new JsonObject { ["name"] = "other" } }; + using var named = await http.PostAsJsonAsync("/openai/v1/chat/completions", body); + await Assert.That(named.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + body.Remove("tools"); body.Remove("tool_choice"); + using var missing = await http.PostAsJsonAsync("/openai/v1/chat/completions", body); + await Assert.That(missing.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + body["tools"] = Tools(new JsonObject { ["type"] = "object" }); body["tool_choice"] = "required"; + using var tool = await http.PostAsJsonAsync("/openai/v1/chat/completions", body); + tool.EnsureSuccessStatusCode(); + var payload = await tool.Content.ReadFromJsonAsync(); + await Assert.That(payload.GetProperty("choices")[0].GetProperty("message").GetProperty("tool_calls")[0].GetProperty("function").GetProperty("name").GetString()).IsEqualTo("weather"); + using var required = await http.PostAsJsonAsync("/openai/v1/chat/completions", body); + await Assert.That(required.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + body["tool_choice"] = "none"; + using var final = await http.PostAsJsonAsync("/openai/v1/chat/completions", body); + final.EnsureSuccessStatusCode(); + await Assert.That(await final.Content.ReadAsStringAsync()).Contains("done"); + } + + [Test] + public async Task LocalSchemaReferences_AreEvaluatedAndInvalidArgumentsDoNotConsumeAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddChatScenario("tools", scenario => scenario.CallsTool("weather", "{\"city\":\"London\"}"))); + using var http = host.GetTestClient(); + var body = Request(); + body["tools"] = Tools(JsonNode.Parse("""{"$defs":{"city":{"const":"Paris"}},"type":"object","properties":{"city":{"$ref":"#/$defs/city"}},"required":["city"]}""")!); + using var rejected = await http.PostAsJsonAsync("/openai/v1/chat/completions", body); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + body["tools"]![0]!["function"]!["parameters"]!["$defs"]!["city"]!["const"] = "London"; + using var accepted = await http.PostAsJsonAsync("/openai/v1/chat/completions", body); + accepted.EnsureSuccessStatusCode(); + await Assert.That(await accepted.Content.ReadAsStringAsync()).Contains("London"); + } + + [Test] + public async Task StructuredStream_ValidatesActualConcatenatedContentAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(builder => builder.AddChatScenario("json", scenario => scenario.Responds("{}", "not json"))); + using var http = host.GetTestClient(); + var body = Request(); body["stream"] = true; body["response_format"] = new JsonObject { ["type"] = "json_object" }; + using var rejected = await http.PostAsJsonAsync("/openai/v1/chat/completions", body); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + body.Remove("response_format"); body["stream"] = false; + using var accepted = await http.PostAsJsonAsync("/openai/v1/chat/completions", body); + accepted.EnsureSuccessStatusCode(); + var payload = await accepted.Content.ReadFromJsonAsync(); + await Assert.That(payload.GetProperty("choices")[0].GetProperty("message").GetProperty("content").GetString()).IsEqualTo("{}"); + } + + private static JsonObject Request() + { + return JsonNode.Parse("""{"model":"gpt-4.1-mini","messages":[{"role":"user","content":"weather"}]}""")!.AsObject(); + } + + private static JsonArray Tools(JsonNode schema) + { + return new JsonArray(new JsonObject + { + ["type"] = "function", + ["function"] = new JsonObject { ["name"] = "weather", ["parameters"] = schema }, + }); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Runtime/LlmTestServerTokenCounterTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Runtime/LlmTestServerTokenCounterTests.cs new file mode 100644 index 0000000..ea7ad95 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Runtime/LlmTestServerTokenCounterTests.cs @@ -0,0 +1,35 @@ +using System.Net.Http.Json; +using System.Text.Json; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.Runtime; + +public sealed class LlmTestServerTokenCounterTests +{ + [Test] + [Arguments("hello,world", 3)] + [Arguments("Text tokenization is the process of splitting a string into a list of tokens.", 16)] + [Arguments("antidisestablishmentarianism", 6)] + public async Task ChatResponse_ReportsConfiguredOutputTokenUsageThroughHttpAsync( + string text, + int expectedTokens + ) + { + using var host = await LlmTestServerTestHost.StartAsync(options => options + .AddChatScenario("usage", scenario => scenario.Responds(text))); + using var http = host.GetTestClient(); + using var response = await http.PostAsJsonAsync("/openai/v1/chat/completions", new + { + model = "gpt-4.1-mini", + messages = new[] { new { role = "user", content = "Count the output tokens." } }, + }); + response.EnsureSuccessStatusCode(); + var body = await response.Content.ReadFromJsonAsync(); + await Assert.That(body.GetProperty("choices")[0].GetProperty("message").GetProperty("content").GetString()) + .IsEqualTo(text); + await Assert.That(body.GetProperty("usage").GetProperty("completion_tokens").GetInt32()) + .IsEqualTo(expectedTokens); + var summary = await http.GetFromJsonAsync("/admin-api/assertions"); + await Assert.That(summary.GetProperty("outputTokens").GetInt32()).IsEqualTo(expectedTokens); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Sessions/SessionBoundaryIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Sessions/SessionBoundaryIntegrationTests.cs new file mode 100644 index 0000000..75e305c --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Sessions/SessionBoundaryIntegrationTests.cs @@ -0,0 +1,91 @@ +using System.Net; +using System.Net.Http.Json; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Configuration; +using ManagedCode.LlmTestServer.Hosting; +using ManagedCode.LlmTestServer.Sessions; +using ManagedCode.LlmTestServer.Tests.TestSupport; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; + +namespace ManagedCode.LlmTestServer.Tests.Sessions; + +public sealed class SessionBoundaryIntegrationTests +{ + [Test] + public async Task InvalidSessionCreate_DoesNotReserveCountOrChangeDefaultRuntimeAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(configureServices: services => services.Configure(options => + { options.MaxSessions = 1; options.MaxConfigurationBytes = 4096; })); + using var http = host.GetTestClient(); + foreach (var configuration in new[] + { + LlmTestServerConfiguration.CreateDefault() with { DefaultTranslationText = new string('x', 8192) }, + LlmTestServerConfiguration.CreateDefault() with { MaxStoredChatResponseBytes = long.MaxValue }, + LlmTestServerConfiguration.CreateDefault() with { MaxPromptCacheEntries = -1 }, + }) + { + using var invalid = await http.PostAsJsonAsync("/admin-api/sessions", new LlmTestServerSessionCreateRequest { Configuration = configuration }); + await Assert.That(invalid.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + } + var invalidFixture = new + { + chatScenarios = new[] { new { id = "invalid", responses = new[] + { new { toolCalls = new[] { new { id = "call", name = "weather", argumentsJson = "{" } } } } } } + }; + using var badFixture = await http.PostAsJsonAsync("/admin-api/sessions", new { configuration = invalidFixture }); + await Assert.That(badFixture.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + await using var session = await new LlmTestServerClient(http).CreateSessionAsync(); + using var badConfigure = await session.HttpClient.PostAsJsonAsync("/admin-api/configure", invalidFixture); + await Assert.That(badConfigure.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + using var nested = await session.HttpClient.PostAsJsonAsync("/admin-api/sessions", new { }); + await Assert.That(nested.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + await Assert.That((await session.Control.GetAssertionsAsync()).TotalEvents).IsEqualTo(0); + } + + [Test] + public async Task ShrinkingSessionCapacity_AllowsAnotherSessionWithinAggregateBudgetAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(configureServices: services => services.Configure(options => options.MaxReservedStateBytes = 180L * 1024 * 1024)); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + await using var first = await root.CreateSessionAsync(); + using var full = await http.PostAsJsonAsync("/admin-api/sessions", new { }); + await Assert.That(full.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + await first.Control.ConfigureAsync(LlmTestServerConfiguration.CreateDefault() with { MaxVideoBytes = 0, MaxStoredChatResponseBytes = 0 }); + await using var second = await root.CreateSessionAsync(); + await Assert.That((await second.Control.GetAssertionsAsync()).TotalEvents).IsEqualTo(0); + } + + [Test] + public async Task InvalidSessionHeaders_DoNotExecuteOrJournalDefaultProviderRequestAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(config => config.AddChatScenario("default", scenario => scenario.Responds("default preserved"))); + using var http = host.GetTestClient(); + using var invalid = new HttpRequestMessage(HttpMethod.Get, "/openai/v1/models"); + invalid.Headers.Add(LlmTestServerSessionHeaders.AccessKey, "key-without-session-id"); + using var rejected = await http.SendAsync(invalid); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.NotFound); + await Assert.That(await rejected.Content.ReadAsStringAsync()).Contains("session_not_found"); + await Assert.That(await http.GetStringAsync("/admin-api/requests")).IsEqualTo("[]"); + using var chat = new LlmTestServerClient(http).CreateChatClient(); + await Assert.That((await chat.GetResponseAsync("hello")).Text).IsEqualTo("default preserved"); + } + + [Test] + public async Task DefaultConfigure_DoesNotChangeSessionStartupTemplateOrExistingQueuesAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(config => config.AddChatScenario("startup", scenario => scenario.Responds("startup response"))); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + await using var first = await root.CreateSessionAsync(); + await root.ConfigureAsync(config => config.UseChatScenario("changed", scenario => scenario.Responds("default changed"))); + await using var second = await root.CreateSessionAsync(); + using var a = first.Control.CreateChatClient(); + using var b = second.Control.CreateChatClient(); + using var defaultChat = root.CreateChatClient(); + await Assert.That((await a.GetResponseAsync("hello")).Text).IsEqualTo("startup response"); + await Assert.That((await b.GetResponseAsync("hello")).Text).IsEqualTo("startup response"); + await Assert.That((await defaultChat.GetResponseAsync("hello")).Text).IsEqualTo("default changed"); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Sessions/SessionIsolationIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Sessions/SessionIsolationIntegrationTests.cs new file mode 100644 index 0000000..23cf29f --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Sessions/SessionIsolationIntegrationTests.cs @@ -0,0 +1,135 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Configuration; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Hosting; +using ManagedCode.LlmTestServer.Sessions; +using ManagedCode.LlmTestServer.Tests.TestSupport; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; + +namespace ManagedCode.LlmTestServer.Tests.Sessions; + +public sealed class SessionIsolationIntegrationTests +{ + [Test] + public async Task ParallelSessions_IsolateIdenticalPromptsQueuesJournalsAndResetAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(config => config.RequireBearerToken("shared-key")); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http, "shared-key"); + await using var first = await root.CreateSessionAsync(config => config.RequireBearerToken("shared-key") + .UseChatScenario("same", scenario => scenario.Responds("A1").Responds("A2"))); + await using var second = await root.CreateSessionAsync(config => config.RequireBearerToken("shared-key") + .UseChatScenario("same", scenario => scenario.Responds("B1").Responds("B2"))); + using var a = first.Control.CreateChatClient(); + using var b = second.Control.CreateChatClient(); + var replies = await Task.WhenAll(a.GetResponseAsync("same prompt"), b.GetResponseAsync("same prompt")); + await Assert.That(replies[0].Text).IsEqualTo("A1"); + await Assert.That(replies[1].Text).IsEqualTo("B1"); + await first.Control.ResetAsync(); + await Assert.That((await a.GetResponseAsync("same prompt")).Text).IsEqualTo("A1"); + await Assert.That((await b.GetResponseAsync("same prompt")).Text).IsEqualTo("B2"); + var firstSummary = await first.Control.GetAssertionsAsync(); + var secondSummary = await second.Control.GetAssertionsAsync(); + await Assert.That(firstSummary.Matched).IsEqualTo(1); + await Assert.That(secondSummary.Matched).IsEqualTo(2); + await Assert.That((await root.GetAssertionsAsync()).TotalEvents).IsEqualTo(0); + foreach (var session in new[] { first, second }) + { + session.HttpClient.DefaultRequestHeaders.Authorization = new("Bearer", "shared-key"); + var journal = (await session.HttpClient.GetFromJsonAsync("/admin-api/requests"))!; + await Assert.That(journal.All(entry => entry.SessionId == session.Session.Id)).IsTrue(); + await Assert.That(journal.Length).IsEqualTo(ReferenceEquals(session, first) ? 1 : 2); + await Assert.That(JsonSerializer.Serialize(journal)).DoesNotContain(session.Session.AccessKey); + } + await Assert.That(http.DefaultRequestHeaders.Contains(LlmTestServerSessionHeaders.Id)).IsFalse(); + await root.ResetAsync(); + await Assert.That((await second.Control.GetAssertionsAsync()).Matched).IsEqualTo(2); + } + + [Test] + public async Task SessionCredentials_RejectUnknownMissingForeignAndDuplicateHeadersAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(config => config.RequireBearerToken("root-key")); + using var http = host.GetTestClient(); + using var unauthorized = await http.PostAsJsonAsync("/admin-api/sessions", new { }); + await Assert.That(unauthorized.StatusCode).IsEqualTo(HttpStatusCode.Unauthorized); + var root = new LlmTestServerClient(http, "root-key"); + await using var first = await root.CreateSessionAsync(config => config.RequireBearerToken("provider-key")); + await using var second = await root.CreateSessionAsync(); + foreach (var pair in new[] { (first.Session.Id, "wrong"), ("missing", first.Session.AccessKey), (first.Session.Id, second.Session.AccessKey), (first.Session.Id, "") }) + { + using var request = new HttpRequestMessage(HttpMethod.Get, "/admin-api/models"); + request.Headers.Authorization = new("Bearer", "root-key"); + request.Headers.Add(LlmTestServerSessionHeaders.Id, pair.Item1); + if (pair.Item2.Length > 0) { request.Headers.Add(LlmTestServerSessionHeaders.AccessKey, pair.Item2); } + using var response = await http.SendAsync(request); + await Assert.That(response.StatusCode).IsEqualTo(HttpStatusCode.NotFound); + } + using var duplicate = new HttpRequestMessage(HttpMethod.Get, "/admin-api/models"); + duplicate.Headers.Add(LlmTestServerSessionHeaders.Id, new[] { first.Session.Id, second.Session.Id }); + duplicate.Headers.Add(LlmTestServerSessionHeaders.AccessKey, first.Session.AccessKey); + using var duplicated = await http.SendAsync(duplicate); + await Assert.That(duplicated.StatusCode).IsEqualTo(HttpStatusCode.NotFound); + using var foreignDelete = await first.HttpClient.DeleteAsync("/admin-api/sessions/" + second.Session.Id); + await Assert.That(foreignDelete.StatusCode).IsEqualTo(HttpStatusCode.NotFound); + await Assert.That((await first.Control.GetAssertionsAsync()).TotalEvents).IsEqualTo(0); + await Assert.That((await second.Control.GetAssertionsAsync()).TotalEvents).IsEqualTo(0); + await first.Control.ConfigureAsync(config => config.RequireBearerToken("changed-provider-key").UseGpt41Mini()); + await Assert.That((await first.Control.WithBearerToken("changed-provider-key").GetAssertionsAsync()).TotalEvents).IsEqualTo(0); + } + + [Test] + public async Task SessionDeletion_ReleasesCountBudgetAndLeavesSharedTransportAliveAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(configureServices: services => services.Configure(options => options.MaxSessions = 1)); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + var first = await root.CreateSessionAsync(); + using var full = await http.PostAsJsonAsync("/admin-api/sessions", new { }); + await Assert.That(full.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + await Assert.That(await full.Content.ReadAsStringAsync()).Contains("session_capacity_exceeded"); + var id = first.Session.Id; + var key = first.Session.AccessKey; + await first.DisposeAsync(); + await first.DisposeAsync(); + using var stale = new HttpRequestMessage(HttpMethod.Get, "/admin-api/models"); + stale.Headers.Add(LlmTestServerSessionHeaders.Id, id); + stale.Headers.Add(LlmTestServerSessionHeaders.AccessKey, key); + using var removed = await http.SendAsync(stale); + await Assert.That(removed.StatusCode).IsEqualTo(HttpStatusCode.NotFound); + await using var replacement = await root.CreateSessionAsync(); + await Assert.That(replacement.Session.Id).IsNotEqualTo(id); + await Assert.That((await root.GetAssertionsAsync()).TotalEvents).IsEqualTo(0); + } + + [Test] + public async Task SessionBudget_RejectsAdmissionAndConfigureOverflowWithoutChangingStateAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(configureServices: services => services.Configure(options => options.MaxReservedStateBytes = 60L * 1024 * 1024)); + using var http = host.GetTestClient(); + using var oversized = await http.PostAsJsonAsync("/admin-api/sessions", new { }); + await Assert.That(oversized.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + var configuration = LlmTestServerConfiguration.CreateDefault() with { MaxVideoBytes = 0, MaxStoredChatResponseBytes = 0 }; + using var created = await http.PostAsJsonAsync("/admin-api/sessions", new LlmTestServerSessionCreateRequest { Configuration = configuration }); + created.EnsureSuccessStatusCode(); + var session = (await created.Content.ReadFromJsonAsync())!; + await Assert.That(created.Headers.CacheControl!.NoStore).IsTrue(); + http.DefaultRequestHeaders.Add(LlmTestServerSessionHeaders.Id, session.Id); + http.DefaultRequestHeaders.Add(LlmTestServerSessionHeaders.AccessKey, session.AccessKey); + using var tooLarge = await http.PostAsJsonAsync("/admin-api/configure", LlmTestServerConfiguration.CreateDefault()); + await Assert.That(tooLarge.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + using var invalid = await http.PostAsJsonAsync("/admin-api/configure", configuration with { MaxVideoBytes = -1 }); + await Assert.That(invalid.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + using var invalidOverBudget = await http.PostAsJsonAsync("/admin-api/configure", configuration with + { Models = null!, MaxVideoBytes = 1024L * 1024 * 1024 }); + await Assert.That(invalidOverBudget.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + using var stillActive = await http.PostAsJsonAsync("/openai/v1/embeddings", new { model = "text-embedding-3-small", input = "active" }); + stillActive.EnsureSuccessStatusCode(); + using var deleted = await http.DeleteAsync("/admin-api/sessions/" + session.Id); + await Assert.That(deleted.StatusCode).IsEqualTo(HttpStatusCode.NoContent); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Sessions/SessionProviderIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Sessions/SessionProviderIntegrationTests.cs new file mode 100644 index 0000000..7c4933c --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Sessions/SessionProviderIntegrationTests.cs @@ -0,0 +1,140 @@ +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Decisions; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Hosting; +using ManagedCode.LlmTestServer.Tests.Decisions; +using ManagedCode.LlmTestServer.Tests.Providers; +using ManagedCode.LlmTestServer.Tests.TestSupport; +using Microsoft.Extensions.DependencyInjection; +using OpenAI; +using OpenAI.Chat; + +namespace ManagedCode.LlmTestServer.Tests.Sessions; + +public sealed class SessionProviderIntegrationTests +{ + [Test] + [Arguments("jev-latest", LlmTestServerDecisionProvider.TypeSafe)] + [Arguments("kev-latest", LlmTestServerDecisionProvider.Kev)] + [Arguments("clef", LlmTestServerDecisionProvider.Cloudflare)] + [Arguments("clef-flash", LlmTestServerDecisionProvider.Cloudflare)] + [Arguments("gpt-6-luna", LlmTestServerDecisionProvider.OpenAI)] + public async Task NativeDecisionProviders_KeepSessionFixturesAndErrorsSeparateAsync(string model, LlmTestServerDecisionProvider provider) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + await using var first = await root.CreateSessionAsync(config => config.UseDecisionModel(model, provider) + .UseDecisionScenario(DecisionEndpointTests.Scenario(model))); + await using var second = await root.CreateSessionAsync(config => config.UseDecisionModel(model, provider) + .UseDecisionScenario(DecisionEndpointTests.Scenario(model) with { Error = new() { StatusCode = 503, Code = "capacity", Message = "session capacity fixture" } })); + using var success = await first.HttpClient.PostAsJsonAsync(DecisionEndpointTests.PathFor(model), DecisionEndpointTests.Body(model)); + success.EnsureSuccessStatusCode(); + using var failure = await second.HttpClient.PostAsJsonAsync(DecisionEndpointTests.PathFor(model), DecisionEndpointTests.Body(model)); + await Assert.That(failure.StatusCode).IsEqualTo(HttpStatusCode.ServiceUnavailable); + await Assert.That((await first.Control.GetAssertionsAsync()).Matched).IsEqualTo(1); + await Assert.That((await second.Control.GetAssertionsAsync()).ErrorsReturned).IsEqualTo(1); + await Assert.That((await root.GetAssertionsAsync()).TotalEvents).IsEqualTo(0); + } + + [Test] + public async Task EveryChatProvider_UsesItsSelectedSessionFaultBudgetAndQueueAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + await using var limited = await root.CreateSessionAsync(config => config.SimulateRateLimitAfter(0)); + await using var active = await root.CreateSessionAsync(config => config.UseChatScenario("providers", scenario => + { + foreach (var provider in ProviderFaultSimulationTests.GetProviderFaultCases()) { scenario.Responds("independent response"); } + })); + foreach (var provider in ProviderFaultSimulationTests.GetProviderFaultCases()) + { + using var rejected = await provider.SendAsync(limited.HttpClient, "same prompt"); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.TooManyRequests); + using var accepted = await provider.SendAsync(active.HttpClient, "same prompt"); + accepted.EnsureSuccessStatusCode(); + await Assert.That(await accepted.Content.ReadAsStringAsync()).Contains("independent response"); + } + await Assert.That((await limited.Control.GetAssertionsAsync()).ErrorsReturned).IsEqualTo(13); + await Assert.That((await active.Control.GetAssertionsAsync()).Matched).IsEqualTo(13); + await Assert.That((await root.GetAssertionsAsync()).TotalEvents).IsEqualTo(0); + } + + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task OfficialOpenAiSdk_UsesSessionTransportAndIndependentProviderCredentialAsync(bool stream) + { + using var host = await LlmTestServerTestHost.StartAsync(config => config.RequireBearerToken("management-key")); + using var http = host.GetTestClient(); + await using var session = await new LlmTestServerClient(http, "management-key").CreateSessionAsync(config => config + .RequireBearerToken("provider-key").UseChatScenario("sdk", scenario => scenario.Responds("SDK session reply", "SDK ", "session reply"))); + var sdk = new ChatClient("gpt-4.1-mini", new ApiKeyCredential("provider-key"), new OpenAIClientOptions + { + Endpoint = new Uri(session.HttpClient.BaseAddress!, "openai/v1"), + Transport = new HttpClientPipelineTransport(session.HttpClient), + }); + var result = string.Empty; + if (stream) + { + await foreach (var chunk in sdk.CompleteChatStreamingAsync([new UserChatMessage("same prompt")])) + { result += string.Concat(chunk.ContentUpdate.Select(part => part.Text)); } + } + else + { result = (await sdk.CompleteChatAsync([new UserChatMessage("same prompt")])).Value.Content[0].Text; } + await Assert.That(result).IsEqualTo("SDK session reply"); + await Assert.That((await session.Control.GetAssertionsAsync()).Matched).IsEqualTo(1); + } + + [Test] + public async Task PromptCaches_AreIndependentAcrossSessionsAndLocalResetAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + await using var first = await root.CreateSessionAsync(config => config.UseChatScenario("cache", scenario => scenario.Responds("reply").Responds("reply").Responds("reply"))); + await using var second = await root.CreateSessionAsync(config => config.UseChatScenario("cache", scenario => scenario.Responds("reply").Responds("reply"))); + var prompt = string.Concat(Enumerable.Repeat("A stable sequence of words for provider prompt cache. ", 250)); + async Task CachedAsync(LlmTestServerSessionClient session) + { + using var response = await session.HttpClient.PostAsJsonAsync("/openai/v1/chat/completions", new { model = "gpt-4.1-mini", messages = new[] { new { role = "user", content = prompt } } }); + response.EnsureSuccessStatusCode(); + var json = await response.Content.ReadFromJsonAsync(); + return json.GetProperty("usage").GetProperty("prompt_tokens_details").GetProperty("cached_tokens").GetInt32(); + } + await Assert.That(await CachedAsync(first)).IsEqualTo(0); + await Assert.That(await CachedAsync(first)).IsGreaterThan(0); + await Assert.That(await CachedAsync(second)).IsEqualTo(0); + await first.Control.ResetAsync(); + await Assert.That(await CachedAsync(first)).IsEqualTo(0); + await Assert.That(await CachedAsync(second)).IsGreaterThan(0); + } + + [Test] + public async Task RequestJournalRetention_IsBoundedIndependentlyInEachSessionAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(configureServices: services => services.Configure(options => options.MaxEntries = 1)); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + await using var first = await root.CreateSessionAsync(); + await using var second = await root.CreateSessionAsync(); + foreach (var session in new[] { first, second }) + { + for (var index = 0; index < 2; index++) + { + using var response = await session.HttpClient.PostAsJsonAsync("/openai/v1/embeddings", new { model = "text-embedding-3-small", input = "bounded" }); + response.EnsureSuccessStatusCode(); + } + var journal = (await session.HttpClient.GetFromJsonAsync("/admin-api/requests"))!; + await Assert.That(journal.Length).IsEqualTo(1); + await Assert.That(journal[0].SessionId).IsEqualTo(session.Session.Id); + await Assert.That((await session.Control.GetAssertionsAsync()).Matched).IsEqualTo(2); + } + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Sessions/SessionStateIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Sessions/SessionStateIntegrationTests.cs new file mode 100644 index 0000000..6e973df --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Sessions/SessionStateIntegrationTests.cs @@ -0,0 +1,170 @@ +using System.Diagnostics; +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Sessions; +using ManagedCode.LlmTestServer.Tests.TestSupport; +using Microsoft.Extensions.AI; + +namespace ManagedCode.LlmTestServer.Tests.Sessions; + +public sealed class SessionStateIntegrationTests +{ + [Test] + public async Task CancellingOneSessionRequest_DoesNotConsumeItsQueueOrStopAnotherSessionAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + await using var first = await root.CreateSessionAsync(config => config.UseChatScenario("cancel", scenario => scenario.Responds("A preserved").DelaysBy(500))); + await using var second = await root.CreateSessionAsync(config => config.UseChatScenario("active", scenario => scenario.Responds("B active"))); + using var cancellation = new CancellationTokenSource(TimeSpan.FromMilliseconds(100)); + var cancelled = false; + try + { + using var response = await first.HttpClient.PostAsJsonAsync("/openai/v1/chat/completions", new + { model = "gpt-4.1-mini", messages = new[] { new { role = "user", content = "hello" } } }, cancellation.Token); + } + catch (OperationCanceledException) when (cancellation.IsCancellationRequested) { cancelled = true; } + await Assert.That(cancelled).IsTrue(); + using var b = second.Control.CreateChatClient(); + await Assert.That((await b.GetResponseAsync("hello")).Text).IsEqualTo("B active"); + using var a = first.Control.CreateChatClient(); + await Assert.That((await a.GetResponseAsync("hello")).Text).IsEqualTo("A preserved"); + await Assert.That((await first.Control.GetAssertionsAsync()).Matched).IsEqualTo(1); + await Assert.That((await second.Control.GetAssertionsAsync()).Matched).IsEqualTo(1); + } + + [Test] + public async Task SessionClients_StreamIndependentResponsesAndKeepDefaultHostUnchangedAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + await using var first = await root.CreateSessionAsync(config => config.UseChatScenario("stream", scenario => scenario.Responds("alpha", "al", "pha"))); + await using var second = await root.CreateSessionAsync(config => config.UseChatScenario("stream", scenario => scenario.Responds("beta", "be", "ta"))); + async Task ReadAsync(LlmTestServerSessionClient session) + { + using var chat = session.Control.CreateChatClient(); + var text = string.Empty; + await foreach (var chunk in chat.GetStreamingResponseAsync("identical")) { text += chunk.Text; } + return text; + } + var replies = await Task.WhenAll(ReadAsync(first), ReadAsync(second)); + await Assert.That(replies).IsEquivalentTo(new[] { "alpha", "beta" }); + await Assert.That((await first.Control.GetAssertionsAsync()).Matched).IsEqualTo(1); + await Assert.That((await second.Control.GetAssertionsAsync()).Matched).IsEqualTo(1); + await Assert.That((await root.GetAssertionsAsync()).Matched).IsEqualTo(0); + } + + [Test] + public async Task Sessions_IsolateStoredResponsesEvenWithIdenticalProviderCredentialsAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + await using var first = await root.CreateSessionAsync(config => config.UseChatScenario("history", scenario => scenario.Responds("first").Responds("continued"))); + await using var second = await root.CreateSessionAsync(config => config.UseChatScenario("history", scenario => scenario.Responds("second"))); + using var created = await first.HttpClient.PostAsJsonAsync("/openai/v1/responses", new { model = "gpt-4.1-mini", input = "hello" }); + created.EnsureSuccessStatusCode(); + var id = (await created.Content.ReadFromJsonAsync()).GetProperty("id").GetString(); + using var foreign = await second.HttpClient.PostAsJsonAsync("/openai/v1/responses", new { model = "gpt-4.1-mini", input = "continue", previous_response_id = id }); + await Assert.That(foreign.StatusCode).IsEqualTo(HttpStatusCode.NotFound); + await Assert.That(await foreign.Content.ReadAsStringAsync()).Contains("response_not_found"); + using var continued = await first.HttpClient.PostAsJsonAsync("/openai/v1/responses", new { model = "gpt-4.1-mini", input = "continue", previous_response_id = id }); + continued.EnsureSuccessStatusCode(); + await Assert.That(await continued.Content.ReadAsStringAsync()).Contains("continued"); + using var untouched = await second.HttpClient.PostAsJsonAsync("/openai/v1/responses", new { model = "gpt-4.1-mini", input = "hello" }); + untouched.EnsureSuccessStatusCode(); + await Assert.That(await untouched.Content.ReadAsStringAsync()).Contains("second"); + } + + [Test] + public async Task Sessions_IsolateVideoModelsFixturesAndFaultBudgetsAcrossConfigureAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + await using var first = await root.CreateSessionAsync(config => config.UseVideo([1, 2, 3], "video/mp4").UseEmbeddingVector(1, 2) + .UseTranscriptionText("A transcript").UseTranslationText("A translation")); + await using var second = await root.CreateSessionAsync(config => config.UseVideo([4, 5, 6], "video/mp4")); + await Assert.That(await first.Control.TranscribeAudioAsync("gpt-4o-mini-tts", [82, 73, 70, 70], "fixture.wav")).IsEqualTo("A transcript"); + await Assert.That(await second.Control.TranscribeAudioAsync("gpt-4o-mini-tts", [82, 73, 70, 70], "fixture.wav")).IsEqualTo("transcribed audio fixture"); + await Assert.That(await first.Control.TranslateAudioAsync("gpt-4o-mini-tts", [82, 73, 70, 70], "fixture.wav")).IsEqualTo("A translation"); + using var embeddings = first.Control.CreateEmbeddingGenerator(); + var vectors = await embeddings.GenerateAsync(["same input"]); + await Assert.That(vectors[0].Vector.ToArray()).IsEquivalentTo(new float[] { 1, 2 }); + using var created = await first.HttpClient.PostAsJsonAsync("/openai/v1/videos", new { model = "sora-2", prompt = "A video" }); + created.EnsureSuccessStatusCode(); + var id = (await created.Content.ReadFromJsonAsync()).GetProperty("id").GetString(); + using var absent = await second.HttpClient.GetAsync("/openai/v1/videos/" + id); + await Assert.That(absent.StatusCode).IsEqualTo(HttpStatusCode.NotFound); + using var another = await second.HttpClient.PostAsJsonAsync("/openai/v1/videos", new { model = "sora-2", prompt = "B video" }); + another.EnsureSuccessStatusCode(); + var otherId = (await another.Content.ReadFromJsonAsync()).GetProperty("id").GetString(); + using var contentA = await first.HttpClient.GetAsync("/openai/v1/videos/" + id + "/content"); + using var contentB = await second.HttpClient.GetAsync("/openai/v1/videos/" + otherId + "/content"); + await Assert.That(await contentA.Content.ReadAsByteArrayAsync()).IsEquivalentTo((byte[])[1, 2, 3]); + await Assert.That(await contentB.Content.ReadAsByteArrayAsync()).IsEquivalentTo((byte[])[4, 5, 6]); + await first.Control.ConfigureAsync(config => config.UseChatModel("session-only-model").SimulateRateLimitAfter(0)); + using var throttled = await first.HttpClient.PostAsJsonAsync("/openai/v1/embeddings", new { model = "text-embedding-3-small", input = "hello" }); + await Assert.That(throttled.StatusCode).IsEqualTo(HttpStatusCode.TooManyRequests); + using var available = await second.HttpClient.PostAsJsonAsync("/openai/v1/embeddings", new { model = "text-embedding-3-small", input = "hello" }); + available.EnsureSuccessStatusCode(); + using var cleared = await first.HttpClient.GetAsync("/openai/v1/videos/" + id); + await Assert.That(cleared.StatusCode).IsEqualTo(HttpStatusCode.NotFound); + using var retained = await second.HttpClient.GetAsync("/openai/v1/videos/" + otherId); + retained.EnsureSuccessStatusCode(); + var modelsA = await first.HttpClient.GetStringAsync("/admin-api/models"); + var modelsB = await second.HttpClient.GetStringAsync("/admin-api/models"); + await Assert.That(modelsA).Contains("session-only-model"); + await Assert.That(modelsB).DoesNotContain("session-only-model"); + } + + [Test] + [Arguments(0)] + [Arguments(1)] + [Arguments(2)] + public async Task SessionMutation_InvalidatesInFlightDelayedResponseWithoutAffectingOtherSessionsAsync(int mutation) + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + await using var session = await root.CreateSessionAsync(config => config.UseChatScenario("delayed", scenario => scenario.Responds("retired response").DelaysBy(1000))); + var pending = session.HttpClient.PostAsJsonAsync("/openai/v1/chat/completions", new { model = "gpt-4.1-mini", messages = new[] { new { role = "user", content = "hello" } } }); + var stopwatch = Stopwatch.StartNew(); + while (true) + { + var entries = (await session.HttpClient.GetFromJsonAsync("/admin-api/requests"))!; + if (entries.Any(entry => !entry.IsCompleted)) { break; } + if (stopwatch.Elapsed > TimeSpan.FromSeconds(5)) { throw new TimeoutException("The server did not observe the delayed request."); } + await Task.Delay(10); + } + if (mutation == 0) + { + using var deletion = new HttpRequestMessage(HttpMethod.Delete, "/admin-api/sessions/" + session.Session.Id); + deletion.Headers.Add(LlmTestServerSessionHeaders.Id, session.Session.Id); + deletion.Headers.Add(LlmTestServerSessionHeaders.AccessKey, session.Session.AccessKey); + using var deleted = await http.SendAsync(deletion); + await Assert.That(deleted.StatusCode).IsEqualTo(HttpStatusCode.NoContent); + } + else if (mutation == 1) + { + await session.Control.ConfigureAsync(config => config.UseChatScenario("replacement", scenario => scenario.Responds("reconfigured"))); + } + else { await session.Control.ResetAsync(); } + using var superseded = await pending; + await Assert.That(superseded.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + await Assert.That(await superseded.Content.ReadAsStringAsync()).Contains("llm_test_server_request_superseded"); + if (mutation != 0) + { + await Assert.That((await session.Control.GetAssertionsAsync()).Matched).IsEqualTo(0); + using var chat = session.Control.CreateChatClient(); + await Assert.That((await chat.GetResponseAsync("hello")).Text).IsEqualTo(mutation == 1 ? "reconfigured" : "retired response"); + } + await using var replacement = await root.CreateSessionAsync(); + await Assert.That((await replacement.Control.GetAssertionsAsync()).TotalEvents).IsEqualTo(0); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Streaming/StreamDeliveryAdmissionIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Streaming/StreamDeliveryAdmissionIntegrationTests.cs new file mode 100644 index 0000000..6d1760c --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Streaming/StreamDeliveryAdmissionIntegrationTests.cs @@ -0,0 +1,150 @@ +using System.ClientModel; +using System.Net; +using System.Net.Http.Json; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Configuration; +using ManagedCode.LlmTestServer.Control; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Scenarios; +using OpenAI.Chat; + +namespace ManagedCode.LlmTestServer.Tests.Streaming; + +public sealed partial class StreamDeliveryIntegrationTests +{ + [Test] + public async Task OfficialSdk_GraphReplayRepeatsBrokenDeliveryWithoutAdvancingAndSessionsRemainIsolatedAsync() + { + await using var app = await StartTcpHostAsync(_ => { }); + using var http = TcpClient(app); + var control = new LlmTestServerClient(http); + static void Configure(LlmTestServerClientConfigurationBuilder config) + { + config.UseConversation("stream-graph", graph => graph + .Step("broken", step => step.WhenLatestUserEquals("go").Responds("first second", "first ", "second") + .WithStreamDelivery(new() { Mode = LlmTestServerStreamDeliveryMode.EarlyCleanEnd, AfterContentFrames = 1 }).Then("recovery")) + .Step("recovery", step => step.WhenLatestUserEquals("next").Responds("graph recovered").Complete())); + } + + await using var firstSession = await control.CreateSessionAsync(Configure); + await using var secondSession = await control.CreateSessionAsync(Configure); + using var firstConversation = firstSession.Control.ForConversation("same-label"); + using var secondConversation = secondSession.Control.ForConversation("same-label"); + using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + var first = await ReadSdkAsync(Sdk(firstConversation.HttpClient), [new UserChatMessage("go")], null, deadline.Token); + await Assert.That(first.Text).IsEqualTo("first "); + await Assert.That(first.Finish).IsNull(); + await Assert.That((await secondSession.Control.GetConversationsAsync()).Count).IsEqualTo(0); + var replay = await ReadSdkAsync(Sdk(firstConversation.HttpClient), [new UserChatMessage("go")], null, deadline.Token); + await Assert.That(replay.Text).IsEqualTo(first.Text); + await Assert.That(replay.Finish).IsNull(); + var firstState = (await firstSession.Control.GetConversationsAsync()).Single(); + await Assert.That(firstState.ExecutedSteps).IsEqualTo(1); + await Assert.That(firstState.NextSteps.Single()).IsEqualTo("recovery"); + var firstEvidence = await firstSession.Control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending }); + await Assert.That(firstEvidence.Requests.Count).IsEqualTo(2); + await Assert.That(firstEvidence.Requests[0].IsReplay).IsFalse(); + await Assert.That(firstEvidence.Requests[1].IsReplay).IsTrue(); + await Assert.That(firstEvidence.Requests.All(trace => trace.StreamDeliveryApplied && trace.EmittedContentFrames == 1 && trace.TerminalMarkerObserved != true)).IsTrue(); + var second = await ReadSdkAsync(Sdk(secondConversation.HttpClient), [new UserChatMessage("go")], null, deadline.Token); + await Assert.That(second.Text).IsEqualTo("first "); + await Assert.That((await secondSession.Control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); + var recovery = (await Sdk(firstConversation.HttpClient).CompleteChatAsync([ + new UserChatMessage("go"), new AssistantChatMessage("first second"), new UserChatMessage("next")])).Value; + await Assert.That(recovery.Content.Single().Text).IsEqualTo("graph recovered"); + await Assert.That((await firstSession.Control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(2); + await Assert.That((await firstSession.Control.GetConversationsAsync()).Single().IsTerminal).IsTrue(); + await Assert.That((await secondSession.Control.GetConversationsAsync()).Single().IsTerminal).IsFalse(); + await Assert.That((await secondSession.Control.GetRequestsAsync()).Requests.Count).IsEqualTo(1); + } + + [Test] + [Arguments("json")] + [Arguments("provider")] + [Arguments("tools")] + [Arguments("schema")] + public async Task OfficialSdk_UnsupportedRequestRejectsDeliveryPlanBeforeConsumingFixtureAsync(string kind) + { + await using var app = await StartTcpHostAsync(config => config.AddChatScenario("delivery-admission", scenario => scenario + .Responds("first second", "first ", "second").WithStreamDelivery(new() + { Mode = LlmTestServerStreamDeliveryMode.EarlyCleanEnd, AfterContentFrames = 1 }).Responds("after actual delivery"))); + using var http = TcpClient(app); + var control = new LlmTestServerClient(http); + var sdk = Sdk(http, kind == "provider" ? "/groq/openai/v1" : "/openai/v1"); + var options = new ChatCompletionOptions(); + if (kind == "tools") { options.Tools.Add(ChatTool.CreateFunctionTool("weather")); } + if (kind == "schema") { options.ResponseFormat = ChatResponseFormat.CreateJsonObjectFormat(); } + ClientResultException? failure = null; + try + { + if (kind == "json") { await sdk.CompleteChatAsync([new UserChatMessage("go")], options); } + else { await foreach (var update in sdk.CompleteChatStreamingAsync([new UserChatMessage("go")], options)) { _ = update; } } + } + catch (ClientResultException exception) { failure = exception; } + await Assert.That(failure?.Status).IsEqualTo(409); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(0); + var rejected = (await control.GetRequestsAsync()).Requests.Single(); + await Assert.That(rejected.StatusCode).IsEqualTo(409); + await Assert.That(rejected.StreamDeliveryApplied).IsFalse(); + await Assert.That(rejected.EmittedContentFrames).IsEqualTo(0); + using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + var actual = await ReadSdkAsync(Sdk(http), [new UserChatMessage("go")], null, deadline.Token); + await Assert.That(actual.Text).IsEqualTo("first "); + await Assert.That(actual.Finish).IsNull(); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(1); + var recovery = (await Sdk(http).CompleteChatAsync([new UserChatMessage("next")])).Value; + await Assert.That(recovery.Content.Single().Text).IsEqualTo("after actual delivery"); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(2); + } + + [Test] + [Arguments(0)] + [Arguments(1)] + [Arguments(2)] + [Arguments(3)] + [Arguments(4)] + [Arguments(5)] + [Arguments(6)] + [Arguments(7)] + [Arguments(8)] + [Arguments(9)] + public async Task Configure_InvalidDeliveryPlanIsAtomicAndLeavesPreviousFixtureUsableAsync(int kind) + { + await using var app = await StartTcpHostAsync(config => config.AddChatScenario("preserved", scenario => scenario.Responds("still available"))); + using var http = TcpClient(app); + var control = new LlmTestServerClient(http); + var delivery = new LlmTestServerStreamDelivery + { Mode = LlmTestServerStreamDeliveryMode.EarlyCleanEnd, AfterContentFrames = 1 }; + var response = new LlmTestServerScenarioResponse { Content = "first second", StreamChunks = ["first ", "second"], StreamDelivery = delivery }; + switch (kind) + { + case 0: delivery = delivery with { Mode = (LlmTestServerStreamDeliveryMode)0 }; break; + case 1: delivery = delivery with { AfterContentFrames = 0 }; break; + case 2: delivery = delivery with { AfterContentFrames = 3 }; break; + case 3: delivery = delivery with { Mode = LlmTestServerStreamDeliveryMode.DelayThenContinue }; break; + case 4: delivery = delivery with { DelayMilliseconds = 1 }; break; + case 5: delivery = delivery with { Mode = LlmTestServerStreamDeliveryMode.DelayThenContinue, DelayMilliseconds = 60001 }; break; + case 6: response = response with { StreamChunks = ["different content"] }; break; + case 7: response = response with { ToolCalls = [new() { Id = "tool", Name = "weather", ArgumentsJson = "{}" }] }; break; + case 8: response = response with { Error = new() { StatusCode = 500, Code = "scripted", Message = "failure" } }; break; + default: + response = response with + { + Content = string.Empty, + StreamChunks = [], + OutputItems = [new LlmTestServerResponseWebSearchItem + { Id = "native", Queries = ["fixture query"] }] + }; + break; + } + response = response with { StreamDelivery = delivery }; + var validBase = new LlmTestServerConfigurationBuilder().Build(); + using var rejected = await http.PostAsJsonAsync(LlmTestServerControlRoutes.Configure, + validBase with { ChatScenarios = [new() { Id = "invalid", Responses = [response] }] }); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(0); + var preserved = (await Sdk(http).CompleteChatAsync([new UserChatMessage("unchanged")])).Value; + await Assert.That(preserved.Content.Single().Text).IsEqualTo("still available"); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(1); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Streaming/StreamDeliveryAnthropicIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Streaming/StreamDeliveryAnthropicIntegrationTests.cs new file mode 100644 index 0000000..de92f6a --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Streaming/StreamDeliveryAnthropicIntegrationTests.cs @@ -0,0 +1,125 @@ +using System.Text; +using System.Text.Json; +using global::Anthropic; +using global::Anthropic.Core; +using global::Anthropic.Models.Messages; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Scenarios; + +namespace ManagedCode.LlmTestServer.Tests.Streaming; + +public sealed partial class StreamDeliveryIntegrationTests +{ + [Test] + [Arguments(LlmTestServerStreamDeliveryMode.EarlyCleanEnd, false, false)] + [Arguments(LlmTestServerStreamDeliveryMode.EarlyCleanEnd, false, true)] + [Arguments(LlmTestServerStreamDeliveryMode.AbortConnection, false, false)] + [Arguments(LlmTestServerStreamDeliveryMode.DelayThenContinue, true, false)] + [Arguments(LlmTestServerStreamDeliveryMode.DelayThenContinue, false, false)] + public async Task OfficialAnthropicSdk_StrictTypedTextEventsExposeDeliveryFaultsAndRecoverAsync(LlmTestServerStreamDeliveryMode mode, bool cancel, bool afterAllText) + { + var model = JsonSerializer.SerializeToElement(Model.ClaudeSonnet5_5).GetString()!; + await AssertProtocolDeliveryAsync(model, "/anthropic/v1/messages", "first ", mode, cancel, + async (http, first, token, keepAlive) => + { + var sdk = AnthropicSdk(http); + keepAlive(sdk); + var text = new StringBuilder(); + var terminal = false; + Exception? failure = null; + try + { + await foreach (var update in sdk.Messages.CreateStreaming(AnthropicTextOptions(), token)) + { + if (update.Value is RawContentBlockDeltaEvent block && block.Delta.Value is TextDelta delta) + { + text.Append(delta.Text); + if (!string.IsNullOrEmpty(delta.Text)) { first(); } + } + if (update.Value is RawMessageStopEvent) { terminal = true; } + } + } + catch (Exception exception) when (ProtocolStreamException(exception)) { failure = exception; } + return new(text.ToString(), terminal, failure); + }, + async (http, token, keepAlive) => + { + var sdk = AnthropicSdk(http); + keepAlive(sdk); + return ((TextBlock)(await sdk.Messages.Create(AnthropicTextOptions(), token)).Content.Single().Value!).Text; + }, afterAllText); + } + + [Test] + public async Task OfficialAnthropicSdk_ZeroTokenPrefillWarmsCacheWithoutConsumingDeliveryFixtureAsync() + { + var model = JsonSerializer.SerializeToElement(Model.ClaudeSonnet5_5).GetString()!; + await using var app = await StartTcpHostAsync(config => config.AddModel(model, LlmTestServerModelKind.Chat) + .AddChatScenario("anthropic-prefill", scenario => scenario.ForModel(model) + .Responds("first second", "first ", "second").WithStreamDelivery(new() + { Mode = LlmTestServerStreamDeliveryMode.EarlyCleanEnd }).Responds("prefill recovery"))); + using var http = TcpClient(app); + var control = new LlmTestServerClient(http); + using var sdk = AnthropicSdk(http); + using var watchdog = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + var options = AnthropicTextOptions() with + { + MaxTokens = 0, + CacheControl = new(), + System = string.Join(" ", Enumerable.Repeat("shared prefill reference", 1600)), + }; + var prefillStopped = false; + var prefillText = new StringBuilder(); + long cacheWritten = 0; + await foreach (var update in sdk.Messages.CreateStreaming(options, watchdog.Token)) + { + if (update.Value is RawMessageStartEvent start) { cacheWritten = start.Message.Usage.CacheCreationInputTokens ?? 0; } + if (update.Value is RawContentBlockDeltaEvent block && block.Delta.Value is TextDelta delta) { prefillText.Append(delta.Text); } + if (update.Value is RawMessageStopEvent) { prefillStopped = true; } + } + await Assert.That(prefillStopped).IsTrue(); + await Assert.That(prefillText.ToString()).IsEqualTo(string.Empty); + await Assert.That(cacheWritten).IsGreaterThan(0); + var prefill = await CompletedTraceAsync(control, 1); + await Assert.That(prefill.TerminalMarkerObserved).IsTrue(); + await Assert.That(prefill.StreamDelivery).IsNull(); + await Assert.That(prefill.StreamDeliveryApplied).IsFalse(); + await Assert.That(prefill.EmittedContentFrames).IsEqualTo(0); + await Assert.That(prefill.Usage!.InputTokens).IsGreaterThanOrEqualTo(1600); + await Assert.That((long)prefill.Usage.CacheCreationInputTokens).IsEqualTo(cacheWritten); + await Assert.That((await control.GetConversationsAsync()).Count).IsEqualTo(0); + var generated = new StringBuilder(); + var generationStopped = false; + long cacheRead = 0; + await foreach (var update in sdk.Messages.CreateStreaming(options with { MaxTokens = 256 }, watchdog.Token)) + { + if (update.Value is RawMessageStartEvent start) { cacheRead = start.Message.Usage.CacheReadInputTokens ?? 0; } + if (update.Value is RawContentBlockDeltaEvent block && block.Delta.Value is TextDelta delta) { generated.Append(delta.Text); } + if (update.Value is RawMessageStopEvent) { generationStopped = true; } + } + await Assert.That(generated.ToString()).IsEqualTo("first "); + await Assert.That(generationStopped).IsFalse(); + await Assert.That(cacheRead).IsGreaterThan(0); + var fault = await CompletedTraceAsync(control, 2); + await Assert.That(fault.StreamDeliveryApplied).IsTrue(); + await Assert.That(fault.StreamDelivery!.Mode).IsEqualTo(LlmTestServerStreamDeliveryMode.EarlyCleanEnd); + await Assert.That(fault.EmittedContentFrames).IsEqualTo(1); + await Assert.That(fault.TerminalMarkerObserved == true).IsFalse(); + var recovered = await sdk.Messages.Create(options with { MaxTokens = 256 }, watchdog.Token); + await Assert.That(((TextBlock)recovered.Content.Single().Value!).Text).IsEqualTo("prefill recovery"); + await Assert.That((await control.GetRequestsAsync()).TotalObserved).IsEqualTo(3); + } + + private static AnthropicClient AnthropicSdk(HttpClient http) + { + return new(new ClientOptions + { HttpClient = http, BaseUrl = new Uri(http.BaseAddress!, "/anthropic").ToString(), ApiKey = "anthropic-delivery-key", MaxRetries = 0, ResponseValidation = true }); + } + + private static MessageCreateParams AnthropicTextOptions() + { + return new() + { Model = Model.ClaudeSonnet5_5, MaxTokens = 256, Messages = [new() { Role = Role.User, Content = "stream" }] }; + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Streaming/StreamDeliveryCacheIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Streaming/StreamDeliveryCacheIntegrationTests.cs new file mode 100644 index 0000000..8547c84 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Streaming/StreamDeliveryCacheIntegrationTests.cs @@ -0,0 +1,38 @@ +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Scenarios; +using OpenAI.Chat; + +namespace ManagedCode.LlmTestServer.Tests.Streaming; + +public sealed partial class StreamDeliveryIntegrationTests +{ + [Test] + public async Task OfficialSdk_PostHeaderEarlyEndCommitsCacheAndExplicitNextAttemptReportsRealCachedUsageAsync() + { + await using var app = await StartTcpHostAsync(config => config.AddChatScenario("stream-cache", scenario => scenario + .Responds("first second", "first ", "second").WithStreamDelivery(new() + { Mode = LlmTestServerStreamDeliveryMode.EarlyCleanEnd, AfterContentFrames = 1 }).Responds("cached recovery"))); + using var http = TcpClient(app); + var control = new LlmTestServerClient(http); + List messages = [new SystemChatMessage(string.Join(" ", Enumerable.Repeat("shared cache reference", 1600))), new UserChatMessage("same request")]; + using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + var partial = await ReadSdkAsync(Sdk(http), messages, null, deadline.Token); + await Assert.That(partial.Text).IsEqualTo("first "); + await Assert.That(partial.Finish).IsNull(); + await Assert.That(partial.Failure).IsNull(); + var initial = await CompletedTraceAsync(control, 1); + await Assert.That(initial.Usage!.InputTokens).IsGreaterThanOrEqualTo(1600); + await Assert.That(initial.Usage.CachedInputTokens).IsEqualTo(0); + await Assert.That(initial.StreamDeliveryApplied).IsTrue(); + await Assert.That(initial.TerminalMarkerObserved == true).IsFalse(); + var recovered = (await Sdk(http).CompleteChatAsync(messages)).Value; + await Assert.That(recovered.Content.Single().Text).IsEqualTo("cached recovery"); + await Assert.That(recovered.Usage.InputTokenDetails.CachedTokenCount).IsGreaterThan(0); + var evidence = await control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending }); + await Assert.That(evidence.Requests.Count).IsEqualTo(2); + await Assert.That(evidence.Requests[^1].Usage!.CachedInputTokens).IsGreaterThan(0); + await Assert.That(recovered.Usage.InputTokenDetails.CachedTokenCount).IsEqualTo(evidence.Requests[^1].Usage!.CachedInputTokens); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(2); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Streaming/StreamDeliveryGeminiIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Streaming/StreamDeliveryGeminiIntegrationTests.cs new file mode 100644 index 0000000..626d1d8 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Streaming/StreamDeliveryGeminiIntegrationTests.cs @@ -0,0 +1,87 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text; +using Google.GenAI.Types; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Scenarios; + +namespace ManagedCode.LlmTestServer.Tests.Streaming; + +public sealed partial class StreamDeliveryIntegrationTests +{ + private const string _geminiDeliveryModel = "gemini-3.8-flash"; + + [Test] + [Arguments(LlmTestServerStreamDeliveryMode.EarlyCleanEnd, false, false)] + [Arguments(LlmTestServerStreamDeliveryMode.EarlyCleanEnd, false, true)] + [Arguments(LlmTestServerStreamDeliveryMode.AbortConnection, false, false)] + [Arguments(LlmTestServerStreamDeliveryMode.DelayThenContinue, true, false)] + [Arguments(LlmTestServerStreamDeliveryMode.DelayThenContinue, false, false)] + public async Task OfficialGoogleSdk_TextSseDeliveryFaultsPreservePrefixAndRecoverAsync(LlmTestServerStreamDeliveryMode mode, bool cancel, bool afterAllText) + { + await AssertProtocolDeliveryAsync(_geminiDeliveryModel, "/gemini/v1beta/models/" + _geminiDeliveryModel + ":streamGenerateContent", "first ", mode, cancel, + async (http, first, token, keepAlive) => + { + var sdk = GoogleSdk(http); + keepAlive(sdk); + var text = new StringBuilder(); + var terminal = false; + Exception? failure = null; + try + { + await foreach (var update in sdk.Models.GenerateContentStreamAsync(_geminiDeliveryModel, "stream", new GenerateContentConfig { AutomaticContinuation = false }, cancellationToken: token)) + { + var delta = update.Text; + text.Append(delta); + if (!string.IsNullOrEmpty(delta)) { first(); } + if (update.Candidates?.Any(candidate => candidate.FinishReason is not null) == true) { terminal = true; } + } + } + catch (Exception exception) when (ProtocolStreamException(exception)) { failure = exception; } + return new(text.ToString(), terminal, failure); + }, + async (http, token, keepAlive) => + { + var sdk = GoogleSdk(http); + keepAlive(sdk); + return (await sdk.Models.GenerateContentAsync(_geminiDeliveryModel, "explicit retry", new GenerateContentConfig { AutomaticContinuation = false }, cancellationToken: token)).Text!; + }, afterAllText); + } + + [Test] + public async Task GoogleJsonArrayStreamRejectsDeliveryPlanBeforeConsumptionThenOfficialSseAndJsonRecoverAsync() + { + await using var app = await StartTcpHostAsync(config => config.AddModel(_geminiDeliveryModel, LlmTestServerModelKind.Chat) + .AddChatScenario("gemini-delivery-admission", scenario => scenario.ForModel(_geminiDeliveryModel) + .Responds("first second", "first ", "second").WithStreamDelivery(new() + { Mode = LlmTestServerStreamDeliveryMode.EarlyCleanEnd }).Responds("google recovered"))); + using var http = TcpClient(app); + var control = new LlmTestServerClient(http); + using var rejected = await http.PostAsJsonAsync("/gemini/v1beta/models/" + _geminiDeliveryModel + ":streamGenerateContent", new + { contents = new[] { new { role = "user", parts = new[] { new { text = "stream" } } } } }); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(0); + using var sdk = GoogleSdk(http); + using var watchdog = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + var text = new StringBuilder(); + var finished = false; + await foreach (var update in sdk.Models.GenerateContentStreamAsync(_geminiDeliveryModel, "stream", new GenerateContentConfig { AutomaticContinuation = false }, cancellationToken: watchdog.Token)) + { + text.Append(update.Text); + finished |= update.Candidates?.Any(candidate => candidate.FinishReason is not null) == true; + } + await Assert.That(text.ToString()).IsEqualTo("first "); + await Assert.That(finished).IsFalse(); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(1); + await Assert.That((await sdk.Models.GenerateContentAsync(_geminiDeliveryModel, "next", new GenerateContentConfig { AutomaticContinuation = false }, cancellationToken: watchdog.Token)).Text).IsEqualTo("google recovered"); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(2); + } + + private static Google.GenAI.Client GoogleSdk(HttpClient http) + { + return new(apiKey: "gemini-delivery-key", enterprise: false, + httpOptions: new() { BaseUrl = new Uri(http.BaseAddress!, "/gemini").ToString(), ApiVersion = "v1beta", RetryOptions = new() { Attempts = 1 } }, + clientOptions: new() { HttpClientFactory = () => http }); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Streaming/StreamDeliveryIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Streaming/StreamDeliveryIntegrationTests.cs new file mode 100644 index 0000000..42e483b --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Streaming/StreamDeliveryIntegrationTests.cs @@ -0,0 +1,289 @@ +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Diagnostics; +using System.Net; +using System.Text; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Configuration; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Hosting; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Scenarios; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Hosting.Server; +using Microsoft.AspNetCore.Hosting.Server.Features; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using OpenAI; +using OpenAI.Chat; + +namespace ManagedCode.LlmTestServer.Tests.Streaming; + +public sealed partial class StreamDeliveryIntegrationTests +{ + [Test] + [Arguments(LlmTestServerStreamDeliveryMode.EarlyCleanEnd, false)] + [Arguments(LlmTestServerStreamDeliveryMode.AbortConnection, false)] + [Arguments(LlmTestServerStreamDeliveryMode.EarlyCleanEnd, true)] + public async Task OfficialSdk_PartialDeliveryHasNoFinishAndFailsTerminalVerificationThenExplicitRetryRecoversAsync(LlmTestServerStreamDeliveryMode mode, bool afterAllText) + { + var firstRead = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var app = await StartTcpHostAsync(config => config.AddChatScenario("broken-stream", scenario => scenario + .Responds("first second", "first ", "second").WithStreamDelivery(new() { Mode = mode, AfterContentFrames = afterAllText ? 2 : 1 }) + .Responds("explicit recovery")), firstRead.Task); + using var http = TcpClient(app); + var control = new LlmTestServerClient(http); + using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + var partial = await ReadSdkAsync(Sdk(http), [new UserChatMessage("stream")], firstRead, deadline.Token); + await Assert.That(partial.Text).IsEqualTo(afterAllText ? "first second" : "first "); + await Assert.That(partial.Finish).IsNull(); + await Assert.That(firstRead.Task.IsCompletedSuccessfully).IsTrue(); + await Assert.That(deadline.IsCancellationRequested).IsFalse(); + if (mode == LlmTestServerStreamDeliveryMode.AbortConnection) + { await Assert.That(partial.Failure is IOException or HttpRequestException or ClientResultException).IsTrue(); } + else { await Assert.That(partial.Failure).IsNull(); } + var trace = await CompletedTraceAsync(control, 1); + await Assert.That(trace.StatusCode).IsEqualTo(200); + await Assert.That(trace.TerminalMarkerObserved == true).IsFalse(); + await Assert.That(trace.StreamDelivery!.Mode).IsEqualTo(mode); + await Assert.That(trace.StreamDeliveryApplied).IsTrue(); + await Assert.That(trace.EmittedContentFrames).IsEqualTo(afterAllText ? 2 : 1); + await Assert.That(trace.ResponsePreview).Contains("first "); + await Assert.That(trace.ResponsePreview!.Contains("[DONE]", StringComparison.Ordinal)).IsFalse(); + if (mode == LlmTestServerStreamDeliveryMode.EarlyCleanEnd) + { await Assert.That(trace.TransportState).IsEqualTo(LlmTestServerProviderHttpTraceTransportState.Completed); } + else + { await Assert.That(trace.TransportState == LlmTestServerProviderHttpTraceTransportState.Completed).IsFalse(); } + var verification = await control.VerifyAsync(new() + { Requests = [new() { Query = new() { RequestId = trace.RequestId }, RequireStreamingTerminalMarker = true }] }); + await Assert.That(verification.Passed).IsFalse(); + await Assert.That(verification.Failures.Any(failure => failure.Code == "stream.terminal" && failure.RequestIds.Contains(trace.RequestId))).IsTrue(); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(1); + using var healthy = await http.GetAsync("/health"); + await Assert.That(healthy.StatusCode).IsEqualTo(HttpStatusCode.OK); + var recovered = (await Sdk(http).CompleteChatAsync([new UserChatMessage("explicit retry")])).Value; + await Assert.That(recovered.Content.Single().Text).IsEqualTo("explicit recovery"); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(2); + await Assert.That((await control.GetRequestsAsync()).TotalObserved).IsEqualTo(2); + } + + [Test] + public async Task OfficialSdk_PostFrameDelayContinuesWithFullTextAndTerminalAsync() + { + const int delayMilliseconds = 160; + var firstRead = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var app = await StartTcpHostAsync(config => config.AddChatScenario("delayed-stream", scenario => scenario + .Responds("first second", "first ", "second").WithStreamDelivery(new() + { Mode = LlmTestServerStreamDeliveryMode.DelayThenContinue, AfterContentFrames = 1, DelayMilliseconds = delayMilliseconds })), firstRead.Task); + using var http = TcpClient(app); + var control = new LlmTestServerClient(http); + using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + var afterFirst = new Stopwatch(); + var completed = await ReadSdkAsync(Sdk(http), [new UserChatMessage("stream")], firstRead, deadline.Token, afterFirst.Start); + await Assert.That(completed.Failure).IsNull(); + await Assert.That(completed.Text).IsEqualTo("first second"); + await Assert.That(completed.Finish).IsEqualTo(ChatFinishReason.Stop); + // The clock starts before acknowledging the server flush; allow only 5 ms for timer resolution. + await Assert.That(afterFirst.ElapsedMilliseconds).IsGreaterThanOrEqualTo(delayMilliseconds - 5); + var trace = await CompletedTraceAsync(control, 1); + await Assert.That(trace.TerminalMarkerObserved).IsTrue(); + await Assert.That(trace.EmittedContentFrames).IsEqualTo(2); + await Assert.That(trace.StreamDeliveryApplied).IsTrue(); + await Assert.That(trace.TransportState).IsEqualTo(LlmTestServerProviderHttpTraceTransportState.Completed); + await Assert.That((await control.VerifyAsync(new() { Requests = [new() { RequireStreamingTerminalMarker = true }] })).Passed).IsTrue(); + } + + [Test] + public async Task OfficialSdk_PostFrameStallCancellationKeepsCommittedInferenceThenExplicitRetryRecoversAsync() + { + var firstRead = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var app = await StartTcpHostAsync(config => config.AddChatScenario("stalled-stream", scenario => scenario + .Responds("first second", "first ", "second").WithStreamDelivery(new() + { Mode = LlmTestServerStreamDeliveryMode.DelayThenContinue, AfterContentFrames = 1, DelayMilliseconds = 30000 }) + .Responds("after cancellation")), firstRead.Task); + using var http = TcpClient(app); + var control = new LlmTestServerClient(http); + using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + var afterFirst = new Stopwatch(); + var partial = await ReadSdkAsync(Sdk(http), [new UserChatMessage("stall")], firstRead, deadline.Token, + () => { afterFirst.Start(); deadline.CancelAfter(TimeSpan.FromMilliseconds(180)); }); + await Assert.That(partial.Text).IsEqualTo("first "); + await Assert.That(partial.Finish).IsNull(); + await Assert.That(partial.Failure is OperationCanceledException || partial.Failure is ClientResultException && deadline.IsCancellationRequested).IsTrue(); + await Assert.That(deadline.IsCancellationRequested).IsTrue(); + await Assert.That(afterFirst.Elapsed < TimeSpan.FromSeconds(5)).IsTrue(); + var trace = await CompletedTraceAsync(control, 1); + await Assert.That(trace.TerminalMarkerObserved == true).IsFalse(); + await Assert.That(trace.EmittedContentFrames).IsEqualTo(1); + await Assert.That(trace.TransportState is LlmTestServerProviderHttpTraceTransportState.Cancelled or LlmTestServerProviderHttpTraceTransportState.Aborted).IsTrue(); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(1); + using var healthy = await http.GetAsync("/health"); + await Assert.That(healthy.StatusCode).IsEqualTo(HttpStatusCode.OK); + var recovered = (await Sdk(http).CompleteChatAsync([new UserChatMessage("retry")])).Value; + await Assert.That(recovered.Content.Single().Text).IsEqualTo("after cancellation"); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(2); + } + + private static ChatClient Sdk(HttpClient http, string provider = "/openai/v1") + { + return new(LlmTestServerKnownModelIds.Gpt41Mini, + new ApiKeyCredential("stream-delivery-key"), new OpenAIClientOptions + { Endpoint = new Uri(http.BaseAddress!, provider), Transport = new HttpClientPipelineTransport(http), RetryPolicy = new ClientRetryPolicy(0) }); + } + + private static async Task ReadSdkAsync(ChatClient sdk, IEnumerable messages, + TaskCompletionSource? firstRead, CancellationToken cancellationToken, Action? onFirst = null) + { + var text = new StringBuilder(); + ChatFinishReason? finish = null; + Exception? failure = null; + var observed = false; + try + { + await foreach (var update in sdk.CompleteChatStreamingAsync(messages, cancellationToken: cancellationToken)) + { + foreach (var part in update.ContentUpdate) + { + text.Append(part.Text); + if (!observed && !string.IsNullOrEmpty(part.Text)) + { + observed = true; + onFirst?.Invoke(); + firstRead?.TrySetResult(); + } + } + if (update.FinishReason is { } reason) { finish = reason; } + } + } + catch (Exception exception) when (exception is IOException or HttpRequestException or ClientResultException or OperationCanceledException) + { failure = exception; } + finally { firstRead?.TrySetResult(); } + return new(text.ToString(), finish, failure); + } + + private static async Task CompletedTraceAsync(LlmTestServerClient control, int count) + { + using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + while (true) + { + var snapshot = await control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending }, deadline.Token); + if (snapshot.Requests.Count == count && snapshot.Requests[^1].IsCompleted) { return snapshot.Requests[^1]; } + deadline.Token.ThrowIfCancellationRequested(); + } + } + + private static async Task StartTcpHostAsync(Action configure, Task? firstRead = null, string firstContentPath = "/openai/v1/chat/completions", string? firstContentMarker = null) + { + var builder = WebApplication.CreateBuilder(new WebApplicationOptions { EnvironmentName = Environments.Development }); + builder.WebHost.ConfigureKestrel(options => options.Listen(IPAddress.Loopback, 0)); + builder.Services.AddLlmTestServer(configure); + builder.AddDefaultHealthChecks(); + var app = builder.Build(); + if (firstRead is not null) + { + var gated = 0; + app.Use(async (context, next) => + { + if (context.Request.Path == firstContentPath && Interlocked.CompareExchange(ref gated, 1, 0) == 0) + { + var original = context.Response.Body; + await using var barrier = new FirstFlushGateStream(original, firstRead, firstContentMarker); + context.Response.Body = barrier; + try { await next(context); } + finally { context.Response.Body = original; } + } + else { await next(context); } + }); + } + app.MapLlmTestServer(); + app.MapDefaultEndpoints(); + var host = new TcpTestHost(app); + using var startup = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + try + { + await app.StartAsync(startup.Token); + // Initialize endpoint metadata through real HTTP before a provider stream's watchdog starts. + using var warmup = TcpClient(host); + using var health = await warmup.GetAsync("/health", startup.Token); + health.EnsureSuccessStatusCode(); + return host; + } + catch + { + await host.DisposeAsync(); + throw; + } + } + + private sealed class TcpTestHost(WebApplication application) : IAsyncDisposable + { + internal IServiceProvider Services => application.Services; + + public async ValueTask DisposeAsync() + { + using var shutdown = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + try { await application.StopAsync(shutdown.Token); } + finally { await application.DisposeAsync(); } + } + } + + private static HttpClient TcpClient(TcpTestHost app) + { + var server = app.Services.GetRequiredService(); + var address = server.Features.Get()!.Addresses.Single(); + return new HttpClient { BaseAddress = new Uri(address), Timeout = TimeSpan.FromSeconds(60) }; + } + + private sealed record ObservedStream(string Text, ChatFinishReason? Finish, Exception? Failure); + + // Real TCP backpressure: bytes are forwarded unchanged, and only the first flush waits for the SDK's acknowledgement. + private sealed class FirstFlushGateStream(Stream inner, Task firstRead, string? contentMarker) : Stream + { + private int _gated; + private long _written; + private readonly StringBuilder _wire = new(); + public override bool CanRead => inner.CanRead; + public override bool CanSeek => inner.CanSeek; + public override bool CanWrite => inner.CanWrite; + public override long Length => inner.Length; + public override long Position { get => inner.Position; set => inner.Position = value; } + public override void Flush() + { + inner.Flush(); + } + + public override async Task FlushAsync(CancellationToken cancellationToken) + { + await inner.FlushAsync(cancellationToken); + if (Interlocked.Read(ref _written) > 0 && (contentMarker is null || _wire.ToString().Contains(contentMarker, StringComparison.Ordinal)) && Interlocked.CompareExchange(ref _gated, 1, 0) == 0) + { await firstRead.WaitAsync(TimeSpan.FromSeconds(60), cancellationToken); } + } + public override int Read(byte[] buffer, int offset, int count) + { + return inner.Read(buffer, offset, count); + } + + public override long Seek(long offset, SeekOrigin origin) + { + return inner.Seek(offset, origin); + } + + public override void SetLength(long value) + { + inner.SetLength(value); + } + + public override void Write(byte[] buffer, int offset, int count) + { inner.Write(buffer, offset, count); RecordWrite(buffer.AsSpan(offset, count)); } + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { await inner.WriteAsync(buffer.AsMemory(offset, count), cancellationToken); RecordWrite(buffer.AsSpan(offset, count)); } + public override async ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) + { await inner.WriteAsync(buffer, cancellationToken); RecordWrite(buffer.Span); } + private void RecordWrite(ReadOnlySpan bytes) + { + Interlocked.Add(ref _written, bytes.Length); + if (contentMarker is not null && Volatile.Read(ref _gated) == 0) { _wire.Append(Encoding.UTF8.GetString(bytes)); } + } + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Streaming/StreamDeliveryProtocolIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Streaming/StreamDeliveryProtocolIntegrationTests.cs new file mode 100644 index 0000000..e1b0515 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Streaming/StreamDeliveryProtocolIntegrationTests.cs @@ -0,0 +1,127 @@ +using System.ClientModel; +using System.Diagnostics; +using System.Net; +using System.Net.Http.Json; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Configuration; +using ManagedCode.LlmTestServer.Control; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Scenarios; + +namespace ManagedCode.LlmTestServer.Tests.Streaming; + +public sealed partial class StreamDeliveryIntegrationTests +{ + private static async Task AssertProtocolDeliveryAsync(string model, string path, string marker, + LlmTestServerStreamDeliveryMode mode, bool cancel, + Func, Task> read, + Func, Task> recover, bool afterAllText = false) + { + var firstRead = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var app = await StartTcpHostAsync(config => config.AddModel(model, LlmTestServerModelKind.Chat) + .AddChatScenario("protocol-delivery", scenario => scenario.ForModel(model) + .Responds("first second", "", "first ", "", "second").WithStreamDelivery(new() + { + Mode = mode, + AfterContentFrames = afterAllText ? 2 : 1, + DelayMilliseconds = mode == LlmTestServerStreamDeliveryMode.DelayThenContinue ? cancel ? 30000 : 160 : 0, + }).Responds("protocol recovered")), firstRead.Task, path, marker); + using var http = TcpClient(app); + var sdkOwners = new List(); + try + { + var control = new LlmTestServerClient(http); + using var watchdog = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + var afterFirst = new Stopwatch(); + var sawFirst = false; + void FirstContent() + { + if (sawFirst) { return; } + sawFirst = true; + afterFirst.Start(); + if (cancel) { watchdog.CancelAfter(TimeSpan.FromMilliseconds(180)); } + firstRead.TrySetResult(); + } + ObservedProtocolStream observed; + try { observed = await read(http, FirstContent, watchdog.Token, sdkOwners.Add); } + finally { firstRead.TrySetResult(); } + var full = mode == LlmTestServerStreamDeliveryMode.DelayThenContinue && !cancel; + var allText = full || afterAllText; + await Assert.That(sawFirst).IsTrue(); + await Assert.That(observed.Text).IsEqualTo(allText ? "first second" : "first "); + await Assert.That(observed.Terminal).IsEqualTo(full); + if (full || mode == LlmTestServerStreamDeliveryMode.EarlyCleanEnd) + { await Assert.That(observed.Failure).IsNull(); } + else { await Assert.That(observed.Failure is not null).IsTrue(); } + if (cancel) + { + await Assert.That(watchdog.IsCancellationRequested).IsTrue(); + await Assert.That(afterFirst.Elapsed < TimeSpan.FromSeconds(5)).IsTrue(); + } + else { await Assert.That(watchdog.IsCancellationRequested).IsFalse(); } + // The clock starts before acknowledging the server flush; allow only 5 ms for timer resolution. + if (full) { await Assert.That(afterFirst.ElapsedMilliseconds).IsGreaterThanOrEqualTo(160 - 5); } + var trace = await CompletedTraceAsync(control, 1); + await Assert.That(trace.Path).IsEqualTo(path); + await Assert.That(trace.StatusCode).IsEqualTo(200); + await Assert.That(trace.StreamDelivery!.Mode).IsEqualTo(mode); + await Assert.That(trace.StreamDeliveryApplied).IsTrue(); + await Assert.That(trace.EmittedContentFrames).IsEqualTo(allText ? 2 : 1); + await Assert.That(trace.TerminalMarkerObserved == true).IsEqualTo(full); + if (cancel) + { await Assert.That(trace.TransportState is LlmTestServerProviderHttpTraceTransportState.Cancelled or LlmTestServerProviderHttpTraceTransportState.Aborted).IsTrue(); } + else if (mode == LlmTestServerStreamDeliveryMode.AbortConnection) + { await Assert.That(trace.TransportState == LlmTestServerProviderHttpTraceTransportState.Completed).IsFalse(); } + else { await Assert.That(trace.TransportState).IsEqualTo(LlmTestServerProviderHttpTraceTransportState.Completed); } + var verification = await control.VerifyAsync(new() + { Requests = [new() { Query = new() { RequestId = trace.RequestId }, RequireStreamingTerminalMarker = true }] }); + await Assert.That(verification.Passed).IsEqualTo(full); + if (!full) + { await Assert.That(verification.Failures.Any(failure => failure.Code == "stream.terminal" && failure.RequestIds.Contains(trace.RequestId))).IsTrue(); } + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(1); + using var healthy = await http.GetAsync("/health"); + await Assert.That(healthy.StatusCode).IsEqualTo(HttpStatusCode.OK); + using var recoveryWatchdog = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + await Assert.That(await recover(http, recoveryWatchdog.Token, sdkOwners.Add)).IsEqualTo("protocol recovered"); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(2); + await Assert.That((await control.GetRequestsAsync()).TotalObserved).IsEqualTo(2); + } + finally + { + foreach (var sdk in sdkOwners) { sdk.Dispose(); } + } + } + + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task Configure_EmptyOrUnreachableNonemptyTextBoundaryIsAtomicAsync(bool empty) + { + await using var app = await StartTcpHostAsync(config => config.AddChatScenario("preserved", scenario => scenario.Responds("still available"))); + using var http = TcpClient(app); + var control = new LlmTestServerClient(http); + var validBase = new LlmTestServerConfigurationBuilder().Build(); + using var rejected = await http.PostAsJsonAsync(LlmTestServerControlRoutes.Configure, validBase with + { + ChatScenarios = [new() { Id = "unreachable", Responses = [new() + { + Content = empty ? string.Empty : "first second", + StreamChunks = empty ? ["", ""] : ["first ", "", "second"], + StreamDelivery = new() { Mode = LlmTestServerStreamDeliveryMode.EarlyCleanEnd, AfterContentFrames = empty ? 1 : 3 } + }] }] + }); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + await Assert.That((await control.GetAssertionsAsync()).Matched).IsEqualTo(0); + var response = (await Sdk(http).CompleteChatAsync([new global::OpenAI.Chat.UserChatMessage("original")])).Value; + await Assert.That(response.Content.Single().Text).IsEqualTo("still available"); + } + + private static bool ProtocolStreamException(Exception exception) + { + return exception is IOException or HttpRequestException + or ClientResultException or OperationCanceledException or global::Anthropic.Exceptions.AnthropicException or Google.GenAI.ApiException; + } + + private sealed record ObservedProtocolStream(string Text, bool Terminal, Exception? Failure); +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Streaming/StreamDeliveryReplayIdentityIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Streaming/StreamDeliveryReplayIdentityIntegrationTests.cs new file mode 100644 index 0000000..44c226f --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Streaming/StreamDeliveryReplayIdentityIntegrationTests.cs @@ -0,0 +1,90 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using Google.GenAI; +using Google.GenAI.Types; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Scenarios; +using ManagedCode.LlmTestServer.Tests.TestSupport; + +namespace ManagedCode.LlmTestServer.Tests.Streaming; + +public sealed class StreamDeliveryReplayIdentityIntegrationTests +{ + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task GeminiGraphReplay_SeparatesInferenceIdentityFromCurrentDeliveryApplicabilityAsync(bool deliveryFault) + { + const string model = "gemini-3.8-flash"; + const string root = "/gemini/v1beta/models/" + model; + using var host = await LlmTestServerTestHost.StartAsync(config => config.AddModel(model, LlmTestServerModelKind.Chat) + .AddConversation("gemini-replay-identity", graph => graph.ForModel(model).Step("answer", step => + { + step.Responds("first second", "first ", "second").Complete(); + if (deliveryFault) { step.WithStreamDelivery(new() { Mode = LlmTestServerStreamDeliveryMode.EarlyCleanEnd }); } + }))); + using var http = host.GetTestClient(); var control = new LlmTestServerClient(http); + using var scoped = control.ForConversation("gemini-replay-identity"); + var body = new { contents = new[] { new { role = "user", parts = new[] { new { text = "same inference" } } } } }; + using var sdk = new Google.GenAI.Client(apiKey: "gemini-replay-key", enterprise: false, + httpOptions: new() { BaseUrl = new Uri(scoped.HttpClient.BaseAddress!, "/gemini").ToString(), ApiVersion = "v1beta", RetryOptions = new() { Attempts = 1 } }, + clientOptions: new() { HttpClientFactory = () => scoped.HttpClient }); + using var watchdog = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + async Task> StreamAsync() + { + var frames = new List(); + await foreach (var frame in sdk.Models.GenerateContentStreamAsync(model, "same inference", new GenerateContentConfig { AutomaticContinuation = false }, cancellationToken: watchdog.Token)) + { frames.Add(frame); } + return frames; + } + string? identity; + if (deliveryFault) + { + var first = (await StreamAsync()).Single(); + identity = first.ResponseId; + await Assert.That(first.Text).IsEqualTo("first "); + ClientError? rejected = null; + try { await sdk.Models.GenerateContentAsync(model, "same inference", new GenerateContentConfig { AutomaticContinuation = false }, cancellationToken: watchdog.Token); } + catch (ClientError exception) { rejected = exception; } + await Assert.That(rejected).IsNotNull(); + await Assert.That(rejected!.StatusCode).IsEqualTo(409); + await Assert.That(rejected.Message).Contains("text SSE"); + } + else + { + var initial = await sdk.Models.GenerateContentAsync(model, "same inference", new GenerateContentConfig { AutomaticContinuation = false }, cancellationToken: watchdog.Token); + identity = initial.ResponseId; + await Assert.That(initial.Text).IsEqualTo("first second"); + } + await Assert.That(identity).IsNotNullOrEmpty(); + using var arrayRequest = new HttpRequestMessage(HttpMethod.Post, root + ":streamGenerateContent") { Content = JsonContent.Create(body) }; + // Match the official SDK credential so this wire-format check addresses the same graph owner. + arrayRequest.Headers.Add("x-goog-api-key", "gemini-replay-key"); + using var array = await scoped.HttpClient.SendAsync(arrayRequest); + await Assert.That(array.StatusCode).IsEqualTo(deliveryFault ? HttpStatusCode.Conflict : HttpStatusCode.OK); + if (deliveryFault) + { + await Assert.That((await array.Content.ReadFromJsonAsync()).GetProperty("error").GetProperty("message").GetString()!).Contains("text SSE"); + } + else + { + var frames = await array.Content.ReadFromJsonAsync(); + await Assert.That(frames[0].GetProperty("responseId").GetString()).IsEqualTo(identity); + await Assert.That(frames[frames.GetArrayLength() - 1].GetProperty("candidates")[0].GetProperty("finishReason").GetString()).IsEqualTo("STOP"); + } + var events = await StreamAsync(); + await Assert.That(events.All(item => item.ResponseId == identity)).IsTrue(); + await Assert.That(string.Concat(events.Select(item => item.Text))).IsEqualTo(deliveryFault ? "first " : "first second"); + await Assert.That(events.Count).IsEqualTo(deliveryFault ? 1 : 3); + await Assert.That(events.Any(item => item.Candidates?.Any(candidate => candidate.FinishReason is not null) == true)).IsEqualTo(!deliveryFault); + var states = await scoped.Control.GetConversationsAsync(); + await Assert.That(states.Single().ExecutedSteps).IsEqualTo(1); + var latest = (await scoped.Control.GetRequestsAsync()).Requests[0]; + await Assert.That(latest.IsReplay).IsTrue(); + await Assert.That(latest.StreamDeliveryApplied).IsEqualTo(deliveryFault); + await Assert.That(latest.TerminalMarkerObserved == true).IsEqualTo(!deliveryFault); + } + +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Streaming/StreamDeliveryResponsesIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Streaming/StreamDeliveryResponsesIntegrationTests.cs new file mode 100644 index 0000000..16d6829 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Streaming/StreamDeliveryResponsesIntegrationTests.cs @@ -0,0 +1,53 @@ +#pragma warning disable OPENAI001 // Exercise the pinned official Responses streaming SDK against the running server. +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Text; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Scenarios; +using OpenAI.Responses; + +namespace ManagedCode.LlmTestServer.Tests.Streaming; + +public sealed partial class StreamDeliveryIntegrationTests +{ + [Test] + [Arguments(LlmTestServerStreamDeliveryMode.EarlyCleanEnd, false, false)] + [Arguments(LlmTestServerStreamDeliveryMode.EarlyCleanEnd, false, true)] + [Arguments(LlmTestServerStreamDeliveryMode.AbortConnection, false, false)] + [Arguments(LlmTestServerStreamDeliveryMode.DelayThenContinue, true, false)] + [Arguments(LlmTestServerStreamDeliveryMode.DelayThenContinue, false, false)] + public async Task OfficialResponsesSdk_TextDeliveryFaultsPreserveObservedPrefixAndRecoveryAsync(LlmTestServerStreamDeliveryMode mode, bool cancel, bool afterAllText) + { + await AssertProtocolDeliveryAsync(LlmTestServerKnownModelIds.Gpt41Mini, "/openai/v1/responses", "first ", mode, cancel, + async (http, first, token, _) => + { + var sdk = ResponsesSdk(http); + var text = new StringBuilder(); + var terminal = false; + Exception? failure = null; + try + { + await foreach (var update in sdk.CreateResponseStreamingAsync(new CreateResponseOptions + { Model = LlmTestServerKnownModelIds.Gpt41Mini, StreamingEnabled = true, StoredOutputEnabled = false, InputItems = { ResponseItem.CreateUserMessageItem("stream") } }, token)) + { + if (update is StreamingResponseOutputTextDeltaUpdate delta) + { + text.Append(delta.Delta); + if (!string.IsNullOrEmpty(delta.Delta)) { first(); } + } + if (update is StreamingResponseCompletedUpdate) { terminal = true; } + } + } + catch (Exception exception) when (ProtocolStreamException(exception)) { failure = exception; } + return new(text.ToString(), terminal, failure); + }, + async (http, token, _) => (await ResponsesSdk(http).CreateResponseAsync(new CreateResponseOptions + { Model = LlmTestServerKnownModelIds.Gpt41Mini, StoredOutputEnabled = false, InputItems = { ResponseItem.CreateUserMessageItem("explicit retry") } }, token)).Value.GetOutputText(), afterAllText); + } + + private static ResponsesClient ResponsesSdk(HttpClient http) + { + return new(new ApiKeyCredential("responses-delivery-key"), new ResponsesClientOptions + { Endpoint = new Uri(http.BaseAddress!, "/openai/v1/"), Transport = new HttpClientPipelineTransport(http), RetryPolicy = new ClientRetryPolicy(0) }); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/TestSupport/LlmTestServerProviderFixtures.cs b/tests/ManagedCode.LlmTestServer.Tests/TestSupport/LlmTestServerProviderFixtures.cs new file mode 100644 index 0000000..e8c20f6 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/TestSupport/LlmTestServerProviderFixtures.cs @@ -0,0 +1,44 @@ +using ManagedCode.LlmTestServer.Anthropic; +using ManagedCode.LlmTestServer.AzureOpenAI; +using ManagedCode.LlmTestServer.Bedrock; +using ManagedCode.LlmTestServer.Cloudflare; +using ManagedCode.LlmTestServer.Cohere; +using ManagedCode.LlmTestServer.DeepSeek; +using ManagedCode.LlmTestServer.Foundry; +using ManagedCode.LlmTestServer.Gemini; +using ManagedCode.LlmTestServer.Groq; +using ManagedCode.LlmTestServer.Mistral; +using ManagedCode.LlmTestServer.Ollama; +using ManagedCode.LlmTestServer.OpenAI; +using ManagedCode.LlmTestServer.OpenRouter; +using ManagedCode.LlmTestServer.Perplexity; +using ManagedCode.LlmTestServer.Providers; +using ManagedCode.LlmTestServer.SystemOne; + +namespace ManagedCode.LlmTestServer.Tests.TestSupport; + +internal static class LlmTestServerProviderFixtures +{ + internal static LlmTestServerProviderProfile[] GetProfiles() + { + return + [ + SystemOneCompatibility.Profile, + CloudflareCompatibility.Profile, + OpenAiCompatibility.Profile, + AzureOpenAiCompatibility.Profile, + MicrosoftFoundryCompatibility.Profile, + AnthropicCompatibility.Profile, + GeminiCompatibility.Profile, + GroqCompatibility.Profile, + MistralCompatibility.Profile, + OllamaCompatibility.Profile, + CohereCompatibility.Profile, + BedrockCompatibility.Profile, + OpenRouterCompatibility.Profile, + DeepSeekCompatibility.Profile, + PerplexityCompatibility.Profile, + ]; + } + +} diff --git a/tests/ManagedCode.LlmTck.Tests/TestSupport/LlmTckTestHost.cs b/tests/ManagedCode.LlmTestServer.Tests/TestSupport/LlmTestServerTestHost.cs similarity index 65% rename from tests/ManagedCode.LlmTck.Tests/TestSupport/LlmTckTestHost.cs rename to tests/ManagedCode.LlmTestServer.Tests/TestSupport/LlmTestServerTestHost.cs index f0846d0..5d33346 100644 --- a/tests/ManagedCode.LlmTck.Tests/TestSupport/LlmTckTestHost.cs +++ b/tests/ManagedCode.LlmTestServer.Tests/TestSupport/LlmTestServerTestHost.cs @@ -1,16 +1,17 @@ -using ManagedCode.LlmTck.Configuration; -using ManagedCode.LlmTck.Hosting; +using ManagedCode.LlmTestServer.Configuration; +using ManagedCode.LlmTestServer.Hosting; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; -namespace ManagedCode.LlmTck.Tests.TestSupport; +namespace ManagedCode.LlmTestServer.Tests.TestSupport; -internal static class LlmTckTestHost +internal static class LlmTestServerTestHost { public static async Task StartAsync( - Action? configure = null, + Action? configure = null, + Action? configureServices = null, CancellationToken cancellationToken = default ) { @@ -20,12 +21,13 @@ public static async Task StartAsync( .ConfigureServices(services => { services.AddRouting(); - services.AddLlmTck(configure); + services.AddLlmTestServer(configure); + configureServices?.Invoke(services); }) .Configure(application => { application.UseRouting(); - application.UseEndpoints(endpoints => endpoints.MapLlmTck()); + application.UseEndpoints(endpoints => endpoints.MapLlmTestServer()); })) .Build(); diff --git a/tests/ManagedCode.LlmTestServer.Tests/Tools/WellKnownToolBoundaryTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Tools/WellKnownToolBoundaryTests.cs new file mode 100644 index 0000000..3ee8a22 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Tools/WellKnownToolBoundaryTests.cs @@ -0,0 +1,119 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Tests.TestSupport; +using ManagedCode.LlmTestServer.Tools.WellKnown; + +namespace ManagedCode.LlmTestServer.Tests.Tools; + +public sealed partial class WellKnownToolIntegrationTests +{ + [Test] + public async Task Catalogue_DefinitionsDriveTwelveActualToolCallsAndKeepLinearExhaustionAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(config => config.UseChatScenario("catalogue", scenario => + { + for (var kind = 0; kind < 12; kind++) { scenario.CallsTools(CreateFixture(kind).Call); } + })); + var tools = LlmTestServerWellKnownTools.All.Select(descriptor => new + { + type = "function", + function = new + { + name = descriptor.Name, + description = descriptor.Description, + parameters = JsonSerializer.Deserialize(descriptor.ParametersJson) + } + }).ToArray(); + var returnedNames = new List(); + for (var index = 0; index < 12; index++) + { + using var response = await http.PostAsJsonAsync("/openai/v1/chat/completions", new + { model = "gpt-4.1-mini", messages = new[] { new { role = "user", content = "catalogue" } }, tools, tool_choice = "required" }); + response.EnsureSuccessStatusCode(); + var call = (await response.Content.ReadFromJsonAsync()).GetProperty("choices")[0].GetProperty("message").GetProperty("tool_calls")[0]; + returnedNames.Add(call.GetProperty("function").GetProperty("name").GetString()!); + } + await Assert.That(returnedNames.Distinct().Count()).IsEqualTo(12); + using var exhausted = await http.PostAsJsonAsync("/openai/v1/chat/completions", new + { model = "gpt-4.1-mini", messages = new[] { new { role = "user", content = "catalogue" } }, tools }); + await Assert.That(exhausted.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + await Assert.That(await exhausted.Content.ReadAsStringAsync()).Contains("llm_test_server_scenario_exhausted"); + var verification = await control.VerifyAsync(new() + { + NoUnexpectedCalls = true, + Requests = [new() { Query = new() { Success = true }, MinimumCount = 12, MaximumCount = 12 }, + new() { Query = new() { Status = 409 } }], + }); + await Assert.That(verification.Passed).IsTrue(); + } + + [Test] + [Arguments(0)] + [Arguments(1)] + [Arguments(2)] + [Arguments(3)] + [Arguments(4)] + [Arguments(5)] + [Arguments(6)] + [Arguments(7)] + [Arguments(8)] + [Arguments(9)] + [Arguments(10)] + [Arguments(11)] + public async Task Preset_RejectsIncompatibleOfferedSchemaWithoutConsumingTheCallAsync(int kind) + { + var fixture = CreateFixture(kind); + using var host = await LlmTestServerTestHost.StartAsync(config => config.AddChatScenario("schema", scenario => scenario.CallsTools(fixture.Call))); + using var http = host.GetTestClient(); + var incompatible = new[] { new { type = "function", function = new { name = fixture.Definition.Name, + parameters = new { type = "object", required = new[] { "absent_fixture_field" } } } } }; + using var rejected = await http.PostAsJsonAsync("/openai/v1/chat/completions", new + { model = "gpt-4.1-mini", messages = new[] { new { role = "user", content = "schema" } }, tools = incompatible }); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + await Assert.That(await rejected.Content.ReadAsStringAsync()).Contains("llm_test_server_fixture_mismatch"); + using var accepted = await http.PostAsJsonAsync("/openai/v1/chat/completions", new + { model = "gpt-4.1-mini", messages = new[] { new { role = "user", content = "schema" } }, tools = Tools(fixture) }); + accepted.EnsureSuccessStatusCode(); + var call = (await accepted.Content.ReadFromJsonAsync()).GetProperty("choices")[0].GetProperty("message").GetProperty("tool_calls")[0]; + await Assert.That(call.GetProperty("id").GetString()).IsEqualTo(fixture.Call.Id); + } + + [Test] + public async Task Preset_ToolResultRequiresExactArgumentsAndResultBeforeGraphCanAdvanceAsync() + { + var fixture = LlmTestServerWellKnownTools.Calculator.Success(new() { Expression = "6 * 7" }, new() { Value = 42m }, "call_exact_calculator"); + using var host = await LlmTestServerTestHost.StartAsync(config => config.AddConversation("exact", graph => graph + .Step("call", step => step.CallsTools(fixture.Call).Then("result")) + .Step("result", step => step.WhenToolResult(fixture.Call.Name, fixture.Call.Id).WithToolArgumentsJson(fixture.Call.ArgumentsJson) + .WhenLatestJson("$", fixture.Result.Content).Responds("accepted").Complete()))); + using var http = host.GetTestClient(); + var tools = Tools(fixture); + using var initial = await http.PostAsJsonAsync("/openai/v1/chat/completions", new + { model = "gpt-4.1-mini", messages = new[] { new { role = "user", content = "calculate" } }, tools }); + initial.EnsureSuccessStatusCode(); + object[] History(string arguments, string result) + { + return [ + new { role = "user", content = "calculate" }, + new { role = "assistant", content = "", tool_calls = new[] { new { id = fixture.Call.Id, type = "function", function = new { name = fixture.Call.Name, arguments } } } }, + new { role = "tool", tool_call_id = fixture.Call.Id, content = result }, + ]; + } + + foreach (var history in new[] { History("{\"expression\":\"7 * 7\"}", fixture.Result.Content), History(fixture.Call.ArgumentsJson, "{\"value\":49}") }) + { + using var rejected = await http.PostAsJsonAsync("/openai/v1/chat/completions", new { model = "gpt-4.1-mini", messages = history, tools }); + await Assert.That(rejected.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + await Assert.That(await rejected.Content.ReadAsStringAsync()).Contains("conversation_step_mismatch"); + } + await Assert.That((await new LlmTestServerClient(http).GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); + using var accepted = await http.PostAsJsonAsync("/openai/v1/chat/completions", new { model = "gpt-4.1-mini", messages = History(fixture.Call.ArgumentsJson, fixture.Result.Content), tools }); + accepted.EnsureSuccessStatusCode(); + await Assert.That(await accepted.Content.ReadAsStringAsync()).Contains("accepted"); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Tools/WellKnownToolFaultIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Tools/WellKnownToolFaultIntegrationTests.cs new file mode 100644 index 0000000..2c98688 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Tools/WellKnownToolFaultIntegrationTests.cs @@ -0,0 +1,121 @@ +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Tests.TestSupport; +using ManagedCode.LlmTestServer.Tools.WellKnown; +using OpenAI; +using OpenAI.Chat; + +namespace ManagedCode.LlmTestServer.Tests.Tools; + +public sealed partial class WellKnownToolFaultIntegrationTests +{ + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task NullAndEmptyCollection_RejectMissingAndWrongResultsThenRecoverWithCacheAndHistoryAsync(bool emptyCollection) + { + var preset = LlmTestServerWellKnownTools.WebSearch; + var arguments = new LlmTestServerWebSearchArguments { Query = "fixture evidence" }; + var fixture = emptyCollection + ? preset.Success(arguments, new() { Results = [] }, "call_search_outcome") + : preset.NullResult(arguments, "call_search_outcome"); + var expectedResult = fixture.Result.Content; + using var host = await LlmTestServerTestHost.StartAsync(config => config.AddConversation("outcome", graph => graph + .Step("call", step => step.WhenLatestUserEquals("lookup").CallsTools(fixture.Call).Then("result")) + .Step("result", step => step.WhenToolResult(fixture.Call.Name, fixture.Call.Id) + .WithToolArgumentsJson(fixture.Call.ArgumentsJson).WhenLatestJson("$", expectedResult) + .Responds("outcome accepted").Complete()))); + using var http = host.GetTestClient(); + using var conversation = new LlmTestServerClient(http).ForConversation("outcome-isolation"); + conversation.HttpClient.DefaultRequestHeaders.Authorization = new("Bearer", "tool-fault-key"); + var sdk = CreateSdk(conversation.HttpClient); + var options = new ChatCompletionOptions + { + Tools = { ChatTool.CreateFunctionTool(fixture.Definition.Name, fixture.Definition.Description, + BinaryData.FromString(fixture.Definition.ParametersJson)) }, + }; + var system = string.Join(' ', Enumerable.Repeat("Retained system reference", 1200)); + List messages = [new SystemChatMessage(system), new UserChatMessage("lookup")]; + var first = (await sdk.CompleteChatAsync(messages, options)).Value; + await Assert.That(first.ToolCalls.Single().Id).IsEqualTo(fixture.Call.Id); + messages.Add(new AssistantChatMessage(first)); + var tools = new[] { new { type = "function", function = new { name = fixture.Definition.Name, + parameters = JsonSerializer.Deserialize(fixture.Definition.ParametersJson) } } }; + var assistant = JsonSerializer.SerializeToElement(new + { + role = "assistant", + tool_calls = new[] { new + { id = first.ToolCalls.Single().Id, type = "function", function = new { name = first.ToolCalls.Single().FunctionName, + arguments = first.ToolCalls.Single().FunctionArguments.ToString() } } } + }); + object[] prefix = [new { role = "system", content = system }, new { role = "user", content = "lookup" }, assistant]; + using var missing = await conversation.HttpClient.PostAsJsonAsync("/openai/v1/chat/completions", new + { model = LlmTestServerKnownModelIds.Gpt41Mini, tools, messages = prefix.Append(new { role = "user", content = "continue without a result" }).ToArray() }); + await Assert.That(missing.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + await Assert.That((await conversation.Control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); + var missingEvidence = await conversation.Control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending }); + await Assert.That(missingEvidence.Requests[^1].Messages.Count(message => message.ToolCallId == fixture.Call.Id)).IsEqualTo(0); + var wrongResult = emptyCollection ? "null" : "{\"results\":[]}"; + using var wrong = await conversation.HttpClient.PostAsJsonAsync("/openai/v1/chat/completions", new + { model = LlmTestServerKnownModelIds.Gpt41Mini, tools, messages = prefix.Append(new { role = "tool", tool_call_id = fixture.Call.Id, content = wrongResult }).ToArray() }); + await Assert.That(wrong.StatusCode).IsEqualTo(HttpStatusCode.Conflict); + await Assert.That((await conversation.Control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); + messages.Add(new ToolChatMessage(fixture.Result.ToolCallId, expectedResult)); + var accepted = (await sdk.CompleteChatAsync(messages, options)).Value; + await Assert.That(accepted.Content.Single().Text).IsEqualTo("outcome accepted"); + var replay = (await sdk.CompleteChatAsync(messages, options)).Value; + await Assert.That(replay.Id).IsEqualTo(accepted.Id); + var evidence = await conversation.Control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending }); + await Assert.That(evidence.Requests.Select(request => request.StatusCode)).IsEquivalentTo(new int?[] { 200, 409, 409, 200, 200 }); + await Assert.That(evidence.Requests[0].Usage!.CachedInputTokens).IsEqualTo(0); + await Assert.That(evidence.Requests[0].Usage!.CacheCreationInputTokens).IsGreaterThan(0); + await Assert.That(evidence.Requests.Where(request => request.StatusCode == 409).All(request => request.Usage!.CacheCreationInputTokens == 0)).IsTrue(); + await Assert.That(evidence.Requests[3].Usage!.CachedInputTokens).IsGreaterThan(0); + await Assert.That(evidence.Requests[3].Messages.Single(message => message.Role == "tool").Content).IsEqualTo(expectedResult); + await Assert.That(evidence.Requests[^1].IsReplay).IsTrue(); + var state = (await conversation.Control.GetConversationsAsync()).Single(); + await Assert.That(state.ExecutedSteps).IsEqualTo(2); + await Assert.That(state.IsTerminal).IsTrue(); + } + + [Test] + public async Task EmptyToolText_IsPresentAndDistinctFromLiteralNullAsync() + { + var preset = LlmTestServerWellKnownTools.ReadFile; + var call = preset.CreateCall(new() { Path = "fixture.txt" }, "call_empty_text"); + using var host = await LlmTestServerTestHost.StartAsync(config => config.AddConversation("empty-text", graph => graph + .Step("call", step => step.CallsTools(call).Then("result")) + .Step("result", step => step.WhenToolResult(call.Name, call.Id).Responds("empty text observed").Complete()))); + using var http = host.GetTestClient(); + using var conversation = new LlmTestServerClient(http).ForConversation("empty-text"); + var sdk = CreateSdk(conversation.HttpClient); + var options = new ChatCompletionOptions + { + Tools = { ChatTool.CreateFunctionTool(preset.Definition.Name, preset.Definition.Description, + BinaryData.FromString(preset.Definition.ParametersJson)) }, + }; + List messages = [new UserChatMessage("read fixture")]; + var first = (await sdk.CompleteChatAsync(messages, options)).Value; + messages.Add(new AssistantChatMessage(first)); + messages.Add(new ToolChatMessage(call.Id, string.Empty)); + var accepted = (await sdk.CompleteChatAsync(messages, options)).Value; + await Assert.That(accepted.Content.Single().Text).IsEqualTo("empty text observed"); + var evidence = await conversation.Control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending }); + var result = evidence.Requests[^1].Messages.Single(message => message.Role == "tool"); + await Assert.That(result.ToolCallId).IsEqualTo(call.Id); + await Assert.That(result.Content).IsEqualTo(string.Empty); + await Assert.That((await conversation.Control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(2); + } + + private static global::OpenAI.Chat.ChatClient CreateSdk(HttpClient http) + { + return new(LlmTestServerKnownModelIds.Gpt41Mini, new ApiKeyCredential("tool-fault-key"), new OpenAIClientOptions + { Endpoint = new Uri(http.BaseAddress!, "/openai/v1"), Transport = new HttpClientPipelineTransport(http), RetryPolicy = new ClientRetryPolicy(0) }); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Tools/WellKnownToolFaultProviderIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Tools/WellKnownToolFaultProviderIntegrationTests.cs new file mode 100644 index 0000000..3a6c2ea --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Tools/WellKnownToolFaultProviderIntegrationTests.cs @@ -0,0 +1,182 @@ +using System.ClientModel; +using System.Net; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Configuration; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Hosting; +using ManagedCode.LlmTestServer.Tools.WellKnown; +using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.Hosting; +using OpenAI.Chat; + +namespace ManagedCode.LlmTestServer.Tests.Tools; + +public sealed partial class WellKnownToolFaultIntegrationTests +{ + [Test] + public async Task Provider500_IsOneScriptedAttemptThenExplicitRetryReturnsToolWithoutWarmingErrorCacheAsync() + { + await using var app = await StartHealthyFaultHostAsync(config => config.AddConversation("provider-failure", graph => graph + .Step("failure", step => step.WhenLatestUserEquals("run tool") + .Fails(500, "fixture_provider_internal_error", "Scripted provider failure").Then("recover")) + .Step("recover", step => step.WhenLatestUserEquals("run tool") + .CallsTool("weather", "{}", "call_after_provider_error").Complete()))); + using var http = app.GetTestClient(); + using var conversation = new LlmTestServerClient(http).ForConversation("provider-failure"); + var sdk = CreateSdk(conversation.HttpClient); + var options = new ChatCompletionOptions { Tools = { ChatTool.CreateFunctionTool("weather") } }; + List messages = [new SystemChatMessage(LongCachePrefix()), new UserChatMessage("run tool")]; + ClientResultException? failure = null; + try { await sdk.CompleteChatAsync(messages, options); } + catch (ClientResultException exception) { failure = exception; } + await Assert.That(failure?.Status).IsEqualTo(500); + var failed = await conversation.Control.GetRequestsAsync(); + await Assert.That(failed.Requests.Count).IsEqualTo(1); + await Assert.That(failed.Requests.Single().Usage!.CacheCreationInputTokens).IsEqualTo(0); + await Assert.That(failed.Requests.Single().ResponseToolCalls.Count).IsEqualTo(0); + await Assert.That((await conversation.Control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); + using var healthy = await http.GetAsync("/health"); + await Assert.That(healthy.StatusCode).IsEqualTo(HttpStatusCode.OK); + var recovered = (await sdk.CompleteChatAsync(messages, options)).Value; + await Assert.That(recovered.ToolCalls.Single().Id).IsEqualTo("call_after_provider_error"); + var evidence = await conversation.Control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending }); + await Assert.That(evidence.Requests.Select(request => request.StatusCode)).IsEquivalentTo(new int?[] { 500, 200 }); + await Assert.That(evidence.Requests[^1].Usage!.CachedInputTokens).IsEqualTo(0); + await Assert.That(evidence.Requests[^1].Usage!.CacheCreationInputTokens).IsGreaterThan(0); + var state = (await conversation.Control.GetConversationsAsync()).Single(); + await Assert.That(state.ExecutedSteps).IsEqualTo(2); + await Assert.That(state.IsTerminal).IsTrue(); + } + + [Test] + public async Task ProviderDelay_CancellationRollsBackQueuedToolAndCacheBeforeExplicitRetryAsync() + { + await using var app = await StartHealthyFaultHostAsync(config => config + .AddConversation("delayed-tool", graph => graph + .Step("call", step => step.WhenLatestUserEquals("run tool").CallsTool("weather", "{}", "call_delayed_fault") + .DelaysBy(10000).Complete())) + .AddConversation("startup", graph => graph + .Step("ready", step => step.WhenLatestUserEquals("startup").Responds("provider ready").Complete()))); + using var http = app.GetTestClient(); + var root = new LlmTestServerClient(http); + // Initialize real routing, tokenization and public state serialization before measuring the pending reservation. + // This unrelated short prompt cannot populate the target's long system-prefix cache. + using (var startup = root.ForConversation("startup")) + { + using var startupWatchdog = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + var ready = (await CreateSdk(startup.HttpClient).CompleteChatAsync([new UserChatMessage("startup")], cancellationToken: startupWatchdog.Token)).Value; + await Assert.That(ready.Content.Single().Text).IsEqualTo("provider ready"); + await Assert.That((await startup.Control.GetConversationsAsync(startupWatchdog.Token)).Single().ExecutedSteps).IsEqualTo(1); + } + using var conversation = root.ForConversation("delayed-tool"); + var sdk = CreateSdk(conversation.HttpClient); + var options = new ChatCompletionOptions { Tools = { ChatTool.CreateFunctionTool("weather") } }; + List messages = [new SystemChatMessage(LongCachePrefix()), new UserChatMessage("run tool")]; + using var watchdog = new CancellationTokenSource(TimeSpan.FromSeconds(60)); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(watchdog.Token); + var pending = sdk.CompleteChatAsync(messages, options, cancellation.Token); + try + { + while (true) + { + var observed = conversation.Control.GetConversationsAsync(watchdog.Token); + if (await Task.WhenAny(pending, observed) == pending) + { + await observed; + await pending; + throw new InvalidOperationException("The delayed SDK request completed before its busy reservation was observed."); + } + if ((await observed).Any(state => state.IsBusy)) { break; } + await Task.Delay(10, watchdog.Token); + } + cancellation.Cancel(); + var cancelled = false; + try { await pending; } + catch (OperationCanceledException) when (cancellation.IsCancellationRequested) { cancelled = true; } + await Assert.That(cancelled).IsTrue(); + while ((await conversation.Control.GetConversationsAsync(watchdog.Token)).Any(state => state.IsBusy)) + { await Task.Delay(10, watchdog.Token); } + await Assert.That((await conversation.Control.GetConversationsAsync(watchdog.Token)).Count).IsEqualTo(0); + // Retry the original input against the SAME graph and cache; no configure/reset can mask rollback. + var recovered = (await sdk.CompleteChatAsync(messages, options, watchdog.Token)).Value; + await Assert.That(recovered.ToolCalls.Single().Id).IsEqualTo("call_delayed_fault"); + var evidence = await conversation.Control.GetRequestsAsync(new() { Conversation = conversation.Id, Order = LlmTestServerRequestOrder.Ascending }, watchdog.Token); + await Assert.That(evidence.Requests.Count).IsEqualTo(2); + await Assert.That((await root.GetRequestsAsync(cancellationToken: watchdog.Token)).TotalObserved).IsEqualTo(3); + await Assert.That(evidence.Requests[0].TransportState is LlmTestServerProviderHttpTraceTransportState.Cancelled + or LlmTestServerProviderHttpTraceTransportState.Aborted).IsTrue(); + await Assert.That(evidence.Requests[0].ResponseToolCalls.Count).IsEqualTo(0); + await Assert.That(evidence.Requests[^1].Usage!.CachedInputTokens).IsEqualTo(0); + await Assert.That(evidence.Requests[^1].Usage!.CacheCreationInputTokens).IsGreaterThan(0); + await Assert.That((await conversation.Control.GetConversationsAsync(watchdog.Token)).Single().ExecutedSteps).IsEqualTo(1); + using var healthy = await http.GetAsync("/health", watchdog.Token); + await Assert.That(healthy.StatusCode).IsEqualTo(HttpStatusCode.OK); + } + finally + { + cancellation.Cancel(); + try { await pending.WaitAsync(TimeSpan.FromSeconds(60)); } + catch (OperationCanceledException) when (cancellation.IsCancellationRequested) { } + } + } + + [Test] + public async Task InputFilter_RejectsToolResultWithoutAdvancingThenAcceptsCleanNullResultAsync() + { + var fixture = LlmTestServerWellKnownTools.Weather.NullResult(new() { Location = "Paris" }, "call_filtered_tool"); + await using var app = await StartHealthyFaultHostAsync(config => config.SimulateContentFilter("blocked_tool_result") + .AddConversation("filtered-tool", graph => graph + .Step("call", step => step.CallsTools(fixture.Call).Then("result")) + .Step("result", step => step.WhenToolResult(fixture.Call.Name, fixture.Call.Id) + .WhenLatestJson("$", fixture.Result.Content).Responds("clean result accepted").Complete()))); + using var http = app.GetTestClient(); + using var conversation = new LlmTestServerClient(http).ForConversation("filtered-tool"); + var sdk = CreateSdk(conversation.HttpClient); + var options = new ChatCompletionOptions + { + Tools = { ChatTool.CreateFunctionTool(fixture.Definition.Name, fixture.Definition.Description, + BinaryData.FromString(fixture.Definition.ParametersJson)) }, + }; + List messages = [new SystemChatMessage(LongCachePrefix()), new UserChatMessage("weather")]; + var call = (await sdk.CompleteChatAsync(messages, options)).Value; + messages.Add(new AssistantChatMessage(call)); + messages.Add(new ToolChatMessage(fixture.Call.Id, "blocked_tool_result")); + ClientResultException? failure = null; + try { await sdk.CompleteChatAsync(messages, options); } + catch (ClientResultException exception) { failure = exception; } + await Assert.That(failure?.Status).IsEqualTo(400); + var filtered = await conversation.Control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending }); + await Assert.That(filtered.Requests[^1].ErrorCode).IsEqualTo("content_filter"); + await Assert.That(filtered.Requests[^1].Usage!.CacheCreationInputTokens).IsEqualTo(0); + await Assert.That(filtered.Requests[^1].Messages.Single(message => message.Role == "tool").Content).IsEqualTo("blocked_tool_result"); + await Assert.That((await conversation.Control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); + messages[^1] = new ToolChatMessage(fixture.Call.Id, fixture.Result.Content); + var accepted = (await sdk.CompleteChatAsync(messages, options)).Value; + await Assert.That(accepted.Content.Single().Text).IsEqualTo("clean result accepted"); + var evidence = await conversation.Control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending }); + await Assert.That(evidence.Requests.Select(request => request.StatusCode)).IsEquivalentTo(new int?[] { 200, 400, 200 }); + await Assert.That(evidence.Requests[^1].Usage!.CachedInputTokens).IsGreaterThan(0); + await Assert.That(evidence.Requests[^1].Messages.Single(message => message.Role == "tool").Content).IsEqualTo("null"); + await Assert.That((await conversation.Control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(2); + using var healthy = await http.GetAsync("/health"); + await Assert.That(healthy.StatusCode).IsEqualTo(HttpStatusCode.OK); + } + + private static async Task StartHealthyFaultHostAsync(Action configure) + { + var builder = WebApplication.CreateBuilder(new WebApplicationOptions { EnvironmentName = Environments.Development }); + builder.WebHost.UseTestServer(); + builder.Services.AddLlmTestServer(configure); + builder.AddDefaultHealthChecks(); + var app = builder.Build(); + app.MapLlmTestServer(); + app.MapDefaultEndpoints(); + await app.StartAsync(); + return app; + } + + private static string LongCachePrefix() + { + return string.Join(' ', Enumerable.Repeat("Retained system reference", 1200)); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Tools/WellKnownToolFaultSdkIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Tools/WellKnownToolFaultSdkIntegrationTests.cs new file mode 100644 index 0000000..dd90d74 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Tools/WellKnownToolFaultSdkIntegrationTests.cs @@ -0,0 +1,164 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Models; +using ManagedCode.LlmTestServer.Tests.TestSupport; +using ManagedCode.LlmTestServer.Tools.WellKnown; +using Microsoft.Extensions.AI; + +namespace ManagedCode.LlmTestServer.Tests.Tools; + +public sealed partial class WellKnownToolFaultIntegrationTests +{ + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task HostedSdkFunctionLoop_ReturnsExplicitNullThroughTypedPresetAsync(bool streaming) + { + var fixture = LlmTestServerWellKnownTools.WebSearch.NullResult(new() { Query = "scripted" }, "call_null_loop"); + using var host = await LlmTestServerTestHost.StartAsync(config => config.AddConversation("null-loop", graph => graph + .Step("call", step => step.CallsTools(fixture.Call).Then("result")) + .Step("result", step => step.WhenToolResult(fixture.Call.Name, fixture.Call.Id) + .WhenLatestJson("$", fixture.Result.Content).Responds("null observed").Complete()))); + using var http = host.GetTestClient(); + using var conversation = new LlmTestServerClient(http).ForConversation("null-loop"); + using var client = new FunctionInvokingChatClient(CreateSdk(conversation.HttpClient).AsIChatClient()); + var invocations = 0; + string? observedQuery = null; + var observedLimit = 0; + object? Search(string query, int limit) + { + invocations++; + observedQuery = query; + observedLimit = limit; + return null; + } + var options = new ChatOptions { Tools = [AIFunctionFactory.Create(Search, fixture.Definition.Name)] }; + var answer = streaming + ? await client.GetStreamingResponseAsync("lookup", options).ToChatResponseAsync() + : await client.GetResponseAsync("lookup", options); + await Assert.That(answer.Text).IsEqualTo("null observed"); + await Assert.That(invocations).IsEqualTo(1); + await Assert.That(observedQuery).IsEqualTo("scripted"); + await Assert.That(observedLimit).IsEqualTo(5); + var evidence = await conversation.Control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending }); + await Assert.That(evidence.Requests.Count).IsEqualTo(2); + await Assert.That(evidence.Requests[^1].Messages.Single(message => message.Role == "tool").Content).IsEqualTo("null"); + await Assert.That((await conversation.Control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(2); + } + + [Test] + [Arguments(0, false)] + [Arguments(0, true)] + [Arguments(1, false)] + [Arguments(1, true)] + [Arguments(2, false)] + [Arguments(2, true)] + public async Task HostedSdkFunctionLoop_RecoversFromActualControlledExceptionsWithoutProviderHttpErrorsAsync(int kind, bool streaming) + { + var marker = kind switch { 0 => "fixture_tool_timeout", 1 => "fixture_tool_internal_crash", _ => "fixture_tool_content_filtered" }; + using var host = await LlmTestServerTestHost.StartAsync(config => config.AddConversation("throwing-tool", graph => graph + .Step("call", step => step.CallsTool("controlled_tool", "{}", "call_throwing_tool").Then("recover")) + .Step("recover", step => step.WhenToolResult("controlled_tool", "call_throwing_tool") + .WhenLatestTextContains(marker).Responds("caller failure recovered").Complete()))); + using var http = host.GetTestClient(); + using var conversation = new LlmTestServerClient(http).ForConversation("throwing-tool"); + using var client = new FunctionInvokingChatClient(CreateSdk(conversation.HttpClient).AsIChatClient()) + { IncludeDetailedErrors = true }; + var invocations = 0; + string Invoke() + { + invocations++; + throw kind switch + { + 0 => new TimeoutException(marker), + 1 => new InvalidOperationException(marker), + _ => new UnauthorizedAccessException(marker), + }; + } + var options = new ChatOptions { Tools = [AIFunctionFactory.Create(Invoke, "controlled_tool")] }; + var answer = streaming + ? await client.GetStreamingResponseAsync("run controlled tool", options).ToChatResponseAsync() + : await client.GetResponseAsync("run controlled tool", options); + await Assert.That(answer.Text).IsEqualTo("caller failure recovered"); + await Assert.That(invocations).IsEqualTo(1); + var evidence = await conversation.Control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending }); + await Assert.That(evidence.Requests.Count).IsEqualTo(2); + await Assert.That(evidence.Requests.All(request => request.StatusCode == 200 && request.ErrorCode is null)).IsTrue(); + var observed = evidence.Requests[^1].Messages.Single(message => message.Role == "tool"); + await Assert.That(observed.ToolCallId).IsEqualTo("call_throwing_tool"); + await Assert.That(observed.Content).Contains(marker); + var state = (await conversation.Control.GetConversationsAsync()).Single(); + await Assert.That(state.ExecutedSteps).IsEqualTo(2); + await Assert.That(state.IsTerminal).IsTrue(); + } + + [Test] + public async Task HostedSdkFunctionCancellation_OmitsResultAndAcceptsExplicitLaterRecoveryAsync() + { + const string callId = "call_cancellable_tool"; + using var host = await LlmTestServerTestHost.StartAsync(config => config.AddConversation("cancellable-tool", graph => graph + .Step("call", step => step.CallsTool("cancellable_tool", "{}", callId).Then("recover")) + .Step("recover", step => step.WhenToolResult("cancellable_tool", callId) + .WhenLatestJson("$.error.code", "\"fixture_tool_timeout\"").Responds("timeout recovered").Complete()))); + using var http = host.GetTestClient(); + using var conversation = new LlmTestServerClient(http).ForConversation("cancellable-tool"); + conversation.HttpClient.DefaultRequestHeaders.Authorization = new("Bearer", "tool-fault-key"); + using var client = new FunctionInvokingChatClient(CreateSdk(conversation.HttpClient).AsIChatClient()); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var invocations = 0; + async Task Invoke(CancellationToken cancellationToken) + { + invocations++; + entered.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + return "unreachable"; + } + var options = new ChatOptions { Tools = [AIFunctionFactory.Create(Invoke, "cancellable_tool")] }; + using var cancellation = new CancellationTokenSource(); + var pending = client.GetResponseAsync("run cancellable tool", options, cancellation.Token); + try + { + // This bounds startup under instrumentation; cancellation is triggered only after the real invocation enters. + var started = await Task.WhenAny(entered.Task, pending).WaitAsync(TimeSpan.FromSeconds(60)); + if (started == pending) { await pending; } + await entered.Task.WaitAsync(TimeSpan.FromSeconds(60)); + } + catch + { + cancellation.Cancel(); + try { await pending; } + catch (OperationCanceledException) when (cancellation.IsCancellationRequested) { } + throw; + } + finally { cancellation.Cancel(); } + var cancelled = false; + try { await pending; } + catch (OperationCanceledException) when (cancellation.IsCancellationRequested) { cancelled = true; } + await Assert.That(cancelled).IsTrue(); + await Assert.That(invocations).IsEqualTo(1); + var before = await conversation.Control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending }); + await Assert.That(before.Requests.Count).IsEqualTo(1); + await Assert.That(before.Requests.Single().Messages.Any(message => message.Role == "tool")).IsFalse(); + await Assert.That((await conversation.Control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); + using var responseBody = JsonDocument.Parse(before.Requests.Single().ResponsePreview!); + var assistant = responseBody.RootElement.GetProperty("choices")[0].GetProperty("message").Clone(); + var toolError = LlmTestServerWellKnownTools.Shell.CreateError(new() + { Code = "fixture_tool_timeout", Message = "Controlled invocation was cancelled", Retryable = true }, callId); + using var recovered = await conversation.HttpClient.PostAsJsonAsync("/openai/v1/chat/completions", new + { + model = LlmTestServerKnownModelIds.Gpt41Mini, + tools = new[] { new { type = "function", function = new { name = "cancellable_tool", parameters = new { type = "object" } } } }, + messages = new object[] { new { role = "user", content = "run cancellable tool" }, assistant, + new { role = "tool", tool_call_id = callId, content = toolError.Content } }, + }); + await Assert.That(recovered.StatusCode).IsEqualTo(HttpStatusCode.OK); + var payload = await recovered.Content.ReadFromJsonAsync(); + await Assert.That(payload.GetProperty("choices")[0].GetProperty("message").GetProperty("content").GetString()).IsEqualTo("timeout recovered"); + var after = await conversation.Control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending }); + await Assert.That(after.Requests[^1].Messages.Single(message => message.Role == "tool").Content).IsEqualTo(toolError.Content); + await Assert.That((await conversation.Control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(2); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Tools/WellKnownToolIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Tools/WellKnownToolIntegrationTests.cs new file mode 100644 index 0000000..84da7e7 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Tools/WellKnownToolIntegrationTests.cs @@ -0,0 +1,136 @@ +using System.Net.Http.Json; +using System.Text.Json; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Tests.TestSupport; +using ManagedCode.LlmTestServer.Tools.WellKnown; +using ManagedCode.LlmTestServer.Verification; + +namespace ManagedCode.LlmTestServer.Tests.Tools; + +public sealed partial class WellKnownToolIntegrationTests +{ + [Test] + [Arguments(0, false)] + [Arguments(1, false)] + [Arguments(2, false)] + [Arguments(3, false)] + [Arguments(4, false)] + [Arguments(5, false)] + [Arguments(6, false)] + [Arguments(7, false)] + [Arguments(8, false)] + [Arguments(9, false)] + [Arguments(10, false)] + [Arguments(11, false)] + [Arguments(0, true)] + [Arguments(1, true)] + [Arguments(2, true)] + [Arguments(3, true)] + [Arguments(4, true)] + [Arguments(5, true)] + [Arguments(6, true)] + [Arguments(7, true)] + [Arguments(8, true)] + [Arguments(9, true)] + [Arguments(10, true)] + [Arguments(11, true)] + public async Task WellKnownPreset_UsesTypedScriptedCallAndExplicitSuccessOrErrorResultAsync(int kind, bool error) + { + var fixture = CreateFixture(kind, error); + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(config => config.UseConversation("tool-preset", graph => graph + .Step("call", step => step.WhenLatestUserEquals("run fixture").CallsTools(fixture.Call).DelaysBy(fixture.DelayMilliseconds).Then("result")) + .Step("result", step => step.WhenToolResult(fixture.Call.Name, fixture.Call.Id).WithToolArgumentsJson(fixture.Call.ArgumentsJson) + .WhenLatestJson("$", fixture.Result.Content).RespondsJson(fixture.Result.Content).Complete()))); + using var conversation = control.ForConversation("typed-preset"); + var tools = Tools(fixture); + using var first = await conversation.HttpClient.PostAsJsonAsync("/openai/v1/chat/completions", new + { model = "gpt-4.1-mini", messages = new[] { new { role = "user", content = "run fixture" } }, tools }); + first.EnsureSuccessStatusCode(); + var message = (await first.Content.ReadFromJsonAsync()).GetProperty("choices")[0].GetProperty("message"); + var call = message.GetProperty("tool_calls")[0]; + await Assert.That(call.GetProperty("id").GetString()).IsEqualTo(fixture.Result.ToolCallId); + await Assert.That(call.GetProperty("function").GetProperty("name").GetString()).IsEqualTo(fixture.Definition.Name); + using var returnedArguments = JsonDocument.Parse(call.GetProperty("function").GetProperty("arguments").GetString()!); + using var expectedArguments = JsonDocument.Parse(fixture.Call.ArgumentsJson); + await Assert.That(JsonElement.DeepEquals(returnedArguments.RootElement, expectedArguments.RootElement)).IsTrue(); + object[] history = + [ + new { role = "user", content = "run fixture" }, message, + new { role = fixture.Result.Role, tool_call_id = fixture.Result.ToolCallId, content = fixture.Result.Content }, + ]; + var descriptor = LlmTestServerWellKnownTools.All.Single(tool => tool.Name == fixture.Definition.Name); + using var second = await conversation.HttpClient.PostAsJsonAsync("/openai/v1/chat/completions", new + { + model = "gpt-4.1-mini", + messages = history, + tools, + response_format = new + { + type = "json_schema", + json_schema = new + { + name = "fixture_result", + schema = + JsonSerializer.Deserialize(error ? descriptor.ErrorSchemaJson : descriptor.ResultSchemaJson) + } + }, + }); + second.EnsureSuccessStatusCode(); + var answer = (await second.Content.ReadFromJsonAsync()).GetProperty("choices")[0].GetProperty("message").GetProperty("content").GetString(); + using var returnedResult = JsonDocument.Parse(answer!); + using var expectedResult = JsonDocument.Parse(fixture.Result.Content); + await Assert.That(JsonElement.DeepEquals(returnedResult.RootElement, expectedResult.RootElement)).IsTrue(); + var verification = await conversation.Control.VerifyAsync(new() + { + NoUnexpectedCalls = true, + Requests = + [ + new() { Query = new() { Step = "call" }, Tools = [new() { Name = fixture.Definition.Name, CallId = fixture.Call.Id, ArgumentsJson = fixture.Call.ArgumentsJson }] }, + new() { Query = new() { Step = "result" }, Tools = [new() { Kind = LlmTestServerToolEvidenceKind.Result, Name = fixture.Definition.Name, + CallId = fixture.Call.Id, ResultJson = fixture.Result.Content }] }, + ], + Conversations = [new() { ScenarioId = "tool-preset", VisitedSteps = ["call", "result"], RequireTerminal = true }], + }); + await Assert.That(verification.Passed).IsTrue(); + var status = (await conversation.Control.GetConversationsAsync()).Single(); + await Assert.That(status.ExecutedSteps).IsEqualTo(2); + await Assert.That(status.LastTurnIndex).IsEqualTo(2); + } + + private static object[] Tools(LlmTestServerWellKnownToolFixture fixture) + { + return [new { type = "function", function = new { name = fixture.Definition.Name, description = fixture.Definition.Description, + parameters = JsonSerializer.Deserialize(fixture.Definition.ParametersJson) } }]; + } + + private static LlmTestServerWellKnownToolFixture CreateFixture(int kind, bool error = false) + { + LlmTestServerWellKnownToolFixture Fixture(LlmTestServerWellKnownTool tool, TArguments arguments, TResult result) + where TArguments : class where TResult : class + { + return error + ? tool.Failure(arguments, new() { Code = "fixture_unavailable", Message = "Scripted tool failure", Retryable = true }) + : tool.Success(arguments, result); + } + + return kind switch + { + 0 => Fixture(LlmTestServerWellKnownTools.Calculator, new() { Expression = "6 * 7" }, new() { Value = 42m }), + 1 => Fixture(LlmTestServerWellKnownTools.CurrentTime, new(), new() { CurrentTime = new DateTimeOffset(2026, 1, 2, 3, 4, 5, TimeSpan.Zero) }), + 2 => Fixture(LlmTestServerWellKnownTools.Weather, new() { Location = "Paris", Unit = "celsius" }, new() { Location = "Paris", Temperature = 21m, Unit = "celsius", Condition = "sunny" }), + 3 => Fixture(LlmTestServerWellKnownTools.WebSearch, new() { Query = "fixture search", Limit = 3 }, new() { Results = [new() { Title = "Fixture", Url = "https://example.com/fixture", Snippet = "scripted search hit" }] }), + 4 => Fixture(LlmTestServerWellKnownTools.FetchUrl, new() { Url = "https://example.com/fixture" }, new() { Url = "https://example.com/fixture", ContentType = "text/plain", Content = "scripted page", StatusCode = 200 }), + 5 => Fixture(LlmTestServerWellKnownTools.FileSearch, new() { Query = "fixture", Limit = 2 }, new() { Results = [new() { FileId = "file_fixture", Path = "docs/fixture.txt", Snippet = "scripted file hit" }] }), + 6 => Fixture(LlmTestServerWellKnownTools.ReadFile, new() { Path = "docs/fixture.txt", Offset = 0, Length = 7 }, new() { Path = "docs/fixture.txt", Content = "fixture", Truncated = true }), + 7 => Fixture(LlmTestServerWellKnownTools.WriteFile, new() { Path = "docs/fixture.txt", Content = "fixture", Append = true }, new() { Path = "docs/fixture.txt", BytesWritten = 7 }), + 8 => Fixture(LlmTestServerWellKnownTools.ListFiles, new() { Path = "docs", Recursive = true }, new() { Entries = [new() { Path = "docs/fixture.txt", IsDirectory = false, Bytes = 7 }] }), + 9 => Fixture(LlmTestServerWellKnownTools.RunCode, new() { Language = "python", Code = "print(42)" }, new() { Stdout = "42\n", Stderr = "", ExitCode = 0 }), + 10 => Fixture(LlmTestServerWellKnownTools.Shell, new() { Command = "echo fixture", WorkingDirectory = "workspace" }, new() { Stdout = "fixture\n", Stderr = "", ExitCode = 0 }), + 11 => Fixture(LlmTestServerWellKnownTools.ApplyPatch, new() { Patch = "*** Begin Patch\n*** Add File: fixture.txt\n+fixture\n*** End Patch" }, new() { UpdatedPaths = ["fixture.txt"] }), + _ => throw new ArgumentOutOfRangeException(nameof(kind)), + }; + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Tools/WellKnownToolLifecycleTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Tools/WellKnownToolLifecycleTests.cs new file mode 100644 index 0000000..8a391de --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Tools/WellKnownToolLifecycleTests.cs @@ -0,0 +1,72 @@ +using System.Net.Http.Json; +using System.Text.Json; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Conversations; +using ManagedCode.LlmTestServer.Tests.TestSupport; +using ManagedCode.LlmTestServer.Tools.WellKnown; + +namespace ManagedCode.LlmTestServer.Tests.Tools; + +public sealed partial class WellKnownToolIntegrationTests +{ + [Test] + public async Task Preset_RepeatsThroughExistingConversationPolicyWithStableCallIdentityAsync() + { + var fixture = LlmTestServerWellKnownTools.CurrentTime.Success(new(), + new() { CurrentTime = new DateTimeOffset(2026, 1, 2, 3, 4, 5, TimeSpan.Zero) }, "call_scripted_time"); + using var host = await LlmTestServerTestHost.StartAsync(config => config.AddConversation("repeat-preset", graph => graph + .OnExhaustion(LlmTestServerConversationExhaustion.RepeatLastResponse) + .Step("call", step => step.CallsTools(fixture.Call).Complete()))); + using var http = host.GetTestClient(); + var tools = Tools(fixture); + using var first = await http.PostAsJsonAsync("/openai/v1/chat/completions", new + { model = "gpt-4.1-mini", messages = new[] { new { role = "user", content = "time" } }, tools }); + first.EnsureSuccessStatusCode(); + var message = (await first.Content.ReadFromJsonAsync()).GetProperty("choices")[0].GetProperty("message"); + object[] messages = [new { role = "user", content = "time" }, message, new { role = "user", content = "time again" }]; + using var repeated = await http.PostAsJsonAsync("/openai/v1/chat/completions", new { model = "gpt-4.1-mini", messages, tools }); + repeated.EnsureSuccessStatusCode(); + var call = (await repeated.Content.ReadFromJsonAsync()).GetProperty("choices")[0].GetProperty("message").GetProperty("tool_calls")[0]; + await Assert.That(call.GetProperty("id").GetString()).IsEqualTo("call_scripted_time"); + await Assert.That(call.GetProperty("function").GetProperty("arguments").GetString()).IsEqualTo(fixture.Call.ArgumentsJson); + var control = new LlmTestServerClient(http); + await Assert.That((await control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); + var journal = await control.GetRequestsAsync(); + await Assert.That(journal.Requests.Single(trace => trace.IsReplay).ResponseToolCalls.Single().Id).IsEqualTo("call_scripted_time"); + } + + [Test] + public async Task Preset_OptionalDelayUsesExistingCancellationAndDoesNotConsumeTheCallAsync() + { + var fixture = LlmTestServerWellKnownTools.Shell.Success(new() { Command = "scripted-command" }, + new() { Stdout = "scripted output", ExitCode = 0 }, "call_delayed_shell", delayMilliseconds: 300); + using var host = await LlmTestServerTestHost.StartAsync(config => config.AddConversation("delay-preset", graph => graph + .Step("call", step => step.CallsTools(fixture.Call).DelaysBy(fixture.DelayMilliseconds).Complete()))); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + using var cancellation = new CancellationTokenSource(); + var tools = Tools(fixture); + var pending = http.PostAsJsonAsync("/openai/v1/chat/completions", new + { model = "gpt-4.1-mini", messages = new[] { new { role = "user", content = "shell fixture" } }, tools }, cancellation.Token); + var observedBusy = false; + for (var attempt = 0; attempt < 100; attempt++) + { + if ((await control.GetConversationsAsync()).Any(state => state.IsBusy)) { observedBusy = true; break; } + if (pending.IsCompleted) { break; } + await Task.Delay(1); + } + await Assert.That(observedBusy).IsTrue(); + cancellation.Cancel(); + var wasCancelled = false; + try { using var response = await pending; } + catch (OperationCanceledException) when (cancellation.IsCancellationRequested) { wasCancelled = true; } + await Assert.That(wasCancelled).IsTrue(); + await Assert.That((await control.GetConversationsAsync()).Count).IsEqualTo(0); + using var retry = await http.PostAsJsonAsync("/openai/v1/chat/completions", new + { model = "gpt-4.1-mini", messages = new[] { new { role = "user", content = "shell fixture" } }, tools }); + retry.EnsureSuccessStatusCode(); + var call = (await retry.Content.ReadFromJsonAsync()).GetProperty("choices")[0].GetProperty("message").GetProperty("tool_calls")[0]; + await Assert.That(call.GetProperty("id").GetString()).IsEqualTo("call_delayed_shell"); + await Assert.That((await control.GetConversationsAsync()).Single().ExecutedSteps).IsEqualTo(1); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Verification/VerificationIntegrationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Verification/VerificationIntegrationTests.cs new file mode 100644 index 0000000..3c3f130 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Verification/VerificationIntegrationTests.cs @@ -0,0 +1,132 @@ +using System.Net; +using System.Net.Http.Json; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Hosting; +using ManagedCode.LlmTestServer.Tests.TestSupport; +using ManagedCode.LlmTestServer.Verification; +using Microsoft.Extensions.DependencyInjection; + +namespace ManagedCode.LlmTestServer.Tests.Verification; + +public sealed partial class VerificationIntegrationTests +{ + [Test] + public async Task Verify_CountsFiltersOrderAndUnexpectedCalls_FromActualProviderRequestsAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(config => config.AddChatScenario("verify", + scenario => scenario.Responds("first").Responds("second").Fails(429, "too_many_requests", "retry later"))); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + using var first = await ChatAsync(http, "/openai/v1/chat/completions"); + using var second = await ChatAsync(http, "/mistral/v1/chat/completions"); + using var third = await ChatAsync(http, "/openai/v1/chat/completions"); + await Assert.That(third.StatusCode).IsEqualTo(HttpStatusCode.TooManyRequests); + var expected = new LlmTestServerVerificationRequest + { + InOrder = true, + NoUnexpectedCalls = true, + Requests = + [ + new() { Name = "first", Query = new() { Provider = "openai", Scenario = "verify", Status = 200 } }, + new() { Name = "second", Query = new() { Provider = "mistral", Model = "gpt-4.1-mini", Success = true } }, + new() { Name = "third", Query = new() { Provider = "openai", Status = 429, ErrorCode = "too_many_requests" } }, + ], + }; + var result = await control.VerifyAsync(expected); + await Assert.That(result.Passed).IsTrue(); + await Assert.That(result.EvidenceComplete).IsTrue(); + await Assert.That(result.Failures).IsEmpty(); + await Assert.That(result.MatchedRequestIds.Count).IsEqualTo(3); + var reversed = await control.VerifyAsync(expected with { Requests = expected.Requests.AsEnumerable().Reverse().ToList() }); + await Assert.That(reversed.Passed).IsFalse(); + await Assert.That(reversed.Failures.Any(failure => failure.Code == "request.order")).IsTrue(); + var missing = await control.VerifyAsync(expected with { Requests = [expected.Requests[0]] }); + await Assert.That(missing.Passed).IsFalse(); + await Assert.That(missing.Failures.Single().Code).IsEqualTo("request.unexpected"); + await Assert.That(missing.Failures.Single().RequestIds.Count).IsEqualTo(2); + var wrongCount = await control.VerifyAsync(new() + { Requests = [new() { Name = "all", MinimumCount = 4, MaximumCount = 4 }] }); + await Assert.That(wrongCount.Failures.Single().Code).IsEqualTo("request.count"); + await Assert.That(wrongCount.Failures.Single().Expected).IsEqualTo("4"); + await Assert.That(wrongCount.Failures.Single().Actual).IsEqualTo("3"); + } + + [Test] + public async Task Verify_ExplicitZeroAndRequestIdScopes_DoNotIncludeOtherRequestsAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(config => config.AddChatScenario("verify", scenario => scenario.Responds("one").Responds("two"))); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + var empty = await control.VerifyAsync(new() { NoUnexpectedCalls = true }); + await Assert.That(empty.Passed).IsTrue(); + using var first = await ChatAsync(http, "/openai/v1/chat/completions"); + using var second = await ChatAsync(http, "/mistral/v1/chat/completions"); + var requests = (await http.GetFromJsonAsync>("/admin-api/requests"))!; + var selected = requests.Single(trace => trace.ProviderId == "openai"); + var result = await control.VerifyAsync(new() + { + Scope = new() { RequestId = selected.RequestId, From = selected.StartedAt, To = selected.StartedAt }, + NoUnexpectedCalls = true, + Requests = [new() { Query = new() { Completed = true, Streaming = false } }, + new() { Query = new() { Provider = "gemini" }, MinimumCount = 0, MaximumCount = 0 }], + }); + await Assert.That(result.Passed).IsTrue(); + await Assert.That(result.MatchedRequestIds).IsEquivalentTo(new[] { selected.RequestId }); + var page = await control.VerifyAsync(new() { Scope = new() { Limit = 1 }, Requests = [new()] }); + await Assert.That(page.Passed).IsFalse(); + await Assert.That(page.EvidenceComplete).IsFalse(); + await Assert.That(page.Failures.Any(failure => failure.Code == "evidence.incomplete")).IsTrue(); + var expectationPage = await control.VerifyAsync(new() { Requests = [new() { Query = new() { Limit = 1 } }] }); + await Assert.That(expectationPage.EvidenceComplete).IsFalse(); + await Assert.That(expectationPage.Failures.Any(failure => failure.Code == "evidence.page")).IsTrue(); + } + + [Test] + [Arguments(true)] + [Arguments(false)] + public async Task Verify_IncompleteJournalNeverProvesNoUnexpectedCallsAsync(bool evict) + { + var content = evict ? "answer" : new string('x', 4096); + using var host = await LlmTestServerTestHost.StartAsync(config => config.AddChatScenario("bounded", scenario => scenario.Responds(content).Responds(content)), + configureServices: services => services.Configure(options => + { + options.MaxEntries = evict ? 1 : 10; + options.MaxPreviewBytes = evict ? 65536 : 64; + options.MaxRetainedBytes = 65536; + })); + using var http = host.GetTestClient(); + using var first = await ChatAsync(http, "/openai/v1/chat/completions"); + using var second = await ChatAsync(http, "/openai/v1/chat/completions"); + var result = await new LlmTestServerClient(http).VerifyAsync(new() + { NoUnexpectedCalls = true, Requests = [new() { MinimumCount = 0, MaximumCount = null }] }); + await Assert.That(result.Passed).IsFalse(); + await Assert.That(result.EvidenceComplete).IsFalse(); + await Assert.That(result.Failures.Any(failure => failure.Code == "evidence.incomplete")).IsTrue(); + } + + [Test] + public async Task Verify_IsolatedSessionsAndConversationClient_RestrictEvidenceAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var root = new LlmTestServerClient(http); + await using var a = await root.CreateSessionAsync(config => config.UseConversation("flow", graph => graph.Step("only", step => step.Responds("A").Complete()))); + await using var b = await root.CreateSessionAsync(config => config.UseConversation("flow", graph => graph.Step("only", step => step.Responds("B").Complete()))); + using var aFirst = a.Control.ForConversation("first"); + using var aSecond = a.Control.ForConversation("second"); + using var first = await ChatAsync(aFirst.HttpClient, "/openai/v1/chat/completions"); + using var second = await ChatAsync(aSecond.HttpClient, "/openai/v1/chat/completions"); + using var other = await ChatAsync(b.HttpClient, "/openai/v1/chat/completions"); + var expected = new LlmTestServerVerificationRequest { NoUnexpectedCalls = true, Requests = [new()] }; + await Assert.That((await aFirst.Control.VerifyAsync(expected)).Passed).IsTrue(); + await Assert.That((await b.Control.VerifyAsync(expected)).Passed).IsTrue(); + await Assert.That((await a.Control.VerifyAsync(expected)).Passed).IsFalse(); + await Assert.That((await root.VerifyAsync(new() { NoUnexpectedCalls = true })).Passed).IsTrue(); + } + + private static Task ChatAsync(HttpClient http, string path, bool stream = false) + { + return http.PostAsJsonAsync(path, new { model = "gpt-4.1-mini", messages = new[] { new { role = "user", content = "verify" } }, stream }); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Verification/VerificationProviderMatrixTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Verification/VerificationProviderMatrixTests.cs new file mode 100644 index 0000000..62f019c --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Verification/VerificationProviderMatrixTests.cs @@ -0,0 +1,86 @@ +using System.Net; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Hosting; +using ManagedCode.LlmTestServer.Tests.Providers; +using ManagedCode.LlmTestServer.Tests.TestSupport; +using ManagedCode.LlmTestServer.Verification; + +namespace ManagedCode.LlmTestServer.Tests.Verification; + +public sealed partial class VerificationIntegrationTests +{ + [Test] + public async Task Verify_ProviderMatrixRetriesRecoverOnSameLogicalTurnWithIndependentConversationStateAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(config => config + .AddConversation("matrix-retry", graph => graph + .Step("error", step => step.WhenTurn(1).Fails(429, "too_many_requests", "Retry the same turn").RetryAfter(0).Then("recover")) + .Step("recover", step => step.WhenTurn(1).Responds("recovered").Complete()))); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + var retries = new List(); + var observedRequestIds = new List(); + var providers = ProviderFaultSimulationTests.GetProviderFaultCases(); + await Assert.That(providers.Length).IsEqualTo(13); + for (var index = 0; index < providers.Length; index++) + { + var provider = providers[index]; + var groupId = $"provider-retry-{index}"; + http.DefaultRequestHeaders.Remove(LlmTestServerRequestHeaders.RetryGroup); + http.DefaultRequestHeaders.Add(LlmTestServerRequestHeaders.RetryGroup, groupId); + using var limited = await provider.SendAsync(http, "matrix retry"); + await Assert.That(limited.StatusCode).IsEqualTo(HttpStatusCode.TooManyRequests); + await Assert.That(limited.Headers.RetryAfter!.Delta).IsEqualTo(TimeSpan.Zero); + await Assert.That(await limited.Content.ReadAsStringAsync()).Contains("Retry the same turn"); + var firstId = limited.Headers.GetValues(LlmTestServerProviderHttpTraceOptions.DefaultRequestIdHeaderName).Single(); + observedRequestIds.Add(firstId); + var firstEvidence = await control.GetRequestsAsync(new() { RequestId = firstId }); + await Assert.That(firstEvidence.TotalMatched).IsEqualTo(1); + await Assert.That(firstEvidence.Requests.Single().RetryGroupId).IsEqualTo(groupId); + await Assert.That(firstEvidence.Requests.Single().StatusCode).IsEqualTo(429); + await Assert.That(firstEvidence.Requests.Single().RetryAfter).IsEqualTo("0"); + + using var recovered = await provider.SendAsync(http, "matrix retry"); + await Assert.That(recovered.StatusCode).IsEqualTo(HttpStatusCode.OK); + await Assert.That(await recovered.Content.ReadAsStringAsync()).Contains("recovered"); + var secondId = recovered.Headers.GetValues(LlmTestServerProviderHttpTraceOptions.DefaultRequestIdHeaderName).Single(); + observedRequestIds.Add(secondId); + var secondEvidence = await control.GetRequestsAsync(new() { RequestId = secondId }); + await Assert.That(secondEvidence.TotalMatched).IsEqualTo(1); + await Assert.That(secondEvidence.Requests.Single().RetryGroupId).IsEqualTo(groupId); + await Assert.That(secondEvidence.Requests.Single().StatusCode).IsEqualTo(200); + retries.Add(new() + { + GroupId = groupId, + MinimumAttempts = 2, + MaximumAttempts = 2, + StatusSequence = [429, 200], + MinimumRetryAfterSeconds = 0, + RespectRetryAfter = true, + }); + } + http.DefaultRequestHeaders.Remove(LlmTestServerRequestHeaders.RetryGroup); + var result = await control.VerifyAsync(new() { NoUnexpectedCalls = true, Retries = retries }); + await Assert.That(result.Passed).IsTrue(); + await Assert.That(result.EvidenceComplete).IsTrue(); + await Assert.That(result.Failures).IsEmpty(); + await Assert.That(result.MatchedRequestIds.Count).IsEqualTo(26); + await Assert.That(result.MatchedRequestIds).IsEquivalentTo(observedRequestIds); + var journal = await control.GetRequestsAsync(new() { Order = LlmTestServerRequestOrder.Ascending }); + await Assert.That(journal.TotalMatched).IsEqualTo(26); + await Assert.That(journal.TotalObserved).IsEqualTo(26); + await Assert.That(journal.Requests.Select(trace => trace.ProviderNamespace).Distinct().Count()).IsEqualTo(13); + var conversations = await control.GetConversationsAsync(); + await Assert.That(conversations.Count).IsEqualTo(13); + await Assert.That(conversations.Select(state => state.ProviderNamespace).Distinct().Count()).IsEqualTo(13); + foreach (var conversation in conversations) + { + await Assert.That(conversation.ScenarioId).IsEqualTo("matrix-retry"); + await Assert.That(conversation.ExecutedSteps).IsEqualTo(2); + await Assert.That(conversation.LastTurnIndex).IsEqualTo(1); + await Assert.That(conversation.IsTerminal).IsTrue(); + await Assert.That(conversation.VisitedSteps).IsEquivalentTo(new[] { "error", "recover" }); + } + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Verification/VerificationSdkRetryTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Verification/VerificationSdkRetryTests.cs new file mode 100644 index 0000000..61d7ab8 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Verification/VerificationSdkRetryTests.cs @@ -0,0 +1,68 @@ +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Net.Http.Json; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Tests.TestSupport; +using OpenAI; +using OpenAI.Chat; + +namespace ManagedCode.LlmTestServer.Tests.Verification; + +public sealed partial class VerificationIntegrationTests +{ + [Test] + public async Task Verify_OfficialOpenAiSdkRetriesRateLimitAndProducesBoundedRetryEvidenceAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(config => config + .RequireBearerToken("sdk-retry-key") + .AddChatScenario("sdk-retry", scenario => scenario + .Fails(429, "too_many_requests", "Retry this request").RetryAfter(0) + .Responds("retried"))); + using var http = host.GetTestClient(); + http.DefaultRequestHeaders.Add("X-Llm-Test-Server-Retry-Group", "official-sdk-retry"); + var sdk = new ChatClient("gpt-4.1-mini", new ApiKeyCredential("sdk-retry-key"), new OpenAIClientOptions + { + Endpoint = new Uri(http.BaseAddress!, "openai/v1"), + Transport = new HttpClientPipelineTransport(http), + }); + var completion = await sdk.CompleteChatAsync([new UserChatMessage("retry verification")]); + await Assert.That(completion.Value.Content.Single().Text).IsEqualTo("retried"); + + var control = new LlmTestServerClient(http, "sdk-retry-key"); + var result = await control.VerifyAsync(new() + { + NoUnexpectedCalls = true, + Retries = [new() + { + GroupId = "official-sdk-retry", + MinimumAttempts = 2, + MaximumAttempts = 2, + StatusSequence = [429, 200], + MinimumRetryAfterSeconds = 0, + RespectRetryAfter = true, + }], + }); + await Assert.That(result.Passed).IsTrue(); + await Assert.That(result.EvidenceComplete).IsTrue(); + await Assert.That(result.Failures).IsEmpty(); + http.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", "sdk-retry-key"); + var traces = (await http.GetFromJsonAsync>("/admin-api/requests"))! + .OrderBy(trace => trace.Sequence).ToArray(); + await Assert.That(traces.Length).IsEqualTo(2); + await Assert.That(traces.Select(trace => trace.StatusCode)).IsEquivalentTo(new int?[] { 429, 200 }); + await Assert.That(traces.All(trace => trace.RetryGroupId == "official-sdk-retry")).IsTrue(); + await Assert.That(traces[0].RetryAfter).IsEqualTo("0"); + await Assert.That(traces[1].RetryAfter).IsNull(); + foreach (var trace in traces) + { + await Assert.That(trace.StartedMonotonicSeconds.HasValue).IsTrue(); + await Assert.That(trace.CompletedMonotonicSeconds.HasValue).IsTrue(); + await Assert.That(double.IsFinite(trace.StartedMonotonicSeconds!.Value)).IsTrue(); + await Assert.That(double.IsFinite(trace.CompletedMonotonicSeconds!.Value)).IsTrue(); + await Assert.That(trace.CompletedMonotonicSeconds.Value).IsGreaterThanOrEqualTo(trace.StartedMonotonicSeconds.Value); + } + await Assert.That(traces[1].StartedMonotonicSeconds!.Value).IsGreaterThanOrEqualTo(traces[0].CompletedMonotonicSeconds!.Value); + await Assert.That(result.MatchedRequestIds).IsEquivalentTo(traces.Select(trace => trace.RequestId).ToArray()); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Verification/VerificationToolAndRetryTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Verification/VerificationToolAndRetryTests.cs new file mode 100644 index 0000000..f304660 --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Verification/VerificationToolAndRetryTests.cs @@ -0,0 +1,182 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Diagnostics; +using ManagedCode.LlmTestServer.Tests.TestSupport; +using ManagedCode.LlmTestServer.Verification; + +namespace ManagedCode.LlmTestServer.Tests.Verification; + +public sealed partial class VerificationIntegrationTests +{ + [Test] + public async Task Verify_InspectsResponseCallsRequestCallsToolResultsAndVisitedGraphStepsAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(config => config.UseConversation("weather", graph => graph + .Step("lookup", step => step.WhenLatestUserEquals("weather").CallsTool("weather", "{\"city\":\"Paris\"}", "call_paris").Then("answer")) + .Step("answer", step => step.WhenToolResult("weather", "call_paris").Responds("21 C").Complete()))); + using var conversation = control.ForConversation("weather-42"); + var tools = new[] { new { type = "function", function = new { name = "weather", parameters = new { type = "object" } } } }; + using var first = await conversation.HttpClient.PostAsJsonAsync("/openai/v1/chat/completions", new + { model = "gpt-4.1-mini", messages = new[] { new { role = "user", content = "weather" } }, tools }); + first.EnsureSuccessStatusCode(); + var initial = (await first.Content.ReadFromJsonAsync()).GetProperty("choices")[0].GetProperty("message"); + object[] messages = + [ + new { role = "user", content = "weather" }, initial, + new { role = "tool", tool_call_id = "call_paris", content = "{\"temperature\":21,\"forecast\":[\"sunny\"]}" }, + ]; + using var second = await conversation.HttpClient.PostAsJsonAsync("/openai/v1/chat/completions", new { model = "gpt-4.1-mini", messages, tools }); + second.EnsureSuccessStatusCode(); + var expected = new LlmTestServerVerificationRequest + { + NoUnexpectedCalls = true, + InOrder = true, + Requests = + [ + new() { Name = "lookup", Query = new() { Step = "lookup" }, Tools = + [new() { Kind = LlmTestServerToolEvidenceKind.ResponseCall, Name = "weather", CallId = "call_paris", ArgumentsJson = "{\"city\":\"Paris\"}" }] }, + new() { Name = "answer", Query = new() { Step = "answer" }, Tools = + [new() { Kind = LlmTestServerToolEvidenceKind.RequestCall, Name = "weather", CallId = "call_paris", ArgumentsJson = "{\"city\":\"Paris\"}" }, + new() { Kind = LlmTestServerToolEvidenceKind.Result, Name = "weather", CallId = "call_paris", ResultJson = "{\"forecast\":[\"sunny\"],\"temperature\":21}", + ResultFields = [new() { Path = "$.forecast[0]", EqualsJson = "\"sunny\"" }, new() { Path = "/temperature", EqualsJson = "21" }] }] }, + ], + Conversations = [new() { ScenarioId = "weather", ConversationId = "weather-42", ProviderNamespace = "/openai", VisitedSteps = ["lookup", "answer"], RequireTerminal = true }], + }; + var result = await conversation.Control.VerifyAsync(expected); + await Assert.That(result.Passed).IsTrue(); + await Assert.That(result.MatchedRequestIds.Count).IsEqualTo(2); + var badTool = await conversation.Control.VerifyAsync(expected with + { Requests = [expected.Requests[0] with { Tools = [new() { Name = "different", Kind = LlmTestServerToolEvidenceKind.ResponseCall }] }], NoUnexpectedCalls = false }); + await Assert.That(badTool.Passed).IsFalse(); + await Assert.That(badTool.Failures.Any(failure => failure.Code == "tool.count")).IsTrue(); + var missingStep = await conversation.Control.VerifyAsync(new() + { Conversations = [new() { ScenarioId = "weather", VisitedSteps = ["nonexistent"] }] }); + await Assert.That(missingStep.Failures.Single().Code).IsEqualTo("conversation.steps"); + var missingGraph = await conversation.Control.VerifyAsync(new() { Conversations = [new() { ScenarioId = "missing" }] }); + await Assert.That(missingGraph.Failures.Single().Code).IsEqualTo("conversation.missing"); + } + + [Test] + public async Task Verify_RequiresVisitedStepsAndTerminalStateBeforeConversationCompletesAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + var control = new LlmTestServerClient(http); + await control.ConfigureAsync(config => config.UseConversation("active", graph => graph + .Step("start", step => step.Responds("ready").Then("finish")) + .Step("finish", step => step.Responds("done").Complete()))); + using var first = await ChatAsync(http, "/openai/v1/chat/completions"); + first.EnsureSuccessStatusCode(); + var result = await control.VerifyAsync(new() + { Conversations = [new() { ScenarioId = "active", VisitedSteps = ["start", "finish"], RequireTerminal = true }] }); + await Assert.That(result.Passed).IsFalse(); + await Assert.That(result.Failures.Select(failure => failure.Code)).IsEquivalentTo(new[] { "conversation.steps", "conversation.terminal" }); + } + + [Test] + [Arguments(true)] + [Arguments(false)] + public async Task Verify_StreamingTerminalMarkerUsesObservedTransportEvidenceAsync(bool stream) + { + using var host = await LlmTestServerTestHost.StartAsync(config => config.AddChatScenario("stream", scenario => scenario.Responds("hello", "hel", "lo"))); + using var http = host.GetTestClient(); + using var response = await ChatAsync(http, "/openai/v1/chat/completions", stream); + response.EnsureSuccessStatusCode(); + var wire = await response.Content.ReadAsStringAsync(); + if (stream) { await Assert.That(wire).Contains("[DONE]"); } + var result = await new LlmTestServerClient(http).VerifyAsync(new() + { Requests = [new() { RequireStreamingTerminalMarker = true }] }); + await Assert.That(result.Passed).IsEqualTo(stream); + await Assert.That(result.Failures.Any(failure => failure.Code == "stream.terminal")).IsEqualTo(!stream); + } + + [Test] + public async Task Verify_BoundedRetriesInspectStatusesRetryAfterAndAttemptCountAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(config => config.AddChatScenario("retries", scenario => scenario + .Fails(429, "too_many_requests", "wait").RetryAfter(0).Fails(503, "unavailable", "wait").RetryAfter(0).Responds("recovered"))); + using var http = host.GetTestClient(); + http.DefaultRequestHeaders.Add("X-Llm-Test-Server-Retry-Group", "retry-group"); + using var first = await ChatAsync(http, "/openai/v1/chat/completions"); + using var second = await ChatAsync(http, "/openai/v1/chat/completions"); + using var third = await ChatAsync(http, "/openai/v1/chat/completions"); + await Assert.That(first.StatusCode).IsEqualTo(HttpStatusCode.TooManyRequests); + await Assert.That(second.StatusCode).IsEqualTo(HttpStatusCode.ServiceUnavailable); + third.EnsureSuccessStatusCode(); + var control = new LlmTestServerClient(http); + var retry = new LlmTestServerRetryExpectation + { GroupId = "retry-group", MinimumAttempts = 3, MaximumAttempts = 3, StatusSequence = [429, 503, 200], MinimumRetryAfterSeconds = 0, RespectRetryAfter = true }; + var result = await control.VerifyAsync(new() { NoUnexpectedCalls = true, Retries = [retry] }); + await Assert.That(result.Passed).IsTrue(); + await Assert.That(result.MatchedRequestIds.Count).IsEqualTo(3); + var excess = await control.VerifyAsync(new() { Retries = [retry with { MinimumAttempts = 1, MaximumAttempts = 2 }] }); + await Assert.That(excess.Failures.Any(failure => failure.Code == "retry.attempts")).IsTrue(); + var wrongStatuses = await control.VerifyAsync(new() { Retries = [retry with { StatusSequence = [429, 200] }] }); + await Assert.That(wrongStatuses.Failures.Any(failure => failure.Code == "retry.statuses")).IsTrue(); + var insufficient = await control.VerifyAsync(new() { Retries = [retry with { MinimumRetryAfterSeconds = 1 }] }); + await Assert.That(insufficient.Failures.Count(failure => failure.Code == "retry.after")).IsEqualTo(2); + var missing = await control.VerifyAsync(new() { Retries = [retry with { GroupId = "not-observed" }] }); + await Assert.That(missing.Failures.Any(failure => failure.Code == "retry.attempts")).IsTrue(); + } + + [Test] + public async Task Verify_RetryAfterComplianceAcceptsObservedWaitBeforeRecoveryAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(config => config.AddChatScenario("waited", scenario => scenario + .Fails(429, "too_many_requests", "wait").RetryAfter(1).Responds("recovered"))); + using var http = host.GetTestClient(); + http.DefaultRequestHeaders.Add("X-Llm-Test-Server-Retry-Group", "waited-retry"); + using var first = await ChatAsync(http, "/openai/v1/chat/completions"); + await Assert.That(first.Headers.RetryAfter!.Delta).IsEqualTo(TimeSpan.FromSeconds(1)); + await Task.Delay(TimeSpan.FromMilliseconds(1100)); + using var second = await ChatAsync(http, "/openai/v1/chat/completions"); + second.EnsureSuccessStatusCode(); + var traces = (await http.GetFromJsonAsync>("/admin-api/requests"))! + .OrderBy(trace => trace.Sequence).ToArray(); + await Assert.That(traces.Length).IsEqualTo(2); + foreach (var trace in traces) + { + await Assert.That(trace.StartedMonotonicSeconds.HasValue).IsTrue(); + await Assert.That(trace.CompletedMonotonicSeconds.HasValue).IsTrue(); + await Assert.That(double.IsFinite(trace.StartedMonotonicSeconds!.Value)).IsTrue(); + await Assert.That(double.IsFinite(trace.CompletedMonotonicSeconds!.Value)).IsTrue(); + await Assert.That(trace.CompletedMonotonicSeconds.Value).IsGreaterThanOrEqualTo(trace.StartedMonotonicSeconds.Value); + } + await Assert.That(traces[1].StartedMonotonicSeconds!.Value - traces[0].CompletedMonotonicSeconds!.Value).IsGreaterThanOrEqualTo(1); + var result = await new LlmTestServerClient(http).VerifyAsync(new() + { + NoUnexpectedCalls = true, + Retries = [new() { GroupId = "waited-retry", MinimumAttempts = 2, MaximumAttempts = 2, + StatusSequence = [429, 200], MinimumRetryAfterSeconds = 1, RespectRetryAfter = true }], + }); + await Assert.That(result.Passed).IsTrue(); + await Assert.That(result.Failures).IsEmpty(); + } + + [Test] + public async Task Verify_RetryAfterComplianceRejectsPrematureRetryAndMissingHeaderAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(config => config.AddChatScenario("early", scenario => scenario + .Fails(429, "too_many_requests", "wait").RetryAfter(60).Responds("too early"))); + using var http = host.GetTestClient(); + http.DefaultRequestHeaders.Add("X-Llm-Test-Server-Retry-Group", "early-retry"); + using var first = await ChatAsync(http, "/openai/v1/chat/completions"); + using var second = await ChatAsync(http, "/openai/v1/chat/completions"); + var control = new LlmTestServerClient(http); + var expected = new LlmTestServerVerificationRequest + { Retries = [new() { GroupId = "early-retry", MinimumAttempts = 2, MaximumAttempts = 2, RespectRetryAfter = true }] }; + var result = await control.VerifyAsync(expected); + await Assert.That(result.Passed).IsFalse(); + await Assert.That(result.Failures.Single().Code).IsEqualTo("retry.delay"); + await control.ConfigureAsync(config => config.UseChatScenario("missing-header", scenario => scenario.Fails(503, "unavailable", "wait").Responds("retry"))); + using var missingHeader = await ChatAsync(http, "/openai/v1/chat/completions"); + using var retry = await ChatAsync(http, "/openai/v1/chat/completions"); + var missing = await control.VerifyAsync(expected with { Retries = [expected.Retries[0] with { MinimumRetryAfterSeconds = 0 }] }); + await Assert.That(missing.Failures.Select(failure => failure.Code)).IsEquivalentTo(new[] { "retry.after", "retry.delay" }); + } +} diff --git a/tests/ManagedCode.LlmTestServer.Tests/Verification/VerificationValidationTests.cs b/tests/ManagedCode.LlmTestServer.Tests/Verification/VerificationValidationTests.cs new file mode 100644 index 0000000..af727dd --- /dev/null +++ b/tests/ManagedCode.LlmTestServer.Tests/Verification/VerificationValidationTests.cs @@ -0,0 +1,161 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json.Nodes; +using ManagedCode.LlmTestServer.Client; +using ManagedCode.LlmTestServer.Hosting; +using ManagedCode.LlmTestServer.Tests.TestSupport; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.DependencyInjection; + +namespace ManagedCode.LlmTestServer.Tests.Verification; + +public sealed partial class VerificationIntegrationTests +{ + [Test] + [Arguments("{}")] + [Arguments("{\"scope\":null,\"noUnexpectedCalls\":true}")] + [Arguments("{\"requests\":null,\"noUnexpectedCalls\":true}")] + [Arguments("{\"conversations\":null,\"noUnexpectedCalls\":true}")] + [Arguments("{\"retries\":null,\"noUnexpectedCalls\":true}")] + [Arguments("{\"requests\":[null]}")] + [Arguments("{\"requests\":[{\"name\":\"\"}]}")] + [Arguments("{\"requests\":[{\"query\":null}]}")] + [Arguments("{\"requests\":[{\"minimumCount\":-1}]}")] + [Arguments("{\"requests\":[{\"minimumCount\":2,\"maximumCount\":1}]}")] + [Arguments("{\"requests\":[{\"query\":{\"status\":99}}]}")] + [Arguments("{\"requests\":[{\"query\":{\"offset\":-1}}]}")] + [Arguments("{\"requests\":[{\"query\":{\"limit\":501}}]}")] + [Arguments("{\"requests\":[{\"query\":{\"order\":99}}]}")] + [Arguments("{\"requests\":[{\"tools\":null}]}")] + [Arguments("{\"requests\":[{\"tools\":[null]}]}")] + [Arguments("{\"requests\":[{\"tools\":[{\"kind\":99}]}]}")] + [Arguments("{\"requests\":[{\"tools\":[{\"argumentsJson\":\"{\"}]}]}")] + [Arguments("{\"requests\":[{\"tools\":[{\"kind\":2,\"resultJson\":\"{\"}]}]}")] + [Arguments("{\"requests\":[{\"tools\":[{\"resultFields\":null}]}]}")] + [Arguments("{\"requests\":[{\"tools\":[{\"kind\":2,\"resultFields\":[null]}]}]}")] + [Arguments("{\"requests\":[{\"tools\":[{\"kind\":2,\"resultFields\":[{\"path\":\"$..invalid\",\"equalsJson\":\"null\"}]}]}]}")] + [Arguments("{\"requests\":[{\"tools\":[{\"resultJson\":\"{}\"}]}]}")] + [Arguments("{\"conversations\":[null]}")] + [Arguments("{\"conversations\":[{}]}")] + [Arguments("{\"conversations\":[{\"scenarioId\":\"graph\",\"visitedSteps\":null}]}")] + [Arguments("{\"conversations\":[{\"scenarioId\":\"graph\",\"visitedSteps\":[\"\"]}]}")] + [Arguments("{\"retries\":[null]}")] + [Arguments("{\"retries\":[{}]}")] + [Arguments("{\"retries\":[{\"groupId\":\"retry\",\"query\":null}]}")] + [Arguments("{\"retries\":[{\"groupId\":\"retry\",\"minimumAttempts\":4,\"maximumAttempts\":3}]}")] + [Arguments("{\"retries\":[{\"groupId\":\"retry\",\"statusSequence\":null}]}")] + [Arguments("{\"retries\":[{\"groupId\":\"retry\",\"statusSequence\":[600]}]}")] + [Arguments("{\"retries\":[{\"groupId\":\"retry\",\"minimumRetryAfterSeconds\":-1}]}")] + public async Task Verify_InvalidExpectationsReturnBadRequestAndDoNotConsumeProviderFixturesAsync(string json) + { + using var host = await LlmTestServerTestHost.StartAsync(config => config.AddChatScenario("preserved", scenario => scenario.Responds("still available"))); + using var http = host.GetTestClient(); + using var invalid = await http.PostAsJsonAsync("/admin-api/verify", JsonNode.Parse(json)); + await Assert.That(invalid.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + using var actual = await ChatAsync(http, "/openai/v1/chat/completions"); + actual.EnsureSuccessStatusCode(); + await Assert.That(await actual.Content.ReadAsStringAsync()).Contains("still available"); + } + + [Test] + public async Task Verify_RequiresControlAuthorizationAndTypedClientSendsCredentialAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(config => config.RequireBearerToken("verify-key")); + using var http = host.GetTestClient(); + using var unauthorized = await http.PostAsJsonAsync("/admin-api/verify", new { noUnexpectedCalls = true }); + await Assert.That(unauthorized.StatusCode).IsEqualTo(HttpStatusCode.Unauthorized); + http.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", "wrong-key"); + using var wrong = await http.PostAsJsonAsync("/admin-api/verify", new { noUnexpectedCalls = true }); + await Assert.That(wrong.StatusCode).IsEqualTo(HttpStatusCode.Unauthorized); + var valid = await new LlmTestServerClient(http, "verify-key").VerifyAsync(new() { NoUnexpectedCalls = true }); + await Assert.That(valid.Passed).IsTrue(); + } + + [Test] + [Arguments("empty")] + [Arguments("duplicate")] + [Arguments("oversized")] + public async Task InvalidRetryGroup_DoesNotConsumeProviderFixtureAsync(string defect) + { + if (defect == "empty") { await VerifyEmptyHeaderOverTcpAsync(ManagedCode.LlmTestServer.Diagnostics.LlmTestServerRequestHeaders.RetryGroup); return; } + using var host = await LlmTestServerTestHost.StartAsync(config => config.AddChatScenario("preserved", scenario => scenario.Responds("still available"))); + using var http = host.GetTestClient(); + using var request = new HttpRequestMessage(HttpMethod.Post, "/openai/v1/chat/completions") + { Content = System.Net.Http.Json.JsonContent.Create(new { model = "gpt-4.1-mini", messages = new[] { new { role = "user", content = "hello" } } }) }; + request.Headers.TryAddWithoutValidation(ManagedCode.LlmTestServer.Diagnostics.LlmTestServerRequestHeaders.RetryGroup, + defect == "empty" ? new[] { "" } : defect == "duplicate" ? new[] { "one", "two" } : new[] { new string('x', 129) }); + using var invalid = await http.SendAsync(request); + await Assert.That(invalid.StatusCode).IsEqualTo(HttpStatusCode.BadRequest); + var evidence = await new LlmTestServerClient(http).GetRequestsAsync(new() { Status = 400 }); + await Assert.That(evidence.TotalMatched).IsEqualTo(1); + var unaccounted = await new LlmTestServerClient(http).VerifyAsync(new() { NoUnexpectedCalls = true }); + await Assert.That(unaccounted.Passed).IsFalse(); + using var actual = await ChatAsync(http, "/openai/v1/chat/completions"); + actual.EnsureSuccessStatusCode(); + await Assert.That(await actual.Content.ReadAsStringAsync()).Contains("still available"); + } + + [Test] + public async Task EmptyConversationHeader_DoesNotConsumeProviderFixtureAsync() + { + await VerifyEmptyHeaderOverTcpAsync(ManagedCode.LlmTestServer.Conversations.LlmTestServerConversationHeaders.Id); + } + + private static async Task VerifyEmptyHeaderOverTcpAsync(string headerName) + { + var builder = Microsoft.AspNetCore.Builder.WebApplication.CreateBuilder(); + builder.WebHost.ConfigureKestrel(options => options.Listen(System.Net.IPAddress.Loopback, 0)); + builder.Services.AddLlmTestServer(config => config.AddChatScenario("preserved", scenario => scenario.Responds("still available"))); + await using var app = builder.Build(); + app.MapLlmTestServer(); + await app.StartAsync(); + try + { + var server = app.Services.GetRequiredService(); + var address = new Uri(server.Features.Get()!.Addresses.Single()); + using var http = new HttpClient { BaseAddress = address }; + var control = new LlmTestServerClient(http); + // Complete lazy endpoint initialization before measuring the raw provider exchange. + var initialEvidence = await control.GetRequestsAsync(new()); + await Assert.That(initialEvidence.TotalObserved).IsEqualTo(0); + using var cancellation = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + using var socket = new System.Net.Sockets.TcpClient(); + await socket.ConnectAsync(address.Host, address.Port, cancellation.Token); + const string body = """{"model":"gpt-4.1-mini","messages":[{"role":"user","content":"hello"}]}"""; + var wire = $"POST /openai/v1/chat/completions HTTP/1.1\r\nHost: {address.Authority}\r\nContent-Type: application/json\r\nContent-Length: {System.Text.Encoding.UTF8.GetByteCount(body)}\r\n{headerName}: \r\nConnection: close\r\n\r\n{body}"; + await socket.GetStream().WriteAsync(System.Text.Encoding.UTF8.GetBytes(wire), cancellation.Token); + using var reader = new StreamReader(socket.GetStream()); + var response = await reader.ReadToEndAsync(cancellation.Token); + await Assert.That(response.StartsWith("HTTP/1.1 400", StringComparison.Ordinal)).IsTrue(); + var rejectedTrace = await new LlmTestServerClient(http).GetRequestsAsync(new() { Status = 400 }); + await Assert.That(rejectedTrace.TotalMatched).IsEqualTo(1); + var unaccounted = await new LlmTestServerClient(http).VerifyAsync(new() { NoUnexpectedCalls = true }); + await Assert.That(unaccounted.Passed).IsFalse(); + using var actual = await ChatAsync(http, "/openai/v1/chat/completions"); + actual.EnsureSuccessStatusCode(); + await Assert.That(await actual.Content.ReadAsStringAsync()).Contains("still available"); + } + finally + { + // Drain active requests before the application service provider is disposed. + await app.StopAsync(); + } + } + + [Test] + public async Task ConversationScopedModelDiscovery_IsIncludedInUnexpectedCallVerificationAsync() + { + using var host = await LlmTestServerTestHost.StartAsync(); + using var http = host.GetTestClient(); + using var conversation = new LlmTestServerClient(http).ForConversation("discovery"); + using var response = await conversation.HttpClient.GetAsync("/openai/v1/models"); + response.EnsureSuccessStatusCode(); + var evidence = await conversation.Control.GetRequestsAsync(new() { Conversation = "discovery" }); + await Assert.That(evidence.TotalMatched).IsEqualTo(1); + var unaccounted = await conversation.Control.VerifyAsync(new() { NoUnexpectedCalls = true }); + await Assert.That(unaccounted.Passed).IsFalse(); + var expected = await conversation.Control.VerifyAsync(new() + { NoUnexpectedCalls = true, Requests = [new() { Name = "model discovery" }] }); + await Assert.That(expected.Passed).IsTrue(); + } +}