From 4fad71b45bb66d4cf1c6002b0ce17d671cc6a0c6 Mon Sep 17 00:00:00 2001 From: Test Date: Fri, 23 Oct 2026 10:51:55 -0600 Subject: [PATCH 1/4] feat(oracles): lint oracle files and return reports to coding agents Add an ESLint plugin for oracle files (scripts/eslint-plugin-oracle/) with the oracle/no-disabled rule. It reports `.only`, a `.skip` or `.todo` that switches off a body, `.skipIf`/`.runIf` conditions that name no replay, Playwright's unconditional runtime skips, and generated properties limited to one unpinned run. The rules have no inline exception. scripts/agent-lint-feedback.mjs runs the oracle rules in a pass that ignores inline directives, so an eslint-disable comment cannot silence them. It serves the Claude Code, Codex, Gemini CLI, and Copilot hooks after each edit and at the end of a turn, `pnpm lint:changed`, `pnpm lint:oracles` in CI, and the pre-commit hook. Pre-commit ESLint now runs in chunks to avoid running out of memory on large commits. Remove the skipped random campaign for the #2139 known gap; the fixed-seed `it.fails` cell still pins it, and its replay name is unregistered so a replay request fails loudly. Refs #2234 Co-authored-by: Isaac --- .claude/settings.json | 27 ++ .codex/hooks.json | 27 ++ .gemini/settings.json | 29 +++ .github/hooks/oracle-lint.json | 20 ++ .github/workflows/pr.yml | 4 + AGENTS.md | 14 + CONTRIBUTING.md | 5 + docs/contributing/oracle-coverage.md | 2 +- eslint.config.js | 2 + lint-staged.config.js | 21 ++ package.json | 8 +- .../tests/persisted-oracle.test.ts | 17 +- packages/db/tests/oracle-config.ts | 1 - packages/db/tests/proxy-oracle.test.ts | 2 +- scripts/agent-lint-feedback.mjs | 243 +++++++++++++++++ scripts/eslint-plugin-oracle/index.js | 35 +++ .../eslint-plugin-oracle/no-disabled.test.js | 137 ++++++++++ .../eslint-plugin-oracle/rules/no-disabled.js | 245 ++++++++++++++++++ 18 files changed, 818 insertions(+), 21 deletions(-) create mode 100644 .claude/settings.json create mode 100644 .codex/hooks.json create mode 100644 .gemini/settings.json create mode 100644 .github/hooks/oracle-lint.json create mode 100644 lint-staged.config.js create mode 100644 scripts/agent-lint-feedback.mjs create mode 100644 scripts/eslint-plugin-oracle/index.js create mode 100644 scripts/eslint-plugin-oracle/no-disabled.test.js create mode 100644 scripts/eslint-plugin-oracle/rules/no-disabled.js diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 0000000000..5883c8c69d --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,27 @@ +{ + "hooks": { + "PostToolUse": [ + { + "matcher": "Edit|MultiEdit|Write", + "hooks": [ + { + "type": "command", + "command": "node \"$CLAUDE_PROJECT_DIR/scripts/agent-lint-feedback.mjs\" --format=claude --event=edit", + "timeout": 30 + } + ] + } + ], + "Stop": [ + { + "hooks": [ + { + "type": "command", + "command": "node \"$CLAUDE_PROJECT_DIR/scripts/agent-lint-feedback.mjs\" --format=claude --event=stop", + "timeout": 120 + } + ] + } + ] + } +} diff --git a/.codex/hooks.json b/.codex/hooks.json new file mode 100644 index 0000000000..7e2f706b19 --- /dev/null +++ b/.codex/hooks.json @@ -0,0 +1,27 @@ +{ + "hooks": { + "PostToolUse": [ + { + "matcher": "apply_patch", + "hooks": [ + { + "type": "command", + "command": "node \"$(git rev-parse --show-toplevel)/scripts/agent-lint-feedback.mjs\" --format=codex --event=edit", + "timeout": 30 + } + ] + } + ], + "Stop": [ + { + "hooks": [ + { + "type": "command", + "command": "node \"$(git rev-parse --show-toplevel)/scripts/agent-lint-feedback.mjs\" --format=codex --event=stop", + "timeout": 120 + } + ] + } + ] + } +} diff --git a/.gemini/settings.json b/.gemini/settings.json new file mode 100644 index 0000000000..c83f4593b0 --- /dev/null +++ b/.gemini/settings.json @@ -0,0 +1,29 @@ +{ + "hooks": { + "AfterTool": [ + { + "matcher": "write_file|replace", + "hooks": [ + { + "type": "command", + "name": "oracle-lint-edit", + "command": "node \"$GEMINI_PROJECT_DIR/scripts/agent-lint-feedback.mjs\" --format=gemini --event=edit", + "timeout": 30000 + } + ] + } + ], + "AfterAgent": [ + { + "hooks": [ + { + "type": "command", + "name": "oracle-lint-turn", + "command": "node \"$GEMINI_PROJECT_DIR/scripts/agent-lint-feedback.mjs\" --format=gemini --event=stop", + "timeout": 120000 + } + ] + } + ] + } +} diff --git a/.github/hooks/oracle-lint.json b/.github/hooks/oracle-lint.json new file mode 100644 index 0000000000..3e4218d927 --- /dev/null +++ b/.github/hooks/oracle-lint.json @@ -0,0 +1,20 @@ +{ + "version": 1, + "hooks": { + "postToolUse": [ + { + "type": "command", + "matcher": "edit|create", + "bash": "node scripts/agent-lint-feedback.mjs --format=copilot --event=edit", + "timeoutSec": 30 + } + ], + "agentStop": [ + { + "type": "command", + "bash": "node scripts/agent-lint-feedback.mjs --format=copilot --event=stop", + "timeoutSec": 120 + } + ] + } +} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index b5eb9dd20d..6f3de15a97 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -26,6 +26,10 @@ jobs: uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 - name: Check lint run: pnpm run lint:check + - name: Check oracle lint without inline directives + run: pnpm run lint:oracles + - name: Test oracle lint rules + run: pnpm run test:lint-rules minified-db: name: Minified DB public API runs-on: ubuntu-latest diff --git a/AGENTS.md b/AGENTS.md index 8238e95815..458b57c2aa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -125,6 +125,20 @@ includes this block: - **Limits:** the states, paths, and observations that the evidence does not cover. +### Lint feedback while you work + +The `oracle/` lint rules in `scripts/eslint-plugin-oracle/` report the +shortcuts that a syntactic check can see. The repository's agent hooks +(`.claude/`, `.codex/`, `.gemini/`, and `.github/hooks/`) return the reports +after each edit and at the end of each turn. A hook sees only the edits that +the harness's own edit tools make. After an edit through a shell command, or +in a harness without these hooks, run `pnpm lint:changed` before you finish. + +Fix a report. Do not turn the rule off. The hooks, `pnpm lint:changed`, the +pre-commit hook, and CI ignore `eslint-disable` comments for these rules. A +true exception needs a maintainer to change the rule in a reviewed pull +request. + ## Required reading: live-query materialization Before reading, analyzing, or modifying correlated live-query materialization diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d34fb357a0..985863341c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -53,6 +53,11 @@ If you have been assigned to fix an issue or develop a new feature, please follo pnpm lint ``` + `pnpm lint:changed` lints only the files that your branch changes. If you + work with a coding agent, the repository's hooks return oracle lint reports + to it as it edits. Codex and Gemini CLI run project hooks only after you + trust them in `/hooks`; review them again when they change. + - Implement your changes and tests in the relevant package or example. - For a production change that adds, removes, or changes observable behavior, extend the primary oracle when it can express the governing law. A focused diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 5afacbd874..a085dd75d9 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -638,7 +638,7 @@ generic visitor behavior outside the named collectors. | Experimental no-peek continuation | [test-only live-window probe](https://github.com/TanStack/db/blob/main/packages/query-db-collection/tests/cursor-pagination-oracle.no-peek.integration.test.ts) | The candidate uses an authoritative fact only for the same complete output prefix and otherwise requests the extra row. It remains test-only; the shipped opaque-pagination owner keeps peek-ahead. See [experiment limits](https://github.com/TanStack/db/blob/main/packages/query-db-collection/tests/cursor-pagination/NO-PEEK-RESULTS.md). | | Electric and TrailBase | [Electric histories](https://github.com/TanStack/db/blob/main/packages/electric-db-collection/tests/electric-oracle.property.test.ts), [recovery histories](https://github.com/TanStack/db/blob/main/packages/electric-db-collection/tests/electric-recovery-oracle.test.ts), [held resume snapshots](https://github.com/TanStack/db/blob/main/packages/electric-db-collection/tests/electric-resume-snapshot-races.test.ts), [PostgreSQL semantics](https://github.com/TanStack/db/blob/main/packages/electric-db-collection/e2e/sql-predicate-semantics-oracle.e2e.test.ts), [TrailBase contract](https://github.com/TanStack/db/blob/main/packages/trailbase-db-collection/tests/ORACLE.md) | Installed SDK delivery/framing, independent predicates, exact subscription arguments, restart/reset lineage, held certification and durability races, source-order publication before durability, and late errors. The queued-presence property runs identical fixed/random generators plus isolated seed-and-path replay across insert, update, delete, and truncate callbacks. The recovery fixtures use a mocked ShapeStream; they do not establish live Electric-service framing or native persistence-host behavior. The TrailBase lifecycle oracle's early-demand lane checks that an on-demand load issued before the change listener runs does not fetch until the listener runs or cannot start, through a real live query and the subset boundary: 16 histories cross listen, subscribe rejection, reader failure, abort, and cleanup with a service change before the subscription; the adapter from before the wait fails all 16. Its page-event lane checks that a page read before a concurrent change event neither reverts nor resurrects that row, and its stream-end lane pins that the adapter does not resubscribe: 17 histories, 6 of which fail on the adapter before the page fence. A generated page lane (`trailbase.page-event`) crosses two-page loads, cursor and offset paging, overlapping on-demand loads, and up to four changes among page releases. A held-commit lane checks that a page never fails a load for an older event that core holds while a user mutation persists; deciding insert or update from visible rows fails it. After a generated history drains, the Collection holds exactly the rows a page returned or an event touched. Both page lanes deliver events for changes made before the list call at once, during the page, or after it applies, and check that the listener survives a late insert for a row a page already holds. The retry-decision property (`electric.retry-decision`) generates histories over full-log and progressive waits, with and without `onError`, crossing synchronous and asynchronous retries and give-ups, a throwing `onError`, a later error after a retry, an Electric client give-up, an external abort before or during a decision, new demands, the first up-to-date, a persisting optimistic transaction that holds the retried snapshot and later settles, and cleanup. The retry-decision grammar also crosses eager and changes-only sources, an external signal already aborted at sync start, and a status listener that cleans up inside the failure; it checks that a readiness wait rejects with the same cause as every waiting demand, and that such a cleanup leaves a rejected demand rejected. It also holds the acceptance of the first up-to-date commit across later errors, retries, give-ups, and aborts, and checks that the Collection becomes ready with the snapshot's rows when the commit is accepted, and fails with the acceptance's cause when it is not. The descriptor-isolation oracle holds that acceptance across a managed-cache rotation and checks that the retired session's acceptance settles nothing, whether it resolves or rejects, and that the replacement session's snapshot decides readiness. A progressive snapshot's truncate applies at once; a demand whose snapshot arrived before an external abort settles when its rows apply, and cleanup after the abort rejects it. An independent model checks the status, every demand's settlement and cause, and row visibility after each step: a source that still tries stays `loading`, and only a give-up fails its waits. `main` before #2127 and mutants that fail before the decision, skip the failure on a throw or give-up, or skip `markError` on a client give-up or an external abort each fail it at an assertion. An installed-client lane in the SDK delivery oracle drives the same model (`electric-retry-decision-model.ts`) through the real Electric client and HTTP: a retried 503, a response without Electric headers, and an exhausted 50-retry budget, in eager, full-log, and progressive modes, with a synchronous or asynchronous retry (18 cells). Skipping `markError` on the client's give-up fails the 12 give-up cells; failing before the decision, or `main` before #2127, fails 15; the 3 survivors give up in the same continuation that decides, so an early failure is not observable there. A mutant that settles a held demand on acceptance instead of visibility fails the held replays. Excluded: stream steps while `onError` decides or after a final failure, which the client cannot produce. | | PowerSync | [tests](https://github.com/TanStack/db/tree/main/packages/powersync-db-collection/tests), `tests/correctness-oracle.test.ts` | Applied receipt positions crossed with held peers, native SQLite/SDK and cleanup evidence. A metadata-bearing `Collection.update` passes through an observed SDK watcher callback to the public and SQLite rows and an exact queued CRUD patch. Held callback ordering and synthetic zero-field cases remain controlled-only, and remote upload has no receiving fixture here. Run the focused owner with the package's `test:oracles` command. A timeout mutant proves a progress failure, not every value assertion. | -| SQLite persistence and native hosts | [persisted histories](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/persisted-oracle.test.ts), [reset/resume histories](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter-oracle.test.ts), [setup reopen](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/collection-setup-reopen-oracle.test.ts), [reload class](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/reload-class-oracle.ts), [dual-adapter resume snapshots](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot-oracle.test.ts), [Browser composed-owner histories](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts), [Browser coordinator RPC](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/browser-coordinator-oracle.test.ts), [shared-driver fairness](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.test.ts), [driver contracts](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/contracts/sqlite-driver-contract.ts), [Node shared-handle scheduling](https://github.com/TanStack/db/blob/main/packages/node-db-sqlite-persistence/tests/node-driver.test.ts), [OP-SQLite shared-handle scheduling](https://github.com/TanStack/db/blob/main/packages/react-native-db-sqlite-persistence/tests/op-sqlite-driver-oracle.test.ts), [browser OPFS lifecycle](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/opfs-page-lifecycle-oracle.test.ts), [worker diagnostics](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/opfs-worker-diagnostics-oracle.test.ts), [request order](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/opfs-request-order-oracle.ts), [Electron IPC and composed owner](https://github.com/TanStack/db/blob/main/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts), [113-law manifest](https://github.com/TanStack/db/blob/main/packages/db-collection-e2e/src/fixtures/persisted-conformance-manifest.ts) | Core cache/remote rejection/peer/reopen histories, atomic reset/resume lineage, key-set evidence, dual-adapter races, queued coordinator reload/gap notifications during startup, unscheduled startup/reset overlap, and exact driver results. The setup-reopen owner checks that a fresh adapter's first read repairs an interrupted setup, every subset of missing indexes, triggers, and bookkeeping rows, and a schema change to the fresh-install schema with rows kept or reset; complete setup costs at most one driver call over a cached read. It excludes cache-generation claims and concurrent adapters; the held-registration resume cases cover a collected generation at the setup DDL and at the registry read. The reload class owner keeps the reload law, model, histories, driver loop, and comparison in one module, with four production drivers: the Node Browser driver (real Browser persistence and coordinator, one connection per tab), the Node Electron driver (real renderers and main handler with an in-process `ipcMain` double), the real Chromium OPFS page, and a real Electron app (renderer over real IPC to a main process running the Node persistence over `node:sqlite`). Histories cover 1 to 4 eager or managed Collections with any subset of missing derived setup facts, a schema change, a database missing a migrated column or index, lock grants 1 to 5 tasks after the request, statement latency where the host controls it, a local write during another startup read (issued before or after its grant), a write held in flight at another Collection's grant, a follower write, a write from a third tab, takeover (also while a follower read is held after an earlier follower or third-tab write, and while a follower that knows the departed leader's route has a source commit queued behind a held subset read, which it flushes after the grant or which the departed leader received and never answered), restart, and a tab closed while its terms wait. It checks skipped and repaired Collection and database setup, that no term runs ahead of work queued on its tab at its grant unless a leader need is live in that election (a need that an earlier election served exempts none), that needs elect and settle, that a takeover's queued source commit settles within one RPC timeout (one timeout and a retry when the departed leader received it), and an exact per-reload budget: one registry read per adapter, one row scan per tab, two startup snapshots per eager Collection (metadata before the source starts, rows at the baseline hydrate, both required by the Electric resume laws), no rescan after a commit the tab already knows, and no database setup statement when it is complete. The implementation on origin/main fails every fixed history: the original 24 on all four drivers, and the 5 that cover per-election needs on the Node Browser driver. Production on this branch passes all 29 on all four drivers. The Node drivers reject 14 production mutants by assertion, including a lasting need latch, an earlier served need from this tab or another, a need still waiting at the grant that is ignored, and a grant that keeps the departed leader's route. The Browser coordinator RPC owner's per-election cells also reject those latches and a delayed heartbeat from the departed leader that restores its route during an election. Real Chromium rejects the never-waiting barrier across its histories; the real Electron app rejects it in 10 of the 29 fixed histories; in the others, real IPC finishes the startup reads before any grant. Hold seams patch the adapter in the process that runs it. Not owned here: a barrier that skips queued but not yet running regular work, which driver transaction order and the writer lock hide in reload histories; the shared logical scheduling oracle owns it. The real Electron app uses `node:sqlite` because Electron's ABI cannot load better-sqlite3; a native better-sqlite3 build for Electron is not run. The shared core-adapter contract checks custom local comparator dispatch for ascending, descending, and comparator-equal public-key ordering at `loadSubset` completion. Host packages register this contract, but the local sqlite3 CLI run does not prove native host execution. Browser composes public source commits with per-collection elected-owner routing and covers the complete committed-transaction wire partition through deterministic Node transport seams. Remote-subset histories distinguish logical demand, physical acquisitions, exact acquisition leases, and released replay tombstones. Electron composes source commits with a per-collection renderer owner, IPC adapter, real SQLite, and reopen checks. The shared-driver fairness owner checks K=1 complete-logical-hydrate scheduling with identical fixed, random, and seed-plus-path campaigns. It records public rows, raw dequeue reach, and a persist-first FIFO hostile control. Its Chromium OPFS fixture refines the provider boundary but does not establish a browser matrix, elapsed-time latency, unbounded eventuality, or multi-process coordination. Same-handle Node and OP-SQLite tests cover transaction admission. Controlled OPFS page/worker histories cover ownership and diagnostic-cause retention. The reset/resume owners use sqlite3 CLI and in-memory node:sqlite seams; they do not prove multi-process WAL, mobile/Tauri, or other native-device execution. Distinct database handles rely on SQLite lock admission rather than one in-process queue. React Native hosts without async-context propagation must use the transaction driver supplied to the callback for nested work. Fake workers and synthetic page events do not prove native handle release or real bfcache admission. The Browser composed seams are not real multi-context/OPFS-worker execution; the Electron harness is not an actual Electron process unless its explicit runtime-bridge mode runs. An ownerless elected node suppresses core routing, while a follower may route demand to the elected owner. The follower generation-notice and rotated-reacquisition lanes state the rotation law: a rotated follower holds the new generation's commit notices until the owner has reacquired every active demand, then ends scoped recovery and applies them. Production does not yet end that recovery, so their cells that require it are pinned as failing under the known gap [#2139](https://github.com/TanStack/db/issues/2139); the random generation-notice variant is skipped, because some of its histories pass today. Cells that still pass cover failed reacquisitions (scoped recovery stays and the failure is reported), a superseding rotation as leader, and a scoped recovery without rotation, with or without an active demand (untrusted durable rows stay hidden). Other cells cover a follower whose claim expires while its ensure is in flight and an owner reset (the follower requests its demand again). The follower remote-demand lane checks that a follower's load, and a live query over its demand, settle only after the owner's broadcast applies here, for a broadcast that applies at once, one queued behind another subset's hydration, and one while the cache claim is expired; the claim-rotation placement is pinned under #2139. A failed apply rejects the load, errors the query, and fails the source. A renewal that fails transiently at the reply rejects the load but keeps and retries the demand, without releasing the owner's acquisition; a rotation during the reply wait is tracked under #2139. The flush-side expiry check and arrival-order queueing have no killing witness yet. Removing the follower's wait on either ensure path, its terminal check, or the queued flush's terminal marking each fails it. The Electron IPC readiness oracle checks a fresh or reopened renderer, a successful or failed IPC hydration, and an upstream load that succeeds, fails, or is released during hydration, against a model of the three readiness checkpoints; a renderer that replies before the main-process read or treats an IPC failure as empty fails it. The owner is a controlled coordinator double, not two real coordinators; host coordinators retry only classified transport or admission failures while demand remains retained. The manifest excludes progressive and move suites; registration and shim runs are not device execution. The per-collection coordinator oracle's real-tab lane runs a follower and an owner tab with real Browser coordinators over Node's BroadcastChannel and Web Locks and one real SQLite file: 16 histories over the owner's source (custom or Query Collection), the broadcast's placement, the follower's apply outcome, and the consumer, plus a rotation cell. Main fails all 16 histories; an owner tab for a rotated follower generation is not reachable there. The Electron IPC readiness oracle adds a follower and an owner renderer with Electron coordinators: 4 histories over the apply outcome and the consumer, in process and through a real Electron main process; main fails all 4. | +| SQLite persistence and native hosts | [persisted histories](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/persisted-oracle.test.ts), [reset/resume histories](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/sqlite-core-adapter-oracle.test.ts), [setup reopen](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/collection-setup-reopen-oracle.test.ts), [reload class](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/reload-class-oracle.ts), [dual-adapter resume snapshots](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/sqlite-resume-snapshot-oracle.test.ts), [Browser composed-owner histories](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts), [Browser coordinator RPC](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/browser-coordinator-oracle.test.ts), [shared-driver fairness](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/shared-driver-fairness-oracle.test.ts), [driver contracts](https://github.com/TanStack/db/blob/main/packages/db-sqlite-persistence-core/tests/contracts/sqlite-driver-contract.ts), [Node shared-handle scheduling](https://github.com/TanStack/db/blob/main/packages/node-db-sqlite-persistence/tests/node-driver.test.ts), [OP-SQLite shared-handle scheduling](https://github.com/TanStack/db/blob/main/packages/react-native-db-sqlite-persistence/tests/op-sqlite-driver-oracle.test.ts), [browser OPFS lifecycle](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/opfs-page-lifecycle-oracle.test.ts), [worker diagnostics](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/opfs-worker-diagnostics-oracle.test.ts), [request order](https://github.com/TanStack/db/blob/main/packages/browser-db-sqlite-persistence/tests/opfs-request-order-oracle.ts), [Electron IPC and composed owner](https://github.com/TanStack/db/blob/main/packages/electron-db-sqlite-persistence/tests/electron-ipc.test.ts), [113-law manifest](https://github.com/TanStack/db/blob/main/packages/db-collection-e2e/src/fixtures/persisted-conformance-manifest.ts) | Core cache/remote rejection/peer/reopen histories, atomic reset/resume lineage, key-set evidence, dual-adapter races, queued coordinator reload/gap notifications during startup, unscheduled startup/reset overlap, and exact driver results. The setup-reopen owner checks that a fresh adapter's first read repairs an interrupted setup, every subset of missing indexes, triggers, and bookkeeping rows, and a schema change to the fresh-install schema with rows kept or reset; complete setup costs at most one driver call over a cached read. It excludes cache-generation claims and concurrent adapters; the held-registration resume cases cover a collected generation at the setup DDL and at the registry read. The reload class owner keeps the reload law, model, histories, driver loop, and comparison in one module, with four production drivers: the Node Browser driver (real Browser persistence and coordinator, one connection per tab), the Node Electron driver (real renderers and main handler with an in-process `ipcMain` double), the real Chromium OPFS page, and a real Electron app (renderer over real IPC to a main process running the Node persistence over `node:sqlite`). Histories cover 1 to 4 eager or managed Collections with any subset of missing derived setup facts, a schema change, a database missing a migrated column or index, lock grants 1 to 5 tasks after the request, statement latency where the host controls it, a local write during another startup read (issued before or after its grant), a write held in flight at another Collection's grant, a follower write, a write from a third tab, takeover (also while a follower read is held after an earlier follower or third-tab write, and while a follower that knows the departed leader's route has a source commit queued behind a held subset read, which it flushes after the grant or which the departed leader received and never answered), restart, and a tab closed while its terms wait. It checks skipped and repaired Collection and database setup, that no term runs ahead of work queued on its tab at its grant unless a leader need is live in that election (a need that an earlier election served exempts none), that needs elect and settle, that a takeover's queued source commit settles within one RPC timeout (one timeout and a retry when the departed leader received it), and an exact per-reload budget: one registry read per adapter, one row scan per tab, two startup snapshots per eager Collection (metadata before the source starts, rows at the baseline hydrate, both required by the Electric resume laws), no rescan after a commit the tab already knows, and no database setup statement when it is complete. The implementation on origin/main fails every fixed history: the original 24 on all four drivers, and the 5 that cover per-election needs on the Node Browser driver. Production on this branch passes all 29 on all four drivers. The Node drivers reject 14 production mutants by assertion, including a lasting need latch, an earlier served need from this tab or another, a need still waiting at the grant that is ignored, and a grant that keeps the departed leader's route. The Browser coordinator RPC owner's per-election cells also reject those latches and a delayed heartbeat from the departed leader that restores its route during an election. Real Chromium rejects the never-waiting barrier across its histories; the real Electron app rejects it in 10 of the 29 fixed histories; in the others, real IPC finishes the startup reads before any grant. Hold seams patch the adapter in the process that runs it. Not owned here: a barrier that skips queued but not yet running regular work, which driver transaction order and the writer lock hide in reload histories; the shared logical scheduling oracle owns it. The real Electron app uses `node:sqlite` because Electron's ABI cannot load better-sqlite3; a native better-sqlite3 build for Electron is not run. The shared core-adapter contract checks custom local comparator dispatch for ascending, descending, and comparator-equal public-key ordering at `loadSubset` completion. Host packages register this contract, but the local sqlite3 CLI run does not prove native host execution. Browser composes public source commits with per-collection elected-owner routing and covers the complete committed-transaction wire partition through deterministic Node transport seams. Remote-subset histories distinguish logical demand, physical acquisitions, exact acquisition leases, and released replay tombstones. Electron composes source commits with a per-collection renderer owner, IPC adapter, real SQLite, and reopen checks. The shared-driver fairness owner checks K=1 complete-logical-hydrate scheduling with identical fixed, random, and seed-plus-path campaigns. It records public rows, raw dequeue reach, and a persist-first FIFO hostile control. Its Chromium OPFS fixture refines the provider boundary but does not establish a browser matrix, elapsed-time latency, unbounded eventuality, or multi-process coordination. Same-handle Node and OP-SQLite tests cover transaction admission. Controlled OPFS page/worker histories cover ownership and diagnostic-cause retention. The reset/resume owners use sqlite3 CLI and in-memory node:sqlite seams; they do not prove multi-process WAL, mobile/Tauri, or other native-device execution. Distinct database handles rely on SQLite lock admission rather than one in-process queue. React Native hosts without async-context propagation must use the transaction driver supplied to the callback for nested work. Fake workers and synthetic page events do not prove native handle release or real bfcache admission. The Browser composed seams are not real multi-context/OPFS-worker execution; the Electron harness is not an actual Electron process unless its explicit runtime-bridge mode runs. An ownerless elected node suppresses core routing, while a follower may route demand to the elected owner. The follower generation-notice and rotated-reacquisition lanes state the rotation law: a rotated follower holds the new generation's commit notices until the owner has reacquired every active demand, then ends scoped recovery and applies them. Production does not yet end that recovery, so their cells that require it are pinned as failing under the known gap [#2139](https://github.com/TanStack/db/issues/2139); the random generation-notice campaign is omitted until the fix, because some of its histories pass today. Cells that still pass cover failed reacquisitions (scoped recovery stays and the failure is reported), a superseding rotation as leader, and a scoped recovery without rotation, with or without an active demand (untrusted durable rows stay hidden). Other cells cover a follower whose claim expires while its ensure is in flight and an owner reset (the follower requests its demand again). The follower remote-demand lane checks that a follower's load, and a live query over its demand, settle only after the owner's broadcast applies here, for a broadcast that applies at once, one queued behind another subset's hydration, and one while the cache claim is expired; the claim-rotation placement is pinned under #2139. A failed apply rejects the load, errors the query, and fails the source. A renewal that fails transiently at the reply rejects the load but keeps and retries the demand, without releasing the owner's acquisition; a rotation during the reply wait is tracked under #2139. The flush-side expiry check and arrival-order queueing have no killing witness yet. Removing the follower's wait on either ensure path, its terminal check, or the queued flush's terminal marking each fails it. The Electron IPC readiness oracle checks a fresh or reopened renderer, a successful or failed IPC hydration, and an upstream load that succeeds, fails, or is released during hydration, against a model of the three readiness checkpoints; a renderer that replies before the main-process read or treats an IPC failure as empty fails it. The owner is a controlled coordinator double, not two real coordinators; host coordinators retry only classified transport or admission failures while demand remains retained. The manifest excludes progressive and move suites; registration and shim runs are not device execution. The per-collection coordinator oracle's real-tab lane runs a follower and an owner tab with real Browser coordinators over Node's BroadcastChannel and Web Locks and one real SQLite file: 16 histories over the owner's source (custom or Query Collection), the broadcast's placement, the follower's apply outcome, and the consumer, plus a rotation cell. Main fails all 16 histories; an owner tab for a rotated follower generation is not reachable there. The Electron IPC readiness oracle adds a follower and an owner renderer with Electron coordinators: 4 histories over the apply outcome and the consumer, in process and through a real Electron main process; main fails all 4. | | SQLite expression-index planning | [Node expression-index oracle](https://github.com/TanStack/db/blob/main/packages/node-db-sqlite-persistence/tests/expression-index-oracle.test.ts) | RFC #1659 invariant 8 owns identical persisted-index and runtime-expression shapes. Independent expected keys are checked against direct captured SQL, adapter results, and named-index plans. Generated BigInts use SQLite's signed range; one fixed case checks legacy oversized-value reads. Other limits: bounded unqualified JSON paths/scalars, Node BetterSQLite, and no null, arbitrary raw SQL, or native-host planning. Run the package's `test:oracles` campaign. | | Offline execution | [scheduler](https://github.com/TanStack/db/blob/main/packages/offline-transactions/tests/KeyScheduler-oracle.property.test.ts), [leadership](https://github.com/TanStack/db/blob/main/packages/offline-transactions/tests/leadership-replay-oracle.property.test.ts), [settlement](https://github.com/TanStack/db/blob/main/packages/offline-transactions/tests/transaction-settlement-oracle.property.test.ts), [serialization](https://github.com/TanStack/db/blob/main/packages/offline-transactions/tests/transaction-serializer-oracle.property.test.ts), [web connectivity replay](https://github.com/TanStack/db/blob/main/packages/offline-transactions/tests/connectivity-replay-oracle.test.ts), [review record](oracle-reviews/pr-1879-visible-replay.md) | Declarative FIFO eligibility, per-transaction outcomes, durable state and typed wire trees. The web connectivity owner checks a persisted write under a false browser online hint, then visible replay by event or explicit retry, at provider-call, outbox, and caller-settlement cuts. It uses controlled browser globals and fake storage; it does not prove actual browser network detection, genuine offline retry timing, multi-tab leadership transfer, or React Native behavior. Issued work may finish after ownership loss, but new work must not start. Exactly-once network execution is not promised. | | Frameworks | [React conformance](https://github.com/TanStack/db/blob/main/packages/react-db/tests/conformance.test.tsx), [React pagination](https://github.com/TanStack/db/blob/main/packages/react-db/tests/infinite-query-conformance.test.tsx), [shared framework contracts](https://github.com/TanStack/db/tree/main/packages/db/tests/conformance), [Vue synchronous publication](https://github.com/TanStack/db/blob/main/packages/vue-db/tests/useLiveQuery-publication-oracle.test.ts) | Exact exposed rows/pages and each framework's own lifecycle cuts. Vue's synchronous watcher checks insert, update, and nonterminal delete through supplied Collection and identity query inputs; it does not cover an empty final result, multiple changes in one transaction, or query recompilation. A React witness does not prove Vue/Solid/Angular/Svelte scheduling. Preserve their receiving registrations. The shared lifecycle histories distinguish adapter-created projections from caller-owned retention: omitted and explicitly `undefined` `gcTime` use the 1 ms adapter policy; `gcTime: 0`, a positive 100 ms retention, and supplied Collections keep caller authority. They observe retirement after unmount, replacement, disabling, and positive expiry, plus survival before that expiry. The shared suite also checks initial-query readiness over a lagging on-demand source, for a list and a `findOne` query and for the infinite query (React, Vue, and Svelte): not ready before the subset settles, ready at settlement while the source is still `loading`, and still ready after the source becomes ready. Solid's Suspense over that source has its own witness. A separate Solid two-resource Suspense witness holds one ready projection past the 50 ms unsubscribed floor while its sibling remains pending, then checks both commit and boundary abandonment release the temporary preload owner and preserve ordinary 1 ms teardown. | diff --git a/eslint.config.js b/eslint.config.js index 34cfc4c5ef..4004fd1a5d 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -1,4 +1,5 @@ import { tanstackConfig } from '@tanstack/eslint-config' +import oraclePlugin from './scripts/eslint-plugin-oracle/index.js' export default [ ...tanstackConfig, @@ -45,4 +46,5 @@ export default [ 'import/no-cycle': `error`, }, }, + oraclePlugin.configs.recommended, ] diff --git a/lint-staged.config.js b/lint-staged.config.js new file mode 100644 index 0000000000..7741941e3a --- /dev/null +++ b/lint-staged.config.js @@ -0,0 +1,21 @@ +// ESLint ran out of memory on a commit of more than about 100 files, so +// staged files are linted in chunks, one ESLint process each. lint-staged +// splits each command without a shell or escape sequences, so plain double +// quotes keep Windows backslashes intact. +const chunkSize = 40 + +export default { + // The oracle rules, without inline directives. lint-staged appends the + // staged oracle files as arguments. + '*oracle*.{ts,tsx}': `node scripts/agent-lint-feedback.mjs`, + '*.{ts,tsx}': (files) => { + const commands = [] + for (let start = 0; start < files.length; start += chunkSize) { + const chunk = files.slice(start, start + chunkSize) + commands.push( + `eslint --fix ${chunk.map((file) => `"${file}"`).join(` `)}`, + ) + } + return commands + }, +} diff --git a/package.json b/package.json index 6729e5169d..85af542017 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,9 @@ "format": "prettier --experimental-cli --ignore-unknown '**/*' --write", "lint": "pnpm --filter \"./packages/**\" lint", "lint:check": "pnpm --filter \"./packages/**\" lint --fix=false", + "lint:changed": "node scripts/agent-lint-feedback.mjs", + "lint:oracles": "node scripts/agent-lint-feedback.mjs --all", + "test:lint-rules": "node --test \"scripts/eslint-plugin-oracle/*.test.js\"", "lint-all": "eslint . --fix", "prepare": "husky", "test": "pnpm --filter \"./packages/**\" test && pnpm --filter @tanstack/db test:indexed-db:package", @@ -67,10 +70,5 @@ "vite": "^7.3.0", "vitest": "^3.2.4", "zod": "^3.25.76" - }, - "lint-staged": { - "*.{ts,tsx}": [ - "eslint --fix" - ] } } diff --git a/packages/db-sqlite-persistence-core/tests/persisted-oracle.test.ts b/packages/db-sqlite-persistence-core/tests/persisted-oracle.test.ts index c4304f0de3..647a9520e4 100644 --- a/packages/db-sqlite-persistence-core/tests/persisted-oracle.test.ts +++ b/packages/db-sqlite-persistence-core/tests/persisted-oracle.test.ts @@ -215,7 +215,6 @@ const describeUnlessOracleReplay = requestedOracleReplayProperty === undefined || requestedOracleReplayProperty === `persistence.retained-demand` || requestedOracleReplayProperty === `persistence.generation-notice` || - requestedOracleReplayProperty === `persistence.follower-generation-notice` || requestedOracleReplayProperty === `persistence.follower-remote-demand` ? describe : describe.skip @@ -26043,7 +26042,10 @@ describeUnlessOracleReplay(`persistedCollectionOptions`, () => { ), }) // Known gap: see #2139. A rotated follower stays in scoped recovery, so - // the owner's rows in the new generation never become visible. + // the owner's rows in the new generation never become visible. Some + // generated histories pass today, so a random draw cannot be pinned as + // failing; this fixed seed pins the gap. The fix for #2139 adds the random + // and replayed campaign back. it.fails( `checks generated follower generation-notice histories (fixed)`, () => @@ -26055,17 +26057,6 @@ describeUnlessOracleReplay(`persistedCollectionOptions`, () => { { seed: 2128, numRuns: oracleRuns(12) }, ), ) - // Known gap: see #2139. Some generated histories pass today, so a random - // draw cannot be pinned as failing; the fixed seed above pins the gap. - it.skip(`checks generated follower generation-notice histories (random or replayed)`, () => - fc.assert( - fc.asyncProperty( - followerGenerationNoticeHistory, - assertFollowerGenerationNoticeHistory, - ), - oraclePropertyOptions(12, `persistence.follower-generation-notice`), - )) - /** * A follower routes demand it cannot serve to the elected owner. The owner * loads the rows upstream, commits them durably, and broadcasts the commit diff --git a/packages/db/tests/oracle-config.ts b/packages/db/tests/oracle-config.ts index 8fe37b9d2e..dffda0f111 100644 --- a/packages/db/tests/oracle-config.ts +++ b/packages/db/tests/oracle-config.ts @@ -46,7 +46,6 @@ const staticOracleProperties = [ `sqlite-reload.class`, `sqlite-reload.class.electron`, `persistence.generation-notice`, - `persistence.follower-generation-notice`, `persistence.follower-remote-demand`, `sqlite-ordinary.write-history`, `sqlite-ordinary.independent-work`, diff --git a/packages/db/tests/proxy-oracle.test.ts b/packages/db/tests/proxy-oracle.test.ts index db57e3901a..a6135a77dc 100644 --- a/packages/db/tests/proxy-oracle.test.ts +++ b/packages/db/tests/proxy-oracle.test.ts @@ -1970,7 +1970,7 @@ describe(`Proxy Library`, () => { delete proxy.arr[2] expect(getChanges()).toEqual({ - // eslint-disable-next-line + // eslint-disable-next-line no-sparse-arrays -- the deleted index is a hole arr: [1, 2, , 4, 5], }) expect(2 in arr).toBe(true) diff --git a/scripts/agent-lint-feedback.mjs b/scripts/agent-lint-feedback.mjs new file mode 100644 index 0000000000..635cfece92 --- /dev/null +++ b/scripts/agent-lint-feedback.mjs @@ -0,0 +1,243 @@ +#!/usr/bin/env node +/** + * Returns oracle lint reports to a coding agent while it works. + * + * AGENTS.md forbids the shortcuts that make an oracle pass without enforcing + * its law. Prose did not stop them, because nothing made a shortcut visible + * while the agent was still working. Each agent harness can run a command + * after an edit and at the end of a turn, and can return the command's output + * to the model. This script is that command for every harness: a thin + * adapter per harness reads its hook input and writes its response format. + * + * --format=claude|codex|gemini|copilot the harness that runs the hook + * --event=edit after an edit: lint the edited files + * --event=stop at the end of a turn: lint every file the branch changed + * --format=text print the reports and exit 1: `pnpm lint:changed` lints + * the changed files, `pnpm lint:oracles` (`--all`) every + * oracle file, and the pre-commit hook the staged files it + * passes as arguments + * + * Every mode runs only the `oracle/` rules, and ignores inline directives. + * The rules have no inline exception, and ESLint cannot protect one rule from + * an `eslint-disable` comment, so this pass is where they are enforced. The + * root ESLint config also runs them, so editors show them, but a directive can + * silence them there. The rules need no type information, so a typical file + * takes tens of milliseconds, and the largest about a second. The repository + * config builds a TypeScript program and takes several seconds per file, + * which is too slow for every edit; `pnpm lint`, the pre-commit hook, and CI + * run it. + * + * An edit hook sees only the harness's own edit tools. An edit made through a + * shell command reaches the model at the end of the turn, when the stop hook + * lints every changed file. When a harness's hook input names no file that + * this script can find, the edit hook lints every changed file as well. + */ +import { execFileSync } from 'node:child_process' +import { existsSync, readFileSync } from 'node:fs' +import { isAbsolute, join, relative, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { parseArgs } from 'node:util' +import tsParser from '@typescript-eslint/parser' +import { ESLint } from 'eslint' +import oraclePlugin, { oracleFiles } from './eslint-plugin-oracle/index.js' + +const root = fileURLToPath(new URL(`../`, import.meta.url)) + +const { values: options, positionals } = parseArgs({ + options: { + format: { type: `string`, default: `text` }, + event: { type: `string`, default: `edit` }, + all: { type: `boolean`, default: false }, + }, + allowPositionals: true, +}) + +const formats = new Set([`claude`, `codex`, `gemini`, `copilot`, `text`]) +if (!formats.has(options.format)) { + throw new Error(`--format must be one of ${[...formats].join(`, `)}`) +} +if (![`edit`, `stop`].includes(options.event)) { + throw new Error(`--event must be edit or stop`) +} +const isHook = options.format !== `text` + +/** Every file that the branch adds or changes, including uncommitted work. */ +function changedFiles() { + const git = (...args) => + execFileSync(`git`, args, { cwd: root, encoding: `utf8` }).trim() + let base = `HEAD` + try { + base = git(`merge-base`, `HEAD`, `origin/main`) + } catch { + // Without origin/main, the uncommitted work is the change. + } + return [ + git(`diff`, `--name-only`, `--diff-filter=ACMR`, base), + git(`ls-files`, `--others`, `--exclude-standard`), + ] + .flatMap((output) => output.split(`\n`)) + .filter(Boolean) + .map((file) => join(root, file)) +} + +const pathKeys = /^(file_?path|filePath|path|absolute_path|target_file)$/i +const patchHeader = /^\*\*\* (?:Add File|Update File|Move to): (.+)$/gm + +/** + * The files that a hook input names. Harnesses put the path in different + * fields, and Codex sends the text of an `apply_patch` call instead, so this + * searches the whole input for path fields and patch headers. + */ +function editedFiles(input) { + const base = typeof input.cwd === `string` ? input.cwd : root + const found = new Set() + const visit = (value, key) => { + if (typeof value === `string`) { + if (key !== undefined && pathKeys.test(key)) found.add(value) + for (const match of value.matchAll(patchHeader)) + found.add(match[1].trim()) + // Copilot's tool arguments can arrive as a JSON string. + if (/^\s*\{/.test(value)) { + try { + visit(JSON.parse(value)) + } catch { + // Not JSON; already searched as text. + } + } + } else if (Array.isArray(value)) { + for (const item of value) visit(item) + } else if (value !== null && typeof value === `object`) { + for (const [childKey, child] of Object.entries(value)) + visit(child, childKey) + } + } + visit(input.tool_input ?? input.toolArgs ?? input) + return [...found].map((file) => + isAbsolute(file) ? file : resolve(base, file), + ) +} + +function readHookInput() { + if (!isHook) return {} + const text = readFileSync(0, `utf8`) + return text.trim() === `` ? {} : JSON.parse(text) +} + +function createLinter() { + return new ESLint({ + cwd: root, + overrideConfigFile: true, + overrideConfig: [ + { ignores: [`**/dist/**`] }, + { files: [`**/*.ts`, `**/*.tsx`], languageOptions: { parser: tsParser } }, + { linterOptions: { noInlineConfig: true } }, + oraclePlugin.configs.recommended, + ], + }) +} + +/** Lints the oracle files among `files`, or every oracle file without them. */ +async function lint(eslint, files) { + const lintable = [] + for (const file of new Set(files ?? [])) { + if (!existsSync(file) || relative(root, file).startsWith(`..`)) continue + if (!/oracle[^/]*\.tsx?$/.test(file)) continue + if (await eslint.isPathIgnored(file)) continue + lintable.push(file) + } + if (files !== undefined && lintable.length === 0) return [] + const results = await eslint.lintFiles( + files === undefined ? oracleFiles : lintable, + ) + // Keep oracle reports and files that cannot be parsed. This drops the + // notices that the ignored directives have no effect. + return results.map((result) => ({ + ...result, + messages: result.messages.filter( + (message) => message.fatal || message.ruleId?.startsWith(`oracle/`), + ), + })) +} + +function reportText(results) { + const lines = [] + for (const result of results) { + for (const message of result.messages) { + if (message.severity < 2) continue + lines.push( + `${relative(root, result.filePath)}:${message.line}:${message.column} ` + + `${message.message} (${message.ruleId ?? `parse`})`, + ) + } + } + return lines.join(`\n`) +} + +/** + * Writes the report in the harness's format. A post-edit report reaches the + * model beside the tool result. A stop report makes the agent take another + * turn, once: when the harness says a stop hook already continued this turn, + * the report is left for the pre-commit hook and CI, so a report the agent + * cannot fix does not loop. + */ +function respond(report, input, scope) { + const { format, event } = options + if (!isHook) { + if (report !== ``) { + process.stderr.write(`${report}\n`) + process.exitCode = 1 + } + return + } + const header = + scope === `edited` + ? `Oracle lint reports in the file you edited:` + : `Oracle lint reports in this branch's changed files. Fix them before finishing:` + const reason = `${header}\n${report}\nSee "How oracle work is scored" in AGENTS.md.` + if (report === `` || (event === `stop` && input.stop_hook_active === true)) { + // Codex requires JSON on stdout when a stop hook exits 0. + if (format === `codex` && event === `stop`) process.stdout.write(`{}\n`) + return + } + if (format === `claude` || format === `codex`) { + // Exit code 2 returns stderr to the model in both harnesses. + process.stderr.write(`${reason}\n`) + process.exitCode = 2 + return + } + if (format === `gemini`) { + const output = + event === `stop` + ? { decision: `deny`, reason } + : { + hookSpecificOutput: { + hookEventName: `AfterTool`, + additionalContext: reason, + }, + } + process.stdout.write(`${JSON.stringify(output)}\n`) + return + } + // Copilot treats a nonzero exit as a warning, so it needs JSON on stdout. + const output = + event === `stop` + ? { decision: `block`, reason } + : { additionalContext: reason } + process.stdout.write(`${JSON.stringify(output)}\n`) +} + +/** The files to lint: named on the command line, edited, or changed. */ +function selectFiles(input) { + if (options.all) return { scope: `all`, files: undefined } + if (positionals.length > 0) { + return { scope: `named`, files: positionals.map((file) => resolve(file)) } + } + const edited = options.event === `edit` && isHook ? editedFiles(input) : [] + return edited.length > 0 + ? { scope: `edited`, files: edited } + : { scope: `changed`, files: changedFiles() } +} + +const input = readHookInput() +const { scope, files } = selectFiles(input) +respond(reportText(await lint(createLinter(), files)), input, scope) diff --git a/scripts/eslint-plugin-oracle/index.js b/scripts/eslint-plugin-oracle/index.js new file mode 100644 index 0000000000..9588d8d324 --- /dev/null +++ b/scripts/eslint-plugin-oracle/index.js @@ -0,0 +1,35 @@ +import noDisabled from './rules/no-disabled.js' + +/** + * Lint rules for oracle files. + * + * AGENTS.md lists the shortcuts that make an oracle pass without enforcing + * its law ("How oracle work is scored"). These rules report the ones that a + * syntactic check can see, in files whose names contain `oracle`. They need no + * type information, so an agent's post-edit hook can run them on one file in + * well under a second. A directive can silence them in the root config, so + * scripts/agent-lint-feedback.mjs enforces them in a pass that ignores inline + * directives. + */ + +export const oracleFiles = [`**/*oracle*.ts`, `**/*oracle*.tsx`] + +const plugin = { + meta: { name: `eslint-plugin-oracle` }, + rules: { + 'no-disabled': noDisabled, + }, +} + +plugin.configs = { + recommended: { + name: `oracle/recommended`, + files: oracleFiles, + plugins: { oracle: plugin }, + rules: { + 'oracle/no-disabled': `error`, + }, + }, +} + +export default plugin diff --git a/scripts/eslint-plugin-oracle/no-disabled.test.js b/scripts/eslint-plugin-oracle/no-disabled.test.js new file mode 100644 index 0000000000..0e8ca8017b --- /dev/null +++ b/scripts/eslint-plugin-oracle/no-disabled.test.js @@ -0,0 +1,137 @@ +import { describe, it } from 'node:test' +import { RuleTester } from 'eslint' +import tsParser from '@typescript-eslint/parser' +import noDisabled from './rules/no-disabled.js' + +RuleTester.describe = describe +RuleTester.it = it + +const ruleTester = new RuleTester({ + languageOptions: { parser: tsParser }, +}) + +ruleTester.run(`oracle/no-disabled`, noDisabled, { + valid: [ + // A replay that does not select this property registers an empty test. + `it.skip('runs only when its replay property is selected', () => {})`, + `it.todo('loads every subset from one generation')`, + // Playwright's runtime skip, conditioned on the environment. + `test.skip(browserName !== 'chromium', 'Chromium-only control')`, + `test.skip(isMobile)`, + `test.skip(false, 'kept for a later browser')`, + // A replay selection may switch a test off. + `it.skipIf(replayRequested)('checks the history', () => check())`, + `it.runIf(replay.replayPath === undefined)('checks the history', () => check())`, + `fc.assert(property, oraclePropertyOptions(12, 'cleanup-queue.history'))`, + `fc.assert(property, { numRuns: oracleRuns(20) })`, + // A pinned replay or mutant witness runs one fixed history. + `fc.check(property, { seed: 205203, numRuns: 1 })`, + `fc.check(property, { seed: fixedSeed, numRuns: 1 })`, + `fc.assert(property, { seed: 2128, numRuns: oracleRuns(1) })`, + `fc.assert(property, replayRequested ? { numRuns: oracleRuns(1) } : {})`, + `fc.assert(property, campaign === 'replay' ? { numRuns: 1 } : {})`, + `fc.assert(property, options.path === undefined ? options : { ...options, numRuns: 1 })`, + // An expectation about a recorded run is not a run option. + `expect(report).toMatchObject({ numRuns: 1, failed: true })`, + // A known gap is pinned as failing on a fixed seed, not skipped. + `it.fails('checks the fixed history', () => fc.assert(property, { seed: 2128, numRuns: oracleRuns(12) }))`, + ], + invalid: [ + { + code: `it.only('checks the history', () => {})`, + errors: [{ messageId: `only` }], + }, + { + code: `describe.only('cleanup queue', () => { it('a', () => {}) })`, + errors: [{ messageId: `only` }], + }, + { + code: `fcTest.only.prop([history])('checks the history', (h) => check(h))`, + errors: [{ messageId: `only` }], + }, + { + code: `it.skip('checks random histories', () => fc.assert(property))`, + errors: [{ messageId: `skip`, data: { name: `skip` } }], + }, + { + code: `describe.skip('cleanup queue', () => { it('a', () => check()) })`, + errors: [{ messageId: `skip` }], + }, + { + code: `fcTest.skip.prop([history])('checks the history', (h) => check(h))`, + errors: [{ messageId: `skip` }], + }, + { + code: `it.skip('checks follower histories', assertFollowerHistory)`, + errors: [{ messageId: `skip` }], + }, + { + code: `fcTest.skip.prop([history])('checks the history', checkHistory)`, + errors: [{ messageId: `skip` }], + }, + { + code: `test.skip()`, + errors: [{ messageId: `skip` }], + }, + { + code: `test.skip(true, 'flaky in CI')`, + errors: [{ messageId: `skip` }], + }, + { + code: `it.skipIf(true)('checks the history', () => check())`, + errors: [{ messageId: `conditional`, data: { name: `skipIf` } }], + }, + { + code: `describe.runIf(false)('cleanup queue', () => { it('a', () => check()) })`, + errors: [{ messageId: `conditional`, data: { name: `runIf` } }], + }, + { + code: `describe.skipIf(process.env.CI)('cleanup queue', () => { it('a', () => check()) })`, + errors: [{ messageId: `conditional` }], + }, + { + code: `it.todo('checks the history', () => { check() })`, + errors: [{ messageId: `skip`, data: { name: `todo` } }], + }, + { + code: `fc.assert(property, { numRuns: 1 })`, + errors: [{ messageId: `singleRun` }], + }, + { + code: `fc.assert(property, { ...oraclePropertyOptions(100, 'p'), numRuns: 1 })`, + errors: [{ messageId: `singleRun` }], + }, + { + code: `fc.assert(property, oraclePropertyOptions(1, 'cleanup-queue.history'))`, + errors: [{ messageId: `singleRun` }], + }, + { + code: `fc.assert(property, { seed: Date.now(), numRuns: 1 })`, + errors: [{ messageId: `singleRun` }], + }, + { + code: `fc.assert(property, { seed: undefined, numRuns: 1 })`, + errors: [{ messageId: `singleRun` }], + }, + { + code: `fc.assert(property, filePath ? { numRuns: 1 } : {})`, + errors: [{ messageId: `singleRun` }], + }, + { + code: `it.skipIf(useFilePath)('checks the history', () => check())`, + errors: [{ messageId: `conditional` }], + }, + { + code: `fc.assert(property, { numRuns: oracleRuns(0) })`, + errors: [{ messageId: `singleRun` }], + }, + { + // A comment does not grant an exception. + code: ` + // oracle-exclusion: known gap + it.skip('checks random histories', () => fc.assert(property)) + `, + errors: [{ messageId: `skip` }], + }, + ], +}) diff --git a/scripts/eslint-plugin-oracle/rules/no-disabled.js b/scripts/eslint-plugin-oracle/rules/no-disabled.js new file mode 100644 index 0000000000..d4bfcacabd --- /dev/null +++ b/scripts/eslint-plugin-oracle/rules/no-disabled.js @@ -0,0 +1,245 @@ +/** + * An oracle enforces its laws only in the tests that run. This rule reports + * the edits that stop a law from running while every test still passes: + * + * - `.only` on a test or suite, which silences every other law in the file. + * - `.skip` or `.todo` on a test or suite that has a body, inline or passed by + * name. An empty body is a placeholder that disables nothing, such as the + * test that a replay registers when it does not select its property. + * - `.skipIf(condition)` or `.runIf(condition)` unless the condition names a + * replay. Only a replay selection may switch an oracle test off. + * - Playwright's `test.skip()` or `test.skip(true, reason)` inside a test, + * which skips it unconditionally. `test.skip(condition, reason)` with an + * environment condition, such as the browser, is allowed. + * - A generated property limited to one unseeded run, by a literal + * `numRuns: 1` (or 0) or a base of 1 passed to the run helpers. One random + * history is not a generated property. A pinned replay passes a fixed + * `seed` beside its `numRuns`, or selects the one run under a condition + * that names the replay, so it is not reported. + * + * The rule has no inline exception. A known gap is pinned with `it.fails` on + * a fixed seed, not skipped. + * + * The rule cannot see a run count lowered to any value above 1, a condition + * that never selects a replay, or a skip reached through an alias such as + * `const run = it.skip`. Reviewers still check those. + */ + +const testRoots = new Set([`describe`, `fcTest`, `it`, `suite`, `test`]) +const conditionalModifiers = new Set([`runIf`, `skipIf`]) +const runHelpers = new Set([ + `oraclePropertyOptions`, + `oracleRandomParameters`, + `oracleRuns`, +]) + +/** The root identifier and member names of a callee such as `it.skip.prop(...)`. */ +function calleeChain(callee) { + const names = [] + let node = callee + for (;;) { + if (node.type === `MemberExpression`) { + if (node.property.type === `Identifier`) names.unshift(node.property.name) + node = node.object + } else if (node.type === `CallExpression`) { + node = node.callee + } else { + break + } + } + return { root: node.type === `Identifier` ? node.name : undefined, names } +} + +function isFunction(node) { + return ( + node?.type === `ArrowFunctionExpression` || + node?.type === `FunctionExpression` + ) +} + +/** A test body that runs something: a named function or a non-empty one. */ +function hasBody(body) { + if (!isFunction(body)) return true + return body.body.type !== `BlockStatement` || body.body.body.length > 0 +} + +function isStringLike(node) { + return ( + node.type === `TemplateLiteral` || + (node.type === `Literal` && typeof node.value === `string`) + ) +} + +function isBooleanLiteral(node) { + return node?.type === `Literal` && typeof node.value === `boolean` +} + +/** The condition of `it.skipIf(condition)` in a callee chain, if there is one. */ +function registrationCondition(callee) { + let node = callee + while (node.type === `CallExpression` || node.type === `MemberExpression`) { + if ( + node.type === `CallExpression` && + node.callee.type === `MemberExpression` && + conditionalModifiers.has(node.callee.property.name) + ) { + return { name: node.callee.property.name, condition: node.arguments[0] } + } + node = node.type === `CallExpression` ? node.callee : node.object + } + return undefined +} + +/** + * Whether a `.skip` or `.todo` call switches off a body that runs something. + * A test registration passes `(name, body)`. Playwright's runtime form passes + * `(condition, reason)`, `(condition)`, or nothing, and skips the test it + * runs in unless the condition is false. + */ +function disablesBody(args) { + if (args.length === 0) return true + const [first, second] = args + if (second === undefined || isStringLike(second)) { + return isBooleanLiteral(first) && first.value + } + return hasBody(second) +} + +/** A numeric literal at most 1: the run counts that check one history or none. */ +function isSingleRunLiteral(node) { + return ( + node?.type === `Literal` && + typeof node.value === `number` && + node.value <= 1 + ) +} + +/** + * Source text that names a replay: `replay` anywhere, or `path` or `seed` as + * a whole name, as in `options.path`. `filePath` names no replay. + */ +const replayText = /replay|(? propertyName(property) === `seed`, + ) + if (seed !== undefined) { + return !unfixedSeedText.test(sourceCode.getText(seed.value)) + } + const { parent } = object + if (parent.type === `ConditionalExpression` && parent.test !== object) { + return replayText.test(sourceCode.getText(parent.test)) + } + if (parent.type === `CallExpression` && parent.arguments.includes(object)) { + return calleeChain(parent.callee).root === `expect` + } + return false +} + +function propertyName(property) { + if (property.type !== `Property` || property.computed) return undefined + return property.key.type === `Identifier` + ? property.key.name + : property.key.value +} + +/** @type {import('eslint').Rule.RuleModule} */ +export default { + meta: { + type: `problem`, + docs: { + description: `Report oracle tests that stop running: .only, a skipped body, or one unseeded run.`, + }, + schema: [], + messages: { + only: `\`.only\` runs only this test, so every other law in the file stops running. Remove it before committing.`, + skip: `\`.{{name}}\` stops this test from enforcing its law. Remove it. To keep a known gap visible, pin a failing history with \`it.fails\` and a fixed seed.`, + conditional: `\`.{{name}}\` can switch this test off, and its condition names no replay. Only a replay selection may switch an oracle test off. Remove the condition.`, + singleRun: `A generated property with one unseeded run checks one random history. Pass a base run count to oracleRuns or oraclePropertyOptions; a pinned replay passes a fixed \`seed\` beside \`numRuns\`.`, + }, + }, + create(context) { + const { sourceCode } = context + + /** Whether `node` is the value of a `numRuns` option that a replay pins. */ + function isPinnedRunCount(node) { + const { parent } = node + return ( + parent.type === `Property` && + propertyName(parent) === `numRuns` && + isReplayOrObservation(parent.parent, sourceCode) + ) + } + + return { + CallExpression(node) { + // A curried call such as `fcTest.prop(arbs)(name, fn)` is reported + // once, at the outermost call, which holds the test body. + if ( + node.parent.type === `CallExpression` && + node.parent.callee === node + ) { + return + } + const { root, names } = calleeChain(node.callee) + if (runHelpers.has(root) && names.length === 0) { + if ( + isSingleRunLiteral(node.arguments[0]) && + !isPinnedRunCount(node) + ) { + context.report({ node: node.arguments[0], messageId: `singleRun` }) + } + return + } + if (!testRoots.has(root)) return + if (names.includes(`only`)) { + context.report({ + node: node.callee, + messageId: `only`, + }) + return + } + const conditional = registrationCondition(node.callee) + if (conditional !== undefined) { + if ( + conditional.condition === undefined || + !replayText.test(sourceCode.getText(conditional.condition)) + ) { + context.report({ + node, + messageId: `conditional`, + data: { name: conditional.name }, + }) + } + return + } + const disabled = names.find( + (name) => name === `skip` || name === `todo`, + ) + if (disabled !== undefined && disablesBody(node.arguments)) { + context.report({ node, messageId: `skip`, data: { name: disabled } }) + } + }, + Property(node) { + if ( + propertyName(node) === `numRuns` && + isSingleRunLiteral(node.value) && + !isPinnedRunCount(node.value) + ) { + context.report({ node, messageId: `singleRun` }) + } + }, + } + }, +} From 012f406d38b193c481930b83f426351dd11774a3 Mon Sep 17 00:00:00 2001 From: Test Date: Fri, 23 Oct 2026 11:20:37 -0600 Subject: [PATCH 2/4] fix(oracles): pin one-run and replay exemptions structurally Replace the rule's "names a replay" heuristic, which wrong-branch conditionals, inverted conditions, the node:path module, and environment seeds all passed. A one-run options object now needs a `path` (a recorded counterexample) or a seed that is a number literal or a constant bound to one. `skipIf` accepts only `oracleReplayRequested()` from oracle-config, and `runIf` is always reported. The rule also reports Playwright `fixme`, context skips such as `ctx.skip()`, and runtime skips other than `browserName !== ''`. oraclePropertyOptions now returns one run for a path replay, so six campaign sites drop their own one-run conditional. The OPFS diagnostics oracle uses the shared replay variables, and the replay calibration fixture overrides its run count after the replay options are chosen. The enforcing pass lists tracked oracle files with git, so it skips untracked worktrees, and it fails when no main branch is available to compare against. CONTRIBUTING.md states that lint:changed runs only the oracle rules. Stale prose about the removed replay name is corrected. Refs #2234 Co-authored-by: Isaac --- CONTRIBUTING.md | 7 +- .../opfs-worker-diagnostics-oracle.test.ts | 26 ++- .../per-collection-coordinator-oracle.test.ts | 16 +- ...tructural-identity-oracle.property.test.ts | 10 +- packages/db/tests/oracle-config.ts | 15 ++ .../db/tests/oracle-replay.fixture.test.ts | 8 +- packages/db/tests/oracle-replay.test.ts | 19 +- ...ad-subset-replay-refinement-oracle.test.ts | 5 +- ...ordered-source-loader-state-oracle.test.ts | 5 +- ...rsor-pagination-oracle.integration.test.ts | 5 +- .../refetch-ordered-prefix-oracle.test.ts | 8 +- scripts/agent-lint-feedback.mjs | 70 ++++-- scripts/eslint-plugin-oracle/index.js | 2 +- .../eslint-plugin-oracle/no-disabled.test.js | 93 +++++++- .../eslint-plugin-oracle/rules/no-disabled.js | 212 ++++++++++++------ 15 files changed, 345 insertions(+), 156 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 985863341c..872abef797 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -53,9 +53,10 @@ If you have been assigned to fix an issue or develop a new feature, please follo pnpm lint ``` - `pnpm lint:changed` lints only the files that your branch changes. If you - work with a coding agent, the repository's hooks return oracle lint reports - to it as it edits. Codex and Gemini CLI run project hooks only after you + `pnpm lint:changed` runs only the `oracle/` rules, on the oracle files that + your branch changes. It does not replace `pnpm lint`. If you work with a + coding agent, the repository's hooks return oracle lint reports to it as it + edits. Codex and Gemini CLI run project hooks only after you trust them in `/hooks`; review them again when they change. - Implement your changes and tests in the relevant package or example. diff --git a/packages/browser-db-sqlite-persistence/tests/opfs-worker-diagnostics-oracle.test.ts b/packages/browser-db-sqlite-persistence/tests/opfs-worker-diagnostics-oracle.test.ts index 67f813beb8..0a2bb16923 100644 --- a/packages/browser-db-sqlite-persistence/tests/opfs-worker-diagnostics-oracle.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/opfs-worker-diagnostics-oracle.test.ts @@ -1,5 +1,10 @@ import { fc } from '@fast-check/vitest' import { afterEach, describe, expect, it, vi } from 'vitest' +import { + oracleReplayRequested, + oracleRuns, + readOracleRunConfig, +} from '../../db/tests/oracle-config' import type { BrowserOPFSWorkerRequest, BrowserOPFSWorkerResponse, @@ -142,13 +147,13 @@ const diagnosticInput = fc.record({ ), }) -const replaySeed = process.env.TANSTACK_DB_OPFS_DIAGNOSTIC_ORACLE_SEED -const diagnosticSeed = Number(replaySeed ?? 1846) -const diagnosticRuns = Number( - process.env.TANSTACK_DB_OPFS_DIAGNOSTIC_ORACLE_RUNS ?? 40, -) -const diagnosticPath = process.env.TANSTACK_DB_OPFS_DIAGNOSTIC_ORACLE_PATH -const replayRequested = replaySeed !== undefined || diagnosticPath !== undefined +const diagnosticProperty = `opfs-worker.diagnostics` +const replay = readOracleRunConfig() +const diagnosticSeed = replay.replaySeed ?? 1846 +const diagnosticRuns = oracleRuns(40) +const diagnosticPath = + replay.replayProperty === diagnosticProperty ? replay.replayPath : undefined +const replayRequested = oracleReplayRequested() const campaigns = replayRequested ? [{ name: `replay`, seed: diagnosticSeed }] : [ @@ -243,8 +248,9 @@ cause; the absent-cause examples reject an invented cause. Pinned examples run separately so both generated campaigns share one property and budget, and a reported shrink path selects its generated case directly. Verbose FastCheck output retains original/reduced traces. Set -TANSTACK_DB_OPFS_DIAGNOSTIC_ORACLE_SEED and -TANSTACK_DB_OPFS_DIAGNOSTIC_ORACLE_PATH for direct replay. +TANSTACK_DB_ORACLE_SEED, TANSTACK_DB_ORACLE_PATH, and +TANSTACK_DB_ORACLE_PROPERTY=opfs-worker.diagnostics for direct replay. +TANSTACK_DB_ORACLE_RUNS_MULTIPLIER scales the generated campaign. */ describe(`OPFS worker diagnostics oracle`, () => { if (!replayRequested) { @@ -254,7 +260,7 @@ describe(`OPFS worker diagnostics oracle`, () => { ) } - it.skipIf(replayRequested)( + it.skipIf(oracleReplayRequested())( `rejects a diagnostic that drops an exposed VFS cause`, async () => { const input = diagnosticExamples[4]![0] diff --git a/packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts b/packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts index d94319f941..3b3b5f4272 100644 --- a/packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/per-collection-coordinator-oracle.test.ts @@ -5001,10 +5001,13 @@ packages/db/src/query/live/ARCHITECTURE.md) for a follower tab. A follower routes on-demand demand it cannot serve to the elected owner. Its load, and so its live query's first ready, settles only when the rows the owner fetched for it are visible in the follower; if applying the owner's commit fails here, the -load rejects and the follower's source enters error. The single-runtime lanes -`persistence.follower-remote-demand` and `persistence.follower-generation-notice` -in db-sqlite-persistence-core/tests/persisted-oracle.test.ts own the runtime -mechanism with a coordinator double. This lane is their receiving witness +load rejects and the follower's source enters error. The single-runtime +`persistence.follower-remote-demand` lane and the follower generation-notice +cells in db-sqlite-persistence-core/tests/persisted-oracle.test.ts own the +runtime mechanism with a coordinator double. The generation-notice cells are +fixed histories: their rotation cell is pinned as failing under the known gap +#2139, and the fix adds their generated campaign back. This lane is their +receiving witness across the real boundary: two tabs, each with its own real BrowserCollectionCoordinator over Node's real BroadcastChannel and Web Locks, and one real SQLite file opened by both tabs. @@ -5433,8 +5436,9 @@ describe(`follower remote-demand settlement across real tabs`, () => { // source. The model owes the same applied settlement there: the rows it // already showed stay visible, and a new demand settles with its rows. // This topology never routes to another tab's owner, so the - // elected-owner notice path stays covered by the single-runtime - // `persistence.follower-generation-notice` lane. + // elected-owner notice path stays with the single-runtime follower + // generation-notice cells in persisted-oracle.test.ts. Their rotation cell + // is pinned as failing under the known gap #2139. it( `keeps a rotated follower's rows and serves its new demand from its own generation`, () => diff --git a/packages/db-ivm/tests/structural-identity-oracle.property.test.ts b/packages/db-ivm/tests/structural-identity-oracle.property.test.ts index 9b0e23d6f0..334c4a2508 100644 --- a/packages/db-ivm/tests/structural-identity-oracle.property.test.ts +++ b/packages/db-ivm/tests/structural-identity-oracle.property.test.ts @@ -462,10 +462,7 @@ describe(`structuralIdentity`, () => { ) if (options.seed === undefined) await fc.assert(property, { numRuns: options.numRuns, seed: 20261014 }) - await fc.assert( - property, - options.path === undefined ? options : { ...options, numRuns: 1 }, - ) + await fc.assert(property, options) }) // Named replays of counterexamples the random campaign found. Nested `deep` @@ -625,9 +622,6 @@ describe(`structuralIdentity`, () => { ) if (options.seed === undefined) await fc.assert(property, { numRuns: options.numRuns, seed: 1158727847 }) - await fc.assert( - property, - options.path === undefined ? options : { ...options, numRuns: 1 }, - ) + await fc.assert(property, options) }) }) diff --git a/packages/db/tests/oracle-config.ts b/packages/db/tests/oracle-config.ts index dffda0f111..df7caae724 100644 --- a/packages/db/tests/oracle-config.ts +++ b/packages/db/tests/oracle-config.ts @@ -24,6 +24,7 @@ const staticOracleProperties = [ `oracle-replay.calibration`, `structural-identity.equal-representations`, `structural-identity.independent-values`, + `opfs-worker.diagnostics`, `sqlite-persistence.source-fifo-order`, `sqlite-persistence.abort-graph`, `sqlite-persistence.owner-isolation`, @@ -408,10 +409,14 @@ export function oracleRandomParameters( return { numRuns, seed: replaySeed, + // A path replays one recorded counterexample, so the replay runs once. + // From a path with one segment, fast-check would otherwise run `numRuns` + // histories that start at that index. ...(property !== undefined && replayPath !== undefined && replayProperty === property ? { + numRuns: 1, path: replayPath, ...oracleReplayReporter(property, replaySeed, replayPath), } @@ -421,6 +426,16 @@ export function oracleRandomParameters( const { multiplier, ...replay } = readOracleRunConfig() +/** + * Whether a replay campaign is selected. A test that is not the replayed + * property stands aside with `it.skipIf(oracleReplayRequested())`. The + * `oracle/no-disabled` lint rule accepts only this condition, so a replay + * selection cannot switch an oracle test off in ordinary runs. + */ +export function oracleReplayRequested(): boolean { + return replay.replaySeed !== undefined +} + /** Keeps ordinary CI bounded while allowing long randomized oracle campaigns. */ export function oracleRuns(baseRuns: number): number { return baseRuns * multiplier diff --git a/packages/db/tests/oracle-replay.fixture.test.ts b/packages/db/tests/oracle-replay.fixture.test.ts index 0d5ff458a5..23e3edb958 100644 --- a/packages/db/tests/oracle-replay.fixture.test.ts +++ b/packages/db/tests/oracle-replay.fixture.test.ts @@ -16,10 +16,10 @@ beforeEach(() => { if (fault === `setup`) throw new Error(`replay setup sentinel`) }) -const options = oraclePropertyOptions( - fault === `zero` ? 0 : 3, - `oracle-replay.calibration`, -) +// A path replay runs once, so the zero-run fault overrides the run count +// after the replay options are chosen, like the seed and path faults. +const options = oraclePropertyOptions(3, `oracle-replay.calibration`) +if (fault === `zero`) options.numRuns = 0 if (fault === `seed`) options.seed = 43 if (fault === `path`) options.path = `1` const propertyTest = diff --git a/packages/db/tests/oracle-replay.test.ts b/packages/db/tests/oracle-replay.test.ts index 539419facd..05da1ba869 100644 --- a/packages/db/tests/oracle-replay.test.ts +++ b/packages/db/tests/oracle-replay.test.ts @@ -201,7 +201,8 @@ describe(`guarded oracle replay`, () => { const result = runReplay(files) expect(result.status, result.output).toBe(0) expect(result.output).toContain(`"executions":1`) - expect(result.output).toContain(`"numRuns":3`) + // A path replay runs its recorded counterexample once. + expect(result.output).toContain(`"numRuns":1`) }, 40_000, ) @@ -300,6 +301,22 @@ describe(`oracle replay reporting`, () => { ) }) + it(`runs a path replay once, and only for its selected property`, () => { + vi.stubEnv(`TANSTACK_DB_ORACLE_REPLAY_WITNESS`, undefined) + const replay = { + replaySeed: 42, + replayPath: `3`, + replayProperty: `oracle-replay.calibration`, + } + expect( + oracleRandomParameters(5, replay, `oracle-replay.calibration`), + ).toEqual({ numRuns: 1, seed: 42, path: `3` }) + expect(oracleRandomParameters(5, replay, `cursor.exact-width`)).toEqual({ + numRuns: 5, + seed: 42, + }) + }) + it.each([false, true])( `preserves native async failure formatting and cause=%s`, async (errorWithCause) => { diff --git a/packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts b/packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts index 32ec955ad6..f537b670d3 100644 --- a/packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts +++ b/packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts @@ -2133,10 +2133,7 @@ describe(`Ordered windows across truncate replays`, () => { ), campaign === `fixed` ? { seed: 20261021, numRuns: oracleRuns(300) } - : { - ...oraclePropertyOptions(300, orderedReplayProperty), - ...(campaign === `replay` ? { numRuns: 1 } : {}), - }, + : oraclePropertyOptions(300, orderedReplayProperty), ) // Reach witnesses: releases throw without a truncate, while a replay // starts, and while it settles; loads fail and stay held across a diff --git a/packages/db/tests/query/ordered-source-loader-state-oracle.test.ts b/packages/db/tests/query/ordered-source-loader-state-oracle.test.ts index 8deecfb889..78475bc857 100644 --- a/packages/db/tests/query/ordered-source-loader-state-oracle.test.ts +++ b/packages/db/tests/query/ordered-source-loader-state-oracle.test.ts @@ -3702,10 +3702,7 @@ describe(`Ordered source request authority`, () => { }), campaign === `fixed` ? { seed: 20261016, numRuns: oracleRuns(10000) } - : { - ...oraclePropertyOptions(10000, authorityProperty), - ...(campaign === `replay` ? { numRuns: 1 } : {}), - }, + : oraclePropertyOptions(10000, authorityProperty), ) // Reach witnesses: superseded requests fail, stale requests settle // after a repair succeeded, the strand law is checkable, a superseded diff --git a/packages/query-db-collection/tests/cursor-pagination-oracle.integration.test.ts b/packages/query-db-collection/tests/cursor-pagination-oracle.integration.test.ts index faa11fc5bb..7dc89ec31d 100644 --- a/packages/query-db-collection/tests/cursor-pagination-oracle.integration.test.ts +++ b/packages/query-db-collection/tests/cursor-pagination-oracle.integration.test.ts @@ -351,10 +351,7 @@ describe(`cursor adapter through production pagination`, () => { ), campaign === `fixed` ? { seed: 20261012, numRuns: oracleRuns(50) } - : { - ...oraclePropertyOptions(50, manualWriteProperty), - ...(campaign === `replay` ? { numRuns: 1 } : {}), - }, + : oraclePropertyOptions(50, manualWriteProperty), ) } catch (error) { lawFailure = error diff --git a/packages/query-db-collection/tests/refetch-ordered-prefix-oracle.test.ts b/packages/query-db-collection/tests/refetch-ordered-prefix-oracle.test.ts index f4d4ad4e90..2adf895b7f 100644 --- a/packages/query-db-collection/tests/refetch-ordered-prefix-oracle.test.ts +++ b/packages/query-db-collection/tests/refetch-ordered-prefix-oracle.test.ts @@ -13,6 +13,7 @@ import fc from 'fast-check' import { describe, expect, it, vi } from 'vitest' import { oraclePropertyOptions, + oracleReplayRequested, oracleRuns, readOracleRunConfig, } from '../../db/tests/oracle-config.js' @@ -1967,7 +1968,7 @@ describe(`query-db on-demand refetch keeps served ordered prefixes whole`, () => // The premise-directed campaign for the skip under a hold. Its reach // witness requires refetches that report while a hold is running. - it.runIf(replay.replayPath === undefined)( + it.skipIf(oracleReplayRequested())( `generated refetch histories under a hold (fixed campaign)`, async () => { let reachedReport = 0 @@ -1996,10 +1997,7 @@ describe(`query-db on-demand refetch keeps served ordered prefixes whole`, () => }), campaign === `fixed` ? { seed: 20261013, numRuns: oracleRuns(80) } - : { - ...oraclePropertyOptions(80, property), - ...(campaign === `replay` ? { numRuns: 1 } : {}), - }, + : oraclePropertyOptions(80, property), ) // Positive reach witness: generated histories serve continuation pages // before a refetch, which is the premise of the gap. diff --git a/scripts/agent-lint-feedback.mjs b/scripts/agent-lint-feedback.mjs index 635cfece92..fb4f22ff2f 100644 --- a/scripts/agent-lint-feedback.mjs +++ b/scripts/agent-lint-feedback.mjs @@ -34,12 +34,12 @@ */ import { execFileSync } from 'node:child_process' import { existsSync, readFileSync } from 'node:fs' -import { isAbsolute, join, relative, resolve } from 'node:path' +import { basename, isAbsolute, join, relative, resolve } from 'node:path' import { fileURLToPath } from 'node:url' import { parseArgs } from 'node:util' import tsParser from '@typescript-eslint/parser' import { ESLint } from 'eslint' -import oraclePlugin, { oracleFiles } from './eslint-plugin-oracle/index.js' +import oraclePlugin from './eslint-plugin-oracle/index.js' const root = fileURLToPath(new URL(`../`, import.meta.url)) @@ -61,23 +61,49 @@ if (![`edit`, `stop`].includes(options.event)) { } const isHook = options.format !== `text` -/** Every file that the branch adds or changes, including uncommitted work. */ +function git(...args) { + return execFileSync(`git`, args, { cwd: root, encoding: `utf8` }).trim() +} + +function repositoryFiles(...outputs) { + return outputs + .flatMap((output) => output.split(`\n`)) + .filter(Boolean) + .map((file) => join(root, file)) +} + +/** + * Every tracked oracle file. Git leaves out builds and untracked checkouts, + * such as other branches' worktrees under `.worktrees/`. + */ +function trackedOracleFiles() { + return repositoryFiles(git(`ls-files`, `*oracle*.ts`, `*oracle*.tsx`)) +} + +/** + * Every file that the branch adds or changes, including uncommitted work. + * Without a main branch to compare against, the branch's own commits cannot + * be found, so this fails instead of linting only the uncommitted work. + */ function changedFiles() { - const git = (...args) => - execFileSync(`git`, args, { cwd: root, encoding: `utf8` }).trim() - let base = `HEAD` - try { - base = git(`merge-base`, `HEAD`, `origin/main`) - } catch { - // Without origin/main, the uncommitted work is the change. + const base = [`origin/main`, `main`] + .map((branch) => { + try { + return git(`merge-base`, `HEAD`, branch) + } catch { + return undefined + } + }) + .find((commit) => commit !== undefined) + if (base === undefined) { + throw new Error( + `Cannot find origin/main or main to compare this branch against. Run \`git fetch origin main\`.`, + ) } - return [ + return repositoryFiles( git(`diff`, `--name-only`, `--diff-filter=ACMR`, base), git(`ls-files`, `--others`, `--exclude-standard`), - ] - .flatMap((output) => output.split(`\n`)) - .filter(Boolean) - .map((file) => join(root, file)) + ) } const pathKeys = /^(file_?path|filePath|path|absolute_path|target_file)$/i @@ -136,19 +162,17 @@ function createLinter() { }) } -/** Lints the oracle files among `files`, or every oracle file without them. */ +/** Lints the oracle files among `files`. */ async function lint(eslint, files) { const lintable = [] - for (const file of new Set(files ?? [])) { + for (const file of new Set(files)) { if (!existsSync(file) || relative(root, file).startsWith(`..`)) continue - if (!/oracle[^/]*\.tsx?$/.test(file)) continue + if (!/oracle.*\.tsx?$/.test(basename(file))) continue if (await eslint.isPathIgnored(file)) continue lintable.push(file) } - if (files !== undefined && lintable.length === 0) return [] - const results = await eslint.lintFiles( - files === undefined ? oracleFiles : lintable, - ) + if (lintable.length === 0) return [] + const results = await eslint.lintFiles(lintable) // Keep oracle reports and files that cannot be parsed. This drops the // notices that the ignored directives have no effect. return results.map((result) => ({ @@ -228,7 +252,7 @@ function respond(report, input, scope) { /** The files to lint: named on the command line, edited, or changed. */ function selectFiles(input) { - if (options.all) return { scope: `all`, files: undefined } + if (options.all) return { scope: `all`, files: trackedOracleFiles() } if (positionals.length > 0) { return { scope: `named`, files: positionals.map((file) => resolve(file)) } } diff --git a/scripts/eslint-plugin-oracle/index.js b/scripts/eslint-plugin-oracle/index.js index 9588d8d324..1362fa2933 100644 --- a/scripts/eslint-plugin-oracle/index.js +++ b/scripts/eslint-plugin-oracle/index.js @@ -12,7 +12,7 @@ import noDisabled from './rules/no-disabled.js' * directives. */ -export const oracleFiles = [`**/*oracle*.ts`, `**/*oracle*.tsx`] +const oracleFiles = [`**/*oracle*.ts`, `**/*oracle*.tsx`] const plugin = { meta: { name: `eslint-plugin-oracle` }, diff --git a/scripts/eslint-plugin-oracle/no-disabled.test.js b/scripts/eslint-plugin-oracle/no-disabled.test.js index 0e8ca8017b..291426cb1e 100644 --- a/scripts/eslint-plugin-oracle/no-disabled.test.js +++ b/scripts/eslint-plugin-oracle/no-disabled.test.js @@ -15,26 +15,26 @@ ruleTester.run(`oracle/no-disabled`, noDisabled, { // A replay that does not select this property registers an empty test. `it.skip('runs only when its replay property is selected', () => {})`, `it.todo('loads every subset from one generation')`, - // Playwright's runtime skip, conditioned on the environment. + // Playwright's runtime skip for a control that one browser supports. `test.skip(browserName !== 'chromium', 'Chromium-only control')`, - `test.skip(isMobile)`, - `test.skip(false, 'kept for a later browser')`, - // A replay selection may switch a test off. - `it.skipIf(replayRequested)('checks the history', () => check())`, - `it.runIf(replay.replayPath === undefined)('checks the history', () => check())`, + // A replay campaign stands the other tests aside. + `it.skipIf(oracleReplayRequested())('checks the history', () => check())`, `fc.assert(property, oraclePropertyOptions(12, 'cleanup-queue.history'))`, `fc.assert(property, { numRuns: oracleRuns(20) })`, - // A pinned replay or mutant witness runs one fixed history. + // A replay passes the recorded path, and a witness a fixed seed. + `fc.check(property, { seed: failed.seed, path: failed.counterexamplePath, numRuns: 1 })`, `fc.check(property, { seed: 205203, numRuns: 1 })`, - `fc.check(property, { seed: fixedSeed, numRuns: 1 })`, + `fc.check(property, { seed: -1659005, numRuns: 1 })`, + `const fixedSeed = 1657019\nfc.check(property, { seed: fixedSeed, numRuns: 1 })`, `fc.assert(property, { seed: 2128, numRuns: oracleRuns(1) })`, - `fc.assert(property, replayRequested ? { numRuns: oracleRuns(1) } : {})`, - `fc.assert(property, campaign === 'replay' ? { numRuns: 1 } : {})`, - `fc.assert(property, options.path === undefined ? options : { ...options, numRuns: 1 })`, - // An expectation about a recorded run is not a run option. + // An expectation about a recorded run is not a run option, and a sample + // runs no property. `expect(report).toMatchObject({ numRuns: 1, failed: true })`, + `fc.sample(historyArbitrary, { numRuns: 1 })`, // A known gap is pinned as failing on a fixed seed, not skipped. `it.fails('checks the fixed history', () => fc.assert(property, { seed: 2128, numRuns: oracleRuns(12) }))`, + // A skip on something other than the test context is not a test skip. + `it('reads a page', (ctx) => { const cursor = open(); cursor.skip() })`, ], invalid: [ { @@ -125,6 +125,75 @@ ruleTester.run(`oracle/no-disabled`, noDisabled, { code: `fc.assert(property, { numRuns: oracleRuns(0) })`, errors: [{ messageId: `singleRun` }], }, + { + code: `test.skip(isMobile)`, + errors: [{ messageId: `skip` }], + }, + { + code: `test.skip(process.env.CI === 'true', 'flaky')`, + errors: [{ messageId: `skip` }], + }, + { + code: `test.fixme('checks the history', check)`, + errors: [{ messageId: `skip`, data: { name: `fixme` } }], + }, + { + code: `test.describe.fixme('cleanup queue', () => { test('a', check) })`, + errors: [{ messageId: `skip`, data: { name: `fixme` } }], + }, + { + code: `test.fixme()`, + errors: [{ messageId: `skip`, data: { name: `fixme` } }], + }, + { + code: `it('checks the history', (ctx) => { ctx.skip(); check() })`, + errors: [{ messageId: `skip` }], + }, + { + // Only the shared helper may select a replay: its polarity is fixed. + code: `it.skipIf(replayRequested)('checks the history', () => check())`, + errors: [{ messageId: `conditional` }], + }, + { + code: `it.runIf(replay.replayPath === undefined)('checks the history', () => check())`, + errors: [{ messageId: `conditional`, data: { name: `runIf` } }], + }, + { + code: `it.runIf(replay.replayPath !== undefined)('checks the history', check)`, + errors: [{ messageId: `conditional` }], + }, + { + code: `describe.skipIf(!oracleReplayRequested())('cleanup queue', () => { it('a', () => check()) })`, + errors: [{ messageId: `conditional` }], + }, + { + code: `describe.skipIf(path.sep === '\\\\')('cleanup queue', () => { it('a', () => check()) })`, + errors: [{ messageId: `conditional` }], + }, + { + code: `fc.assert(property, replayRequested ? {} : { numRuns: 1 })`, + errors: [{ messageId: `singleRun` }], + }, + { + code: `fc.assert(property, campaign === 'replay' ? { numRuns: 1 } : {})`, + errors: [{ messageId: `singleRun` }], + }, + { + code: `fc.assert(property, { seed: process.env.SEED, numRuns: 1 })`, + errors: [{ messageId: `singleRun` }], + }, + { + code: `fc.assert(property, { seed: replay.replaySeed, numRuns: 1 })`, + errors: [{ messageId: `singleRun` }], + }, + { + code: `let seed = 1\nfc.assert(property, { seed, numRuns: 1 })`, + errors: [{ messageId: `singleRun` }], + }, + { + code: `fc.assert(property, { seed: replay.replaySeed, path: undefined, numRuns: 1 })`, + errors: [{ messageId: `singleRun` }], + }, { // A comment does not grant an exception. code: ` diff --git a/scripts/eslint-plugin-oracle/rules/no-disabled.js b/scripts/eslint-plugin-oracle/rules/no-disabled.js index d4bfcacabd..8644bbd65f 100644 --- a/scripts/eslint-plugin-oracle/rules/no-disabled.js +++ b/scripts/eslint-plugin-oracle/rules/no-disabled.js @@ -3,30 +3,38 @@ * the edits that stop a law from running while every test still passes: * * - `.only` on a test or suite, which silences every other law in the file. - * - `.skip` or `.todo` on a test or suite that has a body, inline or passed by - * name. An empty body is a placeholder that disables nothing, such as the - * test that a replay registers when it does not select its property. - * - `.skipIf(condition)` or `.runIf(condition)` unless the condition names a - * replay. Only a replay selection may switch an oracle test off. - * - Playwright's `test.skip()` or `test.skip(true, reason)` inside a test, - * which skips it unconditionally. `test.skip(condition, reason)` with an - * environment condition, such as the browser, is allowed. - * - A generated property limited to one unseeded run, by a literal - * `numRuns: 1` (or 0) or a base of 1 passed to the run helpers. One random - * history is not a generated property. A pinned replay passes a fixed - * `seed` beside its `numRuns`, or selects the one run under a condition - * that names the replay, so it is not reported. + * - `.skip`, `.todo`, or Playwright's `.fixme` on a test or suite that has a + * body, inline or passed by name. An empty body is a placeholder that + * disables nothing, such as the test that a replay registers when it does + * not select its property. + * - A test that skips itself through its context, as in `ctx.skip()`. + * - `.skipIf(condition)` or `.runIf(condition)`, unless the call is exactly + * `.skipIf(oracleReplayRequested())`. A replay may stand a test aside, and + * nothing else may switch an oracle test off. The shared helper fixes the + * polarity, so a condition cannot turn the test off in ordinary runs. + * - Playwright's runtime `test.skip()` or `test.fixme()` inside a test, + * unless its condition is `browserName !== ''`: a control that + * only one browser supports. + * - A generated property limited to one run that pins no history: a literal + * `numRuns` of 1 or 0, or a base of 1 passed to the run helpers. One random + * history is not a generated property. A replay pins its history with a + * `path` beside `numRuns`, and a witness with a seed that is a number + * literal or a constant bound to one. `oraclePropertyOptions` returns one + * run for a path replay, so a campaign needs no one-run option of its own. * * The rule has no inline exception. A known gap is pinned with `it.fails` on * a fixed seed, not skipped. * - * The rule cannot see a run count lowered to any value above 1, a condition - * that never selects a replay, or a skip reached through an alias such as - * `const run = it.skip`. Reviewers still check those. + * The rule cannot see a run count lowered to any value above 1, a `path` or + * seed read from configuration that is absent in ordinary runs, a skip + * reached through an alias such as `const run = it.skip`, or a registration + * inside an `if`. Reviewers still check those. */ const testRoots = new Set([`describe`, `fcTest`, `it`, `suite`, `test`]) +const disablingModifiers = new Set([`fixme`, `skip`, `todo`]) const conditionalModifiers = new Set([`runIf`, `skipIf`]) +const replaySelection = `oracleReplayRequested` const runHelpers = new Set([ `oraclePropertyOptions`, `oracleRandomParameters`, @@ -70,11 +78,7 @@ function isStringLike(node) { ) } -function isBooleanLiteral(node) { - return node?.type === `Literal` && typeof node.value === `boolean` -} - -/** The condition of `it.skipIf(condition)` in a callee chain, if there is one. */ +/** The `skipIf` or `runIf` call in a callee chain, if there is one. */ function registrationCondition(callee) { let node = callee while (node.type === `CallExpression` || node.type === `MemberExpression`) { @@ -83,24 +87,49 @@ function registrationCondition(callee) { node.callee.type === `MemberExpression` && conditionalModifiers.has(node.callee.property.name) ) { - return { name: node.callee.property.name, condition: node.arguments[0] } + return { name: node.callee.property.name, args: node.arguments } } node = node.type === `CallExpression` ? node.callee : node.object } return undefined } +/** Exactly `.skipIf(oracleReplayRequested())`. */ +function isReplaySelection({ name, args }) { + const [condition] = args + return ( + name === `skipIf` && + args.length === 1 && + condition.type === `CallExpression` && + condition.callee.type === `Identifier` && + condition.callee.name === replaySelection && + condition.arguments.length === 0 + ) +} + +/** Playwright's `browserName !== ''`. */ +function isBrowserCondition(node) { + return ( + node.type === `BinaryExpression` && + node.operator === `!==` && + node.left.type === `Identifier` && + node.left.name === `browserName` && + isStringLike(node.right) + ) +} + /** - * Whether a `.skip` or `.todo` call switches off a body that runs something. - * A test registration passes `(name, body)`. Playwright's runtime form passes - * `(condition, reason)`, `(condition)`, or nothing, and skips the test it - * runs in unless the condition is false. + * Whether a `.skip`, `.todo`, or `.fixme` call switches off a body that runs + * something. A test registration passes `(name, body)`, and a todo passes + * only its name. Playwright's runtime form passes `(condition, reason)`, + * `(condition)`, or nothing, and skips the test that it runs in. */ function disablesBody(args) { - if (args.length === 0) return true const [first, second] = args + if (first === undefined) return true + if (second === undefined && isStringLike(first)) return false if (second === undefined || isStringLike(second)) { - return isBooleanLiteral(first) && first.value + return !isBrowserCondition(first) } return hasBody(second) } @@ -114,37 +143,31 @@ function isSingleRunLiteral(node) { ) } -/** - * Source text that names a replay: `replay` anywhere, or `path` or `seed` as - * a whole name, as in `options.path`. `filePath` names no replay. - */ -const replayText = /replay|(? propertyName(property) === `seed`, +/** A number literal, or a constant bound to one. */ +function isFixedSeed(node, sourceCode) { + if (isNumberLiteral(node)) return true + if (node.type !== `Identifier`) return false + const definition = findVariable(sourceCode.getScope(node), node.name)?.defs[0] + return ( + definition?.type === `Variable` && + definition.parent.kind === `const` && + isNumberLiteral(definition.node.init) ) - if (seed !== undefined) { - return !unfixedSeedText.test(sourceCode.getText(seed.value)) - } - const { parent } = object - if (parent.type === `ConditionalExpression` && parent.test !== object) { - return replayText.test(sourceCode.getText(parent.test)) - } - if (parent.type === `CallExpression` && parent.arguments.includes(object)) { - return calleeChain(parent.callee).root === `expect` - } - return false } function propertyName(property) { @@ -154,19 +177,69 @@ function propertyName(property) { : property.key.value } +/** + * A one-run object pins its history when it replays a `path` or passes a + * fixed seed. An object that an `expect` matcher compares holds an observed + * run count, and `fc.sample` draws examples without running a property. + */ +function isPinnedOrObservation(object, sourceCode) { + const option = (name) => + object.properties.find((property) => propertyName(property) === name) + const path = option(`path`) + if (path !== undefined && sourceCode.getText(path.value) !== `undefined`) { + return true + } + const seed = option(`seed`) + if (seed !== undefined && isFixedSeed(seed.value, sourceCode)) return true + const { parent } = object + if (parent.type !== `CallExpression` || !parent.arguments.includes(object)) { + return false + } + const { root, names } = calleeChain(parent.callee) + return root === `expect` || (root === `fc` && names[0] === `sample`) +} + +/** + * Whether a call skips the test that it runs in through the test context, + * as in `it('law', (ctx) => { ctx.skip() })`. + */ +function isContextSkip(node) { + const { callee } = node + if ( + callee.type !== `MemberExpression` || + callee.property.type !== `Identifier` || + callee.property.name !== `skip` || + callee.object.type !== `Identifier` + ) { + return false + } + let fn = node.parent + while (fn && !isFunction(fn)) fn = fn.parent + const [context] = fn?.params ?? [] + if (context?.type !== `Identifier` || context.name !== callee.object.name) { + return false + } + const registration = fn.parent + return ( + registration.type === `CallExpression` && + registration.arguments.includes(fn) && + testRoots.has(calleeChain(registration.callee).root) + ) +} + /** @type {import('eslint').Rule.RuleModule} */ export default { meta: { type: `problem`, docs: { - description: `Report oracle tests that stop running: .only, a skipped body, or one unseeded run.`, + description: `Report oracle tests that stop running: .only, a skipped body, a switch-off condition, or one unpinned run.`, }, schema: [], messages: { only: `\`.only\` runs only this test, so every other law in the file stops running. Remove it before committing.`, skip: `\`.{{name}}\` stops this test from enforcing its law. Remove it. To keep a known gap visible, pin a failing history with \`it.fails\` and a fixed seed.`, - conditional: `\`.{{name}}\` can switch this test off, and its condition names no replay. Only a replay selection may switch an oracle test off. Remove the condition.`, - singleRun: `A generated property with one unseeded run checks one random history. Pass a base run count to oracleRuns or oraclePropertyOptions; a pinned replay passes a fixed \`seed\` beside \`numRuns\`.`, + conditional: `\`.{{name}}\` can switch this test off in ordinary runs. Only \`.skipIf(oracleReplayRequested())\` may switch an oracle test off, so that a replay can stand it aside. Remove the condition.`, + singleRun: `A generated property with one run that pins no history checks one random history. Pass a base run count to oracleRuns or oraclePropertyOptions. A replay passes \`path\`, and a witness passes a seed that is a number literal or a constant.`, }, }, create(context) { @@ -178,7 +251,8 @@ export default { return ( parent.type === `Property` && propertyName(parent) === `numRuns` && - isReplayOrObservation(parent.parent, sourceCode) + parent.parent.type === `ObjectExpression` && + isPinnedOrObservation(parent.parent, sourceCode) ) } @@ -192,6 +266,10 @@ export default { ) { return } + if (isContextSkip(node)) { + context.report({ node, messageId: `skip`, data: { name: `skip` } }) + return + } const { root, names } = calleeChain(node.callee) if (runHelpers.has(root) && names.length === 0) { if ( @@ -204,18 +282,12 @@ export default { } if (!testRoots.has(root)) return if (names.includes(`only`)) { - context.report({ - node: node.callee, - messageId: `only`, - }) + context.report({ node: node.callee, messageId: `only` }) return } const conditional = registrationCondition(node.callee) if (conditional !== undefined) { - if ( - conditional.condition === undefined || - !replayText.test(sourceCode.getText(conditional.condition)) - ) { + if (!isReplaySelection(conditional)) { context.report({ node, messageId: `conditional`, @@ -224,9 +296,7 @@ export default { } return } - const disabled = names.find( - (name) => name === `skip` || name === `todo`, - ) + const disabled = names.find((name) => disablingModifiers.has(name)) if (disabled !== undefined && disablesBody(node.arguments)) { context.report({ node, messageId: `skip`, data: { name: disabled } }) } From 2cf76d5cf066a0bd0c750aee8f6ea0fa2cbc3ae1 Mon Sep 17 00:00:00 2001 From: Test Date: Fri, 23 Oct 2026 11:28:00 -0600 Subject: [PATCH 3/4] test(oracles): expect one run from a path replay in the run-config test The previous commit made oraclePropertyOptions run a path replay once. utils.test.ts still expected the base run count for the named property. Refs #2234 Co-authored-by: Isaac --- packages/db/tests/utils.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/db/tests/utils.test.ts b/packages/db/tests/utils.test.ts index 1bbf4e4730..eff8abb949 100644 --- a/packages/db/tests/utils.test.ts +++ b/packages/db/tests/utils.test.ts @@ -169,9 +169,10 @@ describe(`oracle run configuration`, () => { numRuns: 40, seed: -42, }) + // A path replays one recorded counterexample, so it runs once. expect( oracleRandomParameters(40, replayRun, `includes.incremental-history`), - ).toEqual({ numRuns: 40, seed: -42, path: `1:0:2` }) + ).toEqual({ numRuns: 1, seed: -42, path: `1:0:2` }) }) }) From 73104aea045a42ed29f89a69914e9473c65aad4f Mon Sep 17 00:00:00 2001 From: Test Date: Fri, 23 Oct 2026 11:35:08 -0600 Subject: [PATCH 4/4] fix(oracles): resolve replay helper and test context through scope Address CodeRabbit's review of 012f406d3: - oracleReplayRequested() means a path replay. A seed without a path only reseeds the random campaigns, so fixed tests keep running. - The OPFS diagnostics oracle takes its replay and random options from oraclePropertyOptions, so a path replay runs once and a seed-only run reseeds only the random campaign. - oracle/no-disabled accepts skipIf(oracleReplayRequested()) only when the helper is imported from oracle-config, so a local function of that name does not pass. - The context-skip check resolves the binding through scope, so a context captured by a nested callback is reported and a shadowing variable is not. Refs #2234 Co-authored-by: Isaac --- .../opfs-worker-diagnostics-oracle.test.ts | 43 +++++++++++------- packages/db/tests/oracle-config.ts | 12 ++--- .../eslint-plugin-oracle/no-disabled.test.js | 25 ++++++++++- .../eslint-plugin-oracle/rules/no-disabled.js | 44 ++++++++++++------- 4 files changed, 86 insertions(+), 38 deletions(-) diff --git a/packages/browser-db-sqlite-persistence/tests/opfs-worker-diagnostics-oracle.test.ts b/packages/browser-db-sqlite-persistence/tests/opfs-worker-diagnostics-oracle.test.ts index 0a2bb16923..32b3844169 100644 --- a/packages/browser-db-sqlite-persistence/tests/opfs-worker-diagnostics-oracle.test.ts +++ b/packages/browser-db-sqlite-persistence/tests/opfs-worker-diagnostics-oracle.test.ts @@ -1,6 +1,7 @@ import { fc } from '@fast-check/vitest' import { afterEach, describe, expect, it, vi } from 'vitest' import { + oraclePropertyOptions, oracleReplayRequested, oracleRuns, readOracleRunConfig, @@ -148,18 +149,30 @@ const diagnosticInput = fc.record({ }) const diagnosticProperty = `opfs-worker.diagnostics` +const diagnosticRuns = 40 +// A path replay runs only the selected property's recorded history, once. A +// seed without a path reseeds the random campaign and keeps the fixed one. const replay = readOracleRunConfig() -const diagnosticSeed = replay.replaySeed ?? 1846 -const diagnosticRuns = oracleRuns(40) -const diagnosticPath = - replay.replayProperty === diagnosticProperty ? replay.replayPath : undefined -const replayRequested = oracleReplayRequested() -const campaigns = replayRequested - ? [{ name: `replay`, seed: diagnosticSeed }] - : [ - { name: `fixed`, seed: diagnosticSeed }, - { name: `random`, seed: undefined }, - ] +const campaigns = + replay.replayPath === undefined + ? [ + { + name: `fixed`, + options: { seed: 1846, numRuns: oracleRuns(diagnosticRuns) }, + }, + { + name: `random`, + options: oraclePropertyOptions(diagnosticRuns, diagnosticProperty), + }, + ] + : replay.replayProperty === diagnosticProperty + ? [ + { + name: `replay`, + options: oraclePropertyOptions(diagnosticRuns, diagnosticProperty), + }, + ] + : [] const diagnosticExamples: Array<[DiagnosticInput]> = [ [ @@ -253,7 +266,7 @@ TANSTACK_DB_ORACLE_PROPERTY=opfs-worker.diagnostics for direct replay. TANSTACK_DB_ORACLE_RUNS_MULTIPLIER scales the generated campaign. */ describe(`OPFS worker diagnostics oracle`, () => { - if (!replayRequested) { + if (!oracleReplayRequested()) { it.each(diagnosticExamples)( `preserves the pinned open-failure diagnostic (%#)`, checkDiagnostic, @@ -280,11 +293,9 @@ describe(`OPFS worker diagnostics oracle`, () => { it.each(campaigns)( `matches the independent diagnostic formatter ($name campaign)`, - async ({ seed }) => { + async ({ options }) => { await fc.assert(fc.asyncProperty(diagnosticInput, checkDiagnostic), { - ...(seed === undefined ? {} : { seed }), - numRuns: diagnosticRuns, - ...(diagnosticPath ? { path: diagnosticPath } : {}), + ...options, verbose: true, }) }, diff --git a/packages/db/tests/oracle-config.ts b/packages/db/tests/oracle-config.ts index df7caae724..46e6efb6b2 100644 --- a/packages/db/tests/oracle-config.ts +++ b/packages/db/tests/oracle-config.ts @@ -427,13 +427,15 @@ export function oracleRandomParameters( const { multiplier, ...replay } = readOracleRunConfig() /** - * Whether a replay campaign is selected. A test that is not the replayed - * property stands aside with `it.skipIf(oracleReplayRequested())`. The - * `oracle/no-disabled` lint rule accepts only this condition, so a replay - * selection cannot switch an oracle test off in ordinary runs. + * Whether a path replay is selected. It replays one property's recorded + * history, so a test that is not that property stands aside with + * `it.skipIf(oracleReplayRequested())`. A seed without a path only reseeds + * the random campaigns, so every test still runs. The `oracle/no-disabled` + * lint rule accepts only this condition, so a replay selection cannot switch + * an oracle test off in ordinary runs. */ export function oracleReplayRequested(): boolean { - return replay.replaySeed !== undefined + return replay.replayPath !== undefined } /** Keeps ordinary CI bounded while allowing long randomized oracle campaigns. */ diff --git a/scripts/eslint-plugin-oracle/no-disabled.test.js b/scripts/eslint-plugin-oracle/no-disabled.test.js index 291426cb1e..c241383bd3 100644 --- a/scripts/eslint-plugin-oracle/no-disabled.test.js +++ b/scripts/eslint-plugin-oracle/no-disabled.test.js @@ -18,7 +18,14 @@ ruleTester.run(`oracle/no-disabled`, noDisabled, { // Playwright's runtime skip for a control that one browser supports. `test.skip(browserName !== 'chromium', 'Chromium-only control')`, // A replay campaign stands the other tests aside. - `it.skipIf(oracleReplayRequested())('checks the history', () => check())`, + { + code: `import { oracleReplayRequested } from '../../db/tests/oracle-config.js'\nit.skipIf(oracleReplayRequested())('checks the history', () => check())`, + languageOptions: { sourceType: `module` }, + }, + { + code: `import { oracleReplayRequested } from './oracle-config'\nit.skipIf(oracleReplayRequested())('checks the history', () => check())`, + languageOptions: { sourceType: `module` }, + }, `fc.assert(property, oraclePropertyOptions(12, 'cleanup-queue.history'))`, `fc.assert(property, { numRuns: oracleRuns(20) })`, // A replay passes the recorded path, and a witness a fixed seed. @@ -35,6 +42,9 @@ ruleTester.run(`oracle/no-disabled`, noDisabled, { `it.fails('checks the fixed history', () => fc.assert(property, { seed: 2128, numRuns: oracleRuns(12) }))`, // A skip on something other than the test context is not a test skip. `it('reads a page', (ctx) => { const cursor = open(); cursor.skip() })`, + // An inner variable that shadows the context is not the test context. + `it('reads a page', (ctx) => { pages.forEach((ctx) => ctx.skip()) })`, + `it('reads a page', (_, ctx) => { ctx.skip() })`, ], invalid: [ { @@ -149,6 +159,19 @@ ruleTester.run(`oracle/no-disabled`, noDisabled, { code: `it('checks the history', (ctx) => { ctx.skip(); check() })`, errors: [{ messageId: `skip` }], }, + { + code: `it('checks the history', (ctx) => { steps.forEach(() => ctx.skip()) })`, + errors: [{ messageId: `skip` }], + }, + { + // A local function with the helper's name is not the shared helper. + code: `const oracleReplayRequested = () => process.env.CI === 'true'\nit.skipIf(oracleReplayRequested())('checks the history', () => check())`, + errors: [{ messageId: `conditional` }], + }, + { + code: `it.skipIf(oracleReplayRequested())('checks the history', () => check())`, + errors: [{ messageId: `conditional` }], + }, { // Only the shared helper may select a replay: its polarity is fixed. code: `it.skipIf(replayRequested)('checks the history', () => check())`, diff --git a/scripts/eslint-plugin-oracle/rules/no-disabled.js b/scripts/eslint-plugin-oracle/rules/no-disabled.js index 8644bbd65f..19c0f6313a 100644 --- a/scripts/eslint-plugin-oracle/rules/no-disabled.js +++ b/scripts/eslint-plugin-oracle/rules/no-disabled.js @@ -95,15 +95,26 @@ function registrationCondition(callee) { } /** Exactly `.skipIf(oracleReplayRequested())`. */ -function isReplaySelection({ name, args }) { +function isReplaySelection({ name, args }, sourceCode) { const [condition] = args + if ( + name !== `skipIf` || + args.length !== 1 || + condition.type !== `CallExpression` || + condition.callee.type !== `Identifier` || + condition.callee.name !== replaySelection || + condition.arguments.length > 0 + ) { + return false + } + // The helper must be the shared one, not a local function of that name. + const definition = findVariable( + sourceCode.getScope(condition), + replaySelection, + )?.defs[0] return ( - name === `skipIf` && - args.length === 1 && - condition.type === `CallExpression` && - condition.callee.type === `Identifier` && - condition.callee.name === replaySelection && - condition.arguments.length === 0 + definition?.type === `ImportBinding` && + /(^|\/)oracle-config(\.js)?$/.test(definition.parent.source.value) ) } @@ -203,7 +214,7 @@ function isPinnedOrObservation(object, sourceCode) { * Whether a call skips the test that it runs in through the test context, * as in `it('law', (ctx) => { ctx.skip() })`. */ -function isContextSkip(node) { +function isContextSkip(node, sourceCode) { const { callee } = node if ( callee.type !== `MemberExpression` || @@ -213,12 +224,13 @@ function isContextSkip(node) { ) { return false } - let fn = node.parent - while (fn && !isFunction(fn)) fn = fn.parent - const [context] = fn?.params ?? [] - if (context?.type !== `Identifier` || context.name !== callee.object.name) { - return false - } + // Resolve the binding, so a context captured by a nested callback counts + // and an unrelated variable of the same name does not. + const definition = findVariable(sourceCode.getScope(node), callee.object.name) + ?.defs[0] + if (definition?.type !== `Parameter`) return false + const fn = definition.node + if (fn.params[0] !== definition.name) return false const registration = fn.parent return ( registration.type === `CallExpression` && @@ -266,7 +278,7 @@ export default { ) { return } - if (isContextSkip(node)) { + if (isContextSkip(node, sourceCode)) { context.report({ node, messageId: `skip`, data: { name: `skip` } }) return } @@ -287,7 +299,7 @@ export default { } const conditional = registrationCondition(node.callee) if (conditional !== undefined) { - if (!isReplaySelection(conditional)) { + if (!isReplaySelection(conditional, sourceCode)) { context.report({ node, messageId: `conditional`,