fix: harden V3 isolation and add reproducible VM regression

.github/pull_request_template.md

@@ -0,0 +1,19 @@
+## Change
+
+Describe the concrete problem and resulting behavior. Link the relevant ADR or roadmap stage.
+
+## Validation
+
+List the commands actually run and their results. Link regression artifacts when available. A stub returning zero is not a hardware pass.
+
+## Compatibility and failure behavior
+
+Describe affected identity/layout versions, tenant boundaries, admission limits, buffer/pin lifecycle, and miss/recompute behavior where relevant.
+
+## Evidence limits
+
+List unrun device/engine checks explicitly (`NOT_RUN_HARDWARE` where applicable). Keep `TCO_UNKNOWN` and `NOT_EVALUATED` until paired evidence exists. Distinguish CPU simulation, VM execution, real hardware and proposed work.
+
+## Scope and rollback
+
+State remaining limitations and the rollback/fallback for changed behavior. GPUCache does not add hot-tier erasure coding or durability replicas. Do not include credentials, prompts or KV payloads in this PR.

.github/workflows/ci.yml

@@ -14,28 +14,16 @@ jobs:
       - uses: dtolnay/rust-toolchain@stable
         with:
           components: clippy, rustfmt
-      - name: Format
-        run: cargo fmt --all -- --check
-      - name: Clippy
-        run: cargo clippy --workspace --all-targets -- -D warnings
-      - name: Test
-        run: cargo test --workspace --all-targets
-      - name: Fallback demo
-        run: cargo run -p gpucache-transport --bin fault-demo --quiet
-      - name: Paired baseline stub
-        run: cargo run -p gpucache-bench --bin paired-baseline --quiet
-      - name: Shard and cold demos
-        run: |
-          cargo run -p gpucache-transport --bin shard-demo --quiet
-          cargo run -p gpucache-cold --bin cold-demo --quiet
-      - name: V3 DPU contract and paired gate
-        run: |
-          cargo run -p gpucache-dpu --bin dpu-demo --quiet
-          cargo run -p gpucache-bench --bin v3-paired --quiet
-      - name: Optional RDMA feature still skips hardware
-        run: cargo test -p gpucache-transport --all-targets --features rdma-nixl
-      - name: Optional DPU feature still skips hardware
-        run: cargo test -p gpucache-dpu --all-targets --features dpu-bf3
+      - name: Reproducible CPU VM regression
+        run: python3 scripts/ci/vm_regression.py --profile cpu --output artifacts/vm-regression
+      - name: Retain regression evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: cpu-vm-regression-x86_64
+          path: artifacts/vm-regression/
+          if-no-files-found: error
+          retention-days: 14
       - name: Reject accelerator linkage and erasure coding
         run: |
           python3 - <<'PY'
@@ -83,7 +71,7 @@ jobs:
       - name: Test
         run: |
           echo "aarch64 cargo test is a CPU build. It is not a BlueField or DOCA run."
-          cargo test --workspace --all-targets
+          cargo test --workspace --all-targets --locked
 
   test-aarch64-qemu:
     name: test (aarch64 qemu-user)
@@ -102,4 +90,4 @@ jobs:
         env:
           CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc
           CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUNNER: qemu-aarch64-static -L /usr/aarch64-linux-gnu
-        run: cargo test --workspace --all-targets --target aarch64-unknown-linux-gnu
+        run: cargo test --workspace --all-targets --locked --target aarch64-unknown-linux-gnu

.gitignore

@@ -5,3 +5,5 @@
 /tmp
 .cargo/registry
 .cargo/git
+/artifacts/
+**/__pycache__/

Cargo.toml

@@ -16,7 +16,7 @@ edition = "2021"
 license = "Apache-2.0"
 rust-version = "1.85"
 authors = ["RustFS, Inc."]
-repository = "https://github.com/rustfs/gpucache"
+repository = "https://github.com/rustfs/GPUCache"
 
 [workspace.dependencies]
 sha2 = "=0.10.8"

README.md

@@ -29,7 +29,7 @@ V1 runs the cache contract on CPU. V2 adds a transport trait, a simulated regist
 | V3: one candidate offload (storage queue) and ablation stubs | Interface only. P2P, DPA, compression, and crypto offload stay `NOT_RUN_HARDWARE` when enabled. Hot-tier EC cannot be enabled |
 | V3: authz, replay window, integrity seal, rate limit, secret rotation, audit | CPU hooks in this tree. Not mTLS. Trusted-fabric boundary is [docs/adr/0007-trusted-fabric.md](docs/adr/0007-trusted-fabric.md) |
 | V3: rebalance, rolling upgrade, fault-domain isolation | In-process stubs on the V2 shard directory. They fence fills. They do not copy a replica |
-| V3: degrade matrix (DPU restart, GPU reset, NIC flap, storage loss, NVMe timeout, control-plane loss, RustFS slow/down) | Simulated fault decisions request recompute. Hidden slab bytes remain charged and can exhaust capacity until explicit restart; recovery is not proven on hardware |
+| V3: degrade matrix (DPU restart, GPU reset, NIC flap, storage loss, NVMe timeout, control-plane loss, RustFS slow/down) | Simulated fault decisions request recompute. Unknown completions remain quarantined and charged until explicit drain. Tenant-scoped invalidation/reclaim cannot free referenced slots. Recovery is not proven on hardware |
 | V3: G4 retention, manifest version, reclaim, delete verification, tenant cold keys, backfill budget | Implemented against the mock. Live RustFS is `NOT_RUN_HARDWARE` |
 | V3: paired NIC-RDMA baseline vs DPU | `NOT_RUN_HARDWARE`. `tco=TCO_UNKNOWN`. `go_gate=NOT_EVALUATED`. TCP is not the DPU baseline |
 | V3: second engine (SGLang / HiCache) | Not added. Evaluation note only. See [docs/adr/0008-second-engine.md](docs/adr/0008-second-engine.md) |
@@ -69,6 +69,16 @@ Resource limits cover object size, writer admissions, memory registrations, per-
 
 ## What you can run
 
+For an unattended VM or Grok bot regression, run:
+
+```bash
+python3 scripts/ci/vm_regression.py --profile cpu --output artifacts/vm-regression
+```
+
+The runner checks the locked dependency contract, formatting, Clippy, workspace and doc tests, optional and combined features, and CPU demos. It writes JSON, JUnit, a summary and per-gate logs. Hardware profiles return `NOT_RUN_HARDWARE` and exit 2; a CPU pass does not qualify RDMA, GDS, a GPU engine or BF3. Live RustFS access is disabled for this profile, including Cargo-configured endpoints. See the [VM runbook](docs/VM_REGRESSION.md), [staged roadmap](docs/ROADMAP.md), [replacement strategy](docs/REPLACEMENT_STRATEGY.md), and [English PR plan](docs/PR_PLAN.md).
+
+V3 regression hardening rejects active replay-window eviction, cross-domain replay, retired credential revival and malformed identities. TCP replies must match the operation, epoch and ordered request IDs before a connection can be reused. The DPU queue simulation bounds task/slot counts and retains unknown-completion credits; these checks do not constitute device isolation or hardware drain.
+
 From a clean checkout, with Rust stable (MSRV 1.85):
 
 ```bash

crates/core-types/src/identity.rs

@@ -209,6 +209,15 @@ impl CacheIdentity {
         Self::decode_body(bytes, false)
     }
 
+    /// Validate protocol hard limits independently of tighter store quotas.
+    pub fn validate_wire(&self) -> Result<(), CacheError> {
+        self.validate(&IdentityLimits {
+            max_field_bytes: crate::limits::DEFAULT_MAX_FIELD_BYTES,
+            max_prefix_links: 4096,
+            max_block_tokens: u32::MAX,
+        })
+    }
+
     pub fn validate(&self, limits: &IdentityLimits) -> Result<(), CacheError> {
         if self.version != IDENTITY_VERSION {
             return Err(CacheError::IdentityInvalid {
@@ -483,6 +492,9 @@ impl CacheIdentity {
                 reason: "physical field on logical key",
             });
         }
+        // Syntax alone is insufficient: callers also decode exports and keys
+        // without passing through a store's admission checks.
+        id.validate_wire()?;
         Ok(id)
     }
 }
@@ -667,3 +679,119 @@ pub fn logical_fields_diverge(a: &CacheIdentity, b: &CacheIdentity) -> bool {
         || a.prefix_chain != b.prefix_chain
         || a.multimodal != b.multimodal
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::{decode_physical_key, encode_physical_key, synthetic_identity, ResourceLimits};
+
+    #[test]
+    fn identity_decoder_rejects_structurally_encoded_invalid_identity() {
+        let base = synthetic_identity(1, 1);
+        let mut cases = Vec::new();
+        let mut id = base.clone();
+        id.layout.family = 0;
+        cases.push(id);
+        let mut id = base.clone();
+        id.shard.tp_rank = id.shard.tp_size;
+        cases.push(id);
+        let mut id = base.clone();
+        id.block_token_count = 0;
+        cases.push(id);
+        let mut id = base.clone();
+        id.valid_length = id.block_token_count + 1;
+        cases.push(id);
+        let mut id = base;
+        id.rope.scaling_factor_bits = 1.0f64.to_bits();
+        cases.push(id);
+        for id in cases {
+            let bytes = id.encode_body(false).unwrap();
+            assert!(matches!(
+                CacheIdentity::from_bytes(&bytes),
+                Err(CacheError::IdentityInvalid { .. })
+            ));
+        }
+    }
+
+    #[test]
+    fn wire_validation_preserves_hard_bounds_without_imposing_store_defaults() {
+        let mut id = synthetic_identity(1, 1);
+        id.block_token_count = u32::MAX;
+        id.valid_length = u32::MAX;
+        id.prefix_chain = vec![Digest([1; 32]); 4096];
+        assert_eq!(
+            CacheIdentity::from_bytes(&id.to_bytes().unwrap()).unwrap(),
+            id
+        );
+        id.prefix_chain.push(Digest([2; 32]));
+        assert!(id.validate_wire().is_err());
+        assert!(CacheIdentity::from_bytes(&id.to_bytes().unwrap()).is_err());
+    }
+
+    #[test]
+    fn physical_keys_separate_all_security_and_layout_fields() {
+        let base = synthetic_identity(1, 1);
+        let mut variants = Vec::new();
+        let mut changed = base.clone();
+        changed.tenant = TenantId::new(b"other-tenant".to_vec()).unwrap();
+        variants.push(changed);
+        let mut changed = base.clone();
+        changed.trust_domain = TrustDomain::new(b"other-trust".to_vec()).unwrap();
+        variants.push(changed);
+        let mut changed = base.clone();
+        changed.model_revision = Digest([9; 32]);
+        variants.push(changed);
+        let mut changed = base.clone();
+        changed.model_config_hash = Digest([9; 32]);
+        variants.push(changed);
+        let mut changed = base.clone();
+        changed.tokenizer_revision = Digest([9; 32]);
+        variants.push(changed);
+        let mut changed = base.clone();
+        changed.adapter = AdapterIdentity::Lora {
+            revision: Digest([9; 32]),
+        };
+        variants.push(changed);
+        let mut changed = base.clone();
+        changed.kv_dtype = KvDtype::F16;
+        variants.push(changed);
+        let mut changed = base.clone();
+        changed.quant_format = QuantFormat::Fp8;
+        variants.push(changed);
+        let mut changed = base.clone();
+        changed.layout.minor += 1;
+        variants.push(changed);
+        let mut changed = base.clone();
+        changed.shard.tp_size = 2;
+        changed.shard.tp_rank = 1;
+        variants.push(changed);
+        let mut changed = base.clone();
+        changed.position = PositionEncoding::Absolute;
+        variants.push(changed);
+        let mut changed = base.clone();
+        changed.rope.theta_bits = 500_000.0f64.to_bits();
+        variants.push(changed);
+        let mut changed = base.clone();
+        changed.prefix_chain[0] = Digest([9; 32]);
+        variants.push(changed);
+        let mut changed = base.clone();
+        changed.block_token_count += 1;
+        variants.push(changed);
+        let mut changed = base.clone();
+        changed.valid_length -= 1;
+        variants.push(changed);
+        let mut changed = base.clone();
+        changed.multimodal = Some(MultimodalIdentity {
+            processor_revision: Digest([8; 32]),
+            input_digest: Digest([9; 32]),
+        });
+        variants.push(changed);
+        let limits = IdentityLimits::from_resources(&ResourceLimits::dev_defaults());
+        let key = encode_physical_key(&base, &limits).unwrap();
+        for changed in variants {
+            let other_key = encode_physical_key(&changed, &limits).unwrap();
+            assert_ne!(key, other_key);
+            assert_eq!(decode_physical_key(&other_key).unwrap(), changed);
+        }
+    }
+}

crates/dpu/src/degrade.rs

@@ -4,8 +4,10 @@
 //! Failure outcomes for the V3 degrade matrix.
 //!
 //! Every listed fault is a clean miss. The engine recomputes. The decision
-//! carries no payload, does not report a wrong hit, and does not stick in a
-//! permanent block. Device-side effects live on `StorageQueue::apply_fault`.
+//! carries no payload and does not report a wrong hit. `blocked=false` means
+//! the caller may fall back to recompute; it does not promise queue admission.
+//! Unknown completions retain quarantine credit until explicit drain or a
+//! simulated restart fence. Effects live on `StorageQueue::apply_fault`.
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub enum Fault {