diff --git a/changelog.d/699.fixed.md b/changelog.d/699.fixed.md new file mode 100644 index 000000000..55fc07874 --- /dev/null +++ b/changelog.d/699.fixed.md @@ -0,0 +1,3 @@ +Require a byte-bound input-surface audit when accepting a UK regional model mismatch, refuse dropped columns, and record defaulted inputs and limitations in the bundle, TRACE and execution receipts. Preserve the regional manifest digest and area build model while keeping area receipts uncertified; dropping the regional release removes its exception record. + +Refuse regional releases that explicitly name a different model package, and forward empty CLI reasons so certification rejects them. Document that the UK loader silently skips unknown columns. diff --git a/changelog.d/703.added.md b/changelog.d/703.added.md new file mode 100644 index 000000000..821b167b8 --- /dev/null +++ b/changelog.d/703.added.md @@ -0,0 +1 @@ +Support opt-in local data release manifests pinned by the installed bundle TRO, with fail-closed verification and a distinct runtime certification basis for offline Budget-day runners. Reject conflicting TRO hash assertions and require local and fetched manifest artifacts to agree with the installed bundle. diff --git a/changelog.d/accept-regional-model-mismatch.added.md b/changelog.d/accept-regional-model-mismatch.added.md new file mode 100644 index 000000000..a60f85002 --- /dev/null +++ b/changelog.d/accept-regional-model-mismatch.added.md @@ -0,0 +1 @@ +`certify-data --accept-regional-model-mismatch REASON` (UK only) keeps a local-area release routed when it was built on another policyengine-uk version, and records the reviewed mismatch as `regional_certification` in the bundle instead of refusing; area receipts are uncertified. diff --git a/docs/bundles.md b/docs/bundles.md index 21bcd37a9..55fff09be 100644 --- a/docs/bundles.md +++ b/docs/bundles.md @@ -116,6 +116,68 @@ policyengine bundle manifest 4.19.1 `install` targeted. Use `--venv` or `--python` only to inspect a different target explicitly. +## Local release manifests for Budget-day runners + +Set `POLICYENGINE_UK_DATA_MANIFEST_FILE` before starting a generation process to +read the UK release manifest from that file. The same setting is available as +`POLICYENGINE__DATA_MANIFEST_FILE` for other bundled countries. When set, +it always selects the local file, avoiding a Hugging Face request and its timeout. +An unset setting preserves the existing fetch and certification behavior. + +The trust anchor is the `trov:sha256` on +`composition/1/artifact/data_release_manifest` in the installed country's +`data/bundle/{country}.trace.tro.jsonld`. The file's **raw bytes** must match that +pin. A missing pin, unreadable file, altered bytes, or certification mismatch +stops the process. Every definition of the artifact ID in the TRO must agree on +one digest. After revision normalization, the manifest's certified/default +artifact and all artifacts shared with the bundle must agree with the bundle's +locations and pinned hashes. This artifact check also applies to fetched +manifests. The model, core, build ID and fingerprint checks still apply; +selecting a local file cannot extend the bundle's certification. + +At G0 each morning, while Hugging Face is available, use the exact installed +release that the runner will use and download and verify the manifest before +enabling the setting. Keep the Hugging Face token in `HUGGING_FACE_TOKEN`, +`HF_TOKEN` or `HUGGINGFACE_HUB_TOKEN`: + +```bash +env -u POLICYENGINE_UK_DATA_MANIFEST_FILE POLICYENGINE_SKIP_COUNTRY_IMPORTS=1 python - <<'PY' && +import hashlib +from pathlib import Path + +import requests +from policyengine.provenance.manifest import ( + bundled_data_release_manifest_sha256, get_release_manifest, + hugging_face_auth_headers, https_release_manifest_uri, +) + +expected = bundled_data_release_manifest_sha256("uk") +response = requests.get( + https_release_manifest_uri(get_release_manifest("uk").data_package), + headers=hugging_face_auth_headers(), timeout=30, +) +response.raise_for_status() +actual = hashlib.sha256(response.content).hexdigest() +if actual != expected: + raise RuntimeError(f"Manifest SHA-256: expected {expected}; actual {actual}") +target = Path("data/uk-release-manifest.json") +target.parent.mkdir(parents=True, exist_ok=True) +staged = target.with_suffix(".json.tmp") +staged.write_bytes(response.content) +staged.replace(target) +print(f"Verified {target}: {actual}") +PY +export POLICYENGINE_UK_DATA_MANIFEST_FILE="$PWD/data/uk-release-manifest.json" +``` + +Start a fresh runner process with that setting and record its receipt at G0. +Runtime certification and TRACE output use a distinct basis ending in +`_local_manifest`, such as `exact_build_model_version_local_manifest`; packaged +execution receipts record it as `runtime.data_certification_basis`. The +receipt's certificate keeps the bundle's published basis so certificate +verification remains reproducible. See [execution receipts](receipts.md). +Refresh and verify the file whenever the installed certified release changes. + ## Bundle-only PRs Run: diff --git a/docs/engineering/skills/data-certification.md b/docs/engineering/skills/data-certification.md index 23c5d8133..e4805591e 100644 --- a/docs/engineering/skills/data-certification.md +++ b/docs/engineering/skills/data-certification.md @@ -79,6 +79,80 @@ Microcosm national release with one of: claim first. Its `frs_2024_25` is not carried, so keep it loadable with a `dataset_overlays.uk` entry. +- **An older local-area release, on an accepted mismatch.** When the national + cut moves to a new policyengine-uk before a local-area cut on that version + exists, a reviewer can accept retaining the older regional route. A reason + and a byte-bound input-surface audit are both required. First audit the + actual local H5 with the exact target model installed: + + ```bash + .venv/bin/python scripts/audit_regional_input_surface.py \ + --regional-manifest-uri "" \ + --artifact "microcosm_uk_2024_25_dense=/local/dense.h5" \ + --model-version 2.147.2 \ + --audited-by "" \ + --limitation "the dense cut's VAT predates the policyengine-uk VAT factor fix (#2187, 2.135.7); the dashboard holds a VAT limit" \ + --output regional-input-surface.json + + .venv/bin/python scripts/bundle.py certify-data --country uk --data-producer populace \ + --manifest-uri "hf://dataset/policyengine/populace-uk-private@/releases/microcosm-uk-2024-25-national/release_manifest.json" \ + --regional-manifest-uri "" \ + --accept-regional-model-mismatch "" \ + --regional-input-surface-evidence regional-input-surface.json \ + --model-version 2.147.2 + ``` + + The comparison is ported from Microcosm #1348's + `tools/certify_uk_cut_in_policyengine_py.py`, and strengthened to inspect the + persisted entity-table columns, including each period and country-loader + aliases. Comparing the old model's input inventory alone cannot prove which + columns the file carries. The audit binds the regional release identity, + original manifest bytes, actual H5 bytes and exact target model version. + Certification refuses missing or mismatched evidence and **any dropped + column**. It records every defaulted input and the free-text limitations in + the validated `regional_certification`; the bundle hash binds that record. + + The policyengine-uk 2.147.2 dataset loaders silently skip columns absent + from the model's variables: both the + [generic dataset loader](https://github.com/PolicyEngine/policyengine-uk/blob/fe0b99a36bd309a722b19937345e34290cb6dd73/policyengine_uk/simulation.py#L1113-L1127) + and the + [UK multi-year loader](https://github.com/PolicyEngine/policyengine-uk/blob/fe0b99a36bd309a722b19937345e34290cb6dd73/policyengine_uk/simulation.py#L1186-L1196) + use `continue` without warning. A removed or renamed input can therefore + disappear silently at runtime; the audit must catch that column loss before + certification. + + The national certificate never widens. Area status remains `uncertified`: + the certificate still covers the national dataset only (#685). TRACE and + execution receipts expose the regional identity, build model, acceptance, + reason, input-surface result and limitations separately from the national + certification. The flag remains UK only; an empty reason, a release that + already covers the target model, or a missing regional release refuses. + The regional core checks still apply. Re-certifying with + `--drop-regional-release` removes the regional record, routes and artifact. + + For the Budget route, the measured **model-inventory comparison** from + 2.132.6 to 2.147.2 has no dropped names and these 16 new pure inputs: + `additional_residential_property_purchased`, `defined_benefit_pension_share`, + `directly_held_shares`, `early_years_school_start_deferred`, + `is_full_year_category_a_director`, `local_bus_uncapped_single_fare`, + `non_residential_property_purchased`, `non_residential_property_secured_debt`, + `other_residential_property_secured_debt`, + `pension_annual_allowance_taxable_lump_sum_death_benefits`, + `pension_contributions_via_salary_sacrifice_pre_2015`, `pip_dl_points`, + `pip_m_points`, `salary_sacrifice_returned_to_income`, + `unit_and_investment_trusts`, and `war_widows_pension_reported`. + Both `salary_sacrifice_returned_to_income` and + `local_bus_uncapped_single_fare` default in that comparison. This is not an + audit of cut `microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe`: + its private H5 was unavailable to this implementation job without + authentication. Certification must use the complete list measured from the + actual cut by the audit above, including any further missing stored inputs. + Record the pre-fix VAT limitation shown above for this dense cut. + + After the receipt schema changes, refresh the Budget dashboard's vendored + `tests/fixtures/w4/execution_receipt.v1.schema.json` from + `src/policyengine/data/schemas/execution_receipt.v1.schema.json`. + Either way, certification checks that the regional release serves the pinned model by its own build or compatibility claims (a `--qualified-pair-id` names the primary release only), checks its core claims as for the primary release diff --git a/docs/receipts.md b/docs/receipts.md index a082f449b..48686f6e8 100644 --- a/docs/receipts.md +++ b/docs/receipts.md @@ -47,6 +47,25 @@ receipt_json = receipt.to_canonical_json() For a country `Microsimulation` built with `managed_microsimulation()`, call `build_execution_receipt_for_microsimulation(sim, years=..., reform=..., baseline=...)`. It reads the dataset that was loaded from `sim.policyengine_bundle`. The file must still hash to the SHA-256 verified when it was loaded, and the receipt must name the dataset, URI, revision and package recorded then; otherwise it is refused, because it would describe bytes that did not run. A simulation loaded from an unmanaged file or URI recorded no load-time hash, so it cannot be receipted. The microsimulation does not record its reform, so pass the reform you built it with. +With `POLICYENGINE_UK_DATA_MANIFEST_FILE` enabled, a packaged-bundle receipt +rechecks runtime certification using the local file and records its distinct +`*_local_manifest` basis in `runtime.data_certification_basis`. Local-file +verification errors stop receipt creation. The certificate's +`compatibility_basis` stays the published bundle's assertion, which +`verify_certificate` can reproduce. Like the other runtime context, this source +annotation is excluded from execution and receipt digests. Caller-supplied +bundles do not claim this packaged-bundle runtime annotation. + +The execution-receipt schema accepts the annotation as an optional string; +compatibility bases in the certificate and TRACE TRO are also strings. The UK +Budget dashboard's receipt adapter uses this package's receipt model and +`verify_certificate`, so it needs no basis allowlist change. When adopting this +release, refresh the dashboard's vendored +`tests/fixtures/w4/execution_receipt.v1.schema.json`: its +`tests/test_released_receipt.py` checks exact equality against the installed +schema. Older consumers that forbid extra runtime fields need the updated +schema before reading these receipts. + There are two ways to bind a policyengine-uk `Scenario`: - A scenario built from `parameter_changes` is bound by content with `scenario_reform_identity(scenario)`. - `Scenario.from_reform(d)` wraps `d` in a closure that cannot be bound by its code, so bind `d` itself with `reform_identity(d, "policyengine-uk.reform_dict/v1")`. diff --git a/scripts/audit_regional_input_surface.py b/scripts/audit_regional_input_surface.py new file mode 100644 index 000000000..cbe5cd1a2 --- /dev/null +++ b/scripts/audit_regional_input_surface.py @@ -0,0 +1,56 @@ +"""Write a byte-bound audit of regional H5 columns for an installed UK model. + +Run in an environment with the exact requested policyengine-uk version and +PyTables installed. Supply the H5 files from the immutable regional manifest; +the audit verifies their hashes without installing models or downloading data. +""" + +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +# The target model must be auditable before the current bundle certifies it. +os.environ.setdefault("POLICYENGINE_SKIP_COUNTRY_IMPORTS", "1") +sys.path.insert(0, str(REPO_ROOT / "src")) + +from policyengine.provenance.regional_input_surface import ( # noqa: E402 + measure_regional_input_surface, +) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--regional-manifest-uri", required=True) + parser.add_argument( + "--artifact", required=True, action="append", metavar="NAME=PATH" + ) + parser.add_argument("--model-version", required=True) + parser.add_argument("--audited-by", required=True) + parser.add_argument("--limitation", required=True, action="append") + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args(argv) + paths: dict[str, Path] = {} + for entry in args.artifact: + name, separator, path = entry.partition("=") + if not separator or not name.strip() or not path.strip() or name in paths: + parser.error("Each --artifact must be a unique nonblank NAME=PATH.") + paths[name] = Path(path) + evidence = measure_regional_input_surface( + regional_manifest_uri=args.regional_manifest_uri, + artifact_paths=paths, + model_version=args.model_version, + audited_by=args.audited_by, + limitations=args.limitation, + token=os.environ.get("HUGGING_FACE_TOKEN") or os.environ.get("HF_TOKEN"), + ) + args.output.write_text(evidence.model_dump_json(indent=2) + "\n") + print(f"Saved {len(evidence.artifacts)} artifact input audits to {args.output}.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/bundle.py b/scripts/bundle.py index 53b74a26a..9dd4040a4 100644 --- a/scripts/bundle.py +++ b/scripts/bundle.py @@ -48,6 +48,14 @@ def _certify_data(args: argparse.Namespace) -> int: argv.extend(["--regional-release-kind", args.regional_release_kind]) if args.drop_regional_release: argv.append("--drop-regional-release") + if args.accept_regional_model_mismatch is not None: + argv.extend( + ["--accept-regional-model-mismatch", args.accept_regional_model_mismatch] + ) + if args.regional_input_surface_evidence: + argv.extend( + ["--regional-input-surface-evidence", args.regional_input_surface_evidence] + ) if args.no_generate: argv.append("--no-generate") if args.no_changelog: @@ -226,6 +234,19 @@ def _parser() -> argparse.ArgumentParser: "Cannot be combined with --regional-manifest-uri." ), ) + certify.add_argument( + "--accept-regional-model-mismatch", + metavar="REASON", + help=( + "UK only: route regions to a regional release on another model " + "version, recording this reviewed reason; forwarded to " + "scripts/certify_data_release.py." + ), + ) + certify.add_argument( + "--regional-input-surface-evidence", + help="Audited regional input-surface JSON; required for an accepted mismatch.", + ) certify.add_argument( "--regional-artifact-prefix", help="Legacy per-state artifact prefix to import. Defaults to states/.", diff --git a/scripts/certify_data_release.py b/scripts/certify_data_release.py index f3a720ee4..7cdc1a5a1 100644 --- a/scripts/certify_data_release.py +++ b/scripts/certify_data_release.py @@ -24,6 +24,7 @@ from __future__ import annotations import argparse +import json import os import sys from pathlib import Path @@ -115,7 +116,22 @@ def main(argv=None) -> int: "qualification of this pair (requires --core-version)." ), ) + parser.add_argument( + "--accept-regional-model-mismatch", + metavar="REASON", + help=( + "UK only: route regions to a regional release that does not cover " + "the pinned model, recording this reviewed reason in the bundle " + "instead of refusing. Requires --regional-input-surface-evidence; " + "dropped columns refuse. The core must still be covered." + ), + ) parser.add_argument("--no-generate", action="store_true") + parser.add_argument( + "--regional-input-surface-evidence", + type=Path, + help="Audited JSON evidence bound to the regional bytes and exact target model.", + ) parser.add_argument("--no-changelog", action="store_true") parser.add_argument( "--skip-artifact-check", @@ -149,6 +165,12 @@ def main(argv=None) -> int: drop_regional_release=args.drop_regional_release, core_version=args.core_version, qualified_pair_id=args.qualified_pair_id, + regional_model_mismatch_reason=args.accept_regional_model_mismatch, + regional_input_surface_evidence=( + json.loads(args.regional_input_surface_evidence.read_text()) + if args.regional_input_surface_evidence is not None + else None + ), ) print(result.summary()) diff --git a/src/policyengine/data/schemas/execution_receipt.v1.schema.json b/src/policyengine/data/schemas/execution_receipt.v1.schema.json index 12fc69861..f2e467cc3 100644 --- a/src/policyengine/data/schemas/execution_receipt.v1.schema.json +++ b/src/policyengine/data/schemas/execution_receipt.v1.schema.json @@ -457,6 +457,273 @@ "title": "ReformIdentity", "type": "object" }, + "RegionalCertification": { + "description": "A reviewer's acceptance of a regional release on another model version.\n\nWritten only by ``certify-data --accept-regional-model-mismatch``: the\nregional release neither was built with nor claims the certified model,\nand area runs on it are uncertified.", + "properties": { + "accepted_reason": { + "title": "Accepted Reason", + "type": "string" + }, + "area_receipts": { + "const": "uncertified", + "default": "uncertified", + "title": "Area Receipts", + "type": "string" + }, + "built_with_model_version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Built With Model Version" + }, + "certified_for_model_version": { + "title": "Certified For Model Version", + "type": "string" + }, + "compatibility_basis": { + "const": "accepted_model_mismatch", + "title": "Compatibility Basis", + "type": "string" + }, + "defaulted_inputs": { + "items": { + "type": "string" + }, + "title": "Defaulted Inputs", + "type": "array" + }, + "dropped_columns": { + "items": { + "type": "string" + }, + "title": "Dropped Columns", + "type": "array" + }, + "evidence": { + "$ref": "#/$defs/RegionalInputSurfaceEvidence" + }, + "limitations": { + "items": { + "type": "string" + }, + "title": "Limitations", + "type": "array" + }, + "regional_manifest_sha256": { + "pattern": "^[0-9a-f]{64}$", + "title": "Regional Manifest Sha256", + "type": "string" + }, + "regional_manifest_uri": { + "title": "Regional Manifest Uri", + "type": "string" + } + }, + "required": [ + "compatibility_basis", + "certified_for_model_version", + "accepted_reason", + "regional_manifest_uri", + "regional_manifest_sha256", + "defaulted_inputs", + "dropped_columns", + "limitations", + "evidence" + ], + "title": "RegionalCertification", + "type": "object" + }, + "RegionalInputArtifactEvidence": { + "additionalProperties": false, + "description": "Measured columns of one byte-verified regional H5 artifact.", + "properties": { + "columns": { + "items": { + "type": "string" + }, + "minItems": 1, + "title": "Columns", + "type": "array" + }, + "columns_by_period": { + "additionalProperties": { + "items": { + "type": "string" + }, + "type": "array" + }, + "minProperties": 1, + "title": "Columns By Period", + "type": "object" + }, + "path": { + "minLength": 1, + "title": "Path", + "type": "string" + }, + "repo_id": { + "minLength": 1, + "title": "Repo Id", + "type": "string" + }, + "repo_type": { + "anyOf": [ + { + "enum": [ + "model", + "dataset" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Repo Type" + }, + "revision": { + "minLength": 1, + "title": "Revision", + "type": "string" + }, + "sha256": { + "pattern": "^[0-9a-f]{64}$", + "title": "Sha256", + "type": "string" + } + }, + "required": [ + "repo_id", + "path", + "revision", + "sha256", + "columns", + "columns_by_period" + ], + "title": "RegionalInputArtifactEvidence", + "type": "object" + }, + "RegionalInputSurfaceEvidence": { + "additionalProperties": false, + "description": "An operator's audit of actual regional columns against an exact model.\n\nSurfaces are retained so certification recomputes the comparison rather\nthan trusting a declared zero-loss result. The artifact hashes identify\nthe bytes inspected by the auditor, not merely a model's possible inputs.", + "properties": { + "artifacts": { + "additionalProperties": { + "$ref": "#/$defs/RegionalInputArtifactEvidence" + }, + "minProperties": 1, + "title": "Artifacts", + "type": "object" + }, + "audited_by": { + "minLength": 1, + "pattern": "\\S", + "title": "Audited By", + "type": "string" + }, + "defaulted_inputs": { + "items": { + "type": "string" + }, + "title": "Defaulted Inputs", + "type": "array" + }, + "dropped_columns": { + "items": { + "type": "string" + }, + "title": "Dropped Columns", + "type": "array" + }, + "input_aliases": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Input Aliases" + }, + "limitations": { + "items": { + "type": "string" + }, + "minItems": 1, + "title": "Limitations", + "type": "array" + }, + "model_inputs": { + "items": { + "type": "string" + }, + "minItems": 1, + "title": "Model Inputs", + "type": "array" + }, + "model_package": { + "minLength": 1, + "title": "Model Package", + "type": "string" + }, + "model_variables": { + "items": { + "type": "string" + }, + "minItems": 1, + "title": "Model Variables", + "type": "array" + }, + "model_version": { + "minLength": 1, + "title": "Model Version", + "type": "string" + }, + "regional_manifest_sha256": { + "pattern": "^[0-9a-f]{64}$", + "title": "Regional Manifest Sha256", + "type": "string" + }, + "regional_manifest_uri": { + "minLength": 1, + "title": "Regional Manifest Uri", + "type": "string" + }, + "schema_version": { + "const": 1, + "default": 1, + "title": "Schema Version", + "type": "integer" + } + }, + "required": [ + "regional_manifest_uri", + "regional_manifest_sha256", + "model_package", + "model_version", + "artifacts", + "model_variables", + "model_inputs", + "dropped_columns", + "defaulted_inputs", + "audited_by", + "limitations" + ], + "title": "RegionalInputSurfaceEvidence", + "type": "object" + }, "RuntimeContext": { "additionalProperties": false, "description": "Where the execution ran. Part of no digest.", @@ -473,6 +740,19 @@ "default": null, "title": "Ci Run Url" }, + "data_certification_basis": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Runtime certification basis when an opt-in local data manifest was verified against the packaged bundle TRACE TRO. The certificate's compatibility_basis remains the published bundle's assertion.", + "title": "Data Certification Basis" + }, "platform": { "title": "Platform", "type": "string" @@ -642,6 +922,18 @@ "reform": { "$ref": "#/$defs/ReformIdentity" }, + "regional_certification": { + "anyOf": [ + { + "$ref": "#/$defs/RegionalCertification" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The bundle's accepted regional model mismatch, including its exact regional manifest and artifact identity, measured input surface, acceptance reason and limitations. This records an exception for uncertified area runs and does not widen the national certificate." + }, "runtime": { "anyOf": [ { @@ -698,7 +990,7 @@ "x-policyengine-digests": { "canonicalization": "RFC 8785 (JSON Canonicalization Scheme) over the stated value after removing object members whose value is null (array elements are kept). Numbers are IEEE 754 doubles; integers must lie within +/-(2**53 - 1).", "certificate.certificate_sha256": "the bundle manifest's data_releases. record", - "execution_identity_sha256": "{schema_version, country_id, wrapper, model, core, data, certification_status, certificate?} where certificate keeps only bundle_version, country_id, certificate_sha256, compatibility_basis, certified_for_model_version, certified_for_core_version, certified_dataset_sha256 and certified_by", + "execution_identity_sha256": "{schema_version, country_id, wrapper, model, core, data, certification_status, certificate?, regional_certification?} where certificate keeps only bundle_version, country_id, certificate_sha256, compatibility_basis, certified_for_model_version, certified_for_core_version, certified_dataset_sha256 and certified_by", "outputs_sha256": "the canonical output payload the receipt binds", "receipt_sha256": "the whole receipt without runtime and without reform.name and baseline.name", "reform.reform_sha256": "RFC 8785 over {representation, payload} where payload is the reform exactly as given in that representation, nulls kept (a null in a reform can mean something, unlike an absent receipt field); for policyengine-uk.scenario_composition/v1 the payload is {applied_before_data_load, application: parameter_changes_then_modifiers, members: [{representation, reform_sha256, binding}, ...]} with members in application order; an infinite number in a payload is the object {\"$nonfinite\": \"Infinity\"} or {\"$nonfinite\": \"-Infinity\"}, and a payload may not use the key $nonfinite itself" diff --git a/src/policyengine/provenance/certification.py b/src/policyengine/provenance/certification.py index dc773e215..5ddec8e47 100644 --- a/src/policyengine/provenance/certification.py +++ b/src/policyengine/provenance/certification.py @@ -47,6 +47,8 @@ from typing import Literal, Optional, cast, get_args import requests +from packaging.version import InvalidVersion, Version +from pydantic import ValidationError from policyengine.data.uk_geography_assets import ( CONSTITUENCY_ASSET_SPEC, @@ -57,11 +59,18 @@ HF_REQUEST_TIMEOUT_SECONDS, DataReleaseArtifact, DataReleaseManifest, + RegionalInputSurfaceEvidence, RegionalReleaseKind, _specifier_matches, fetch_pypi_wheel_metadata, release_build_id, ) +from policyengine.provenance.manifest import ( + RELEASE_SCOPED_ARTIFACT_KINDS as RELEASE_SCOPED_ARTIFACT_KINDS, +) +from policyengine.provenance.manifest import ( + artifact_path_for_country_manifest as artifact_path_for_country_manifest, +) MANIFEST_URI_PATTERN = re.compile( r"^hf://(?Pdataset|model)/(?P[^@]+)@(?P[^/]+)" @@ -113,9 +122,6 @@ CONSTITUENCY_ASSET_SPEC.weight_matrix_filename, LOCAL_AUTHORITY_ASSET_SPEC.weight_matrix_filename, ) -# Artifact kinds published beside the release manifest rather than at the repo -# root, so their manifest paths are relative to the release directory. -RELEASE_SCOPED_ARTIFACT_KINDS = frozenset({"diagnostics", "evidence"}) US_STATE_CODES = ( "AL", "AK", @@ -410,6 +416,185 @@ def model_compatibility_basis( return basis, [] if warning is None else [warning] +#: A regional release's model coverage accepted by a reviewer, not by the +#: release's own build or claims (``--accept-regional-model-mismatch``). +REGIONAL_BASIS_ACCEPTED_MISMATCH = "accepted_model_mismatch" +REGIONAL_VAT_LIMITATION = ( + "the dense cut's VAT predates the policyengine-uk VAT factor fix " + "(#2187, 2.135.7); the dashboard holds a VAT limit" +) + + +def _regional_build_model_version( + regional_manifest: DataReleaseManifest, model_package: str +) -> Optional[str]: + """Return the recorded regional build version; refuse a different package.""" + built = ( + regional_manifest.build.built_with_model_package + if regional_manifest.build is not None + else None + ) + if built is not None and built.name != model_package: + raise CertificationError( + f"Regional release was built with {built.name}, not {model_package}; " + "regional releases must use the target model package." + ) + return built.version if built is not None else None + + +def accepted_regional_mismatch( + regional_manifest: DataReleaseManifest, + model_package: str, + model_version: str, + reason: str, + *, + regional_manifest_uri: str, + regional_manifest_sha256: str, + evidence: Optional[RegionalInputSurfaceEvidence], +) -> tuple[dict, str]: + """Record a reviewed decision to route regions to a release on another model. + + Area runs on such a release are uncertified twice over: the certificate + covers the national dataset only, and the regional file was built and + calibrated under another model version. The record names both versions + and the reason so the bundle, its TRO and every consumer can say so. + Returns ``(record, warning)``. + """ + + reason = reason.strip() + if not reason: + raise CertificationError( + "--accept-regional-model-mismatch needs a reason a reviewer can read." + ) + built_with = _regional_build_model_version(regional_manifest, model_package) + if evidence is None: + raise CertificationError( + "An accepted regional model mismatch requires audited input-surface " + "evidence (--regional-input-surface-evidence)." + ) + try: + audit = RegionalInputSurfaceEvidence.model_validate(evidence) + except ValidationError as exc: + raise CertificationError( + f"Invalid regional input-surface evidence: {exc}" + ) from exc + if ( + audit.regional_manifest_uri != regional_manifest_uri + or audit.regional_manifest_sha256 != regional_manifest_sha256 + or audit.model_package != model_package + or audit.model_version != model_version + ): + raise CertificationError( + "Regional input-surface evidence must bind the exact regional cut, " + "raw manifest bytes and target model package/version." + ) + expected_artifacts = { + name: artifact + for name, artifact in regional_manifest.artifacts.items() + if artifact.kind == "microdata" and artifact.path.endswith(".h5") + } + if not expected_artifacts or set(audit.artifacts) != set(expected_artifacts): + raise CertificationError( + "Evidence must measure every regional microdata artifact." + ) + for name, artifact in expected_artifacts.items(): + measured = audit.artifacts[name] + if any( + getattr(measured, key) != getattr(artifact, key) + for key in ("repo_id", "path", "revision", "sha256") + ) or ( + measured.repo_type is not None + and measured.repo_type + != ( + artifact.repo_type + or parse_manifest_uri(regional_manifest_uri)["repo_type"] + ) + ): + raise CertificationError( + f"Regional evidence artifact {name!r} does not match its identity/bytes." + ) + if not all(column.strip() for column in measured.columns): + raise CertificationError("Regional evidence has an empty column name.") + if measured.columns_by_period is not None: + if ( + not measured.columns_by_period + or any(not columns for columns in measured.columns_by_period.values()) + or set(measured.columns) + != { + column + for columns in measured.columns_by_period.values() + for column in columns + } + ): + raise CertificationError( + "Regional evidence has inconsistent period columns." + ) + if not set(audit.model_inputs).issubset(audit.model_variables): + raise CertificationError( + "Evidence model inputs must belong to its variable surface." + ) + if not all(item.strip() for item in audit.limitations): + raise CertificationError("Regional evidence limitations must not be blank.") + from policyengine.provenance.regional_input_surface import compare_input_surface + + try: + comparison = compare_input_surface( + audit.artifacts, + audit.model_variables, + audit.model_inputs, + audit.input_aliases, + ) + except ValueError as exc: + raise CertificationError( + f"Invalid regional input-surface evidence: {exc}" + ) from exc + if ( + audit.dropped_columns != comparison["dropped_columns"] + or audit.defaulted_inputs != comparison["defaulted_inputs"] + ): + raise CertificationError( + "Regional evidence comparison disagrees with its measured surfaces." + ) + if comparison["dropped_columns"]: + raise CertificationError( + "Dropped regional input columns are refused: " + + ", ".join(comparison["dropped_columns"]) + ) + limitations = list(audit.limitations) + try: + predates_vat_fix = built_with is not None and Version(built_with) < Version( + "2.135.7" + ) <= Version(model_version) + except InvalidVersion as exc: + raise CertificationError( + "Regional model versions must be valid versions." + ) from exc + if predates_vat_fix and REGIONAL_VAT_LIMITATION not in limitations: + limitations.append(REGIONAL_VAT_LIMITATION) + record = { + "compatibility_basis": REGIONAL_BASIS_ACCEPTED_MISMATCH, + "certified_for_model_version": model_version, + "built_with_model_version": built_with, + "accepted_reason": reason, + "area_receipts": "uncertified", + "regional_manifest_uri": regional_manifest_uri, + "regional_manifest_sha256": regional_manifest_sha256, + "defaulted_inputs": comparison["defaulted_inputs"], + "dropped_columns": comparison["dropped_columns"], + "limitations": limitations, + "evidence": audit.model_dump(mode="json", exclude_none=True), + } + warning = ( + f"regional release: built with {model_package} " + f"{built_with or 'an unrecorded version'} and routed under " + f"{model_version} on an accepted mismatch ({reason}); area runs are " + f"uncertified; audited dropped columns: none; defaulted inputs: " + f"{', '.join(comparison['defaulted_inputs']) or 'none'}. " + + " ".join(limitations) + ) + return record, warning + + def _check_core_compatibility( manifest: DataReleaseManifest, core_version: str, @@ -526,19 +711,6 @@ def head_release_file( return response.status_code == 200 -def artifact_path_for_country_manifest(artifact, uri_parts: dict) -> str: - path = artifact.path - release_dir = release_manifest_dir(uri_parts) - if ( - artifact.kind in RELEASE_SCOPED_ARTIFACT_KINDS - and artifact.repo_id == uri_parts["repo_id"] - and artifact.revision == uri_parts["revision"] - and not path.startswith(f"{release_dir}/") - ): - return f"{release_dir}/{path}" - return path - - def artifact_reference_url( reference: dict, uri_parts: dict, prefix: str = "datasets/" ) -> str: @@ -1107,6 +1279,8 @@ def build_bundle_data_release_payload( regional_manifest_uri: Optional[str] = None, regional_uri_parts: Optional[dict] = None, regional_release_kind: Optional[RegionalReleaseKind] = None, + regional_certification: Optional[dict] = None, + regional_manifest_sha256: Optional[str] = None, ) -> dict: """Map the country certification payload into bundle ``data_releases``.""" payload = copy.deepcopy(country_payload) @@ -1120,12 +1294,16 @@ def build_bundle_data_release_payload( payload["release_manifest_uri"] = https_manifest_url(uri_parts) if regional_manifest_uri is not None: payload["regional_source_manifest_uri"] = regional_manifest_uri + if regional_manifest_sha256 is not None: + payload["regional_source_manifest_sha256"] = regional_manifest_sha256 if regional_uri_parts is not None: payload["regional_release_manifest_uri"] = https_manifest_url( regional_uri_parts ) if regional_release_kind is not None: payload["regional_release_kind"] = regional_release_kind + if regional_certification is not None: + payload["regional_certification"] = regional_certification payload["default_dataset_uri"] = certified_artifact.get("uri") return payload @@ -1149,6 +1327,8 @@ def certify( core_version: Optional[str] = None, qualified_pair_id: Optional[str] = None, bundle_core_version: Optional[str] = None, + regional_model_mismatch_reason: Optional[str] = None, + regional_input_surface_evidence: Optional[RegionalInputSurfaceEvidence] = None, ) -> tuple[dict, str, list[str]]: raise NotImplementedError @@ -1172,6 +1352,8 @@ def certify( core_version: Optional[str] = None, qualified_pair_id: Optional[str] = None, bundle_core_version: Optional[str] = None, + regional_model_mismatch_reason: Optional[str] = None, + regional_input_surface_evidence: Optional[RegionalInputSurfaceEvidence] = None, ) -> tuple[dict, str, list[str]]: raise CertificationError( "Legacy data-producer certification updates are not implemented in " @@ -1198,20 +1380,36 @@ def certify( core_version: Optional[str] = None, qualified_pair_id: Optional[str] = None, bundle_core_version: Optional[str] = None, + regional_model_mismatch_reason: Optional[str] = None, + regional_input_surface_evidence: Optional[RegionalInputSurfaceEvidence] = None, ) -> tuple[dict, str, list[str]]: manifest, manifest_sha256, uri_parts = fetch_release_manifest( manifest_uri, token=token ) primary_manifest = manifest regional_manifest = None + regional_manifest_sha256 = None regional_uri_parts = None recorded_release_kind = None regional_artifacts: dict[str, DataReleaseArtifact] = {} regional_warnings: list[str] = [] + regional_certification: Optional[dict] = None + mismatch_warning: Optional[str] = None + if regional_model_mismatch_reason is not None and regional_manifest_uri is None: + raise CertificationError( + "--accept-regional-model-mismatch applies to a regional release; " + "pass --regional-manifest-uri." + ) + if regional_model_mismatch_reason is not None and country != "uk": + raise CertificationError( + "--accept-regional-model-mismatch is only supported for the UK, " + "whose area receipts are uncertified." + ) if regional_manifest_uri is not None: - regional_manifest, _, regional_uri_parts = fetch_release_manifest( - regional_manifest_uri, token=token + regional_manifest, regional_manifest_sha256, regional_uri_parts = ( + fetch_release_manifest(regional_manifest_uri, token=token) ) + _regional_build_model_version(regional_manifest, model_package) # The regional release runs on the same model and core, so it must # cover the pinned versions by its own build or claims. A qualified # pair names the primary release only, so it does not extend here. @@ -1219,6 +1417,30 @@ def certify( _, regional_warnings = model_compatibility_basis( regional_manifest, model_package, model_version ) + except CertificationError as exc: + if regional_model_mismatch_reason is None: + raise CertificationError( + f"Regional release {regional_manifest_uri}: {exc}" + ) from exc + regional_certification, mismatch_warning = accepted_regional_mismatch( + regional_manifest, + model_package, + model_version, + regional_model_mismatch_reason, + regional_manifest_uri=regional_manifest_uri, + regional_manifest_sha256=regional_manifest_sha256, + evidence=regional_input_surface_evidence, + ) + regional_warnings = [] + else: + if regional_model_mismatch_reason is not None: + raise CertificationError( + f"Regional release {regional_manifest_uri} serves " + f"{model_package} {model_version} by its own build or " + "claims; drop --accept-regional-model-mismatch so the " + "certificate does not record a mismatch that is not there." + ) + try: if core_version is not None: _check_core_compatibility( regional_manifest, core_version, None, regional_warnings @@ -1230,6 +1452,8 @@ def certify( regional_warnings = [ f"regional release: {warning}" for warning in regional_warnings ] + if mismatch_warning is not None: + regional_warnings.append(mismatch_warning) primary_artifact_names = set(manifest.artifacts) manifest, recorded_release_kind = merge_regional_release_manifest( country, @@ -1323,6 +1547,8 @@ def certify( regional_manifest_uri=regional_manifest_uri, regional_uri_parts=regional_uri_parts, regional_release_kind=recorded_release_kind, + regional_certification=regional_certification, + regional_manifest_sha256=regional_manifest_sha256, ), manifest_sha256, warnings, @@ -1368,6 +1594,8 @@ def certify_data_release( drop_regional_release: bool = False, core_version: Optional[str] = None, qualified_pair_id: Optional[str] = None, + regional_model_mismatch_reason: Optional[str] = None, + regional_input_surface_evidence: Optional[RegionalInputSurfaceEvidence] = None, ) -> CertificationResult: """Certify a data release into the bundle manifest. @@ -1383,8 +1611,22 @@ def certify_data_release( given, by its own build or claims: a qualified pair names the primary release only. While the bundle routes regions, a run without ``regional_manifest_uri`` is refused unless ``drop_regional_release``. + + ``regional_model_mismatch_reason`` (UK only) requires + ``regional_input_surface_evidence`` audited against the exact target model + and regional manifest/artifact bytes. Dropped columns are always refused; + every defaulted input and limitation is recorded in + ``regional_certification`` alongside the reason and both model versions. + The core must still be covered; area receipts remain uncertified. """ certified_country = _certified_country(country) + if ( + regional_input_surface_evidence is not None + and regional_model_mismatch_reason is None + ): + raise CertificationError( + "Regional input-surface evidence requires an accepted mismatch reason." + ) if drop_regional_release and regional_manifest_uri is not None: raise CertificationError( "--drop-regional-release removes the bundle's regional routing, so it " @@ -1429,6 +1671,8 @@ def certify_data_release( bundle_core_version=( (bundle.get("packages") or {}).get("policyengine-core") or {} ).get("version"), + regional_model_mismatch_reason=regional_model_mismatch_reason, + regional_input_surface_evidence=regional_input_surface_evidence, ) previous_release = (bundle.get("data_releases") or {}).get(country) or {} bundle.setdefault("data_releases", {})[country] = data_release diff --git a/src/policyengine/provenance/manifest.py b/src/policyengine/provenance/manifest.py index f3b15fb5d..1e00f5fce 100644 --- a/src/policyengine/provenance/manifest.py +++ b/src/policyengine/provenance/manifest.py @@ -1,14 +1,15 @@ import hashlib import json import os +import re from functools import lru_cache from importlib.resources import files from pathlib import Path -from typing import Literal, Optional +from typing import Literal, NoReturn, Optional from urllib.parse import quote import requests -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, PrivateAttr from policyengine.provenance.bundle_validation import ( shadowed_dataset_overlays, @@ -17,6 +18,8 @@ HF_REQUEST_TIMEOUT_SECONDS = 30 PYPI_REQUEST_TIMEOUT_SECONDS = 30 +# Producer paths for these artifacts are relative to the release directory. +RELEASE_SCOPED_ARTIFACT_KINDS = frozenset({"diagnostics", "evidence"}) class DataReleaseManifestUnavailableError(ValueError): @@ -140,6 +143,7 @@ def uri(self) -> str: class DataReleaseManifest(BaseModel): + _source_is_local: bool = PrivateAttr(default=False) schema_version: int data_package: PackageVersion dataset_role: Optional[str] = None @@ -187,7 +191,12 @@ class DataCertification(BaseModel): #: Bases whose runtime model legitimately differs from the data's build-time #: model, so the build fingerprint is not expected to match. CLAIM_BASED_COMPATIBILITY = frozenset( - {"legacy_compatible_model_package", "qualified_exact_pair"} + { + "legacy_compatible_model_package", + "qualified_exact_pair", + "legacy_compatible_model_package_local_manifest", + "qualified_exact_pair_local_manifest", + } ) @@ -202,6 +211,66 @@ class CertifiedDataArtifact(BaseModel): RegionalReleaseKind = Literal["local_area", "area_coded_national"] +class RegionalInputArtifactEvidence(BaseModel): + """Measured columns of one byte-verified regional H5 artifact.""" + + model_config = {"extra": "forbid"} + + repo_id: str = Field(min_length=1) + path: str = Field(min_length=1) + revision: str = Field(min_length=1) + sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + repo_type: Optional[Literal["model", "dataset"]] = None + columns: list[str] = Field(min_length=1) + columns_by_period: dict[str, list[str]] = Field(min_length=1) + + +class RegionalInputSurfaceEvidence(BaseModel): + """An operator's audit of actual regional columns against an exact model. + + Surfaces are retained so certification recomputes the comparison rather + than trusting a declared zero-loss result. The artifact hashes identify + the bytes inspected by the auditor, not merely a model's possible inputs. + """ + + model_config = {"extra": "forbid"} + + schema_version: Literal[1] = 1 + regional_manifest_uri: str = Field(min_length=1) + regional_manifest_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + model_package: str = Field(min_length=1) + model_version: str = Field(min_length=1) + artifacts: dict[str, RegionalInputArtifactEvidence] = Field(min_length=1) + model_variables: list[str] = Field(min_length=1) + model_inputs: list[str] = Field(min_length=1) + input_aliases: Optional[dict[str, str]] = None + dropped_columns: list[str] + defaulted_inputs: list[str] + audited_by: str = Field(min_length=1, pattern=r"\S") + limitations: list[str] = Field(min_length=1) + + +class RegionalCertification(BaseModel): + """A reviewer's acceptance of a regional release on another model version. + + Written only by ``certify-data --accept-regional-model-mismatch``: the + regional release neither was built with nor claims the certified model, + and area runs on it are uncertified. + """ + + compatibility_basis: Literal["accepted_model_mismatch"] + certified_for_model_version: str + built_with_model_version: Optional[str] = None + accepted_reason: str + area_receipts: Literal["uncertified"] = "uncertified" + regional_manifest_uri: str + regional_manifest_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + defaulted_inputs: list[str] + dropped_columns: list[str] + limitations: list[str] + evidence: RegionalInputSurfaceEvidence + + class CountryReleaseManifest(BaseModel): schema_version: int = 1 bundle_id: Optional[str] = None @@ -218,6 +287,11 @@ class CountryReleaseManifest(BaseModel): ``local_area`` for a non-default local-area release, and ``area_coded_national`` for another release's national dataset that carries area codes (and its area weight matrices).""" + regional_certification: Optional[RegionalCertification] = None + """Present only when the regional release was routed on an accepted + model mismatch rather than its own build or claims.""" + regional_source_manifest_sha256: Optional[str] = None + """SHA-256 of the original regional release manifest bytes.""" certified_data_artifact: Optional[CertifiedDataArtifact] = None certification: Optional[DataCertification] = None source_sha256: Optional[str] = Field(default=None, exclude=True) @@ -367,8 +441,116 @@ def get_release_manifest(country_id: str) -> CountryReleaseManifest: return manifest +def _tro_nodes_with_id(value: object, node_id: str) -> list[dict]: + """Collect node definitions and references wherever JSON-LD nests them.""" + nodes: list[dict] = [] + if isinstance(value, dict): + if value.get("@id") == node_id: + nodes.append(value) + for child in value.values(): + nodes.extend(_tro_nodes_with_id(child, node_id)) + elif isinstance(value, list): + for child in value: + nodes.extend(_tro_nodes_with_id(child, node_id)) + return nodes + + +def bundled_data_release_manifest_sha256(country_id: str) -> str: + """Read the manifest's byte digest from the installed country bundle TRO. + + The artifact and its arrangement must describe the release this bundle + serves. All assertions for the artifact's JSON-LD identifier must agree, + including definitions outside the inline composition. A missing or stale + sidecar cannot authorise a local manifest. + """ + country_manifest = get_release_manifest(country_id) + tro_path = files("policyengine").joinpath( + "data", "bundle", f"{country_id}.trace.tro.jsonld" + ) + artifact_id = "composition/1/artifact/data_release_manifest" + try: + tro = json.loads(tro_path.read_bytes()) + artifacts = [ + artifact + for node in tro["@graph"] + for artifact in node.get("trov:hasComposition", {}).get( + "trov:hasArtifact", [] + ) + if artifact.get("@id") == artifact_id + ] + locations = [ + location.get("trov:hasLocation") + for node in tro["@graph"] + for arrangement in node.get("trov:hasArrangement", []) + for location in arrangement.get("trov:hasArtifactLocation", []) + if location.get("trov:hasArtifact", {}).get("@id") == artifact_id + ] + digest = artifacts[0].get("trov:sha256") if len(artifacts) == 1 else None + if ( + not isinstance(digest, str) + or re.fullmatch(r"[0-9a-fA-F]{64}", digest) is None + or locations != [https_release_manifest_uri(country_manifest.data_package)] + ): + raise ValueError("missing, ambiguous, malformed, or stale manifest pin") + asserted_digests = set() + for node in _tro_nodes_with_id(tro, artifact_id): + if "trov:sha256" not in node: + continue + assertion = node["trov:sha256"] + if ( + not isinstance(assertion, str) + or re.fullmatch(r"[0-9a-fA-F]{64}", assertion) is None + ): + raise ValueError("malformed manifest pin assertion") + asserted_digests.add(assertion.lower()) + if asserted_digests != {digest.lower()}: + raise ValueError("conflicting manifest pin assertions") + except (OSError, ValueError, KeyError, TypeError, AttributeError) as exc: + raise ValueError( + f"Cannot use a local data release manifest for country '{country_id}': " + f"no valid SHA-256 pin in the installed bundle TRO {tro_path}. " + "Expected SHA-256: unavailable; actual SHA-256: unavailable." + ) from exc + return digest.lower() + + +def _load_data_release_manifest(country_id: str) -> DataReleaseManifest: + """Load an explicitly configured, bundle-pinned file or the cached fetch. + + Local files are read and verified on every call, including after a remote + manifest was cached. Configure the setting before importing country models. + """ + env_name = f"POLICYENGINE_{country_id.upper()}_DATA_MANIFEST_FILE" + local_file = os.environ.get(env_name) + if local_file is None: + return _fetch_data_release_manifest(country_id) + + expected = bundled_data_release_manifest_sha256(country_id) + try: + source_bytes = Path(local_file).expanduser().read_bytes() + except (OSError, ValueError, RuntimeError) as exc: + raise ValueError( + f"Cannot read {env_name}={local_file!r}: {exc}. " + f"Expected SHA-256: {expected}; actual SHA-256: unavailable." + ) from exc + actual = hashlib.sha256(source_bytes).hexdigest() + if actual != expected: + raise ValueError( + f"Local data release manifest {env_name}={local_file!r} " + f"does not match the installed bundle. Expected SHA-256: {expected}; " + f"actual SHA-256: {actual}." + ) + manifest = DataReleaseManifest.model_validate_json(source_bytes) + manifest.source_sha256 = actual + manifest._source_is_local = True + country_manifest = get_release_manifest(country_id) + _apply_release_revision(manifest, country_manifest) + _validate_bundle_artifacts(manifest, country_manifest) + return manifest + + @lru_cache -def get_data_release_manifest(country_id: str) -> DataReleaseManifest: +def _fetch_data_release_manifest(country_id: str) -> DataReleaseManifest: country_manifest = get_release_manifest(country_id) try: @@ -402,6 +584,15 @@ def get_data_release_manifest(country_id: str) -> DataReleaseManifest: if not isinstance(source_bytes, bytes): source_bytes = response.text.encode("utf-8") data_release_manifest.source_sha256 = hashlib.sha256(source_bytes).hexdigest() + _apply_release_revision(data_release_manifest, country_manifest) + _validate_bundle_artifacts(data_release_manifest, country_manifest) + return data_release_manifest + + +def _apply_release_revision( + data_release_manifest: DataReleaseManifest, + country_manifest: CountryReleaseManifest, +) -> DataReleaseManifest: release_revision = country_manifest.data_package.release_manifest_revision if release_revision is not None: for artifact in data_release_manifest.artifacts.values(): @@ -413,6 +604,113 @@ def get_data_release_manifest(country_id: str) -> DataReleaseManifest: return data_release_manifest +def artifact_path_for_country_manifest( + artifact: DataReleaseArtifact, uri_parts: dict +) -> str: + """Resolve producer-relative evidence paths as bundle certification does.""" + path = artifact.path + release_dir = uri_parts["path"].rsplit("/", 1)[0] + if ( + artifact.kind in RELEASE_SCOPED_ARTIFACT_KINDS + and artifact.repo_id == uri_parts["repo_id"] + and artifact.revision == uri_parts["revision"] + and not path.startswith(f"{release_dir}/") + ): + return f"{release_dir}/{path}" + return path + + +def _validate_bundle_artifacts( + manifest: DataReleaseManifest, country: CountryReleaseManifest +) -> None: + """Bind a loaded manifest to the artifacts this installed bundle serves. + + Run after revision rewriting. Regional releases and additive overlays may + supply records absent from the primary manifest; every overlapping record + must agree, and the certified/default artifact must always be present. + """ + + def refuse(dataset: str, detail: str) -> NoReturn: + raise ValueError( + f"Data release manifest artifact {dataset!r} does not match the " + f"installed bundle for country {country.country_id!r}: {detail}." + ) + + default = country.default_dataset + declared_default = manifest.default_datasets.get("national") + if declared_default is not None and declared_default != default: + refuse(default, f"manifest declares default {declared_default!r}") + if default not in manifest.artifacts: + refuse(default, "missing default dataset artifact") + certified = country.certified_data_artifact + if certified is not None: + artifact = manifest.artifacts.get(certified.dataset) + if artifact is None: + refuse(certified.dataset, "missing certified dataset artifact") + try: + location, revision = certified.uri.removeprefix("hf://").rsplit("@", 1) + owner, repo, path = location.split("/", 2) + except ValueError: + refuse(certified.dataset, "invalid bundled certified artifact URI") + expected_identity = (f"{owner}/{repo}", path, revision) + actual_identity = (artifact.repo_id, artifact.path, artifact.revision) + if ( + not certified.uri.startswith("hf://") + or actual_identity != expected_identity + ): + refuse( + certified.dataset, + f"expected {expected_identity!r}, got {actual_identity!r}", + ) + if certified.sha256 is not None and artifact.sha256 != certified.sha256: + refuse(certified.dataset, "certified SHA-256 pin differs") + if default not in country.datasets and ( + certified is None or certified.dataset != default + ): + refuse(default, "missing bundled default artifact record") + + uri_parts = { + "repo_id": country.data_package.repo_id, + "revision": _artifact_revision(country.data_package), + "path": country.data_package.release_manifest_path, + } + for name, artifact in manifest.artifacts.items(): + reference = country.datasets.get(name) + if reference is None: + continue + expected_identity = ( + reference.repo_id or country.data_package.repo_id, + reference.path, + reference.revision or _artifact_revision(country.data_package), + ) + actual_identity = ( + artifact.repo_id, + artifact_path_for_country_manifest(artifact, uri_parts), + artifact.revision, + ) + if actual_identity != expected_identity: + refuse(name, f"expected {expected_identity!r}, got {actual_identity!r}") + if (artifact.repo_type or country.data_package.repo_type) != ( + reference.repo_type or country.data_package.repo_type + ): + refuse(name, "repository type differs") + if reference.sha256 is not None and artifact.sha256 != reference.sha256: + refuse(name, "managed SHA-256 pin differs") + + +class _DataReleaseManifestLoader: + """Callable loader retaining the fetched manifest's typed cache controls.""" + + def __call__(self, country_id: str) -> DataReleaseManifest: + return _load_data_release_manifest(country_id) + + cache_clear = staticmethod(_fetch_data_release_manifest.cache_clear) + cache_info = staticmethod(_fetch_data_release_manifest.cache_info) + + +get_data_release_manifest = _DataReleaseManifestLoader() + + def _specifier_matches(version: str, specifier: str) -> bool: """Match a PEP 440 version specifier (``==``, ``>=``, ``<``, ``,``-joined).""" from packaging.specifiers import InvalidSpecifier, SpecifierSet @@ -433,7 +731,10 @@ def _runtime_core_is_certified( tested with one core, so it holds only for that core. """ - if certification.compatibility_basis != "qualified_exact_pair": + if certification.compatibility_basis not in { + "qualified_exact_pair", + "qualified_exact_pair_local_manifest", + }: return True if runtime_core_version is None: from importlib import metadata @@ -492,6 +793,33 @@ def certify_data_release_compatibility( ) raise exc + certification = _certify_available_data_release( + country_id, + country_manifest, + data_release_manifest, + runtime_model_version, + runtime_data_build_fingerprint, + runtime_core_version, + ) + if data_release_manifest._source_is_local: + return certification.model_copy( + update={ + "compatibility_basis": ( + f"{certification.compatibility_basis}_local_manifest" + ) + } + ) + return certification + + +def _certify_available_data_release( + country_id: str, + country_manifest: CountryReleaseManifest, + data_release_manifest: DataReleaseManifest, + runtime_model_version: str, + runtime_data_build_fingerprint: Optional[str], + runtime_core_version: Optional[str], +) -> DataCertification: # A remote build-model, fingerprint or publisher claim cannot broaden an # exact-pair certificate beyond the model, core and recorded data build # that qualification tested. Check them before any acceptance path returns. diff --git a/src/policyengine/provenance/receipt.py b/src/policyengine/provenance/receipt.py index 12bcbcdbd..2492e706f 100644 --- a/src/policyengine/provenance/receipt.py +++ b/src/policyengine/provenance/receipt.py @@ -76,6 +76,7 @@ import json import math import operator +import os import platform import re import struct @@ -113,7 +114,14 @@ from urllib.request import url2pathname import psutil -from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from pydantic import ( + BaseModel, + ConfigDict, + Field, + field_validator, + model_serializer, + model_validator, +) from policyengine.provenance.bytecode import code_shape as _code_shape from policyengine.provenance.bytecode import compile_source @@ -125,6 +133,7 @@ drop_null_members, ) from policyengine.provenance.loaded import loaded_versions +from policyengine.provenance.manifest import RegionalCertification from policyengine.utils.hashing import sha256_file __all__ = [ @@ -215,7 +224,7 @@ #: How each digest is computed. DIGEST_RULES: dict[str, str] = { "canonicalization": "RFC 8785 (JSON Canonicalization Scheme) over the stated value after removing object members whose value is null (array elements are kept). Numbers are IEEE 754 doubles; integers must lie within +/-(2**53 - 1).", - "execution_identity_sha256": "{schema_version, country_id, wrapper, model, core, data, certification_status, certificate?} where certificate keeps only bundle_version, country_id, certificate_sha256, compatibility_basis, certified_for_model_version, certified_for_core_version, certified_dataset_sha256 and certified_by", + "execution_identity_sha256": "{schema_version, country_id, wrapper, model, core, data, certification_status, certificate?, regional_certification?} where certificate keeps only bundle_version, country_id, certificate_sha256, compatibility_basis, certified_for_model_version, certified_for_core_version, certified_dataset_sha256 and certified_by", "receipt_sha256": "the whole receipt without runtime and without reform.name and baseline.name", "certificate.certificate_sha256": "the bundle manifest's data_releases. record", "reform.reform_sha256": 'RFC 8785 over {representation, payload} where payload is the reform exactly as given in that representation, nulls kept (a null in a reform can mean something, unlike an absent receipt field); for policyengine-uk.scenario_composition/v1 the payload is {applied_before_data_load, application: parameter_changes_then_modifiers, members: [{representation, reform_sha256, binding}, ...]} with members in application order; an infinite number in a payload is the object {"$nonfinite": "Infinity"} or {"$nonfinite": "-Infinity"}, and a payload may not use the key $nonfinite itself', @@ -488,6 +497,21 @@ class RuntimeContext(_Contract): python_version: str platform: str ci_run_url: Optional[str] = None + data_certification_basis: Optional[str] = Field( + default=None, + description=( + "Runtime certification basis when an opt-in local data manifest was " + "verified against the packaged bundle TRACE TRO. The certificate's " + "compatibility_basis remains the published bundle's assertion." + ), + ) + + @model_serializer(mode="wrap") + def _omit_absent_local_basis(self, handler): + payload = handler(self) + if self.data_certification_basis is None: + payload.pop("data_certification_basis", None) + return payload class ExecutionReceipt(_Contract): @@ -506,6 +530,15 @@ class ExecutionReceipt(_Contract): data: DataIdentity certification_status: Literal["certified", "uncertified"] certificate: Optional[CertificateIdentity] = None + regional_certification: Optional[RegionalCertification] = Field( + default=None, + description=( + "The bundle's accepted regional model mismatch, including its exact " + "regional manifest and artifact identity, measured input surface, " + "acceptance reason and limitations. This records an exception for " + "uncertified area runs and does not widen the national certificate." + ), + ) reform: ReformIdentity baseline: ReformIdentity fiscal_year: FiscalYearConvention @@ -526,6 +559,13 @@ class ExecutionReceipt(_Contract): ) runtime: Optional[RuntimeContext] = None + @model_serializer(mode="wrap") + def _omit_absent_regional_certification(self, handler): + payload = handler(self) + if self.regional_certification is None: + payload.pop("regional_certification", None) + return payload + @field_validator("schema_version", mode="before") @classmethod def _schema_version_is_an_integer(cls, value: Any) -> Any: @@ -604,6 +644,10 @@ def execution_identity(self) -> dict[str, Any]: for key in CertificateIdentity.IDENTITY_FIELDS if key in payload } + if self.regional_certification is not None: + identity["regional_certification"] = self.regional_certification.model_dump( + mode="json", exclude_none=True + ) return identity @property @@ -3788,12 +3832,25 @@ def _data_identity( datasets = record.get("datasets") or {} entry = datasets.get(dataset_name) primary = datasets.get(record.get("default_dataset")) or {} - # A regional release's artifacts are certified beside the primary release - # but are not part of it, so only entries at the primary release's own - # revision take its build and manifest identity. - release = entry is not None and entry.get("revision") in ( - None, - primary.get("revision"), + # A regional release's artifacts keep their own build provenance, even + # when acceptance allows them to run beside another national model. + regional_certification = manifest.regional_certification + regional = regional_certification is not None and any( + artifact.repo_id == spec.repo_id + and artifact.path == spec.path + and artifact.revision == spec.revision + and artifact.sha256 == spec.sha256 + and artifact.repo_type in (None, spec.repo_type) + for artifact in regional_certification.evidence.artifacts.values() + ) + release = ( + not regional + and entry is not None + and entry.get("revision") + in ( + None, + primary.get("revision"), + ) ) data_package = record["data_package"] certified_artifact = record.get("certified_data_artifact") or {} @@ -3809,9 +3866,13 @@ def _data_identity( build_fingerprint=certification.get("data_build_fingerprint") if release else None, - build_model_version=certification.get("built_with_model_version") - if release - else None, + build_model_version=( + regional_certification.built_with_model_version + if regional and regional_certification is not None + else certification.get("built_with_model_version") + if release + else None + ), release_manifest_revision=data_package.get("release_manifest_revision") if release else None, @@ -3871,6 +3932,21 @@ def identity(name: str) -> dict[str, Any]: actual_sha256 = sha256_file(dataset_path) data = _data_identity(country_id, record, bundle, dataset_name, actual_sha256) + runtime_basis = None + if source == "packaged" and ( + f"POLICYENGINE_{country_id.upper()}_DATA_MANIFEST_FILE" in os.environ + ): + from policyengine.provenance.manifest import ( + certify_data_release_compatibility, + ) + + runtime_basis = certify_data_release_compatibility( + country_id, + model.version, + runtime_data_build_fingerprint=data.build_fingerprint, + runtime_core_version=core.version, + ).compatibility_basis + fields = _certificate_fields(country_id, record, bundle) certified_artifact = record.get("certified_data_artifact") or {} certificate = None @@ -3908,6 +3984,7 @@ def identity(name: str) -> dict[str, Any]: data=data, certification_status=status, certificate=certificate, + regional_certification=record.get("regional_certification"), reform=reform, baseline=baseline, fiscal_year=uk_fiscal_year_convention(), @@ -3920,6 +3997,7 @@ def identity(name: str) -> dict[str, Any]: python_version=platform.python_version(), platform=sys.platform, ci_run_url=ci_run_url, + data_certification_basis=runtime_basis, ), ) @@ -3954,6 +4032,25 @@ def verify_certificate( if not isinstance(record, Mapping): return [f"the bundle has no {receipt.country_id} data release"] problems = [] + regional_record = record.get("regional_certification") + try: + expected_regional = ( + RegionalCertification.model_validate(regional_record).model_dump( + mode="json", exclude_none=True + ) + if regional_record is not None + else None + ) + except ValueError as exc: + problems.append(f"the bundle cannot determine regional_certification: {exc}") + else: + claimed_regional = ( + receipt.regional_certification.model_dump(mode="json", exclude_none=True) + if receipt.regional_certification is not None + else None + ) + if claimed_regional != expected_regional: + problems.append("regional_certification differs from the bundle's record") for package in (receipt.wrapper, receipt.model, receipt.core): if not _package_can_be_certified(package): problems.append( diff --git a/src/policyengine/provenance/regional_input_surface.py b/src/policyengine/provenance/regional_input_surface.py new file mode 100644 index 000000000..460c53337 --- /dev/null +++ b/src/policyengine/provenance/regional_input_surface.py @@ -0,0 +1,351 @@ +"""Audit regional H5 bytes against the exact installed UK country model. + +The set comparison ports Microcosm PR #1348's handoff comparison, replacing +its older-model inventory proxy with the columns actually stored in every +artifact and period. Country-model input moves are read from that installed +model's constructor, so an alias cannot make a supplied input look defaulted. +No household values are included in the resulting audit. + +policyengine-uk 2.147.2 silently skips unknown columns in both dataset loaders; +the audit must detect that loss before certification. See the ``continue`` +branches in ``build_from_dataset`` and ``build_from_multi_year_dataset``: +https://github.com/PolicyEngine/policyengine-uk/blob/fe0b99a36bd309a722b19937345e34290cb6dd73/policyengine_uk/simulation.py#L1113-L1127 +https://github.com/PolicyEngine/policyengine-uk/blob/fe0b99a36bd309a722b19937345e34290cb6dd73/policyengine_uk/simulation.py#L1186-L1196 +""" + +from __future__ import annotations + +import ast +import hashlib +import inspect +import re +import textwrap +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path +from typing import Any + +from policyengine.provenance.manifest import ( + RegionalInputArtifactEvidence, + RegionalInputSurfaceEvidence, +) + +UK_ENTITIES = ("person", "benunit", "household") + + +class RegionalInputSurfaceError(ValueError): + """A complete, byte-bound regional surface cannot be established.""" + + +def _names(values: list[str], label: str) -> set[str]: + if not values or any( + not isinstance(value, str) or not value.strip() or value != value.strip() + for value in values + ): + raise RegionalInputSurfaceError(f"{label} must contain nonblank names.") + names = set(values) + if len(names) != len(values): + raise RegionalInputSurfaceError(f"{label} contains duplicate names.") + return names + + +def compare_input_surface( + artifacts: dict[str, RegionalInputArtifactEvidence], + model_variables: list[str], + model_inputs: list[str], + input_aliases: dict[str, str] | None = None, +) -> dict[str, list[str]]: + """Recompute losses and every default, including defaults in just one year. + + Inputs supplied by the model's unconditional ``move_values`` calls use the + destination name after the move. Raw columns still have to be variables + defined by the target model. Every artifact and period must pass: the + union of supplied columns cannot hide a missing input in another period. + """ + variables = _names(model_variables, "model_variables") + inputs = _names(model_inputs, "model_inputs") + if not inputs <= variables: + raise RegionalInputSurfaceError("model_inputs must be model_variables.") + aliases = input_aliases or {} + for donor, target in aliases.items(): + if ( + donor not in variables + or target not in inputs + or donor == target + or target in aliases + ): + raise RegionalInputSurfaceError( + f"Unsupported input alias {donor!r} -> {target!r}." + ) + if not artifacts: + raise RegionalInputSurfaceError("No regional artifact columns measured.") + dropped: set[str] = set() + defaulted: set[str] = set() + for name, artifact in artifacts.items(): + columns = _names(artifact.columns, f"{name} columns") + if not artifact.columns_by_period: + raise RegionalInputSurfaceError(f"{name} has no measured periods.") + periods = [] + for period, period_columns in artifact.columns_by_period.items(): + if not re.fullmatch(r"[0-9]{4}", period): + raise RegionalInputSurfaceError( + f"{name} has an invalid period {period!r}." + ) + periods.append(_names(period_columns, f"{name}/{period} columns")) + if set().union(*periods) != columns: + raise RegionalInputSurfaceError( + f"{name} columns must equal the union of columns_by_period." + ) + for supplied in periods: + dropped.update(supplied - variables) + effective = set(supplied) + for donor, target in aliases.items(): + if donor in effective: + effective.remove(donor) + effective.add(target) + defaulted.update(inputs - effective) + return { + "dropped_columns": sorted(dropped), + "defaulted_inputs": sorted(defaulted), + } + + +def _input_aliases(simulation_class: Any) -> dict[str, str]: + """Read unconditional literal moves; refuse unsupported dynamic semantics. + + policyengine-uk has no public alias inventory. Its public Simulation + constructor declares the input moves. Reading that source avoids running + a simulation or encoding one version's aliases as a permanent allowlist. + """ + try: + tree = ast.parse(textwrap.dedent(inspect.getsource(simulation_class.__init__))) + except (OSError, TypeError, SyntaxError) as exc: + raise RegionalInputSurfaceError( + "Cannot inspect the installed UK Simulation input aliases." + ) from exc + constructor = tree.body[0] + if not isinstance(constructor, (ast.FunctionDef, ast.AsyncFunctionDef)): + raise RegionalInputSurfaceError("Cannot locate the UK Simulation constructor.") + unconditional = { + id(statement.value) + for statement in constructor.body + if isinstance(statement, ast.Expr) + } + aliases: dict[str, str] = {} + for node in ast.walk(constructor): + if not ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and isinstance(node.func.value, ast.Name) + and node.func.value.id == "self" + and node.func.attr == "move_values" + ): + continue + if ( + id(node) not in unconditional + or len(node.args) != 2 + or node.keywords + or any( + not isinstance(argument, ast.Constant) + or not isinstance(argument.value, str) + for argument in node.args + ) + ): + raise RegionalInputSurfaceError( + "UK Simulation has a conditional or dynamic move_values input alias; " + "an explicit audit of its semantics is required." + ) + donor = str(node.args[0].value) # type: ignore[attr-defined] + target = str(node.args[1].value) # type: ignore[attr-defined] + if donor in aliases: + raise RegionalInputSurfaceError(f"Duplicate move_values alias {donor!r}.") + aliases[donor] = target + return aliases + + +def _model_surface(model_version: str) -> tuple[list[str], list[str], dict[str, str]]: + try: + installed = version("policyengine-uk") + except PackageNotFoundError as exc: + raise RegionalInputSurfaceError( + "Install the exact policyengine-uk target first." + ) from exc + if installed != model_version: + raise RegionalInputSurfaceError( + f"Installed policyengine-uk {installed} differs from requested {model_version}." + ) + # Keep country-model loading out of certification's evidence-validation path. + import policyengine_uk + + system = policyengine_uk.CountryTaxBenefitSystem() + variables = sorted(system.variables) + inputs = sorted( + name + for name, variable in system.variables.items() + if variable.is_input_variable() + ) + return variables, inputs, _input_aliases(policyengine_uk.Simulation) + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as file: + for chunk in iter(lambda: file.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _hdf_columns(path: Path) -> dict[str, list[str]]: + import pandas as pd + + try: + with pd.HDFStore(path, mode="r") as store: + keys = set(store.keys()) + flat = {f"/{entity}" for entity in UK_ENTITIES} & keys + nested = { + key + for key in keys + if any(key.startswith(f"/{entity}/") for entity in UK_ENTITIES) + } + if flat and nested: + raise RegionalInputSurfaceError( + "H5 mixes single-year and multi-year entity keys." + ) + entity_keys: dict[str, dict[str, str]] = {} + if flat: + if "/time_period" not in keys: + raise RegionalInputSurfaceError( + "Single-year H5 requires /time_period." + ) + metadata = store.select("/time_period") + values = metadata.to_numpy().ravel().tolist() + if len(values) != 1 or not re.fullmatch(r"[0-9]{4}", str(values[0])): + raise RegionalInputSurfaceError( + "H5 /time_period must name one year." + ) + entity_keys[str(values[0])] = { + entity: f"/{entity}" + for entity in UK_ENTITIES + if f"/{entity}" in flat + } + else: + for key in sorted(nested): + parts = key.strip("/").split("/") + if len(parts) != 2 or not re.fullmatch(r"[0-9]{4}", parts[1]): + raise RegionalInputSurfaceError( + f"Invalid H5 entity period key {key!r}." + ) + entity_keys.setdefault(parts[1], {})[parts[0]] = key + if not entity_keys: + raise RegionalInputSurfaceError("H5 has no UK entity periods.") + columns_by_period = {} + for period, entities in sorted(entity_keys.items()): + missing = set(UK_ENTITIES) - set(entities) + if missing: + raise RegionalInputSurfaceError( + f"H5 period {period} is missing entity tables: {sorted(missing)}." + ) + columns: set[str] = set() + for entity in UK_ENTITIES: + # This obtains the same column metadata for fixed and table + # stores without selecting household or person rows. + frame = store.select(entities[entity], start=0, stop=0) + columns.update( + _names(list(frame.columns), f"{entity}/{period} columns") + ) + columns_by_period[period] = sorted(columns) + return columns_by_period + except RegionalInputSurfaceError: + raise + except (OSError, KeyError, ValueError, ImportError) as exc: + raise RegionalInputSurfaceError( + f"Cannot read H5 entity metadata from {path.name}." + ) from exc + + +def measure_regional_input_surface( + regional_manifest_uri: str, + artifact_paths: dict[str, Path], + model_version: str, + audited_by: str, + limitations: list[str], + token: str | None = None, +) -> RegionalInputSurfaceEvidence: + """Create an audit from actual H5 columns and an exact installed UK model. + + The operator supplies local files named by the immutable regional release. + Every microdata H5 must be measured and match its manifest's SHA-256 both + before and after metadata inspection. This reads no unit values for output. + """ + # Imported lazily: the certification gate reuses compare_input_surface. + from policyengine.provenance.certification import fetch_release_manifest + + if ( + not audited_by.strip() + or not limitations + or any(not item.strip() for item in limitations) + ): + raise RegionalInputSurfaceError( + "An auditor and nonblank limitations are required." + ) + manifest, manifest_sha256, parts = fetch_release_manifest( + regional_manifest_uri, token=token + ) + expected = { + name: artifact + for name, artifact in manifest.artifacts.items() + if artifact.kind == "microdata" and Path(artifact.path).suffix.lower() == ".h5" + } + if not expected or set(artifact_paths) != set(expected): + raise RegionalInputSurfaceError( + "Supplied artifact paths must exactly cover the regional microdata H5 artifacts: " + f"{sorted(expected)}." + ) + variables, inputs, aliases = _model_surface(model_version) + artifacts = {} + for name, artifact in sorted(expected.items()): + path = Path(artifact_paths[name]) + if not artifact.sha256: + raise RegionalInputSurfaceError( + f"Artifact {name!r} has no manifest sha256." + ) + before = _sha256_file(path) + if before != artifact.sha256: + raise RegionalInputSurfaceError( + f"Artifact {name!r} bytes differ from its manifest sha256." + ) + periods = _hdf_columns(path) + after = _sha256_file(path) + if after != before: + raise RegionalInputSurfaceError( + f"Artifact {name!r} bytes changed during metadata inspection." + ) + artifacts[name] = RegionalInputArtifactEvidence( + repo_id=artifact.repo_id, + repo_type=artifact.repo_type or parts["repo_type"], + path=artifact.path, + revision=artifact.revision, + sha256=after, + columns=sorted( + {column for columns in periods.values() for column in columns} + ), + columns_by_period=periods, + ) + comparison = compare_input_surface(artifacts, variables, inputs, aliases) + if comparison["dropped_columns"]: + raise RegionalInputSurfaceError( + f"Dropped regional input columns refuse acceptance: {comparison['dropped_columns']}." + ) + return RegionalInputSurfaceEvidence( + regional_manifest_uri=regional_manifest_uri, + regional_manifest_sha256=manifest_sha256, + model_package="policyengine-uk", + model_version=model_version, + artifacts=artifacts, + model_variables=variables, + model_inputs=inputs, + input_aliases=aliases, + dropped_columns=comparison["dropped_columns"], + defaulted_inputs=comparison["defaulted_inputs"], + audited_by=audited_by, + limitations=limitations, + ) diff --git a/src/policyengine/provenance/trace.py b/src/policyengine/provenance/trace.py index bcffda78a..0e79b34e8 100644 --- a/src/policyengine/provenance/trace.py +++ b/src/policyengine/provenance/trace.py @@ -556,6 +556,15 @@ def _build_bundle_performance( performance["pe:dataBuildId"] = certification.data_build_id if certification.certified_by is not None: performance["pe:certifiedBy"] = certification.certified_by + if country_manifest.regional_certification is not None: + # An explicit JSON literal preserves evidence member names through + # JSON-LD expansion; they are JSON contract keys, not RDF predicates. + performance["pe:regionalCertification"] = { + "@type": "@json", + "@value": country_manifest.regional_certification.model_dump( + mode="json", exclude_none=True + ), + } performance.update(dict(emission_context or _emission_context())) return performance diff --git a/tests/fixtures/local_manifest/main_485dc2330_certification.py.txt b/tests/fixtures/local_manifest/main_485dc2330_certification.py.txt new file mode 100644 index 000000000..b439117fb --- /dev/null +++ b/tests/fixtures/local_manifest/main_485dc2330_certification.py.txt @@ -0,0 +1,167 @@ +# Unmodified certification function from policyengine.py main 485dc2330. +def certify_data_release_compatibility( + country_id: str, + runtime_model_version: str, + runtime_data_build_fingerprint: Optional[str] = None, + runtime_core_version: Optional[str] = None, +) -> DataCertification: + country_manifest = get_release_manifest(country_id) + try: + data_release_manifest = get_data_release_manifest(country_id) + except DataReleaseManifestUnavailableError as exc: + bundled_certification = country_manifest.certification + if bundled_certification is not None: + if ( + runtime_data_build_fingerprint is not None + and bundled_certification.data_build_fingerprint is not None + and bundled_certification.compatibility_basis + not in CLAIM_BASED_COMPATIBILITY + and runtime_data_build_fingerprint + != bundled_certification.data_build_fingerprint + ): + raise ValueError( + "Runtime data build fingerprint does not match the bundled " + "data certification." + ) + if ( + bundled_certification.certified_for_model_version + == runtime_model_version + and _runtime_core_is_certified( + bundled_certification, runtime_core_version + ) + ): + return bundled_certification + return DataCertification( + compatibility_basis="unverified_data_release_manifest_unavailable", + certified_for_model_version=runtime_model_version, + data_build_id=bundled_certification.data_build_id, + built_with_model_version=( + bundled_certification.built_with_model_version + ), + built_with_model_git_sha=( + bundled_certification.built_with_model_git_sha + ), + data_build_fingerprint=(bundled_certification.data_build_fingerprint), + certified_by=bundled_certification.certified_by, + ) + raise exc + + # A remote build-model, fingerprint or publisher claim cannot broaden an + # exact-pair certificate beyond the model, core and recorded data build + # that qualification tested. Check them before any acceptance path returns. + bundled_certification = country_manifest.certification + # Named as certification names it: the cut tag when the release has one. + remote_build_id = release_build_id(data_release_manifest.build) + if ( + bundled_certification is not None + and bundled_certification.compatibility_basis == "qualified_exact_pair" + ): + if bundled_certification.certified_for_model_version != runtime_model_version: + raise ValueError( + "Data release manifest is not certified for the runtime model " + f"version {runtime_model_version} in country '{country_id}'; " + "the exact pair requires " + f"{bundled_certification.certified_for_model_version}." + ) + if not _runtime_core_is_certified(bundled_certification, runtime_core_version): + raise ValueError( + "Data release manifest is not certified for the runtime core " + f"version {runtime_core_version} in country '{country_id}'; " + "the exact pair requires " + f"{bundled_certification.certified_for_core_version}." + ) + if ( + bundled_certification.data_build_id is not None + and bundled_certification.data_build_id != remote_build_id + ): + raise ValueError( + "Data release manifest is not certified for the remote data " + f"build {remote_build_id} in country '{country_id}'; the exact " + f"pair requires {bundled_certification.data_build_id}." + ) + built_with_model = ( + data_release_manifest.build.built_with_model_package + if data_release_manifest.build is not None + else None + ) + + if ( + built_with_model is not None + and built_with_model.name != country_manifest.model_package.name + ): + raise ValueError( + "Data release manifest was built with a different model package: " + f"expected {country_manifest.model_package.name}, " + f"got {built_with_model.name}." + ) + + if ( + built_with_model is not None + and built_with_model.version == runtime_model_version + ): + return DataCertification( + compatibility_basis="exact_build_model_version", + certified_for_model_version=runtime_model_version, + data_build_id=release_build_id(data_release_manifest.build), + built_with_model_version=built_with_model.version, + built_with_model_git_sha=built_with_model.git_sha, + data_build_fingerprint=built_with_model.data_build_fingerprint, + ) + + if ( + built_with_model is not None + and built_with_model.data_build_fingerprint is not None + and runtime_data_build_fingerprint is not None + and built_with_model.data_build_fingerprint == runtime_data_build_fingerprint + ): + return DataCertification( + compatibility_basis="matching_data_build_fingerprint", + certified_for_model_version=runtime_model_version, + data_build_id=release_build_id(data_release_manifest.build), + built_with_model_version=built_with_model.version, + built_with_model_git_sha=built_with_model.git_sha, + data_build_fingerprint=built_with_model.data_build_fingerprint, + ) + + for compatible_model_package in data_release_manifest.compatible_model_packages: + if compatible_model_package.name != country_manifest.model_package.name: + continue + if _specifier_matches( + version=runtime_model_version, + specifier=compatible_model_package.specifier, + ): + return DataCertification( + compatibility_basis="legacy_compatible_model_package", + certified_for_model_version=runtime_model_version, + data_build_id=release_build_id(data_release_manifest.build), + built_with_model_version=( + built_with_model.version if built_with_model is not None else None + ), + built_with_model_git_sha=( + built_with_model.git_sha if built_with_model is not None else None + ), + data_build_fingerprint=( + built_with_model.data_build_fingerprint + if built_with_model is not None + else None + ), + ) + + # An automated qualification of exactly this model and data build: the + # bundled certificate names the pair, and the ledger evidence for it is + # enforced when the bundle is built (tests/test_certification_evidence.py). + bundled_certification = country_manifest.certification + if ( + bundled_certification is not None + and bundled_certification.compatibility_basis == "qualified_exact_pair" + and bundled_certification.certified_for_model_version == runtime_model_version + and bundled_certification.qualification_pair_id is not None + and bundled_certification.data_build_id == remote_build_id + and _runtime_core_is_certified(bundled_certification, runtime_core_version) + ): + return bundled_certification + + raise ValueError( + "Data release manifest is not certified for the runtime model version " + f"{runtime_model_version} in country '{country_id}'." + ) diff --git a/tests/test_certify_data_release.py b/tests/test_certify_data_release.py index 092420f61..2b9e00975 100644 --- a/tests/test_certify_data_release.py +++ b/tests/test_certify_data_release.py @@ -1412,6 +1412,222 @@ def test__given_regional_release_without_a_claim__then_refuses(self, tmp_path): regional_release_kind="area_coded_national", ) + def _older_dense(self) -> dict: + dense = _microcosm_uk_dense_payload() + dense["build"]["built_with_model_package"]["version"] = "2.120.0" + dense["compatible_model_packages"] = [ + {"name": "policyengine-uk", "specifier": "==2.120.0"} + ] + return dense + + def test__given_older_dense_cut__then_refuses_without_an_accepted_mismatch( + self, tmp_path + ): + with pytest.raises(CertificationError, match="Regional release .*2.126.12"): + _certify_uk( + _tmp_bundle(tmp_path), + [ + _response(_microcosm_uk_national_payload()), + _response(self._older_dense()), + ], + regional_manifest_uri=UK_DENSE_URI, + ) + + def test__given_accepted_mismatch__then_routes_and_records_it(self, tmp_path): + bundle_path = _tmp_bundle(tmp_path) + reason = "area receipts are uncertified; the dense cut lags one release" + from tests.test_regional_input_evidence import input_evidence + + evidence = input_evidence(self._older_dense()) + + result = _certify_uk( + bundle_path, + [ + _response(_microcosm_uk_national_payload()), + _response(self._older_dense()), + ], + regional_manifest_uri=UK_DENSE_URI, + regional_model_mismatch_reason=reason, + regional_input_surface_evidence=evidence, + ) + + release = json.loads(bundle_path.read_text())["data_releases"]["uk"] + assert release["regional_release_kind"] == "local_area" + assert release["regional_certification"] == { + "compatibility_basis": "accepted_model_mismatch", + "certified_for_model_version": UK_MODEL_VERSION, + "built_with_model_version": "2.120.0", + "accepted_reason": reason, + "area_receipts": "uncertified", + "regional_manifest_uri": UK_DENSE_URI, + "regional_manifest_sha256": evidence["regional_manifest_sha256"], + "defaulted_inputs": evidence["defaulted_inputs"], + "dropped_columns": [], + "limitations": evidence["limitations"], + "evidence": evidence, + } + # The national certificate itself is untouched by the regional record. + assert release["certification"]["compatibility_basis"] == ( + "built_with_model_package" + ) + assert _sim_api_routes_both_area_types(release) + manifest = CountryReleaseManifest.model_validate(release) + assert manifest.regional_certification is not None + assert manifest.regional_certification.built_with_model_version == "2.120.0" + mismatch = [w for w in result.warnings if "accepted mismatch" in w] + assert len(mismatch) == 1 and "2.120.0" in mismatch[0] + + def test__given_a_covering_regional_release__then_an_accepted_mismatch_refuses( + self, tmp_path + ): + with pytest.raises(CertificationError, match="drop --accept-regional"): + _certify_uk( + _tmp_bundle(tmp_path), + [ + _response(_microcosm_uk_national_payload()), + _response(_microcosm_uk_dense_payload()), + ], + regional_manifest_uri=UK_DENSE_URI, + regional_model_mismatch_reason="not needed", + ) + + @pytest.mark.parametrize("reason", ["", " "]) + def test__given_an_empty_reason__then_refuses(self, tmp_path, reason): + with pytest.raises(CertificationError, match="needs a reason"): + _certify_uk( + _tmp_bundle(tmp_path), + [ + _response(_microcosm_uk_national_payload()), + _response(self._older_dense()), + ], + regional_manifest_uri=UK_DENSE_URI, + regional_model_mismatch_reason=reason, + ) + + def test__given_an_accepted_mismatch__then_the_core_must_still_be_covered( + self, tmp_path + ): + from tests.test_regional_input_evidence import input_evidence + + national = _microcosm_uk_national_payload() + national["compatible_core_packages"] = [ + {"name": "policyengine-core", "specifier": "==3.33.0"} + ] + with pytest.raises( + CertificationError, match="Regional release .*policyengine-core 3.33.0" + ): + _certify_uk( + _tmp_bundle(tmp_path), + [_response(national), _response(self._older_dense())], + regional_manifest_uri=UK_DENSE_URI, + regional_model_mismatch_reason="area receipts are uncertified", + regional_input_surface_evidence=input_evidence(self._older_dense()), + core_version="3.33.0", + ) + + def test__given_an_accepted_mismatch_without_a_regional_release__then_refuses( + self, tmp_path + ): + with pytest.raises(CertificationError, match="pass --regional-manifest-uri"): + _certify_uk( + _tmp_bundle(tmp_path), + [_response(_microcosm_uk_national_payload())], + drop_regional_release=True, + regional_model_mismatch_reason="area receipts are uncertified", + ) + + def test__given_bundle_wrapper_accepted_mismatch__then_forwards_it( + self, monkeypatch + ): + bundle_script = _load_bundle_script(monkeypatch) + captured: dict[str, list[str]] = {} + fake = ModuleType("certify_data_release") + + def fake_main(argv): + captured["argv"] = argv + return 0 + + fake.main = fake_main + monkeypatch.setitem(sys.modules, "certify_data_release", fake) + reason = "area receipts are uncertified" + assert ( + bundle_script.main( + [ + "certify-data", + "--country", + "uk", + "--manifest-uri", + UK_NATIONAL_URI, + "--regional-manifest-uri", + UK_DENSE_URI, + "--accept-regional-model-mismatch", + reason, + "--regional-input-surface-evidence", + "regional-audit.json", + ] + ) + == 0 + ) + argv = captured["argv"] + assert argv[argv.index("--accept-regional-model-mismatch") + 1] == reason + assert ( + argv[argv.index("--regional-input-surface-evidence") + 1] + == "regional-audit.json" + ) + + def test__given_bundle_wrapper_empty_reason__then_certifier_refuses_it( + self, monkeypatch, tmp_path + ): + bundle_script = _load_bundle_script(monkeypatch) + spec = importlib.util.spec_from_file_location( + "certify_data_release", + Path(__file__).resolve().parents[1] / "scripts" / "certify_data_release.py", + ) + certifier = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(certifier) + monkeypatch.setitem(sys.modules, "certify_data_release", certifier) + bundle_path = _tmp_bundle(tmp_path) + original_bundle = bundle_path.read_bytes() + captured = {} + + def certify_temporary_bundle(**kwargs): + captured.update(kwargs) + kwargs["bundle_path"] = bundle_path + return certify_data_release(**kwargs) + + monkeypatch.setattr(certifier, "certify_data_release", certify_temporary_bundle) + with ( + patch( + "policyengine.provenance.certification.requests.get", + side_effect=[ + _response(_microcosm_uk_national_payload()), + _response(self._older_dense()), + ], + ), + pytest.raises(CertificationError, match="needs a reason"), + ): + bundle_script.main( + [ + "certify-data", + "--country", + "uk", + "--manifest-uri", + UK_NATIONAL_URI, + "--model-version", + UK_MODEL_VERSION, + "--regional-manifest-uri", + UK_DENSE_URI, + "--accept-regional-model-mismatch", + "", + "--no-generate", + "--no-changelog", + "--skip-artifact-check", + ] + ) + assert captured["regional_model_mismatch_reason"] == "" + assert bundle_path.read_bytes() == original_bundle + def test__given_pinned_core__then_the_regional_release_must_cover_it( self, tmp_path ): diff --git a/tests/test_execution_receipt.py b/tests/test_execution_receipt.py index 66fe4d220..6589bff47 100644 --- a/tests/test_execution_receipt.py +++ b/tests/test_execution_receipt.py @@ -433,6 +433,206 @@ def test_a_regional_release_dataset_keeps_its_own_identity(dataset_file): assert receipt.data.release_manifest_revision is None +def _bundle_with_regional_acceptance(dataset_file): + """A synthetic, audited area exception beside a national certificate.""" + + bundle = _bundle_for(dataset_file.read_bytes()) + record = bundle["data_releases"]["uk"] + artifact = { + "path": "area_file.h5", + "repo_id": record["data_package"]["repo_id"], + "revision": "regional-release-cut", + "sha256": hashlib.sha256(dataset_file.read_bytes()).hexdigest(), + } + record["datasets"]["area_file"] = artifact + manifest_uri = ( + "hf://dataset/policyengine/populace-uk-private@regional-release-cut/" + "releases/dense/release_manifest.json" + ) + defaulted = [ + "local_bus_uncapped_single_fare", + "salary_sacrifice_returned_to_income", + ] + limitations = [ + "The dense cut's VAT predates the policyengine-uk VAT factor fix " + "(#2187, 2.135.7); the dashboard holds a VAT limit." + ] + record["regional_certification"] = { + "compatibility_basis": "accepted_model_mismatch", + "certified_for_model_version": INSTALLED_UK, + "built_with_model_version": "2.132.6", + "accepted_reason": "UK hub accepted this measured input surface.", + "area_receipts": "uncertified", + "regional_manifest_uri": manifest_uri, + "regional_manifest_sha256": "b" * 64, + "defaulted_inputs": defaulted, + "dropped_columns": [], + "limitations": limitations, + "evidence": { + "model_package": "policyengine-uk", + "model_version": INSTALLED_UK, + "regional_manifest_uri": manifest_uri, + "regional_manifest_sha256": "b" * 64, + "artifacts": { + "area_file": { + **artifact, + "columns": ["employment_income", "household_weight"], + "columns_by_period": { + "2025": ["employment_income", "household_weight"] + }, + } + }, + "model_variables": sorted( + ["employment_income", "household_weight", *defaulted] + ), + "model_inputs": sorted( + ["employment_income", "household_weight", *defaulted] + ), + "dropped_columns": [], + "defaulted_inputs": defaulted, + "audited_by": "UK hub review", + "limitations": limitations, + }, + } + return bundle + + +@pytest.mark.parametrize("area", [False, True], ids=["national", "area"]) +def test_receipt_exposes_regional_acceptance_without_widening_certification( + dataset_file, schema_validator, area +): + from policyengine.provenance.manifest import RegionalCertification + + bundle = _bundle_with_regional_acceptance(dataset_file) + record = bundle["data_releases"]["uk"] + + receipt = _build(dataset_file, bundle, dataset="area_file" if area else None) + + payload = json.loads(receipt.model_dump_json(exclude_none=True)) + assert payload["regional_certification"] == RegionalCertification.model_validate( + record["regional_certification"] + ).model_dump(mode="json", exclude_none=True) + assert receipt.certification_status == ("uncertified" if area else "certified") + assert (receipt.certificate is None) is area + assert receipt.data.build_model_version == ( + "2.132.6" if area else record["certification"]["built_with_model_version"] + ) + schema_validator.validate(payload) + restored = ExecutionReceipt.model_validate_json(receipt.model_dump_json()) + assert restored.regional_certification == receipt.regional_certification + assert restored.receipt_sha256 == receipt.receipt_sha256 + + +def test_regional_limitations_are_bound_to_receipt_identity(dataset_file): + bundle = _bundle_with_regional_acceptance(dataset_file) + receipt = _build(dataset_file, bundle, dataset="area_file") + changed_bundle = copy.deepcopy(bundle) + record = changed_bundle["data_releases"]["uk"]["regional_certification"] + record["limitations"].append("Additional accepted area limitation.") + record["evidence"]["limitations"].append("Additional accepted area limitation.") + + changed = _build(dataset_file, changed_bundle, dataset="area_file") + + assert changed.execution_identity_sha256 != receipt.execution_identity_sha256 + assert changed.receipt_sha256 != receipt.receipt_sha256 + + +def test_certificate_verification_checks_the_exposed_regional_record(dataset_file): + bundle = _bundle_with_regional_acceptance(dataset_file) + receipt = _build(dataset_file, bundle) + payload = receipt.model_dump(mode="json") + payload["regional_certification"]["accepted_reason"] = "A different acceptance." + changed = ExecutionReceipt.model_validate(payload) + + assert any( + "regional_certification" in issue + for issue in verify_certificate(changed, bundle) + ) + + +def test_trace_exposes_regional_acceptance_separately_from_national(dataset_file): + from policyengine.provenance.manifest import ( + CountryReleaseManifest, + RegionalCertification, + ) + from policyengine.provenance.trace import build_trace_tro_from_release_bundle + + record = _bundle_with_regional_acceptance(dataset_file)["data_releases"]["uk"] + tro = build_trace_tro_from_release_bundle( + CountryReleaseManifest.model_validate(record), + None, + fetch_pypi=lambda name, version: {}, + ) + performance = tro["@graph"][0]["trov:hasPerformance"] + + assert performance["pe:regionalCertification"]["@type"] == "@json" + assert performance["pe:regionalCertification"][ + "@value" + ] == RegionalCertification.model_validate( + record["regional_certification"] + ).model_dump(mode="json", exclude_none=True) + assert ( + performance["pe:compatibilityBasis"] + == record["certification"]["compatibility_basis"] + ) + assert ( + performance["pe:regionalCertification"]["@value"]["area_receipts"] + == "uncertified" + ) + + +def test_trace_regional_acceptance_survives_jsonld_expansion(dataset_file): + rdflib = pytest.importorskip("rdflib") + from policyengine.provenance.manifest import CountryReleaseManifest + from policyengine.provenance.trace import ( + POLICYENGINE_TRACE_NAMESPACE, + build_trace_tro_from_release_bundle, + ) + + record = _bundle_with_regional_acceptance(dataset_file)["data_releases"]["uk"] + manifest = CountryReleaseManifest.model_validate(record) + tro = build_trace_tro_from_release_bundle( + manifest, + None, + fetch_pypi=lambda name, version: {}, + ) + + graph = rdflib.Graph().parse(data=json.dumps(tro), format="json-ld") + values = list( + graph.objects( + None, rdflib.URIRef(POLICYENGINE_TRACE_NAMESPACE + "regionalCertification") + ) + ) + + assert len(values) == 1 + assert isinstance(values[0], rdflib.Literal) + assert values[0].datatype == rdflib.RDF.JSON + assert json.loads(str(values[0])) == manifest.regional_certification.model_dump( + mode="json", exclude_none=True + ) + + +def test_removing_regional_acceptance_removes_downstream_record(dataset_file): + from policyengine.provenance.manifest import CountryReleaseManifest + from policyengine.provenance.trace import build_trace_tro_from_release_bundle + + bundle = _bundle_with_regional_acceptance(dataset_file) + record = bundle["data_releases"]["uk"] + del record["regional_certification"] + del record["datasets"]["area_file"] + + receipt = _build(dataset_file, bundle) + tro = build_trace_tro_from_release_bundle( + CountryReleaseManifest.model_validate(record), + None, + fetch_pypi=lambda name, version: {}, + ) + + assert "regional_certification" not in receipt.canonical_payload() + assert "pe:regionalCertification" not in tro["@graph"][0]["trov:hasPerformance"] + assert receipt.certification_status == "certified" + + def test_an_omitted_artifact_revision_is_the_release_manifest_revision(dataset_file): """Round-4 review M5: the receipt names what the materializer downloads.""" diff --git a/tests/test_local_data_manifest.py b/tests/test_local_data_manifest.py new file mode 100644 index 000000000..b63cbdf14 --- /dev/null +++ b/tests/test_local_data_manifest.py @@ -0,0 +1,793 @@ +"""The local manifest is an opt-in transport for the bundle's exact bytes. + +The unset cases characterize certification on main (485dc2330), including +fetched, bundled-offline, and refused pairs. No credentials or network are used. +""" + +from __future__ import annotations + +import copy +import hashlib +import json +from pathlib import Path +from unittest.mock import Mock, patch + +import pytest +import requests +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +from policyengine.provenance import manifest as manifests + +ENVIRONMENT_VARIABLE = "POLICYENGINE_UK_DATA_MANIFEST_FILE" +MODEL = "2.130.1" +OTHER_MODEL = "2.130.2" +CORE = "3.33.0" +BUILD = "test-data-build" +FINGERPRINT = "sha256:test-build" +_DEFAULT = object() + + +def _payload(): + return { + "schema_version": 1, + "data_package": {"name": "policyengine-uk-data", "version": "test-release"}, + "build": { + "build_id": BUILD, + "built_with_core_package": {"name": "policyengine-core", "version": CORE}, + "built_with_model_package": { + "name": "policyengine-uk", + "version": MODEL, + "git_sha": "abc123", + "data_build_fingerprint": FINGERPRINT, + }, + }, + "default_datasets": {"national": "test_dataset"}, + "artifacts": { + "test_dataset": { + "kind": "microdata", + "path": "test_dataset.h5", + "repo_id": "policyengine/test-data", + "revision": "test-release", + "sha256": "d" * 64, + } + }, + } + + +def _manifest_bytes(payload): + # Whitespace and the final newline are deliberately part of the pin. + return (json.dumps(payload, indent=2) + "\n").encode("utf-8") + + +def _qualified_certification(): + return { + "compatibility_basis": "qualified_exact_pair", + "certified_for_model_version": MODEL, + "certified_for_core_version": CORE, + "data_build_id": BUILD, + "built_with_model_version": MODEL, + "built_with_model_git_sha": "abc123", + "data_build_fingerprint": FINGERPRINT, + "qualification_pair_id": "a" * 64, + "certified_by": "test-bundle", + } + + +def _install_bundle(tmp_path, monkeypatch, payload=None, certification=_DEFAULT): + """Install synthetic package resources, exercising the real TRO pin reader.""" + manifests.get_data_release_manifest.cache_clear() + manifests.get_release_manifest.cache_clear() + payload = payload if payload is not None else _payload() + raw_bytes = _manifest_bytes(payload) + local_path = tmp_path / "release_manifest.json" + local_path.write_bytes(raw_bytes) + resource_root = tmp_path / "package" + bundle_dir = resource_root / "data" / "bundle" + bundle_dir.mkdir(parents=True, exist_ok=True) + country = { + "schema_version": 1, + "country_id": "uk", + "policyengine_version": "6.10.1", + "model_package": {"name": "policyengine-uk", "version": MODEL}, + "data_package": { + "name": "policyengine-uk-data", + "version": "test-release", + "repo_id": "policyengine/test-data", + "release_manifest_revision": "f" * 40, + }, + "default_dataset": "test_dataset", + "certified_data_artifact": { + "dataset": "test_dataset", + "uri": f"hf://policyengine/test-data/test_dataset.h5@{'f' * 40}", + "sha256": "d" * 64, + }, + "datasets": { + "test_dataset": { + "path": "test_dataset.h5", + "revision": "f" * 40, + "sha256": "d" * 64, + } + }, + "certification": ( + _qualified_certification() if certification is _DEFAULT else certification + ), + } + (bundle_dir / "manifest.json").write_text( + json.dumps({"data_releases": {"uk": country}}) + ) + tro = { + "@graph": [ + { + "@id": "tro", + "trov:hasArrangement": [ + { + "trov:hasArtifactLocation": [ + { + "trov:hasArtifact": { + "@id": "composition/1/artifact/data_release_manifest" + }, + "trov:hasLocation": ( + "https://huggingface.co/policyengine/test-data/resolve/" + f"{'f' * 40}/release_manifest.json" + ), + } + ] + } + ], + "trov:hasComposition": { + "@id": "composition/1", + "trov:hasArtifact": [ + { + "@id": "composition/1/artifact/data_release_manifest", + "trov:sha256": hashlib.sha256(raw_bytes).hexdigest(), + } + ], + }, + } + ] + } + trace_path = bundle_dir / "uk.trace.tro.jsonld" + trace_path.write_text(json.dumps(tro)) + monkeypatch.setattr(manifests, "files", lambda package: resource_root) + return local_path, trace_path, raw_bytes + + +@pytest.fixture(autouse=True) +def _isolated_manifest_settings(monkeypatch): + monkeypatch.delenv(ENVIRONMENT_VARIABLE, raising=False) + monkeypatch.delenv("POLICYENGINE_US_DATA_MANIFEST_FILE", raising=False) + manifests.get_data_release_manifest.cache_clear() + manifests.get_release_manifest.cache_clear() + yield + manifests.get_data_release_manifest.cache_clear() + manifests.get_release_manifest.cache_clear() + + +def _certify(**kwargs): + implementation = kwargs.pop( + "_implementation", manifests.certify_data_release_compatibility + ) + return implementation( + "uk", + runtime_model_version=kwargs.pop("runtime_model_version", MODEL), + runtime_core_version=kwargs.pop("runtime_core_version", CORE), + **kwargs, + ) + + +def test_matching_local_manifest_certifies_without_a_network_call( + tmp_path, monkeypatch +): + local_path, _, raw_bytes = _install_bundle(tmp_path, monkeypatch) + monkeypatch.setenv(ENVIRONMENT_VARIABLE, str(local_path)) + + with patch.object( + manifests.requests, "get", side_effect=requests.ConnectionError("HF is down") + ) as network: + certification = _certify() + loaded = manifests.get_data_release_manifest("uk") + + network.assert_not_called() + assert ( + certification.compatibility_basis == "exact_build_model_version_local_manifest" + ) + assert certification.certified_for_model_version == MODEL + assert certification.data_build_id == BUILD + assert certification.data_build_fingerprint == FINGERPRINT + assert loaded.source_sha256 == hashlib.sha256(raw_bytes).hexdigest() + assert loaded.artifacts["test_dataset"].revision == "f" * 40 + # Source information stays out of public manifest JSON round trips. + expected = manifests.DataReleaseManifest.model_validate_json(raw_bytes) + expected.artifacts["test_dataset"].revision = "f" * 40 + assert loaded.model_dump(mode="json") == expected.model_dump(mode="json") + + +@settings( + max_examples=80, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@given( + position=st.integers(min_value=0, max_value=len(_manifest_bytes(_payload())) - 1) +) +def test_any_one_byte_mutation_is_refused(tmp_path, monkeypatch, position): + local_path, _, raw_bytes = _install_bundle(tmp_path, monkeypatch) + altered = bytearray(raw_bytes) + altered[position] ^= 1 + local_path.write_bytes(altered) + monkeypatch.setenv(ENVIRONMENT_VARIABLE, str(local_path)) + + with patch.object(manifests.requests, "get") as network: + with pytest.raises(ValueError) as error: + _certify() + + network.assert_not_called() + assert hashlib.sha256(raw_bytes).hexdigest() in str(error.value) + assert hashlib.sha256(altered).hexdigest() in str(error.value) + + +def test_another_release_is_refused_even_with_an_environment_digest( + tmp_path, monkeypatch +): + local_path, _, raw_bytes = _install_bundle(tmp_path, monkeypatch) + another_release = _payload() + another_release["build"]["build_id"] = "another-release" + altered_bytes = _manifest_bytes(another_release) + local_path.write_bytes(altered_bytes) + monkeypatch.setenv(ENVIRONMENT_VARIABLE, str(local_path)) + monkeypatch.setenv( + "POLICYENGINE_UK_DATA_MANIFEST_SHA256", + hashlib.sha256(altered_bytes).hexdigest(), + ) + + with pytest.raises(ValueError) as error: + _certify() + + assert hashlib.sha256(raw_bytes).hexdigest() in str(error.value) + assert hashlib.sha256(altered_bytes).hexdigest() in str(error.value) + + +@pytest.mark.parametrize("failure", ["missing", "unreadable", "directory", "empty"]) +def test_bad_local_path_is_a_hard_error(tmp_path, monkeypatch, failure): + local_path, _, raw_bytes = _install_bundle(tmp_path, monkeypatch) + setting = str(local_path) + if failure == "missing": + local_path.unlink() + elif failure == "unreadable": + read_bytes = Path.read_bytes + + def read_unless_local(path): + if path == local_path: + raise PermissionError("test permission denied") + return read_bytes(path) + + monkeypatch.setattr(Path, "read_bytes", read_unless_local) + elif failure == "directory": + setting = str(tmp_path) + else: + setting = "" + monkeypatch.setenv(ENVIRONMENT_VARIABLE, setting) + + with patch.object(manifests.requests, "get") as network: + with pytest.raises(ValueError) as error: + _certify() + + network.assert_not_called() + assert ENVIRONMENT_VARIABLE in str(error.value) + assert hashlib.sha256(raw_bytes).hexdigest() in str(error.value) + assert "actual" in str(error.value).lower() + + +@pytest.mark.parametrize( + "failure", + [ + "absent-trace", + "absent-pin", + "invalid-pin", + "other-artifact", + "absent-location", + "stale-release-location", + "duplicate-pin", + "duplicate-location", + "malformed-graph", + "malformed-composition", + ], +) +def test_local_manifest_requires_the_bundled_manifest_artifact_pin( + tmp_path, monkeypatch, failure +): + local_path, trace_path, _ = _install_bundle(tmp_path, monkeypatch) + if failure == "absent-trace": + trace_path.unlink() + else: + tro = json.loads(trace_path.read_text()) + artifact = tro["@graph"][0]["trov:hasComposition"]["trov:hasArtifact"][0] + if failure == "absent-pin": + del artifact["trov:sha256"] + elif failure == "invalid-pin": + artifact["trov:sha256"] = "not-a-sha256" + elif failure == "other-artifact": + artifact["@id"] = "composition/1/artifact/dataset" + elif failure == "absent-location": + del tro["@graph"][0]["trov:hasArrangement"] + elif failure == "stale-release-location": + location = tro["@graph"][0]["trov:hasArrangement"][0][ + "trov:hasArtifactLocation" + ][0] + location["trov:hasLocation"] = ( + "https://huggingface.co/policyengine/test-data/resolve/" + "old-release/release_manifest.json" + ) + elif failure == "duplicate-pin": + tro["@graph"][0]["trov:hasComposition"]["trov:hasArtifact"].append( + copy.deepcopy(artifact) + ) + elif failure == "duplicate-location": + locations = tro["@graph"][0]["trov:hasArrangement"][0][ + "trov:hasArtifactLocation" + ] + locations.append(copy.deepcopy(locations[0])) + elif failure == "malformed-graph": + tro["@graph"] = None + else: + tro["@graph"][0]["trov:hasComposition"] = [] + trace_path.write_text(json.dumps(tro)) + monkeypatch.setenv(ENVIRONMENT_VARIABLE, str(local_path)) + + with patch.object(manifests.requests, "get") as network: + with pytest.raises(ValueError, match="(?i)(bundle|pin|digest|sha256)"): + _certify() + + network.assert_not_called() + + +@pytest.mark.parametrize("difference", ["model", "core", "build", "model-package"]) +def test_a_valid_digest_cannot_widen_exact_pair_certification( + tmp_path, monkeypatch, difference +): + payload = _payload() + runtime = {} + if difference == "model": + runtime["runtime_model_version"] = OTHER_MODEL + elif difference == "core": + runtime["runtime_core_version"] = "99.0.0" + elif difference == "build": + payload["build"]["build_id"] = "another-build" + else: + payload["build"]["built_with_model_package"]["name"] = "another-model" + local_path, _, _ = _install_bundle(tmp_path, monkeypatch, payload) + monkeypatch.setenv(ENVIRONMENT_VARIABLE, str(local_path)) + + with pytest.raises(ValueError, match="not certified|different model package"): + _certify(**runtime) + + +@pytest.mark.parametrize( + ("mode", "expected_basis"), + [ + ("fingerprint", "matching_data_build_fingerprint_local_manifest"), + ("publisher-claim", "legacy_compatible_model_package_local_manifest"), + ("qualified", "qualified_exact_pair_local_manifest"), + ], +) +def test_local_transport_retains_existing_certification_bases( + tmp_path, monkeypatch, mode, expected_basis +): + payload = _payload() + payload["build"]["built_with_model_package"]["version"] = OTHER_MODEL + runtime = {} + if mode == "fingerprint": + runtime["runtime_data_build_fingerprint"] = FINGERPRINT + elif mode == "publisher-claim": + payload["compatible_model_packages"] = [ + {"name": "policyengine-uk", "specifier": f"=={MODEL}"} + ] + local_path, _, _ = _install_bundle(tmp_path, monkeypatch, payload) + monkeypatch.setenv(ENVIRONMENT_VARIABLE, str(local_path)) + + assert _certify(**runtime).compatibility_basis == expected_basis + + +def test_local_fingerprint_mismatch_does_not_create_a_new_acceptance_path( + tmp_path, monkeypatch +): + payload = _payload() + payload["build"]["built_with_model_package"]["version"] = OTHER_MODEL + local_path, _, _ = _install_bundle( + tmp_path, monkeypatch, payload, certification=None + ) + monkeypatch.setenv(ENVIRONMENT_VARIABLE, str(local_path)) + + with pytest.raises(ValueError, match="not certified"): + _certify(runtime_data_build_fingerprint="sha256:different") + + +def _response(raw_bytes): + response = Mock() + response.status_code = 200 + response.content = raw_bytes + response.text = raw_bytes.decode("utf-8") + return response + + +def _edit_bundle(trace_path, edit): + bundle_path = trace_path.with_name("manifest.json") + bundle = json.loads(bundle_path.read_text()) + edit(bundle["data_releases"]["uk"]) + bundle_path.write_text(json.dumps(bundle)) + manifests.get_release_manifest.cache_clear() + + +@pytest.mark.parametrize("transport", ["local", "fetched"]) +@pytest.mark.parametrize( + ("field", "value"), + [ + ("repo_id", "adversary/elsewhere"), + ("path", "elsewhere.h5"), + ("revision", "unqualified-revision"), + ("sha256", "0" * 64), + ("sha256", None), + ("repo_type", "dataset"), + ("location", "adversary/elsewhere/elsewhere.h5"), + ], +) +def test_pinned_manifest_cannot_change_the_bundled_default_artifact( + tmp_path, monkeypatch, transport, field, value +): + payload = _payload() + artifact = payload["artifacts"]["test_dataset"] + if field == "location": + artifact.update(repo_id="adversary/elsewhere", path="elsewhere.h5") + else: + artifact[field] = value + # Install a TRO matching these exact conflicting bytes while keeping the + # bundle's original certified URI, build, model and fingerprint unchanged. + local_path, _, raw_bytes = _install_bundle(tmp_path, monkeypatch, payload) + if transport == "local": + monkeypatch.setenv(ENVIRONMENT_VARIABLE, str(local_path)) + + with patch.object( + manifests.requests, "get", return_value=_response(raw_bytes) + ) as network: + with pytest.raises(ValueError, match="artifact.*installed bundle"): + _certify() + assert network.call_count == (transport == "fetched") + + +@pytest.mark.parametrize("transport", ["local", "fetched"]) +@pytest.mark.parametrize("failure", ["missing-artifact", "other-default"]) +def test_manifest_requires_the_bundled_default_dataset( + tmp_path, monkeypatch, transport, failure +): + payload = _payload() + if failure == "missing-artifact": + payload["artifacts"].clear() + else: + payload["default_datasets"]["national"] = "another_dataset" + local_path, _, raw_bytes = _install_bundle(tmp_path, monkeypatch, payload) + if transport == "local": + monkeypatch.setenv(ENVIRONMENT_VARIABLE, str(local_path)) + with patch.object(manifests.requests, "get", return_value=_response(raw_bytes)): + with pytest.raises(ValueError, match="artifact.*installed bundle"): + _certify() + + +@pytest.mark.parametrize("transport", ["local", "fetched"]) +@pytest.mark.parametrize("certified_record", [True, False]) +def test_consistent_default_uses_rewritten_revision( + tmp_path, monkeypatch, transport, certified_record +): + local_path, trace_path, raw_bytes = _install_bundle(tmp_path, monkeypatch) + if not certified_record: + _edit_bundle(trace_path, lambda country: country.pop("certified_data_artifact")) + if transport == "local": + monkeypatch.setenv(ENVIRONMENT_VARIABLE, str(local_path)) + with patch.object(manifests.requests, "get", return_value=_response(raw_bytes)): + assert _certify().compatibility_basis == ( + "exact_build_model_version" + + ("_local_manifest" if transport == "local" else "") + ) + assert ( + manifests.get_data_release_manifest("uk").artifacts["test_dataset"].revision + == "f" * 40 + ) + + +@pytest.mark.parametrize("transport", ["local", "fetched"]) +def test_a_different_certified_artifact_cannot_leave_the_default_unpinned( + tmp_path, monkeypatch, transport +): + payload = _payload() + payload["artifacts"]["certified_other"] = dict(payload["artifacts"]["test_dataset"]) + local_path, trace_path, raw_bytes = _install_bundle(tmp_path, monkeypatch, payload) + + def edit(country): + country["datasets"].clear() + country["certified_data_artifact"]["dataset"] = "certified_other" + + _edit_bundle(trace_path, edit) + if transport == "local": + monkeypatch.setenv(ENVIRONMENT_VARIABLE, str(local_path)) + with patch.object(manifests.requests, "get", return_value=_response(raw_bytes)): + with pytest.raises(ValueError, match="missing bundled default artifact record"): + _certify() + + +@pytest.mark.parametrize("transport", ["local", "fetched"]) +@pytest.mark.parametrize( + "difference", ["none", "repo_id", "path", "revision", "sha256", "repo_type"] +) +def test_every_overlapping_managed_artifact_must_match( + tmp_path, monkeypatch, transport, difference +): + payload = _payload() + payload["artifacts"]["supplement"] = { + "kind": "microdata", + "path": "inherited.h5", + "repo_id": "policyengine/inherited-data", + "revision": "inherited-release", + "repo_type": "dataset", + "sha256": "b" * 64, + } + reference = dict(payload["artifacts"]["supplement"]) + reference.pop("kind") + if difference != "none": + payload["artifacts"]["supplement"][difference] = ( + "model" if difference == "repo_type" else "conflicting-pin" + ) + local_path, trace_path, raw_bytes = _install_bundle(tmp_path, monkeypatch, payload) + _edit_bundle( + trace_path, + lambda country: country["datasets"].update( + supplement=reference, + regional_only={"path": "regional.h5", "revision": "regional-release"}, + ), + ) + if transport == "local": + monkeypatch.setenv(ENVIRONMENT_VARIABLE, str(local_path)) + with patch.object(manifests.requests, "get", return_value=_response(raw_bytes)): + if difference == "none": + _certify() + else: + with pytest.raises( + ValueError, match="artifact 'supplement'.*installed bundle" + ): + _certify() + + +@pytest.mark.parametrize("transport", ["local", "fetched"]) +@pytest.mark.parametrize("record", ["certified", "managed"]) +def test_artifact_repository_and_path_are_distinct_identity_fields( + tmp_path, monkeypatch, transport, record +): + payload = _payload() + name = "test_dataset" if record == "certified" else "supplement" + payload["artifacts"][name] = { + "kind": "microdata", + "repo_id": "policyengine/test-data/nested", + "path": "test_dataset.h5", + "revision": "f" * 40, + "sha256": "d" * 64, + } + local_path, trace_path, raw_bytes = _install_bundle(tmp_path, monkeypatch, payload) + + def edit(country): + if record == "certified": + country["datasets"].clear() + country["certified_data_artifact"]["uri"] = ( + f"hf://policyengine/test-data/nested/test_dataset.h5@{'f' * 40}" + ) + else: + country["datasets"][name] = { + "path": "nested/test_dataset.h5", + "revision": "f" * 40, + "sha256": "d" * 64, + } + + _edit_bundle(trace_path, edit) + if transport == "local": + monkeypatch.setenv(ENVIRONMENT_VARIABLE, str(local_path)) + with patch.object(manifests.requests, "get", return_value=_response(raw_bytes)): + # Concatenated URI strings agree, but the asserted repo/path do not. + with pytest.raises(ValueError, match="artifact.*installed bundle"): + _certify() + + +@pytest.mark.parametrize("kind", ["diagnostics", "evidence"]) +@pytest.mark.parametrize("prefixed", [False, True]) +def test_release_relative_paths_match_certification_normalization( + tmp_path, monkeypatch, kind, prefixed +): + payload = _payload() + relative_path = "report.json" + bundled_path = "releases/test-release/report.json" + payload["artifacts"]["report"] = { + "kind": kind, + "path": bundled_path if prefixed else relative_path, + "repo_id": "policyengine/test-data", + "revision": "test-release", + "sha256": "b" * 64, + } + local_path, trace_path, _ = _install_bundle(tmp_path, monkeypatch, payload) + + def edit(country): + country["data_package"]["release_manifest_path"] = ( + "releases/test-release/release_manifest.json" + ) + country["datasets"]["report"] = { + "path": bundled_path, + "revision": "f" * 40, + "sha256": "b" * 64, + } + + _edit_bundle(trace_path, edit) + tro = json.loads(trace_path.read_text()) + tro["@graph"][0]["trov:hasArrangement"][0]["trov:hasArtifactLocation"][0][ + "trov:hasLocation" + ] = ( + "https://huggingface.co/policyengine/test-data/resolve/" + f"{'f' * 40}/releases/test-release/release_manifest.json" + ) + trace_path.write_text(json.dumps(tro)) + monkeypatch.setenv(ENVIRONMENT_VARIABLE, str(local_path)) + with patch.object(manifests.requests, "get") as network: + _certify() + network.assert_not_called() + + +def test_setting_local_manifest_after_a_cached_fetch_still_checks_its_pin( + tmp_path, monkeypatch +): + local_path, _, raw_bytes = _install_bundle(tmp_path, monkeypatch) + with patch.object(manifests.requests, "get", return_value=_response(raw_bytes)): + assert _certify().compatibility_basis == "exact_build_model_version" + local_path.write_bytes(raw_bytes + b" ") + monkeypatch.setenv(ENVIRONMENT_VARIABLE, str(local_path)) + + with patch.object(manifests.requests, "get") as network: + with pytest.raises(ValueError): + _certify() + + network.assert_not_called() + + +@pytest.mark.parametrize("change", ["mutation", "deletion"]) +def test_a_successful_local_load_does_not_cache_unchecked_bytes( + tmp_path, monkeypatch, change +): + local_path, _, raw_bytes = _install_bundle(tmp_path, monkeypatch) + monkeypatch.setenv(ENVIRONMENT_VARIABLE, str(local_path)) + with patch.object(manifests.requests, "get") as network: + assert ( + _certify().compatibility_basis == "exact_build_model_version_local_manifest" + ) + if change == "mutation": + local_path.write_bytes(raw_bytes + b" ") + else: + local_path.unlink() + with pytest.raises(ValueError): + _certify() + network.assert_not_called() + + +@pytest.fixture(scope="module") +def main_certification(): + """Keep the exact main implementation as a differential test oracle.""" + snapshot = ( + Path(__file__).parent + / "fixtures" + / "local_manifest" + / "main_485dc2330_certification.py.txt" + ) + namespace = vars(manifests).copy() + exec(compile(snapshot.read_text(), str(snapshot), "exec"), namespace) + return namespace["certify_data_release_compatibility"] + + +@pytest.mark.parametrize( + "case", + [ + "fetched-exact", + "fetched-fingerprint", + "fetched-claim", + "fetched-qualified", + "fetched-model-mismatch", + "fetched-core-mismatch", + "fetched-build-mismatch", + "offline-bundled", + "offline-model-mismatch", + "offline-fingerprint-mismatch", + ], +) +def test_unset_setting_matches_main_certification_cases( + tmp_path, monkeypatch, main_certification, case +): + """Golden differential of existing release/qualified-pair test scenarios. + + Compare complete results or exact exception types/messages with an unchanged + main 485dc2330 implementation, then check the characterized expected result. + Synthetic resources isolate the comparison from future country release pins. + """ + payload = _payload() + bundled = _qualified_certification() + runtime = {} + expected_error = None + expected = manifests.DataCertification( + compatibility_basis="exact_build_model_version", + certified_for_model_version=MODEL, + data_build_id=BUILD, + built_with_model_version=MODEL, + built_with_model_git_sha="abc123", + data_build_fingerprint=FINGERPRINT, + ) + if case in {"fetched-fingerprint", "fetched-claim", "fetched-qualified"}: + payload["build"]["built_with_model_package"]["version"] = OTHER_MODEL + expected.built_with_model_version = OTHER_MODEL + if case == "fetched-fingerprint": + runtime["runtime_data_build_fingerprint"] = FINGERPRINT + expected.compatibility_basis = "matching_data_build_fingerprint" + elif case == "fetched-claim": + payload["compatible_model_packages"] = [ + {"name": "policyengine-uk", "specifier": f">={MODEL},<3.0.0"} + ] + expected.compatibility_basis = "legacy_compatible_model_package" + elif case in {"fetched-qualified", "offline-bundled"}: + expected = manifests.DataCertification.model_validate(bundled) + elif case == "fetched-model-mismatch": + runtime["runtime_model_version"] = OTHER_MODEL + expected_error = "Data release manifest is not certified for the runtime model" + elif case == "fetched-core-mismatch": + runtime["runtime_core_version"] = "99.0.0" + expected_error = "Data release manifest is not certified for the runtime core" + elif case == "fetched-build-mismatch": + payload["build"]["build_id"] = "another-build" + expected_error = ( + "Data release manifest is not certified for the remote data build" + ) + elif case == "offline-model-mismatch": + runtime["runtime_model_version"] = OTHER_MODEL + expected = manifests.DataCertification( + compatibility_basis="unverified_data_release_manifest_unavailable", + certified_for_model_version=OTHER_MODEL, + data_build_id=BUILD, + built_with_model_version=MODEL, + built_with_model_git_sha="abc123", + data_build_fingerprint=FINGERPRINT, + certified_by="test-bundle", + ) + elif case == "offline-fingerprint-mismatch": + bundled["compatibility_basis"] = "matching_data_build_fingerprint" + runtime["runtime_data_build_fingerprint"] = "sha256:different" + expected_error = "Runtime data build fingerprint does not match the bundled" + _, _, raw_bytes = _install_bundle( + tmp_path, monkeypatch, payload, certification=copy.deepcopy(bundled) + ) + assert ENVIRONMENT_VARIABLE not in manifests.os.environ + network_settings = ( + {"side_effect": requests.Timeout("offline")} + if case.startswith("offline-") + else {"return_value": _response(raw_bytes)} + ) + + def outcome(implementation): + try: + return ( + "ok", + _certify(_implementation=implementation, **runtime).model_dump(), + ) + except ValueError as error: + return ("error", type(error), str(error)) + + with patch.object(manifests.requests, "get", **network_settings) as network: + main_outcome = outcome(main_certification) + manifests.get_data_release_manifest.cache_clear() + current_outcome = outcome(manifests.certify_data_release_compatibility) + + assert current_outcome == main_outcome + if expected_error: + assert current_outcome[0] == "error" + assert expected_error in current_outcome[2] + else: + assert current_outcome == ("ok", expected.model_dump()) + assert network.call_count == 2 diff --git a/tests/test_local_manifest_receipt.py b/tests/test_local_manifest_receipt.py new file mode 100644 index 000000000..f89a9bbf0 --- /dev/null +++ b/tests/test_local_manifest_receipt.py @@ -0,0 +1,176 @@ +"""Local-manifest runtime evidence preserves the bundle certificate's identity.""" + +import hashlib +import json +from pathlib import Path + +import jsonschema +import pytest + +from policyengine.provenance import certification, manifest, receipt + +REPO_ROOT = Path(__file__).resolve().parents[1] +SCHEMA_PATH = ( + REPO_ROOT / "src/policyengine/data/schemas/execution_receipt.v1.schema.json" +) +LOCAL_BASIS = "exact_build_model_version_local_manifest" + + +@pytest.fixture +def packaged_receipt(monkeypatch, tmp_path): + bundle = json.loads( + (REPO_ROOT / "src/policyengine/data/bundle/manifest.json").read_text() + ) + record = bundle["data_releases"]["uk"] + dataset_path = tmp_path / "dataset.h5" + dataset_path.write_bytes(b"synthetic certified dataset") + digest = hashlib.sha256(dataset_path.read_bytes()).hexdigest() + record["datasets"][record["default_dataset"]]["sha256"] = digest + record["certified_data_artifact"]["sha256"] = digest + bundle_path = tmp_path / "manifest.json" + bundle_path.write_text(json.dumps(bundle)) + versions = { + "policyengine": bundle["policyengine_version"], + "policyengine-uk": record["certification"]["certified_for_model_version"], + "policyengine-core": bundle["packages"]["policyengine-core"]["version"], + } + monkeypatch.setattr( + certification, "default_bundle_source_path", lambda: bundle_path + ) + monkeypatch.setattr( + receipt, + "installed_package_identity", + lambda name, **kwargs: receipt.PackageIdentity( + name=name, version=versions[name] + ), + ) + monkeypatch.setattr(receipt, "loaded_versions", lambda: {}) + monkeypatch.setattr( + receipt, "_refuse_package_changed_after_process_start", lambda name: None + ) + monkeypatch.setattr(receipt, "_projected_years", lambda *args: (2025, 2026)) + monkeypatch.delenv("POLICYENGINE_UK_DATA_MANIFEST_FILE", raising=False) + + def build(*, caller_supplied=False): + current_law = receipt.ReformIdentity(kind="current_law") + return receipt.build_execution_receipt( + country_id="uk", + dataset_path=dataset_path, + years=[2026], + reform=current_law, + baseline=current_law, + bundle=bundle if caller_supplied else None, + ) + + return build, bundle + + +def test_packaged_receipt_records_local_basis_without_changing_certificate( + packaged_receipt, monkeypatch +): + build, bundle = packaged_receipt + calls = [] + + def certify(country_id, model_version, **kwargs): + calls.append((country_id, model_version, kwargs)) + return manifest.DataCertification( + compatibility_basis=LOCAL_BASIS, + certified_for_model_version=model_version, + ) + + monkeypatch.setenv("POLICYENGINE_UK_DATA_MANIFEST_FILE", "/verified/manifest.json") + monkeypatch.setattr(manifest, "certify_data_release_compatibility", certify) + result = build() + assert result.runtime.data_certification_basis == LOCAL_BASIS + assert calls == [ + ( + "uk", + result.model.version, + { + "runtime_core_version": result.core.version, + "runtime_data_build_fingerprint": result.data.build_fingerprint, + }, + ) + ] + assert ( + result.certificate.compatibility_basis + == (bundle["data_releases"]["uk"]["certification"]["compatibility_basis"]) + ) + assert receipt.verify_certificate(result, bundle) == [] + payload = json.loads(result.model_dump_json()) + jsonschema.Draft202012Validator(json.loads(SCHEMA_PATH.read_text())).validate( + payload + ) + assert ( + receipt.ExecutionReceipt.model_validate_json(result.model_dump_json()) == result + ) + + +def test_local_manifest_errors_stop_receipt_building(packaged_receipt, monkeypatch): + build, _ = packaged_receipt + monkeypatch.setenv("POLICYENGINE_UK_DATA_MANIFEST_FILE", "/missing/manifest.json") + + def refuse(*args, **kwargs): + raise ValueError("local manifest: expected SHA-256 aaa; actual SHA-256 bbb") + + monkeypatch.setattr(manifest, "certify_data_release_compatibility", refuse) + with pytest.raises(ValueError, match="expected SHA-256 aaa; actual SHA-256 bbb"): + build() + + +@pytest.mark.parametrize("caller_supplied", [False, True]) +def test_unset_setting_keeps_receipt_building_offline( + packaged_receipt, monkeypatch, caller_supplied +): + build, _ = packaged_receipt + + def unexpected(*args, **kwargs): + raise AssertionError( + "unset local manifest must not cause certification or fetches" + ) + + monkeypatch.setattr(manifest, "certify_data_release_compatibility", unexpected) + monkeypatch.setattr(manifest.requests, "get", unexpected) + result = build(caller_supplied=caller_supplied) + assert result.runtime.data_certification_basis is None + assert ( + "data_certification_basis" + not in json.loads(result.model_dump_json())["runtime"] + ) + + +def test_local_runtime_context_does_not_change_certificate_or_result_digests( + packaged_receipt, monkeypatch +): + build, bundle = packaged_receipt + original = build() + monkeypatch.setenv("POLICYENGINE_UK_DATA_MANIFEST_FILE", "/verified/manifest.json") + monkeypatch.setattr( + manifest, + "certify_data_release_compatibility", + lambda *args, **kwargs: manifest.DataCertification( + compatibility_basis=LOCAL_BASIS, + certified_for_model_version=original.model.version, + ), + ) + local = build() + assert local.runtime.data_certification_basis == LOCAL_BASIS + assert local.certificate == original.certificate + assert local.execution_identity_sha256 == original.execution_identity_sha256 + assert local.receipt_sha256 == original.receipt_sha256 + assert receipt.verify_certificate(local, bundle) == [] + + +def test_caller_supplied_bundle_does_not_claim_packaged_manifest_provenance( + packaged_receipt, monkeypatch +): + build, _ = packaged_receipt + monkeypatch.setenv("POLICYENGINE_UK_DATA_MANIFEST_FILE", "/verified/manifest.json") + + def unexpected(*args, **kwargs): + raise AssertionError("a caller's bundle is not the local file's trust anchor") + + monkeypatch.setattr(manifest, "certify_data_release_compatibility", unexpected) + result = build(caller_supplied=True) + assert result.certificate.certificate_source == "caller_supplied" + assert result.runtime.data_certification_basis is None diff --git a/tests/test_local_manifest_tro_pins.py b/tests/test_local_manifest_tro_pins.py new file mode 100644 index 000000000..fed5a9786 --- /dev/null +++ b/tests/test_local_manifest_tro_pins.py @@ -0,0 +1,174 @@ +"""A TRO's manifest pin includes every assertion for the JSON-LD node ID.""" + +from __future__ import annotations + +import json + +import pytest +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +from policyengine.provenance import manifest as manifests + +ARTIFACT_ID = "composition/1/artifact/data_release_manifest" +PIN = "a" * 64 + + +def _tro(): + return { + "@graph": [ + { + "@id": "tro", + "trov:hasComposition": { + "trov:hasArtifact": [{"@id": ARTIFACT_ID, "trov:sha256": PIN}] + }, + "trov:hasArrangement": [ + { + "trov:hasArtifactLocation": [ + { + "trov:hasArtifact": {"@id": ARTIFACT_ID}, + "trov:hasLocation": ( + "https://huggingface.co/policyengine/test-data/" + "resolve/test-release/release_manifest.json" + ), + } + ] + } + ], + } + ] + } + + +@pytest.fixture +def read_pin(tmp_path, monkeypatch): + bundle_dir = tmp_path / "data" / "bundle" + bundle_dir.mkdir(parents=True) + country_manifest = manifests.CountryReleaseManifest( + country_id="uk", + policyengine_version="6.10.1", + model_package=manifests.PackageVersion(name="policyengine-uk", version="test"), + data_package=manifests.DataPackageVersion( + name="policyengine-uk-data", + version="test-release", + repo_id="policyengine/test-data", + ), + default_dataset="test_dataset", + ) + monkeypatch.setattr(manifests, "files", lambda package: tmp_path) + monkeypatch.setattr( + manifests, "get_release_manifest", lambda country: country_manifest + ) + + def read(tro): + (bundle_dir / "uk.trace.tro.jsonld").write_text(json.dumps(tro)) + return manifests.bundled_data_release_manifest_sha256("uk") + + return read + + +@pytest.mark.parametrize("before_inline_definition", [False, True]) +def test_direct_graph_conflicting_manifest_pin_is_refused( + read_pin, before_inline_definition +): + tro = _tro() + duplicate = {"@id": ARTIFACT_ID, "trov:sha256": "0" * 64} + if before_inline_definition: + tro["@graph"].insert(0, duplicate) + else: + tro["@graph"].append(duplicate) + + with pytest.raises(ValueError, match="no valid SHA-256 pin"): + read_pin(tro) + + +@pytest.mark.parametrize( + "assertion", + [ + [PIN], + [PIN, "0" * 64], + {"@value": PIN}, + {"@value": "0" * 64}, + "not-a-sha256", + None, + 42, + ], +) +def test_noncanonical_duplicate_hash_assertions_fail_closed(read_pin, assertion): + tro = _tro() + tro["@graph"].append({"@id": ARTIFACT_ID, "trov:sha256": assertion}) + + with pytest.raises(ValueError, match="no valid SHA-256 pin"): + read_pin(tro) + + +def test_identical_external_definitions_and_references_do_not_add_a_hash(read_pin): + tro = _tro() + tro["@graph"].extend( + [ + {"@id": ARTIFACT_ID, "trov:sha256": PIN.upper()}, + {"extension": {"@id": ARTIFACT_ID, "trov:sha256": PIN}}, + {"@id": ARTIFACT_ID}, + ] + ) + + assert read_pin(tro) == PIN + + +def test_hash_on_an_existing_artifact_reference_is_checked(read_pin): + tro = _tro() + reference = tro["@graph"][0]["trov:hasArrangement"][0]["trov:hasArtifactLocation"][ + 0 + ]["trov:hasArtifact"] + reference["trov:sha256"] = "0" * 64 + + with pytest.raises(ValueError, match="no valid SHA-256 pin"): + read_pin(tro) + + +@settings( + max_examples=80, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@given( + placement=st.sampled_from( + ["graph", "document", "tro", "composition", "arrangement", "location"] + ), + wrappers=st.lists( + st.sampled_from(["list", "property", "graph"]), min_size=0, max_size=8 + ), + conflicting_hash=st.binary(min_size=32, max_size=32) + .map(bytes.hex) + .filter(lambda digest: digest != PIN), +) +def test_conflicting_pin_is_refused_at_any_duplicate_placement( + read_pin, placement, wrappers, conflicting_hash +): + tro = _tro() + duplicate = {"@id": ARTIFACT_ID, "trov:sha256": conflicting_hash} + for wrapper in wrappers: + if wrapper == "list": + duplicate = [duplicate] + elif wrapper == "graph": + duplicate = {"@graph": [duplicate]} + else: + duplicate = {"extension": duplicate} + + root = tro["@graph"][0] + if placement == "graph": + # Keep the surrounding graph canonical so refusal must inspect the + # nested assertion, rather than fail on an unrelated malformed node. + tro["@graph"].append({"extension": duplicate}) + else: + target = { + "document": tro, + "tro": root, + "composition": root["trov:hasComposition"], + "arrangement": root["trov:hasArrangement"][0], + "location": root["trov:hasArrangement"][0]["trov:hasArtifactLocation"][0], + }[placement] + target["extension"] = duplicate + + with pytest.raises(ValueError, match="no valid SHA-256 pin"): + read_pin(tro) diff --git a/tests/test_measure_regional_inputs.py b/tests/test_measure_regional_inputs.py new file mode 100644 index 000000000..4bff297b6 --- /dev/null +++ b/tests/test_measure_regional_inputs.py @@ -0,0 +1,375 @@ +"""Measure the actual dense H5 surface, never an older model's inventory.""" + +from __future__ import annotations + +import hashlib +import importlib +import importlib.util +import json +import sys +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pandas as pd +import pytest + +from policyengine.provenance.manifest import RegionalInputArtifactEvidence + +MODEL_VERSION = "2.147.2" +URI = ( + "hf://dataset/policyengine/regional@dense-cut/releases/dense/release_manifest.json" +) +LIMITATION = "Reviewed dense VAT limitation; dashboard holds a VAT limit" +DEFAULTS = ["local_bus_uncapped_single_fare", "salary_sacrifice_returned_to_income"] + + +class FakeSimulation: + def __init__(self): + self.move_values("employment_income", "employment_income_before_lsr") + + def move_values(self, source, target): + pass + + +class FakeVariable: + def __init__(self, is_input): + self.is_input = is_input + + def is_input_variable(self): + return self.is_input + + +class FakeCountryTaxBenefitSystem: + def __init__(self): + inputs = { + "employment_income_before_lsr", + "household_weight", + "benunit_id", + *DEFAULTS, + } + self.variables = { + name: FakeVariable(name in inputs) + for name in sorted(inputs | {"employment_income"}) + } + + +def _module(): + return importlib.import_module("policyengine.provenance.regional_input_surface") + + +def _h5(path, *, periods=None, fmt="fixed", missing=None): + with pd.HDFStore(path, mode="w") as store: + if periods is None: + store.put("time_period", pd.Series([2025])) + periods = {None: True} + for period, has_income in periods.items(): + for entity, columns in { + "person": {"employment_income": [1.0]} if has_income else {}, + "benunit": {"benunit_id": [0]}, + "household": {"household_weight": [1.0]}, + }.items(): + if (period, entity) == missing: + continue + key = entity if period is None else f"{entity}/{period}" + if not columns: + columns = {"benunit_id": [0]} + store.put(key, pd.DataFrame(columns), format=fmt) + return path + + +def _response(artifact_paths): + payload = { + "schema_version": 1, + "data_package": {"name": "microcosm-data", "version": "1"}, + "dataset_role": "non_default_local_area", + "is_default": False, + "default_datasets": {}, + "artifacts": { + name: { + "kind": "microdata", + "path": f"{name}.h5", + "repo_id": "policyengine/regional", + "revision": "dense-cut", + "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + } + for name, path in artifact_paths.items() + }, + } + response = MagicMock(status_code=200) + response.content = json.dumps(payload, indent=2).encode() + return response + + +@pytest.fixture +def model(monkeypatch): + module = _module() + monkeypatch.setattr(module, "version", lambda name: MODEL_VERSION) + monkeypatch.setitem( + sys.modules, + "policyengine_uk", + SimpleNamespace( + CountryTaxBenefitSystem=FakeCountryTaxBenefitSystem, + Simulation=FakeSimulation, + ), + ) + return module + + +def _measure(module, paths, response=None, **kwargs): + response = _response(paths) if response is None else response + with patch( + "policyengine.provenance.certification.requests.get", return_value=response + ): + return module.measure_regional_input_surface( + URI, + paths, + MODEL_VERSION, + "Budget operator", + [LIMITATION], + **kwargs, + ) + + +@pytest.mark.parametrize("fmt", ["fixed", "table"]) +def test_actual_h5_columns_and_aliases_produce_bound_audit(tmp_path, model, fmt): + paths = {"dense": _h5(tmp_path / "dense.h5", fmt=fmt)} + response = _response(paths) + result = _measure(model, paths, response) + assert result.model_version == MODEL_VERSION + assert result.model_package == "policyengine-uk" + assert result.regional_manifest_uri == URI + assert ( + result.regional_manifest_sha256 == hashlib.sha256(response.content).hexdigest() + ) + artifact = result.artifacts["dense"] + assert artifact.sha256 == hashlib.sha256(paths["dense"].read_bytes()).hexdigest() + assert artifact.repo_type == "dataset" + assert artifact.columns == ["benunit_id", "employment_income", "household_weight"] + assert artifact.columns_by_period == {"2025": artifact.columns} + assert result.input_aliases == {"employment_income": "employment_income_before_lsr"} + assert result.defaulted_inputs == DEFAULTS + assert result.dropped_columns == [] + assert result.limitations == [LIMITATION] + assert result.audited_by == "Budget operator" + + +def test_a_default_in_one_period_is_not_hidden_by_another_period(tmp_path, model): + paths = {"dense": _h5(tmp_path / "dense.h5", periods={"2025": True, "2026": False})} + result = _measure(model, paths) + assert result.defaulted_inputs == [ + "employment_income_before_lsr", + *DEFAULTS, + ] + assert "employment_income" in result.artifacts["dense"].columns_by_period["2025"] + assert ( + "employment_income" not in result.artifacts["dense"].columns_by_period["2026"] + ) + + +@pytest.mark.parametrize("entity", ["person", "benunit", "household"]) +def test_each_period_requires_every_entity(tmp_path, model, entity): + paths = { + "dense": _h5( + tmp_path / "dense.h5", + periods={"2025": True, "2026": True}, + missing=("2026", entity), + ) + } + with pytest.raises(ValueError, match="2026|entity"): + _measure(model, paths) + + +def test_single_year_layout_requires_a_time_period(tmp_path, model): + path = _h5(tmp_path / "dense.h5") + with pd.HDFStore(path, mode="a") as store: + store.remove("time_period") + with pytest.raises(ValueError, match="time_period|period"): + _measure(model, {"dense": path}) + + +def test_extra_metadata_is_not_mistaken_for_a_model_column(tmp_path, model): + path = _h5(tmp_path / "dense.h5") + with pd.HDFStore(path, mode="a") as store: + store.put("metadata", pd.Series(["producer build notes"])) + assert _measure(model, {"dense": path}).dropped_columns == [] + + +def test_an_actual_dropped_column_blocks_audit(tmp_path, model): + path = _h5(tmp_path / "dense.h5") + with pd.HDFStore(path, mode="a") as store: + store.put("person", pd.DataFrame({"removed_column": [1]})) + with pytest.raises(ValueError, match="[Dd]ropped|removed_column"): + _measure(model, {"dense": path}) + + +def test_installed_model_must_be_the_requested_exact_version( + tmp_path, model, monkeypatch +): + monkeypatch.setattr(model, "version", lambda name: "2.146.4") + paths = {"dense": _h5(tmp_path / "dense.h5")} + with pytest.raises(ValueError, match="2.147.2|2.146.4"): + _measure(model, paths) + + +def test_artifact_bytes_must_match_the_manifest(tmp_path, model): + paths = {"dense": _h5(tmp_path / "dense.h5")} + response = _response(paths) + payload = json.loads(response.content) + payload["artifacts"]["dense"]["sha256"] = "0" * 64 + response.content = json.dumps(payload).encode() + with pytest.raises(ValueError, match="sha256|[Hh]ash|bytes"): + _measure(model, paths, response) + + +def test_file_changing_during_metadata_read_refuses(tmp_path, model, monkeypatch): + paths = {"dense": _h5(tmp_path / "dense.h5")} + response = _response(paths) + original = pd.HDFStore.select + mutated = False + + def select_and_modify(store, *args, **kwargs): + nonlocal mutated + result = original(store, *args, **kwargs) + if not mutated and args[0] != "/time_period": + with paths["dense"].open("ab") as file: + file.write(b"changed while inspecting metadata") + mutated = True + return result + + monkeypatch.setattr(pd.HDFStore, "select", select_and_modify) + with pytest.raises(ValueError, match="chang|sha256|[Hh]ash|bytes"): + _measure(model, paths, response) + + +@pytest.mark.parametrize("names", [[], ["other"], ["dense", "other"]]) +def test_every_routed_h5_requires_exactly_one_supplied_path(tmp_path, model, names): + path = _h5(tmp_path / "dense.h5") + response = _response({"dense": path}) + with pytest.raises(ValueError, match="artifact|[Hh]5"): + _measure(model, {name: path for name in names}, response) + + +def test_comparison_uses_each_artifact_and_every_period(): + module = _module() + artifact = RegionalInputArtifactEvidence( + repo_id="r", + path="dense.h5", + revision="cut", + sha256="a" * 64, + columns=["income", "weight"], + columns_by_period={"2025": ["income", "weight"], "2026": ["weight"]}, + ) + result = module.compare_input_surface( + {"dense": artifact}, + ["income", "income_reported", "weight", "new_input"], + ["income_reported", "weight", "new_input"], + {"income": "income_reported"}, + ) + assert result == { + "dropped_columns": [], + "defaulted_inputs": ["income_reported", "new_input"], + } + + +def test_comparison_refuses_a_period_union_that_hides_columns(): + module = _module() + artifact = RegionalInputArtifactEvidence( + repo_id="r", + path="dense.h5", + revision="cut", + sha256="a" * 64, + columns=["weight"], + columns_by_period={"2025": ["weight", "removed"]}, + ) + with pytest.raises(ValueError, match="columns|union|period"): + module.compare_input_surface({"dense": artifact}, ["weight"], ["weight"]) + + +def test_conditional_alias_cannot_silently_claim_a_supplied_input( + tmp_path, model, monkeypatch +): + class ConditionalSimulation: + def __init__(self): + if self.condition: + self.move_values("employment_income", "employment_income_before_lsr") + + monkeypatch.setattr( + sys.modules["policyengine_uk"], "Simulation", ConditionalSimulation + ) + paths = {"dense": _h5(tmp_path / "dense.h5")} + with pytest.raises(ValueError, match="alias|move_values|conditional"): + _measure(model, paths) + + +def test_audit_cli_writes_pydantic_evidence_json(tmp_path, model, monkeypatch): + script_path = ( + Path(__file__).parents[1] / "scripts" / "audit_regional_input_surface.py" + ) + spec = importlib.util.spec_from_file_location( + "audit_regional_input_surface", script_path + ) + script = importlib.util.module_from_spec(spec) + spec.loader.exec_module(script) + path = _h5(tmp_path / "dense.h5") + result = _measure(model, {"dense": path}) + captured = {} + + def measure(**kwargs): + captured.update(kwargs) + return result + + monkeypatch.setattr(script, "measure_regional_input_surface", measure) + output = tmp_path / "evidence.json" + assert ( + script.main( + [ + "--regional-manifest-uri", + URI, + "--artifact", + f"dense={path}", + "--model-version", + MODEL_VERSION, + "--audited-by", + "Budget operator", + "--limitation", + LIMITATION, + "--output", + str(output), + ] + ) + == 0 + ) + assert captured["artifact_paths"] == {"dense": path} + assert captured["limitations"] == [LIMITATION] + assert json.loads(output.read_text()) == result.model_dump(mode="json") + + +def test_audit_cli_bootstraps_before_target_model_is_certified(tmp_path): + """The target must be auditable before it matches the current bundle.""" + import subprocess + + script = Path(__file__).parents[1] / "scripts" / "audit_regional_input_surface.py" + code = """ +import importlib.util +import os +import runpy +import sys +os.environ.pop('POLICYENGINE_SKIP_COUNTRY_IMPORTS', None) +original = importlib.util.find_spec +def refuse_country_autoload(name, *args, **kwargs): + if name in {'policyengine_uk', 'policyengine_us'}: + raise RuntimeError('audit triggered automatic country import') + return original(name, *args, **kwargs) +importlib.util.find_spec = refuse_country_autoload +sys.argv = [sys.argv[1], '--help'] +runpy.run_path(sys.argv[0], run_name='__main__') +""" + result = subprocess.run( + [sys.executable, "-c", code, str(script)], + capture_output=True, + text=True, + cwd=tmp_path, + ) + assert result.returncode == 0, result.stderr + assert "--model-version" in result.stdout diff --git a/tests/test_regional_input_evidence.py b/tests/test_regional_input_evidence.py new file mode 100644 index 000000000..71f08401f --- /dev/null +++ b/tests/test_regional_input_evidence.py @@ -0,0 +1,315 @@ +"""A regional model exception requires an audit of the exact dense bytes.""" + +import copy +import hashlib +import json + +import pytest + +from policyengine.provenance.certification import CertificationError +from policyengine.provenance.manifest import CountryReleaseManifest +from tests.test_certify_data_release import ( + UK_DENSE_URI, + UK_MODEL_VERSION, + _certify_uk, + _microcosm_uk_dense_payload, + _microcosm_uk_national_payload, + _response, + _tmp_bundle, +) + +REASON = "Budget accepts the audited dense model gap; area runs remain uncertified" +VAT_LIMIT = ( + "the dense cut's VAT predates the policyengine-uk VAT factor fix " + "(#2187, 2.135.7); the dashboard holds a VAT limit" +) +DEFAULTED_INPUTS = [ + "local_bus_uncapped_single_fare", + "salary_sacrifice_returned_to_income", +] + + +def older_dense() -> dict: + dense = _microcosm_uk_dense_payload() + dense["build"]["built_with_model_package"]["version"] = "2.120.0" + dense["compatible_model_packages"] = [ + {"name": "policyengine-uk", "specifier": "==2.120.0"} + ] + return dense + + +def input_evidence(dense: dict | None = None) -> dict: + dense = older_dense() if dense is None else dense + name = "microcosm_uk_2024_25_dense" + artifact = dense["artifacts"][name] + return { + "schema_version": 1, + "regional_manifest_uri": UK_DENSE_URI, + "regional_manifest_sha256": hashlib.sha256( + json.dumps(dense).encode() + ).hexdigest(), + "model_package": "policyengine-uk", + "model_version": UK_MODEL_VERSION, + "artifacts": { + name: { + key: artifact[key] for key in ("repo_id", "path", "revision", "sha256") + } + | { + "columns": ["employment_income"], + "columns_by_period": {"2025": ["employment_income"]}, + } + }, + "model_variables": ["employment_income", *DEFAULTED_INPUTS], + "model_inputs": ["employment_income", *DEFAULTED_INPUTS], + "dropped_columns": [], + "defaulted_inputs": DEFAULTED_INPUTS.copy(), + "audited_by": "Budget regional input-surface review", + "limitations": [VAT_LIMIT], + } + + +def certify_with_evidence(bundle_path, evidence: dict, dense: dict | None = None): + dense = older_dense() if dense is None else dense + return _certify_uk( + bundle_path, + [_response(_microcosm_uk_national_payload()), _response(dense)], + regional_manifest_uri=UK_DENSE_URI, + regional_model_mismatch_reason=REASON, + regional_input_surface_evidence=evidence, + ) + + +def test_accepted_mismatch_without_audited_evidence_refuses(tmp_path): + with pytest.raises(CertificationError, match="[Ee]vidence|[Aa]udit"): + _certify_uk( + _tmp_bundle(tmp_path), + [ + _response(_microcosm_uk_national_payload()), + _response(older_dense()), + ], + regional_manifest_uri=UK_DENSE_URI, + regional_model_mismatch_reason=REASON, + ) + + +@pytest.mark.parametrize("model_package", ["policyengine-us", "another-model"]) +@pytest.mark.parametrize("accept_mismatch", [False, True]) +def test_regional_release_built_with_another_model_package_refuses( + tmp_path, model_package, accept_mismatch +): + dense = older_dense() + dense["build"]["built_with_model_package"]["name"] = model_package + if not accept_mismatch: + dense["compatible_model_packages"] = [ + {"name": "policyengine-uk", "specifier": f"=={UK_MODEL_VERSION}"} + ] + bundle_path = _tmp_bundle(tmp_path) + before = bundle_path.read_bytes() + + with pytest.raises( + CertificationError, + match=f"built with {model_package}, not policyengine-uk", + ): + if accept_mismatch: + certify_with_evidence(bundle_path, input_evidence(dense), dense) + else: + _certify_uk( + bundle_path, + [_response(_microcosm_uk_national_payload()), _response(dense)], + regional_manifest_uri=UK_DENSE_URI, + ) + + assert bundle_path.read_bytes() == before + + +@pytest.mark.parametrize( + ("field", "wrong"), + [ + ("regional_manifest_uri", UK_DENSE_URI.replace("dense", "other-dense")), + ("regional_manifest_sha256", "0" * 64), + ("model_package", "policyengine-us"), + ("model_version", "2.147.1"), + ], +) +def test_evidence_for_another_cut_bytes_or_model_refuses(tmp_path, field, wrong): + evidence = input_evidence() + evidence[field] = wrong + with pytest.raises(CertificationError): + certify_with_evidence(_tmp_bundle(tmp_path), evidence) + + +@pytest.mark.parametrize( + ("field", "wrong"), + [ + ("repo_id", "policyengine/another-dense"), + ("path", "different_dense.h5"), + ("revision", "different-cut"), + ("sha256", "0" * 64), + ], +) +def test_evidence_for_other_artifact_bytes_or_identity_refuses(tmp_path, field, wrong): + evidence = input_evidence() + evidence["artifacts"]["microcosm_uk_2024_25_dense"][field] = wrong + with pytest.raises(CertificationError): + certify_with_evidence(_tmp_bundle(tmp_path), evidence) + + +def test_a_dropped_column_refuses_even_with_an_acceptance_reason(tmp_path): + evidence = input_evidence() + evidence["artifacts"]["microcosm_uk_2024_25_dense"]["columns"].append( + "removed_model_variable" + ) + evidence["artifacts"]["microcosm_uk_2024_25_dense"]["columns_by_period"][ + "2025" + ].append("removed_model_variable") + evidence["dropped_columns"] = ["removed_model_variable"] + with pytest.raises(CertificationError, match="[Dd]ropped|removed_model_variable"): + certify_with_evidence(_tmp_bundle(tmp_path), evidence) + + +@pytest.mark.parametrize("field", ["dropped_columns", "defaulted_inputs"]) +def test_the_declared_comparison_must_match_the_measured_surfaces(tmp_path, field): + evidence = input_evidence() + evidence[field] = ["invented_variable"] + with pytest.raises(CertificationError): + certify_with_evidence(_tmp_bundle(tmp_path), evidence) + + +def test_a_missing_artifact_measurement_refuses(tmp_path): + evidence = input_evidence() + evidence["artifacts"] = {} + with pytest.raises(CertificationError): + certify_with_evidence(_tmp_bundle(tmp_path), evidence) + + +def test_every_default_and_limitation_survives_validation_and_roundtrip(tmp_path): + bundle_path = _tmp_bundle(tmp_path) + evidence = input_evidence() + certify_with_evidence(bundle_path, evidence) + + release = json.loads(bundle_path.read_text())["data_releases"]["uk"] + manifest = CountryReleaseManifest.model_validate(release) + assert manifest.regional_certification is not None + record = manifest.regional_certification.model_dump(mode="json", exclude_none=True) + assert record["defaulted_inputs"] == DEFAULTED_INPUTS + assert record["dropped_columns"] == [] + assert record["limitations"] == [VAT_LIMIT] + assert record["evidence"] == evidence + assert record["area_receipts"] == "uncertified" + assert record["accepted_reason"] == REASON + assert ( + release["regional_source_manifest_sha256"] + == evidence["regional_manifest_sha256"] + ) + rebuilt = CountryReleaseManifest.model_validate_json(manifest.model_dump_json()) + assert rebuilt.regional_certification == manifest.regional_certification + assert release["certification"]["compatibility_basis"] == "built_with_model_package" + + +def test_evidence_digest_binds_original_manifest_bytes(tmp_path): + dense = older_dense() + response = _response(dense) + response.content = json.dumps(dense, indent=2).encode() + evidence = input_evidence(dense) + with pytest.raises(CertificationError): + _certify_uk( + _tmp_bundle(tmp_path), + [_response(_microcosm_uk_national_payload()), response], + regional_manifest_uri=UK_DENSE_URI, + regional_model_mismatch_reason=REASON, + regional_input_surface_evidence=evidence, + ) + + evidence["regional_manifest_sha256"] = hashlib.sha256(response.content).hexdigest() + bundle_path = _tmp_bundle(tmp_path) + _certify_uk( + bundle_path, + [_response(_microcosm_uk_national_payload()), response], + regional_manifest_uri=UK_DENSE_URI, + regional_model_mismatch_reason=REASON, + regional_input_surface_evidence=evidence, + ) + release = json.loads(bundle_path.read_text())["data_releases"]["uk"] + assert ( + release["regional_source_manifest_sha256"] + == evidence["regional_manifest_sha256"] + ) + + +def test_drop_regional_release_removes_acceptance_and_preserves_national_cert(tmp_path): + bundle_path = _tmp_bundle(tmp_path) + certify_with_evidence(bundle_path, input_evidence()) + before = json.loads(bundle_path.read_text())["data_releases"]["uk"] + _certify_uk( + bundle_path, + [_response(_microcosm_uk_national_payload())], + drop_regional_release=True, + ) + after = json.loads(bundle_path.read_text())["data_releases"]["uk"] + assert after["certification"] == before["certification"] + assert "regional_certification" not in after + assert "regional_source_manifest_sha256" not in after + assert "regional_source_manifest_uri" not in after + assert "regional_release_kind" not in after + assert "microcosm_uk_2024_25_dense" not in after["datasets"] + assert set(after["region_datasets"]) == {"national"} + + +def test_limitations_and_evidence_are_in_hashed_bundle_material(tmp_path): + from policyengine.provenance.canonical import canonical_sha256 + + bundle_path = _tmp_bundle(tmp_path) + certify_with_evidence(bundle_path, input_evidence()) + before = json.loads(bundle_path.read_text()) + after = copy.deepcopy(before) + after["data_releases"]["uk"]["regional_certification"]["limitations"].append( + "An additional reviewed regional limitation" + ) + + def digest(bundle): + manifest = CountryReleaseManifest.model_validate(bundle["data_releases"]["uk"]) + return canonical_sha256(manifest.model_dump(mode="json")) + + assert digest(before) != digest(after) + + +def test_missing_period_measurements_refuse(tmp_path): + evidence = input_evidence() + del evidence["artifacts"]["microcosm_uk_2024_25_dense"]["columns_by_period"] + with pytest.raises(CertificationError): + certify_with_evidence(_tmp_bundle(tmp_path), evidence) + + +def test_budget_target_records_vat_limit_and_both_named_defaults(tmp_path, monkeypatch): + from tests import test_certify_data_release as fixtures + + monkeypatch.setattr(fixtures, "UK_MODEL_VERSION", "2.147.2") + dense = older_dense() + dense["build"]["built_with_model_package"]["version"] = "2.132.6" + dense["compatible_model_packages"][0]["specifier"] = "==2.132.6" + audit = input_evidence(dense) + audit["model_version"] = "2.147.2" + audit["limitations"] = ["The dense cut was calibrated under an older model."] + path = _tmp_bundle(tmp_path) + certify_with_evidence(path, audit, dense) + release = json.loads(path.read_text())["data_releases"]["uk"] + record = release["regional_certification"] + assert record["certified_for_model_version"] == "2.147.2" + assert record["built_with_model_version"] == "2.132.6" + assert VAT_LIMIT in record["limitations"] + assert set(DEFAULTED_INPUTS) <= set(record["defaulted_inputs"]) + assert release["certification"]["compatibility_basis"] == "built_with_model_package" + + +def test_regional_acceptance_never_widens_national_coverage(tmp_path): + national = _microcosm_uk_national_payload() + national["build"]["built_with_model_package"]["version"] = "2.120.0" + national["compatible_model_packages"][0]["specifier"] = "==2.120.0" + with pytest.raises(CertificationError, match="new data build|compatibility claim"): + _certify_uk( + _tmp_bundle(tmp_path), + [_response(national), _response(older_dense())], + regional_manifest_uri=UK_DENSE_URI, + regional_model_mismatch_reason=REASON, + regional_input_surface_evidence=input_evidence(), + ) diff --git a/tests/test_release_manifests.py b/tests/test_release_manifests.py index aff21577a..006070e2a 100644 --- a/tests/test_release_manifests.py +++ b/tests/test_release_manifests.py @@ -154,6 +154,52 @@ def _response_with_json(payload: dict) -> MagicMock: return response +def _legacy_us_country_manifest( + version: str = "1.73.0", package_name: str = "policyengine-us-data" +) -> CountryReleaseManifest: + """Pin the legacy fixture independently of the installed Populace bundle.""" + return CountryReleaseManifest.model_validate( + { + "country_id": "us", + "policyengine_version": POLICYENGINE_VERSION, + "model_package": {"name": "policyengine-us", "version": "1.653.3"}, + "data_package": { + "name": package_name, + "version": version, + "repo_id": "policyengine/policyengine-us-data", + }, + "default_dataset": "enhanced_cps_2024", + "datasets": { + "enhanced_cps_2024": { + "path": "enhanced_cps_2024.h5", + "revision": version, + "sha256": "abc", + } + }, + "certified_data_artifact": { + "dataset": "enhanced_cps_2024", + "uri": ( + "hf://policyengine/policyengine-us-data/" + f"enhanced_cps_2024.h5@{version}" + ), + "sha256": "abc", + }, + } + ) + + +def _legacy_us_artifacts(version: str = "1.73.0") -> dict: + return { + "enhanced_cps_2024": { + "kind": "microdata", + "path": "enhanced_cps_2024.h5", + "repo_id": "policyengine/policyengine-us-data", + "revision": version, + "sha256": "abc", + } + } + + def _country_modules_with_microsimulation( name: str, microsimulation: MagicMock, @@ -535,10 +581,16 @@ def test__given_country__then_can_fetch_data_release_manifest(self): }, } - with patch( - "policyengine.provenance.manifest.requests.get", - return_value=_response_with_json(payload), - ) as mock_get: + with ( + patch( + "policyengine.provenance.manifest.get_release_manifest", + return_value=_legacy_us_country_manifest("1.78.2"), + ), + patch( + "policyengine.provenance.manifest.requests.get", + return_value=_response_with_json(payload), + ) as mock_get, + ): manifest = get_data_release_manifest("us") assert manifest.schema_version == 1 @@ -561,8 +613,8 @@ def test__given_country__then_can_fetch_data_release_manifest(self): ) mock_get.assert_called_once() assert mock_get.call_args.args[0] == ( - "https://huggingface.co/datasets/policyengine/populace-us/resolve/" - f"{US_DATA_RELEASE_REVISION}/{US_DATA_RELEASE_PATH}" + "https://huggingface.co/policyengine/policyengine-us-data/resolve/" + "1.78.2/release_manifest.json" ) def test__given_explicit_manifest_revision__then_builds_manifest_url(self): @@ -589,7 +641,9 @@ def test__given_release_manifest_artifact_uses_version_tag__then_rewrites_to_com "path": "populace_us_2024.h5", "repo_id": "policyengine/populace-us", "revision": US_DATA_RELEASE_VERSION, - "sha256": "abc", + "sha256": ( + "6496cc4393d4d3c6574f76eca231de5898c803b9067645591fd5c4d3e65aee84" + ), "size_bytes": 123, } }, @@ -651,12 +705,18 @@ def test__given_range_specifier__then_certification_accepts_compatible_version( } ], "default_datasets": {"national": "enhanced_cps_2024"}, - "artifacts": {}, + "artifacts": _legacy_us_artifacts(), } - with patch( - "policyengine.provenance.manifest.requests.get", - return_value=_response_with_json(payload), + with ( + patch( + "policyengine.provenance.manifest.get_release_manifest", + return_value=_legacy_us_country_manifest(), + ), + patch( + "policyengine.provenance.manifest.requests.get", + return_value=_response_with_json(payload), + ), ): certification = certify_data_release_compatibility( "us", @@ -685,12 +745,18 @@ def test__given_matching_fingerprint__then_certification_allows_reuse(self): }, "compatible_model_packages": [], "default_datasets": {"national": "enhanced_cps_2024"}, - "artifacts": {}, + "artifacts": _legacy_us_artifacts(), } - with patch( - "policyengine.provenance.manifest.requests.get", - return_value=_response_with_json(payload), + with ( + patch( + "policyengine.provenance.manifest.get_release_manifest", + return_value=_legacy_us_country_manifest(), + ), + patch( + "policyengine.provenance.manifest.requests.get", + return_value=_response_with_json(payload), + ), ): certification = certify_data_release_compatibility( "us", @@ -720,12 +786,18 @@ def test__given_release_with_cut_tag__then_cut_tag_is_the_data_build_id(self): }, "compatible_model_packages": [], "default_datasets": {"national": "enhanced_cps_2024"}, - "artifacts": {}, + "artifacts": _legacy_us_artifacts("0.1.0"), } - with patch( - "policyengine.provenance.manifest.requests.get", - return_value=_response_with_json(payload), + with ( + patch( + "policyengine.provenance.manifest.get_release_manifest", + return_value=_legacy_us_country_manifest("0.1.0", "microcosm-data"), + ), + patch( + "policyengine.provenance.manifest.requests.get", + return_value=_response_with_json(payload), + ), ): certification = certify_data_release_compatibility( "us", @@ -914,12 +986,18 @@ def test__given_mismatched_version_and_fingerprint__then_certification_fails(sel }, "compatible_model_packages": [], "default_datasets": {"national": "enhanced_cps_2024"}, - "artifacts": {}, + "artifacts": _legacy_us_artifacts(), } - with patch( - "policyengine.provenance.manifest.requests.get", - return_value=_response_with_json(payload), + with ( + patch( + "policyengine.provenance.manifest.get_release_manifest", + return_value=_legacy_us_country_manifest(), + ), + patch( + "policyengine.provenance.manifest.requests.get", + return_value=_response_with_json(payload), + ), ): try: certify_data_release_compatibility( diff --git a/tests/test_release_tro_generation.py b/tests/test_release_tro_generation.py index d0aa9410d..fb4b6b33c 100644 --- a/tests/test_release_tro_generation.py +++ b/tests/test_release_tro_generation.py @@ -253,7 +253,7 @@ def test_strict_missing_remote_dataset_never_writes_limited_output(release, caps data = json.loads(release[1].content) del data["artifacts"][data["default_datasets"]["national"]] release[1]._content = json.dumps(data).encode() - with pytest.raises(ValueError, match="Default dataset"): + with pytest.raises(ValueError, match="missing default dataset artifact"): generator.regenerate_all(strict=True) assert path.read_bytes() == before assert "limited TRO" not in capsys.readouterr().err