From 9a5286c236bb7d8a3ee2701610113babeb7616d0 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 16:55:12 +0400 Subject: [PATCH 01/12] Read a build published in its own release folder A release manifest in Microcosm's side-by-side layout names its release and build as strings, and lives at releases///. Parse it: the string build becomes DataBuildInfo with only build_id set, so release_build_id names the build and two builds of one release differ, and dumps write it back as the string. Such a manifest may leave its own files' revision to certification. Manifests whose build is an object, with or without cut_tag, are read as before. Co-Authored-By: Claude Opus 5.5 --- src/policyengine/provenance/manifest.py | 110 +++++++++++++- tests/test_release_build_folders.py | 186 ++++++++++++++++++++++++ 2 files changed, 291 insertions(+), 5 deletions(-) create mode 100644 tests/test_release_build_folders.py diff --git a/src/policyengine/provenance/manifest.py b/src/policyengine/provenance/manifest.py index d43e3f74..0a7fe758 100644 --- a/src/policyengine/provenance/manifest.py +++ b/src/policyengine/provenance/manifest.py @@ -1,18 +1,36 @@ import hashlib import json import os +import re from functools import lru_cache from importlib.resources import files from pathlib import Path -from typing import Literal, Optional +from typing import Any, Literal, Optional from urllib.parse import quote import requests -from pydantic import BaseModel, Field +from pydantic import ( + BaseModel, + Field, + SerializerFunctionWrapHandler, + model_serializer, + model_validator, +) HF_REQUEST_TIMEOUT_SECONDS = 30 PYPI_REQUEST_TIMEOUT_SECONDS = 30 +#: Microcosm publishes each build once, in its own folder +#: ``releases///``, and never changes it afterwards; a root +#: ``releases.json`` records which build of each release is current +#: (microcosm-emitter's ``microcosm_emitter_interface.releases``). +RELEASES_FOLDER = "releases" +RELEASE_MANIFEST_NAME = "release_manifest.json" +#: The publisher refuses any other release or build name +#: (``microcosm_emitter_interface.identifiers.SAFE_IDENTIFIER``), and both +#: are interpolated into repository paths. +RELEASE_FOLDER_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") + class DataReleaseManifestUnavailableError(ValueError): """Raised when a data release manifest cannot be fetched or is absent.""" @@ -47,17 +65,25 @@ class BuiltWithModelPackage(PackageVersion): class DataBuildInfo(BaseModel): build_id: Optional[str] = None + """The build's name. For a build published in its own release folder + (``DataReleaseManifest.release`` set), this is the manifest's ``build`` + string, the attempt ID that names the folder.""" cut_tag: Optional[str] = None """The immutable tag of one cut of a release line. Microcosm keeps ``build_id`` equal to the line name for every cut, so the cut tag is what - tells two cuts apart (see ``release_build_id``).""" + tells two cuts apart (see ``release_build_id``). Builds published in their + own release folders have no tag.""" built_at: Optional[str] = None built_with_core_package: Optional[PackageVersion] = None built_with_model_package: Optional[BuiltWithModelPackage] = None def release_build_id(build: Optional[DataBuildInfo]) -> Optional[str]: - """Identify one data build: its cut tag when it has one, else its build id.""" + """Identify one data build: its cut tag when it has one, else its build id. + + A build published in its own release folder has no cut tag, and its build + id is the folder's build name, so two builds of one release differ. + """ if build is None: return None return build.cut_tag or build.build_id @@ -116,7 +142,10 @@ class DataReleaseArtifact(BaseModel): kind: str path: str repo_id: str - revision: str + revision: Optional[str] = None + """Required unless the manifest is a build in its own release folder, + whose own files are pinned at the revision its manifest is certified at + (``pin_build_folder_artifacts``).""" repo_type: Optional[Literal["model", "dataset"]] = None sha256: Optional[str] = None size_bytes: Optional[int] = None @@ -127,6 +156,8 @@ class DataReleaseArtifact(BaseModel): @property def uri(self) -> str: + if self.revision is None: + raise ValueError(f"Artifact {self.path!r} has no revision pin.") return build_hf_uri( repo_id=self.repo_id, path_in_repo=self.path, @@ -144,6 +175,12 @@ class DataReleaseManifest(BaseModel): compatible_model_packages: list[CompatiblePackage] = Field(default_factory=list) compatible_core_packages: list[CompatiblePackage] = Field(default_factory=list) default_datasets: dict[str, str] = Field(default_factory=dict) + release: Optional[str] = None + """The release of a build published in its own folder, + ``releases///``. Such a manifest names its ``release`` + and ``build`` as strings; ``build`` is read as ``DataBuildInfo`` with only + ``build_id`` set, and written back as the string. ``None`` for a manifest + published before that layout, whose ``build`` is an object.""" build: Optional[DataBuildInfo] = None artifacts: dict[str, DataReleaseArtifact] = Field(default_factory=dict) preservation_dois: list[str] = Field(default_factory=list) @@ -157,6 +194,69 @@ class DataReleaseManifest(BaseModel): source_sha256: Optional[str] = Field(default=None, exclude=True) """Byte sha256 of the fetched manifest before runtime URI rewrites.""" + @model_validator(mode="before") + @classmethod + def _read_build_folder_identity(cls, data: Any) -> Any: + """Read a string ``build`` with its ``release`` as the build folder.""" + + if not isinstance(data, dict): + return data + release = data.get("release") + build = data.get("build") + if release is None and not isinstance(build, str): + return data + if release is None: + raise ValueError( + "A release manifest that names its build as a string must " + "name its release." + ) + if not isinstance(build, str): + raise ValueError( + "A release manifest that names its release must name its " + "build as a string." + ) + for label, name in (("release", release), ("build", build)): + if not isinstance(name, str) or not RELEASE_FOLDER_NAME.fullmatch(name): + raise ValueError(f"{label} must match {RELEASE_FOLDER_NAME.pattern}.") + return {**data, "build": {"build_id": build}} + + @model_validator(mode="after") + def _require_revision_pins_outside_build_folders(self) -> "DataReleaseManifest": + if self.release is None: + unpinned = sorted( + name + for name, artifact in self.artifacts.items() + if artifact.revision is None + ) + if unpinned: + raise ValueError( + f"Artifacts {unpinned} have no revision; only a build in " + "its own release folder may leave it to certification." + ) + return self + + @model_serializer(mode="wrap") + def _write_build_folder_identity( + self, handler: SerializerFunctionWrapHandler + ) -> Any: + data = handler(self) + if ( + self.release is not None + and self.build is not None + and isinstance(data, dict) + and "build" in data + ): + data["build"] = self.build.build_id + return data + + @property + def build_folder(self) -> Optional[str]: + """``releases//`` for a build in its own folder.""" + + if self.release is None or self.build is None or self.build.build_id is None: + return None + return f"{RELEASES_FOLDER}/{self.release}/{self.build.build_id}" + class DataCertification(BaseModel): compatibility_basis: str diff --git a/tests/test_release_build_folders.py b/tests/test_release_build_folders.py new file mode 100644 index 00000000..4cd0dafb --- /dev/null +++ b/tests/test_release_build_folders.py @@ -0,0 +1,186 @@ +"""Microcosm builds published side by side in their own release folders. + +In this layout each build is published once, in ``releases///``, +and never changed afterwards; a root ``releases.json`` records which build of +each release is current, and no new Hugging Face tags are made. Each build's +``release_manifest.json`` names its ``release`` and ``build`` as strings +(microcosm-emitter#24 checks that they equal the folder's names). + +The manifest shape below is assumed until Microcosm's release writer change +(H2c) defines it: top-level ``release`` and ``build`` strings, the existing +``artifacts`` entries with bare file names, and no ``cut_tag``. + +Invariants: + +- a build is named by its ``build``, so two builds of one release never match; +- a manifest published before the layout (``build`` an object, maybe with + ``cut_tag``) is read exactly as before. +""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from policyengine.provenance.manifest import ( + DataReleaseManifest, + release_build_id, +) + +REPOSITORY = "policyengine/populace-uk-private" +NATIONAL_RELEASE = "microcosm-uk-2024-25-national" +NATIONAL_BUILD = "uk-frs-calibration-attempt-20261019T083323Z-6cba5d71" +EARLIER_NATIONAL_BUILD = "uk-frs-calibration-attempt-20261014T214443Z-d12ff9b3" +UK_MODEL_VERSION = "2.126.12" + + +def folder_layout_manifest_payload( + release: str = NATIONAL_RELEASE, + build: str = NATIONAL_BUILD, +) -> dict: + """A national release manifest in the side-by-side layout.""" + + return { + "schema_version": 1, + "release": release, + "build": build, + "data_package": {"name": "microcosm-data", "version": "0.1.0"}, + "default_datasets": {"national": "microcosm_uk_2024_25"}, + "compatible_model_packages": [ + {"name": "policyengine-uk", "specifier": f"=={UK_MODEL_VERSION}"} + ], + "artifacts": { + "microcosm_uk_2024_25": { + "kind": "microdata", + "role": "dataset", + "path": "microcosm_uk_2024_25.h5", + "repo_id": REPOSITORY, + "revision": "main", + "sha256": "a" * 64, + }, + "calibration_diagnostics": { + "kind": "diagnostics", + "role": "calibration_diagnostics", + "path": "calibration_diagnostics.json", + "repo_id": REPOSITORY, + "revision": "main", + "sha256": "b" * 64, + }, + }, + } + + +def tagged_layout_manifest_payload() -> dict: + """A manifest published before the layout: ``build`` is an object.""" + + tag = f"{NATIONAL_RELEASE}-20261019T083323Z-6cba5d71" + payload = folder_layout_manifest_payload() + del payload["release"] + payload["build"] = { + "build_id": NATIONAL_RELEASE, + "cut_tag": tag, + "built_with_model_package": { + "name": "policyengine-uk", + "version": UK_MODEL_VERSION, + }, + } + for artifact in payload["artifacts"].values(): + artifact["revision"] = tag + return payload + + +class TestBuildFolderManifest: + def test__given_folder_layout_manifest__then_build_is_the_data_build_id(self): + manifest = DataReleaseManifest.model_validate(folder_layout_manifest_payload()) + + assert manifest.release == NATIONAL_RELEASE + assert release_build_id(manifest.build) == NATIONAL_BUILD + + def test__given_folder_layout_manifest__then_names_its_build_folder(self): + manifest = DataReleaseManifest.model_validate(folder_layout_manifest_payload()) + + assert manifest.build_folder == ( + f"releases/{NATIONAL_RELEASE}/{NATIONAL_BUILD}" + ) + + def test__given_two_builds_of_one_release__then_their_build_ids_differ(self): + newer = DataReleaseManifest.model_validate(folder_layout_manifest_payload()) + earlier = DataReleaseManifest.model_validate( + folder_layout_manifest_payload(build=EARLIER_NATIONAL_BUILD) + ) + + assert release_build_id(newer.build) != release_build_id(earlier.build) + + def test__given_folder_layout_manifest__then_records_no_build_details(self): + # The string build carries no built_with_* packages; until Microcosm + # says where they go, certification relies on compatibility claims. + manifest = DataReleaseManifest.model_validate(folder_layout_manifest_payload()) + + assert manifest.build is not None + assert manifest.build.built_with_model_package is None + assert manifest.build.cut_tag is None + + def test__given_folder_layout_manifest__then_dump_keeps_build_a_string(self): + manifest = DataReleaseManifest.model_validate(folder_layout_manifest_payload()) + + dumped = manifest.model_dump(mode="json", exclude_none=True) + + assert dumped["release"] == NATIONAL_RELEASE + assert dumped["build"] == NATIONAL_BUILD + assert DataReleaseManifest.model_validate(dumped) == manifest + + def test__given_folder_layout_artifact_without_revision__then_parses(self): + payload = folder_layout_manifest_payload() + for artifact in payload["artifacts"].values(): + del artifact["revision"] + + manifest = DataReleaseManifest.model_validate(payload) + + assert manifest.artifacts["microcosm_uk_2024_25"].revision is None + + def test__given_tagged_layout_artifact_without_revision__then_refuses(self): + payload = tagged_layout_manifest_payload() + del payload["artifacts"]["microcosm_uk_2024_25"]["revision"] + + with pytest.raises(ValidationError, match="revision"): + DataReleaseManifest.model_validate(payload) + + def test__given_string_build_without_release__then_refuses(self): + payload = folder_layout_manifest_payload() + del payload["release"] + + with pytest.raises(ValidationError, match="release"): + DataReleaseManifest.model_validate(payload) + + def test__given_release_with_object_build__then_refuses(self): + payload = tagged_layout_manifest_payload() + payload["release"] = NATIONAL_RELEASE + + with pytest.raises(ValidationError, match="build"): + DataReleaseManifest.model_validate(payload) + + @pytest.mark.parametrize( + ("field_name", "unsafe_name"), + [("release", "../microcosm-uk"), ("build", "attempt/../other")], + ) + def test__given_unsafe_release_or_build_name__then_refuses( + self, field_name, unsafe_name + ): + payload = folder_layout_manifest_payload() + payload[field_name] = unsafe_name + + with pytest.raises(ValidationError, match=field_name): + DataReleaseManifest.model_validate(payload) + + def test__given_tagged_layout_manifest__then_cut_tag_stays_the_data_build_id( + self, + ): + manifest = DataReleaseManifest.model_validate(tagged_layout_manifest_payload()) + + assert manifest.release is None + assert manifest.build_folder is None + assert release_build_id(manifest.build) == ( + f"{NATIONAL_RELEASE}-20261019T083323Z-6cba5d71" + ) + dumped = manifest.model_dump(mode="json", exclude_none=True) + assert dumped["build"]["build_id"] == NATIONAL_RELEASE From 158b5aee9ca9d741c1b408ee16ff8fbdbe672b37 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 16:57:56 +0400 Subject: [PATCH 02/12] Certify a build from its own release folder A build published side by side has no Hugging Face tag, so the certificate pins it by its folder. fetch_release_manifest now requires a manifest that names its release and build to be read from releases///release_manifest.json, and pins the build's own files at releases/// and at the manifest URI's revision: the publisher writes a build's files and manifest in one commit and never changes them, while an artifact's own revision field cannot name that commit. Artifacts from other repositories keep their own pins; a file of the build outside its folder is refused. The bundle's datasets, region templates, certified dataset URI and the reachability checks all use the pinned paths, and the build names the certificate. Manifests published before the layout are unchanged. Co-Authored-By: Claude Opus 5.5 --- docs/engineering/skills/data-certification.md | 28 ++ src/policyengine/provenance/certification.py | 22 +- src/policyengine/provenance/manifest.py | 62 +++- tests/test_release_build_folders.py | 277 ++++++++++++++++++ 4 files changed, 387 insertions(+), 2 deletions(-) diff --git a/docs/engineering/skills/data-certification.md b/docs/engineering/skills/data-certification.md index c3f3477d..914f1b13 100644 --- a/docs/engineering/skills/data-certification.md +++ b/docs/engineering/skills/data-certification.md @@ -34,6 +34,34 @@ uses the cut tag as the build id (`data_releases..version`, the certification's `data_build_id` and the changelog fragment name), so two cuts of one line never look identical. +### Builds in their own release folders + +Once Microcosm publishes each build in its own folder, +`releases///`, a build is never changed after publishing and +gets no Hugging Face tag. The repository's root `releases.json` records which +build of each release is current. That build's `release_manifest.json` names +its `release` and `build` as strings, for example +`microcosm-uk-2024-25-national` and +`uk-frs-calibration-attempt-20261019T083323Z-6cba5d71`. Certify the build from +its own folder, at the commit that published it: + +```bash +python scripts/bundle.py certify-data --country uk --data-producer populace \ + --manifest-uri "hf://dataset/policyengine/populace-uk-private@/releases///release_manifest.json" +``` + +- The URI must be the build's own folder: a manifest that names another + release or build than the folder it was read from is refused. +- The build's own files are pinned at `releases///` and + at the URI's revision, whatever revision their manifest entries name. + Artifacts from other repositories keep their own pins. +- The certificate names the build by `build` + (`data_releases..version`, the certification's `data_build_id` and + the changelog fragment name), so two builds of one release never match. + +Manifests published before this layout, whose `build` is an object, are +certified as described above. + ### UK constituency and local-authority regions UK constituency and local-authority regions row-filter the dataset a regional diff --git a/src/policyengine/provenance/certification.py b/src/policyengine/provenance/certification.py index 04d1cd7d..cd498742 100644 --- a/src/policyengine/provenance/certification.py +++ b/src/policyengine/provenance/certification.py @@ -29,6 +29,11 @@ Pin the cut tag (``@``), never a branch: a release line's directory (``releases//``) is rewritten by each cut. +A build published in its own folder (``releases///``, named +by its manifest's ``release`` and ``build`` strings) has no tag. Certify it +from that folder at the commit that published it; its own files are pinned +inside the folder at that revision, and its ``build`` names the certificate. + ``scripts/bundle.py certify-data`` is the operator-facing wrapper. Network access is required (HF manifest fetch + PyPI wheel metadata). Countries whose data release predates release manifests need a dedicated data-producer strategy @@ -56,8 +61,10 @@ HF_REQUEST_TIMEOUT_SECONDS, DataReleaseManifest, RegionalReleaseKind, + ReleaseBuildFolderError, _specifier_matches, fetch_pypi_wheel_metadata, + read_build_folder, release_build_id, ) @@ -245,7 +252,11 @@ def fetch_release_manifest( token: Optional[str] = None, ) -> tuple[DataReleaseManifest, str, dict]: """Fetch and parse the data release manifest; returns - (manifest, sha256 of the fetched bytes, parsed URI parts).""" + (manifest, sha256 of the fetched bytes, parsed URI parts). + + A build published in its own release folder must be read from that + folder, and its own files are pinned inside it at the URI's revision + (``read_build_folder``).""" parts = parse_manifest_uri(manifest_uri) url = https_manifest_url(parts) headers = {"Authorization": f"Bearer {token}"} if token else {} @@ -258,6 +269,15 @@ def fetch_release_manifest( sha256 = hashlib.sha256(response.content).hexdigest() payload = json.loads(response.content) manifest = DataReleaseManifest.model_validate(payload) + try: + manifest = read_build_folder( + manifest, + repo_id=parts["repo_id"], + manifest_path=parts["path"], + revision=parts["revision"], + ) + except ReleaseBuildFolderError as exc: + raise CertificationError(str(exc)) from exc return manifest, sha256, parts diff --git a/src/policyengine/provenance/manifest.py b/src/policyengine/provenance/manifest.py index 0a7fe758..71837be3 100644 --- a/src/policyengine/provenance/manifest.py +++ b/src/policyengine/provenance/manifest.py @@ -36,6 +36,15 @@ class DataReleaseManifestUnavailableError(ValueError): """Raised when a data release manifest cannot be fetched or is absent.""" +class ReleaseBuildFolderError(ValueError): + """A build in its own release folder that does not match that folder. + + Its manifest names another release or build than the folder it was read + from, or one of its own files lies outside the folder. Unlike an + unavailable manifest, this is never covered by a bundled certificate. + """ + + class PackageVersion(BaseModel): name: str version: str @@ -145,7 +154,7 @@ class DataReleaseArtifact(BaseModel): revision: Optional[str] = None """Required unless the manifest is a build in its own release folder, whose own files are pinned at the revision its manifest is certified at - (``pin_build_folder_artifacts``).""" + (``read_build_folder``).""" repo_type: Optional[Literal["model", "dataset"]] = None sha256: Optional[str] = None size_bytes: Optional[int] = None @@ -320,6 +329,57 @@ def default_dataset_uri(self) -> str: return resolve_dataset_reference(self.country_id, self.default_dataset) +def read_build_folder( + manifest: DataReleaseManifest, + *, + repo_id: str, + manifest_path: str, + revision: str, +) -> DataReleaseManifest: + """Pin a build published in its own release folder to that folder. + + ``manifest_path`` and ``revision`` say where the manifest was read from in + ``repo_id``. The manifest must be the one in its own build folder, + ``releases///release_manifest.json``. Its own files (the + artifacts in ``repo_id``) are then pinned at + ``releases///`` and at ``revision``: the publisher + writes a build's files and its manifest in one commit and never changes + them afterwards, so any revision holding the manifest holds the same + files, while an artifact's own ``revision`` field cannot name that commit. + Artifacts in other repositories keep their own paths and pins. + + A manifest published before that layout is returned unchanged. + + Raises: + ReleaseBuildFolderError: If the manifest was read from another folder, + or one of its own files lies outside the folder. + """ + + folder = manifest.build_folder + if folder is None: + return manifest + expected_path = f"{folder}/{RELEASE_MANIFEST_NAME}" + if manifest_path != expected_path: + raise ReleaseBuildFolderError( + f"The release manifest at {manifest_path} names release " + f"{manifest.release} and build {release_build_id(manifest.build)}, " + f"whose manifest is {expected_path}." + ) + pinned = manifest.model_copy(deep=True) + for name, artifact in pinned.artifacts.items(): + if artifact.repo_id != repo_id: + continue + if "/" not in artifact.path: + artifact.path = f"{folder}/{artifact.path}" + elif not artifact.path.startswith(f"{folder}/"): + raise ReleaseBuildFolderError( + f"Artifact {name!r} ({artifact.path}) is outside its build " + f"folder {folder}/." + ) + artifact.revision = revision + return pinned + + def build_hf_uri(repo_id: str, path_in_repo: str, revision: str) -> str: return f"hf://{repo_id}/{path_in_repo}@{revision}" diff --git a/tests/test_release_build_folders.py b/tests/test_release_build_folders.py index 4cd0dafb..90050a02 100644 --- a/tests/test_release_build_folders.py +++ b/tests/test_release_build_folders.py @@ -19,9 +19,17 @@ from __future__ import annotations +import json +from pathlib import Path +from unittest.mock import MagicMock, patch + import pytest from pydantic import ValidationError +from policyengine.provenance.certification import ( + CertificationError, + certify_data_release, +) from policyengine.provenance.manifest import ( DataReleaseManifest, release_build_id, @@ -184,3 +192,272 @@ def test__given_tagged_layout_manifest__then_cut_tag_stays_the_data_build_id( ) dumped = manifest.model_dump(mode="json", exclude_none=True) assert dumped["build"]["build_id"] == NATIONAL_RELEASE + + +NATIONAL_COMMIT = "0123456789abcdef0123456789abcdef01234567" +DENSE_RELEASE = "microcosm-uk-2024-25-dense" +DENSE_BUILD = "uk-frs-calibration-attempt-20261019T000432Z-4e5f6a7b" +DENSE_COMMIT = "89abcdef0123456789abcdef0123456789abcdef" +NATIONAL_FOLDER = f"releases/{NATIONAL_RELEASE}/{NATIONAL_BUILD}" +DENSE_FOLDER = f"releases/{DENSE_RELEASE}/{DENSE_BUILD}" + + +def manifest_uri(folder: str, revision: str) -> str: + return f"hf://dataset/{REPOSITORY}@{revision}/{folder}/release_manifest.json" + + +NATIONAL_URI = manifest_uri(NATIONAL_FOLDER, NATIONAL_COMMIT) +DENSE_URI = manifest_uri(DENSE_FOLDER, DENSE_COMMIT) + + +def folder_layout_dense_payload() -> dict: + """A local-area release manifest in the side-by-side layout.""" + + return { + "schema_version": 1, + "release": DENSE_RELEASE, + "build": DENSE_BUILD, + "data_package": {"name": "microcosm-data", "version": "0.1.0"}, + "dataset_role": "non_default_local_area", + "is_default": False, + "default_datasets": {}, + "compatible_model_packages": [ + {"name": "policyengine-uk", "specifier": f"=={UK_MODEL_VERSION}"} + ], + "artifacts": { + "microcosm_uk_2024_25_dense": { + "kind": "microdata", + "role": "dataset", + "path": "microcosm_uk_2024_25_dense.h5", + "repo_id": REPOSITORY, + "sha256": "c" * 64, + }, + }, + } + + +def manifest_response(payload: dict) -> MagicMock: + response = MagicMock() + response.status_code = 200 + response.content = json.dumps(payload).encode() + response.text = response.content.decode() + response.raise_for_status.return_value = None + return response + + +def empty_bundle(tmp_path: Path) -> Path: + bundle_path = tmp_path / "manifest.json" + bundle_path.write_text( + json.dumps( + { + "schema_version": 2, + "bundle_version": "9.9.9", + "policyengine_version": "9.9.9", + "packages": { + "policyengine": {"name": "policyengine", "version": "9.9.9"}, + "policyengine-uk": {"name": "policyengine-uk", "version": "2.0.0"}, + }, + "extras": {}, + "countries": {"uk": {"model_package": "policyengine-uk"}}, + "data_releases": {}, + } + ) + + "\n" + ) + return bundle_path + + +def certify_uk( + bundle_path: Path, + manifests: list[dict], + manifest_uri: str = NATIONAL_URI, + **options, +): + """Certify with fetched manifests and reachability checks replaced.""" + + with ( + patch( + "policyengine.provenance.certification.requests.get", + side_effect=[manifest_response(payload) for payload in manifests], + ), + patch( + "policyengine.provenance.certification.head_artifact", + return_value=True, + ) as head_artifact, + patch( + "policyengine.provenance.certification.head_artifact_reference", + return_value=True, + ), + patch( + "policyengine.provenance.certification.fetch_pypi_wheel_metadata", + return_value={"sha256": "d" * 64, "url": "https://example"}, + ), + patch( + "policyengine.provenance.certification.policyengine_version", + return_value="9.9.9", + ), + ): + result = certify_data_release( + country="uk", + data_producer="populace", + manifest_uri=manifest_uri, + model_version=UK_MODEL_VERSION, + bundle_path=bundle_path, + **options, + ) + release = json.loads(bundle_path.read_text())["data_releases"]["uk"] + return result, release, head_artifact + + +class TestCertifyBuildFolder: + def test__given_folder_layout_release__then_datasets_are_pinned_in_its_folder( + self, tmp_path + ): + _, release, _ = certify_uk( + empty_bundle(tmp_path), [folder_layout_manifest_payload()] + ) + + assert release["datasets"]["microcosm_uk_2024_25"] == { + "path": f"{NATIONAL_FOLDER}/microcosm_uk_2024_25.h5", + "revision": NATIONAL_COMMIT, + "repo_id": REPOSITORY, + "sha256": "a" * 64, + } + assert release["datasets"]["calibration_diagnostics"]["path"] == ( + f"{NATIONAL_FOLDER}/calibration_diagnostics.json" + ) + assert release["region_datasets"]["national"] == { + "path_template": f"{NATIONAL_FOLDER}/microcosm_uk_2024_25.h5" + } + assert release["default_dataset_uri"] == ( + f"hf://{REPOSITORY}/{NATIONAL_FOLDER}/microcosm_uk_2024_25.h5" + f"@{NATIONAL_COMMIT}" + ) + + def test__given_folder_layout_release__then_runtime_reads_its_folder_manifest( + self, tmp_path + ): + _, release, _ = certify_uk( + empty_bundle(tmp_path), [folder_layout_manifest_payload()] + ) + + assert release["data_package"]["release_manifest_path"] == ( + f"{NATIONAL_FOLDER}/release_manifest.json" + ) + assert release["data_package"]["release_manifest_revision"] == (NATIONAL_COMMIT) + + def test__given_folder_layout_release__then_its_build_names_the_certificate( + self, tmp_path + ): + result, release, _ = certify_uk( + empty_bundle(tmp_path), [folder_layout_manifest_payload()] + ) + + assert release["version"] == NATIONAL_BUILD + assert release["certification"]["data_build_id"] == NATIONAL_BUILD + assert release["certified_data_artifact"]["build_id"] == NATIONAL_BUILD + assert result.build_id == NATIONAL_BUILD + + def test__given_folder_layout_release__then_checks_the_dataset_in_its_folder( + self, tmp_path + ): + _, _, head_artifact = certify_uk( + empty_bundle(tmp_path), [folder_layout_manifest_payload()] + ) + + checked = head_artifact.call_args.args[0] + assert (checked.path, checked.revision) == ( + f"{NATIONAL_FOLDER}/microcosm_uk_2024_25.h5", + NATIONAL_COMMIT, + ) + + def test__given_artifacts_without_revision__then_certified_revision_pins_them( + self, tmp_path + ): + payload = folder_layout_manifest_payload() + for artifact in payload["artifacts"].values(): + del artifact["revision"] + + _, release, _ = certify_uk(empty_bundle(tmp_path), [payload]) + + assert {dataset["revision"] for dataset in release["datasets"].values()} == { + NATIONAL_COMMIT + } + + def test__given_manifest_read_from_another_build_folder__then_refuses( + self, tmp_path + ): + other_folder = f"releases/{NATIONAL_RELEASE}/{EARLIER_NATIONAL_BUILD}" + + with pytest.raises(CertificationError, match=NATIONAL_BUILD): + certify_uk( + empty_bundle(tmp_path), + [folder_layout_manifest_payload()], + manifest_uri=manifest_uri(other_folder, NATIONAL_COMMIT), + ) + + def test__given_artifact_outside_its_build_folder__then_refuses(self, tmp_path): + payload = folder_layout_manifest_payload() + payload["artifacts"]["microcosm_uk_2024_25"]["path"] = ( + f"releases/{NATIONAL_RELEASE}/{EARLIER_NATIONAL_BUILD}/" + "microcosm_uk_2024_25.h5" + ) + + with pytest.raises(CertificationError, match="outside"): + certify_uk(empty_bundle(tmp_path), [payload]) + + def test__given_artifact_from_another_repository__then_keeps_its_own_pins( + self, tmp_path + ): + payload = folder_layout_manifest_payload() + payload["artifacts"]["frs_2024_25"] = { + "kind": "microdata", + "path": "frs_2024_25.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "revision": "1.58.0", + "sha256": "e" * 64, + } + + _, release, _ = certify_uk(empty_bundle(tmp_path), [payload]) + + inherited = release["datasets"]["frs_2024_25"] + assert (inherited["path"], inherited["revision"]) == ( + "frs_2024_25.h5", + "1.58.0", + ) + + def test__given_folder_layout_local_area_release__then_areas_use_its_folder( + self, tmp_path + ): + _, release, _ = certify_uk( + empty_bundle(tmp_path), + [folder_layout_manifest_payload(), folder_layout_dense_payload()], + regional_manifest_uri=DENSE_URI, + ) + + dense_path = f"{DENSE_FOLDER}/microcosm_uk_2024_25_dense.h5" + assert release["regional_release_kind"] == "local_area" + for region_type in ("constituency", "local_authority"): + assert release["region_datasets"][region_type] == { + "path_template": dense_path + } + dense = release["datasets"]["microcosm_uk_2024_25_dense"] + assert (dense["path"], dense["revision"]) == (dense_path, DENSE_COMMIT) + + def test__given_tagged_release__then_paths_and_tag_are_as_before(self, tmp_path): + tag = f"{NATIONAL_RELEASE}-20261019T083323Z-6cba5d71" + + result, release, _ = certify_uk( + empty_bundle(tmp_path), + [tagged_layout_manifest_payload()], + manifest_uri=manifest_uri(f"releases/{NATIONAL_RELEASE}", tag), + ) + + assert release["datasets"]["microcosm_uk_2024_25"]["path"] == ( + "microcosm_uk_2024_25.h5" + ) + assert release["datasets"]["microcosm_uk_2024_25"]["revision"] == tag + assert release["datasets"]["calibration_diagnostics"]["path"] == ( + f"releases/{NATIONAL_RELEASE}/calibration_diagnostics.json" + ) + assert result.build_id == tag From 0320e5cc48ce5121e4876b252583be4e0e0238c2 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 16:59:43 +0400 Subject: [PATCH 03/12] Refuse at import a manifest that names another build than its folder The import-time compatibility check reads the remote release manifest from the bundle's pinned folder. For a build in its own release folder, get_data_release_manifest now requires the manifest to name that folder's release and build, and pins the build's own files inside the folder at the bundle's revision, as certification does. A mismatch raises ReleaseBuildFolderError, which is not an unavailable manifest, so the bundled certificate never covers it. Manifests published before the layout keep the existing revision rewrite. Co-Authored-By: Claude Opus 5.5 --- changelog.d/682.added.md | 1 + docs/engineering/skills/data-certification.md | 3 + src/policyengine/provenance/manifest.py | 10 ++ tests/test_release_build_folders.py | 142 ++++++++++++++++++ 4 files changed, 156 insertions(+) create mode 100644 changelog.d/682.added.md diff --git a/changelog.d/682.added.md b/changelog.d/682.added.md new file mode 100644 index 00000000..d8858cce --- /dev/null +++ b/changelog.d/682.added.md @@ -0,0 +1 @@ +Certify Microcosm builds published side by side in their own release folders (`releases///`, with no Hugging Face tag): a release manifest that names its `release` and `build` as strings is certified from that folder, its own files are pinned inside it at the certified revision, and its `build` names the certificate, so two builds of one release never match. At import, a remote manifest that names another release or build than the bundle's pinned folder is refused. Manifests published before this layout are read as before. diff --git a/docs/engineering/skills/data-certification.md b/docs/engineering/skills/data-certification.md index 914f1b13..82af4089 100644 --- a/docs/engineering/skills/data-certification.md +++ b/docs/engineering/skills/data-certification.md @@ -58,6 +58,9 @@ python scripts/bundle.py certify-data --country uk --data-producer populace \ - The certificate names the build by `build` (`data_releases..version`, the certification's `data_build_id` and the changelog fragment name), so two builds of one release never match. +- At import, a remote manifest that names another release or build than the + bundle's pinned folder is refused, even when the bundle carries a + certificate. Manifests published before this layout, whose `build` is an object, are certified as described above. diff --git a/src/policyengine/provenance/manifest.py b/src/policyengine/provenance/manifest.py index 71837be3..ce2d8550 100644 --- a/src/policyengine/provenance/manifest.py +++ b/src/policyengine/provenance/manifest.py @@ -559,6 +559,16 @@ def get_data_release_manifest(country_id: str) -> DataReleaseManifest: if not isinstance(source_bytes, bytes): source_bytes = response.text.encode("utf-8") data_release_manifest.source_sha256 = hashlib.sha256(source_bytes).hexdigest() + if data_release_manifest.build_folder is not None: + # A build in its own release folder must be the build the bundle + # pinned; ReleaseBuildFolderError is not an unavailable manifest, so + # no bundled certificate covers it. + return read_build_folder( + data_release_manifest, + repo_id=country_manifest.data_package.repo_id, + manifest_path=country_manifest.data_package.release_manifest_path, + revision=_artifact_revision(country_manifest.data_package), + ) release_revision = country_manifest.data_package.release_manifest_revision if release_revision is not None: for artifact in data_release_manifest.artifacts.values(): diff --git a/tests/test_release_build_folders.py b/tests/test_release_build_folders.py index 90050a02..3e446a38 100644 --- a/tests/test_release_build_folders.py +++ b/tests/test_release_build_folders.py @@ -31,7 +31,11 @@ certify_data_release, ) from policyengine.provenance.manifest import ( + CountryReleaseManifest, DataReleaseManifest, + ReleaseBuildFolderError, + certify_data_release_compatibility, + get_data_release_manifest, release_build_id, ) @@ -461,3 +465,141 @@ def test__given_tagged_release__then_paths_and_tag_are_as_before(self, tmp_path) f"releases/{NATIONAL_RELEASE}/calibration_diagnostics.json" ) assert result.build_id == tag + + +PAIR_ID = "a" * 64 + + +def folder_layout_country_manifest( + certification: dict | None = None, +) -> CountryReleaseManifest: + """A bundle entry certified from the national build's own folder.""" + + return CountryReleaseManifest.model_validate( + { + "country_id": "uk", + "policyengine_version": "9.9.9", + "model_package": {"name": "policyengine-uk", "version": UK_MODEL_VERSION}, + "data_package": { + "name": "microcosm-data", + "version": "0.1.0", + "repo_id": REPOSITORY, + "repo_type": "dataset", + "release_manifest_path": f"{NATIONAL_FOLDER}/release_manifest.json", + "release_manifest_revision": NATIONAL_COMMIT, + }, + "default_dataset": "microcosm_uk_2024_25", + "datasets": { + "microcosm_uk_2024_25": { + "path": f"{NATIONAL_FOLDER}/microcosm_uk_2024_25.h5", + "revision": NATIONAL_COMMIT, + "repo_id": REPOSITORY, + } + }, + **({"certification": certification} if certification else {}), + } + ) + + +class TestImportCheckBuildFolder: + def setup_method(self): + get_data_release_manifest.cache_clear() + + def teardown_method(self): + get_data_release_manifest.cache_clear() + + def _patched(self, remote_payload: dict, country_manifest=None): + return ( + patch( + "policyengine.provenance.manifest.get_release_manifest", + return_value=country_manifest or folder_layout_country_manifest(), + ), + patch( + "policyengine.provenance.manifest.requests.get", + return_value=manifest_response(remote_payload), + ), + ) + + def test__given_remote_manifest_of_its_folder__then_artifacts_resolve_in_it( + self, + ): + bundle_patch, request_patch = self._patched(folder_layout_manifest_payload()) + with bundle_patch, request_patch as get: + manifest = get_data_release_manifest("uk") + + assert get.call_args.args[0] == ( + f"https://huggingface.co/datasets/{REPOSITORY}/resolve/" + f"{NATIONAL_COMMIT}/{NATIONAL_FOLDER}/release_manifest.json" + ) + assert manifest.artifacts["microcosm_uk_2024_25"].uri == ( + f"hf://{REPOSITORY}/{NATIONAL_FOLDER}/microcosm_uk_2024_25.h5" + f"@{NATIONAL_COMMIT}" + ) + + def test__given_remote_manifest_naming_another_build__then_refuses(self): + bundle_patch, request_patch = self._patched( + folder_layout_manifest_payload(build=EARLIER_NATIONAL_BUILD) + ) + with ( + bundle_patch, + request_patch, + pytest.raises(ReleaseBuildFolderError, match=EARLIER_NATIONAL_BUILD), + ): + get_data_release_manifest("uk") + + def test__given_mismatched_remote_manifest__then_bundled_certificate_is_not_used( + self, + ): + # A manifest naming another build is wrong, not unavailable: the + # fallback to the bundled certificate must not hide it. + bundled = folder_layout_country_manifest( + certification={ + "compatibility_basis": "legacy_compatible_model_package", + "certified_for_model_version": UK_MODEL_VERSION, + "data_build_id": NATIONAL_BUILD, + } + ) + bundle_patch, request_patch = self._patched( + folder_layout_manifest_payload(build=EARLIER_NATIONAL_BUILD), bundled + ) + with ( + bundle_patch, + request_patch, + pytest.raises(ValueError, match=EARLIER_NATIONAL_BUILD), + ): + certify_data_release_compatibility( + "uk", runtime_model_version=UK_MODEL_VERSION + ) + + def test__given_publisher_claim_on_folder_build__then_build_names_it(self): + bundle_patch, request_patch = self._patched(folder_layout_manifest_payload()) + with bundle_patch, request_patch: + certification = certify_data_release_compatibility( + "uk", runtime_model_version=UK_MODEL_VERSION + ) + + assert certification.compatibility_basis == "legacy_compatible_model_package" + assert certification.data_build_id == NATIONAL_BUILD + + def test__given_qualified_folder_build__then_import_check_honours_its_build( + self, + ): + qualified = { + "compatibility_basis": "qualified_exact_pair", + "certified_for_model_version": "2.130.1", + "certified_for_core_version": "3.33.0", + "data_build_id": NATIONAL_BUILD, + "qualification_pair_id": PAIR_ID, + } + remote = folder_layout_manifest_payload() + remote["compatible_model_packages"] = [] + bundle_patch, request_patch = self._patched( + remote, folder_layout_country_manifest(certification=qualified) + ) + with bundle_patch, request_patch: + certification = certify_data_release_compatibility( + "uk", runtime_model_version="2.130.1", runtime_core_version="3.33.0" + ) + + assert certification.compatibility_basis == "qualified_exact_pair" + assert certification.data_build_id == NATIONAL_BUILD From edc5356bf5f83c51ae8805ee6c503f5dc3106194 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 22:35:53 +0400 Subject: [PATCH 04/12] Test that certification refuses a withdrawn or unlisted build A build published in its own release folder is certified only while the repository's releases.json at main lists it and has not withdrawn it. The certification helper now serves releases.json by URL, so these tests can set its contents; the earlier tests keep their behaviour. Red at this commit: 5 of the 6 new tests fail (the record is never read, so a withdrawn, unlisted or missing record raises nothing); the tagged release test passes, since tagged releases have no record to read. Co-Authored-By: Claude Opus 5.5 --- tests/test_release_build_folders.py | 146 +++++++++++++++++++++++++++- 1 file changed, 144 insertions(+), 2 deletions(-) diff --git a/tests/test_release_build_folders.py b/tests/test_release_build_folders.py index 3e446a38..7c71723c 100644 --- a/tests/test_release_build_folders.py +++ b/tests/test_release_build_folders.py @@ -249,6 +249,52 @@ def manifest_response(payload: dict) -> MagicMock: return response +def missing_file_response() -> MagicMock: + response = MagicMock() + response.status_code = 404 + response.content = b"Entry not found" + response.text = response.content.decode() + return response + + +RELEASES_RECORD_URL = ( + f"https://huggingface.co/datasets/{REPOSITORY}/resolve/main/releases.json" +) + + +def releases_record_payload( + withdrawn: frozenset[tuple[str, str]] = frozenset(), + unlisted: frozenset[tuple[str, str]] = frozenset(), +) -> dict: + """The repository's ``releases.json``, in the release writer's format. + + Lists both national builds (the newer one current) and the dense build + (no current build), none withdrawn, except the ``(release, build)`` pairs + named in ``withdrawn`` or ``unlisted``. Certification never reads + ``current``, so whether the dense release has one does not matter here. + """ + + builds_by_release = { + NATIONAL_RELEASE: [EARLIER_NATIONAL_BUILD, NATIONAL_BUILD], + DENSE_RELEASE: [DENSE_BUILD], + } + releases = {} + for release_name, build_names in builds_by_release.items(): + releases[release_name] = { + "role": "national_default" if release_name == NATIONAL_RELEASE else "dense", + "current": NATIONAL_BUILD if release_name == NATIONAL_RELEASE else None, + "builds": { + build_name: { + "published_at": "2026-10-19T08:33:23Z", + "withdrawn": (release_name, build_name) in withdrawn, + } + for build_name in build_names + if (release_name, build_name) not in unlisted + }, + } + return {"schema_version": 1, "default": NATIONAL_RELEASE, "releases": releases} + + def empty_bundle(tmp_path: Path) -> Path: bundle_path = tmp_path / "manifest.json" bundle_path.write_text( @@ -275,14 +321,36 @@ def certify_uk( bundle_path: Path, manifests: list[dict], manifest_uri: str = NATIONAL_URI, + releases_record: dict | None = None, + requested_urls: list[str] | None = None, **options, ): - """Certify with fetched manifests and reachability checks replaced.""" + """Certify with fetched files and reachability checks replaced. + + Requests for the repository's ``releases.json`` at ``main`` get + ``releases_record`` (``releases_record_payload()`` when ``None``, a 404 + when empty); every other request gets the next of ``manifests``. Each + requested URL is appended to ``requested_urls`` when given. + """ + + record_payload = ( + releases_record_payload() if releases_record is None else releases_record + ) + manifests_to_serve = list(manifests) + + def fake_get(url: str, headers=None, timeout=None) -> MagicMock: + if requested_urls is not None: + requested_urls.append(url) + if url == RELEASES_RECORD_URL: + if not record_payload: + return missing_file_response() + return manifest_response(record_payload) + return manifest_response(manifests_to_serve.pop(0)) with ( patch( "policyengine.provenance.certification.requests.get", - side_effect=[manifest_response(payload) for payload in manifests], + side_effect=fake_get, ), patch( "policyengine.provenance.certification.head_artifact", @@ -467,6 +535,80 @@ def test__given_tagged_release__then_paths_and_tag_are_as_before(self, tmp_path) assert result.build_id == tag +class TestCertifyOnlyListedBuilds: + """``releases.json`` at ``main`` is the record of every published build + and of which ones are withdrawn; certification reads it.""" + + def test__given_listed_build__then_reads_releases_record_at_main(self, tmp_path): + requested_urls: list[str] = [] + + certify_uk( + empty_bundle(tmp_path), + [folder_layout_manifest_payload()], + requested_urls=requested_urls, + ) + + assert RELEASES_RECORD_URL in requested_urls + + def test__given_build_withdrawn_in_releases_record__then_refuses(self, tmp_path): + record = releases_record_payload( + withdrawn=frozenset({(NATIONAL_RELEASE, NATIONAL_BUILD)}) + ) + + with pytest.raises(CertificationError, match="withdrawn"): + certify_uk( + empty_bundle(tmp_path), + [folder_layout_manifest_payload()], + releases_record=record, + ) + + def test__given_build_not_listed_in_releases_record__then_refuses(self, tmp_path): + record = releases_record_payload( + unlisted=frozenset({(NATIONAL_RELEASE, NATIONAL_BUILD)}) + ) + + with pytest.raises(CertificationError, match="does not list"): + certify_uk( + empty_bundle(tmp_path), + [folder_layout_manifest_payload()], + releases_record=record, + ) + + def test__given_withdrawn_local_area_build__then_refuses(self, tmp_path): + record = releases_record_payload( + withdrawn=frozenset({(DENSE_RELEASE, DENSE_BUILD)}) + ) + + with pytest.raises(CertificationError, match=f"{DENSE_BUILD}.*withdrawn"): + certify_uk( + empty_bundle(tmp_path), + [folder_layout_manifest_payload(), folder_layout_dense_payload()], + releases_record=record, + regional_manifest_uri=DENSE_URI, + ) + + def test__given_no_releases_record__then_refuses(self, tmp_path): + with pytest.raises(CertificationError, match="releases.json"): + certify_uk( + empty_bundle(tmp_path), + [folder_layout_manifest_payload()], + releases_record={}, + ) + + def test__given_tagged_release__then_releases_record_is_not_read(self, tmp_path): + tag = f"{NATIONAL_RELEASE}-20261019T083323Z-6cba5d71" + requested_urls: list[str] = [] + + certify_uk( + empty_bundle(tmp_path), + [tagged_layout_manifest_payload()], + manifest_uri=manifest_uri(f"releases/{NATIONAL_RELEASE}", tag), + requested_urls=requested_urls, + ) + + assert RELEASES_RECORD_URL not in requested_urls + + PAIR_ID = "a" * 64 From ea01e115f975220b987eba805f455b4e07883277 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 22:36:59 +0400 Subject: [PATCH 05/12] Certify a build in its own folder only while releases.json lists it fetch_release_manifest now reads the repository's releases.json at main for a build published in its own release folder, and refuses the build when the record is missing or unreadable, has another schema version, does not list the build, or marks it withdrawn. The record is read at main rather than at the URI's revision, because a build is withdrawn after the commit that published it. The primary and the regional manifest both go through this check. Tagged releases are unchanged and read no record. Only certification reads the record. Whether an installed bundle should refuse a build withdrawn after it was certified is not decided here. Green at this commit: 33 passed in tests/test_release_build_folders.py. Co-Authored-By: Claude Opus 5.5 --- src/policyengine/provenance/certification.py | 43 +++++++++++++- src/policyengine/provenance/manifest.py | 61 ++++++++++++++++++++ 2 files changed, 103 insertions(+), 1 deletion(-) diff --git a/src/policyengine/provenance/certification.py b/src/policyengine/provenance/certification.py index 8726f239..7ecc4ce6 100644 --- a/src/policyengine/provenance/certification.py +++ b/src/policyengine/provenance/certification.py @@ -59,11 +59,14 @@ ) from policyengine.provenance.manifest import ( HF_REQUEST_TIMEOUT_SECONDS, + RELEASES_RECORD_NAME, + RELEASES_RECORD_REVISION, DataReleaseArtifact, DataReleaseManifest, RegionalReleaseKind, ReleaseBuildFolderError, _specifier_matches, + check_build_listed, fetch_pypi_wheel_metadata, read_build_folder, release_build_id, @@ -246,6 +249,41 @@ def release_manifest_dir(parts: dict) -> str: return parts["path"].rsplit("/", 1)[0] +def https_releases_record_url(parts: dict) -> str: + """The repository's ``releases.json`` at ``main``, its current record.""" + prefix = "datasets/" if parts["repo_type"] == "dataset" else "" + return ( + f"https://huggingface.co/{prefix}{parts['repo_id']}/resolve/" + f"{RELEASES_RECORD_REVISION}/{RELEASES_RECORD_NAME}" + ) + + +def _require_listed_build( + manifest: DataReleaseManifest, parts: dict, headers: dict +) -> None: + """Refuse a build in its own folder that ``releases.json`` at ``main`` + does not list, or has withdrawn. + + The record is read at ``main``, not at the URI's revision: a build is + withdrawn after the commit that published it. + """ + release, build = manifest.release, release_build_id(manifest.build) + if release is None or build is None: + return + url = https_releases_record_url(parts) + response = requests.get(url, headers=headers, timeout=HF_REQUEST_TIMEOUT_SECONDS) + if response.status_code != 200: + raise CertificationError( + f"Could not fetch {RELEASES_RECORD_NAME} ({response.status_code}) " + f"from {url}; a build in its own release folder is certified only " + "while that record lists it." + ) + try: + check_build_listed(response.content, release=release, build=build) + except ReleaseBuildFolderError as exc: + raise CertificationError(str(exc)) from exc + + def fetch_release_manifest( manifest_uri: str, token: Optional[str] = None, @@ -255,7 +293,8 @@ def fetch_release_manifest( A build published in its own release folder must be read from that folder, and its own files are pinned inside it at the URI's revision - (``read_build_folder``).""" + (``read_build_folder``). The repository's ``releases.json`` at ``main`` + must list the build and not have withdrawn it.""" parts = parse_manifest_uri(manifest_uri) url = https_manifest_url(parts) headers = {"Authorization": f"Bearer {token}"} if token else {} @@ -277,6 +316,8 @@ def fetch_release_manifest( ) except ReleaseBuildFolderError as exc: raise CertificationError(str(exc)) from exc + if manifest.build_folder is not None: + _require_listed_build(manifest, parts, headers) return manifest, sha256, parts diff --git a/src/policyengine/provenance/manifest.py b/src/policyengine/provenance/manifest.py index ce2d8550..54d8e4f5 100644 --- a/src/policyengine/provenance/manifest.py +++ b/src/policyengine/provenance/manifest.py @@ -26,6 +26,12 @@ #: (microcosm-emitter's ``microcosm_emitter_interface.releases``). RELEASES_FOLDER = "releases" RELEASE_MANIFEST_NAME = "release_manifest.json" +#: The record of every published build and of which ones are withdrawn, at +#: the repository's root. Only the release writer changes it, always on +#: ``main``, so ``main`` holds the current record. +RELEASES_RECORD_NAME = "releases.json" +RELEASES_RECORD_REVISION = "main" +RELEASES_RECORD_SCHEMA_VERSION = 1 #: The publisher refuses any other release or build name #: (``microcosm_emitter_interface.identifiers.SAFE_IDENTIFIER``), and both #: are interpolated into repository paths. @@ -380,6 +386,61 @@ def read_build_folder( return pinned +def check_build_listed(record_bytes: bytes, *, release: str, build: str) -> None: + """Refuse a build that ``releases.json`` does not list, or has withdrawn. + + ``record_bytes`` is the repository's ``releases.json`` in the release + writer's format (``microcosm_emitter_interface.releases``):: + + {"schema_version": 1, "default": ..., "releases": {: { + "role": ..., "current": ..., "builds": {: { + "published_at": ..., "withdrawn": }}}}} + + Which build is current does not matter here: a certificate pins one + build, current or not. + + Raises: + ReleaseBuildFolderError: If the record is unreadable, has another + schema version, does not list the build, or marks it withdrawn. + """ + + try: + record = json.loads(record_bytes) + except (ValueError, UnicodeDecodeError, RecursionError): + raise ReleaseBuildFolderError( + f"{RELEASES_RECORD_NAME} is not a readable JSON document." + ) from None + schema_version = record.get("schema_version") if isinstance(record, dict) else None + # ``True`` and ``1.0`` equal 1 in Python; only the integer is the version. + if ( + type(schema_version) is not int + or schema_version != RELEASES_RECORD_SCHEMA_VERSION + ): + raise ReleaseBuildFolderError( + f"{RELEASES_RECORD_NAME} must have schema_version " + f"{RELEASES_RECORD_SCHEMA_VERSION}." + ) + releases = record.get("releases") + release_entry = releases.get(release) if isinstance(releases, dict) else None + builds = release_entry.get("builds") if isinstance(release_entry, dict) else None + build_entry = builds.get(build) if isinstance(builds, dict) else None + if not isinstance(build_entry, dict): + raise ReleaseBuildFolderError( + f"{RELEASES_RECORD_NAME} does not list build {build} of release {release}." + ) + withdrawn = build_entry.get("withdrawn") + if not isinstance(withdrawn, bool): + raise ReleaseBuildFolderError( + f"{RELEASES_RECORD_NAME} gives build {build} of release {release} " + "no withdrawn flag." + ) + if withdrawn: + raise ReleaseBuildFolderError( + f"Build {build} of release {release} is withdrawn in " + f"{RELEASES_RECORD_NAME}." + ) + + def build_hf_uri(repo_id: str, path_in_repo: str, revision: str) -> str: return f"hf://{repo_id}/{path_in_repo}@{revision}" From ef297cb67027084efbfd1cff8bb7cd6c630549a4 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 22:37:43 +0400 Subject: [PATCH 06/12] Test that a build's own files must be directly inside its folder The release writer publishes only bare file names, directly inside releases///. A path that starts with the folder and then leaves it through "..", a nested path, a dot segment or a backslash must be refused, at certification and at import. HTTP clients remove dot segments from URLs, so "/..//" would otherwise fetch another build's file under this build's certificate. Red at this commit: all 7 new tests fail (nothing is raised). Co-Authored-By: Claude Opus 5.5 --- tests/test_release_build_folders.py | 37 +++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/test_release_build_folders.py b/tests/test_release_build_folders.py index 7c71723c..cee8ddf4 100644 --- a/tests/test_release_build_folders.py +++ b/tests/test_release_build_folders.py @@ -478,6 +478,30 @@ def test__given_artifact_outside_its_build_folder__then_refuses(self, tmp_path): with pytest.raises(CertificationError, match="outside"): certify_uk(empty_bundle(tmp_path), [payload]) + @pytest.mark.parametrize( + "artifact_path", + [ + # Starts with the folder, then leaves it: HTTP clients remove the + # dot segments, so this would fetch the earlier build's file. + f"{NATIONAL_FOLDER}/../{EARLIER_NATIONAL_BUILD}/microcosm_uk_2024_25.h5", + f"{NATIONAL_FOLDER}/nested/microcosm_uk_2024_25.h5", + f"{NATIONAL_FOLDER}/..", + "..", + ".", + "microcosm_uk_2024_25\\..\\other.h5", + ], + ) + def test__given_artifact_path_not_directly_inside_its_folder__then_refuses( + self, tmp_path, artifact_path + ): + # The release writer publishes only bare file names, directly inside + # the build folder. + payload = folder_layout_manifest_payload() + payload["artifacts"]["microcosm_uk_2024_25"]["path"] = artifact_path + + with pytest.raises(CertificationError, match="directly inside"): + certify_uk(empty_bundle(tmp_path), [payload]) + def test__given_artifact_from_another_repository__then_keeps_its_own_pins( self, tmp_path ): @@ -689,6 +713,19 @@ def test__given_remote_manifest_naming_another_build__then_refuses(self): ): get_data_release_manifest("uk") + def test__given_remote_artifact_leaving_its_folder__then_refuses(self): + remote = folder_layout_manifest_payload() + remote["artifacts"]["microcosm_uk_2024_25"]["path"] = ( + f"{NATIONAL_FOLDER}/../{EARLIER_NATIONAL_BUILD}/microcosm_uk_2024_25.h5" + ) + bundle_patch, request_patch = self._patched(remote) + with ( + bundle_patch, + request_patch, + pytest.raises(ReleaseBuildFolderError, match="directly inside"), + ): + get_data_release_manifest("uk") + def test__given_mismatched_remote_manifest__then_bundled_certificate_is_not_used( self, ): From 624198056a307fbbc4db301b0a13d5789ea35090 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 22:38:23 +0400 Subject: [PATCH 07/12] Accept only bare file names directly inside a build's folder read_build_folder checked a build's own files with a string prefix, so "/..//" passed, and HTTP clients then fetched the other build's file. An own file's path must now be a bare file name, alone or after "/": one segment, not "." or "..", with no slash or backslash, which is what the release writer publishes (microcosm_emitter_interface.identifiers.bare_file_name). Certification and the import-time check share this rule. Green at this commit: 40 passed in tests/test_release_build_folders.py. Co-Authored-By: Claude Opus 5.5 --- src/policyengine/provenance/manifest.py | 26 ++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/src/policyengine/provenance/manifest.py b/src/policyengine/provenance/manifest.py index 54d8e4f5..4187597e 100644 --- a/src/policyengine/provenance/manifest.py +++ b/src/policyengine/provenance/manifest.py @@ -354,11 +354,14 @@ def read_build_folder( files, while an artifact's own ``revision`` field cannot name that commit. Artifacts in other repositories keep their own paths and pins. + The publisher writes only bare file names, directly inside the folder, so + an own file's ``path`` must be such a name, alone or after the folder. + A manifest published before that layout is returned unchanged. Raises: ReleaseBuildFolderError: If the manifest was read from another folder, - or one of its own files lies outside the folder. + or one of its own files is not directly inside the folder. """ folder = manifest.build_folder @@ -376,16 +379,33 @@ def read_build_folder( if artifact.repo_id != repo_id: continue if "/" not in artifact.path: - artifact.path = f"{folder}/{artifact.path}" - elif not artifact.path.startswith(f"{folder}/"): + file_name = artifact.path + elif artifact.path.startswith(f"{folder}/"): + file_name = artifact.path.removeprefix(f"{folder}/") + else: raise ReleaseBuildFolderError( f"Artifact {name!r} ({artifact.path}) is outside its build " f"folder {folder}/." ) + if not _is_bare_file_name(file_name): + # HTTP clients remove dot segments, so "/..//" + # would fetch another build's file. + raise ReleaseBuildFolderError( + f"Artifact {name!r} ({artifact.path}) is not a file directly " + f"inside its build folder {folder}/." + ) + artifact.path = f"{folder}/{file_name}" artifact.revision = revision return pinned +def _is_bare_file_name(name: str) -> bool: + """One path segment that names a file: what the publisher accepts + (``microcosm_emitter_interface.identifiers.bare_file_name``).""" + + return bool(name) and name not in {".", ".."} and not set(name) & {"/", "\\"} + + def check_build_listed(record_bytes: bytes, *, release: str, build: str) -> None: """Refuse a build that ``releases.json`` does not list, or has withdrawn. From 6b72ec01affcc12d859cf6798630f05933f47451 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 22:38:48 +0400 Subject: [PATCH 08/12] Document the releases.json and file-name checks for build folders The certification guide, the certification module's docstring and the changelog fragment now say that a build in its own folder is certified only while releases.json at main lists it and has not withdrawn it, that its own files must be bare file names directly inside the folder, and that import does not re-read releases.json. Documentation only, so no test step. Co-Authored-By: Claude Opus 5.5 --- changelog.d/682.added.md | 2 +- docs/engineering/skills/data-certification.md | 13 ++++++++++--- src/policyengine/provenance/certification.py | 2 ++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/changelog.d/682.added.md b/changelog.d/682.added.md index d8858cce..56a58aa0 100644 --- a/changelog.d/682.added.md +++ b/changelog.d/682.added.md @@ -1 +1 @@ -Certify Microcosm builds published side by side in their own release folders (`releases///`, with no Hugging Face tag): a release manifest that names its `release` and `build` as strings is certified from that folder, its own files are pinned inside it at the certified revision, and its `build` names the certificate, so two builds of one release never match. At import, a remote manifest that names another release or build than the bundle's pinned folder is refused. Manifests published before this layout are read as before. +Certify Microcosm builds published side by side in their own release folders (`releases///`, with no Hugging Face tag): a release manifest that names its `release` and `build` as strings is certified from that folder, its own files are pinned inside it at the certified revision, and its `build` names the certificate, so two builds of one release never match. Certification refuses such a build unless the repository's `releases.json` at `main` lists it and has not withdrawn it, and refuses any of its own files that is not a bare file name directly inside its folder. At import, a remote manifest that names another release or build than the bundle's pinned folder is refused. Manifests published before this layout are read as before. diff --git a/docs/engineering/skills/data-certification.md b/docs/engineering/skills/data-certification.md index dd13d3b1..ed3ed488 100644 --- a/docs/engineering/skills/data-certification.md +++ b/docs/engineering/skills/data-certification.md @@ -52,15 +52,22 @@ python scripts/bundle.py certify-data --country uk --data-producer populace \ - The URI must be the build's own folder: a manifest that names another release or build than the folder it was read from is refused. +- The repository's `releases.json` at `main` must list the build and must not + mark it withdrawn, or certification refuses it. A local-area release passed + with `--regional-manifest-uri` is checked the same way. Which build is + `current` doesn't matter: a certificate pins one build. - The build's own files are pinned at `releases///` and - at the URI's revision, whatever revision their manifest entries name. - Artifacts from other repositories keep their own pins. + at the URI's revision, whatever revision their manifest entries name. Each + one's `path` must be a bare file name, alone or after the folder, as the + release writer publishes it; a nested path, a `.` or `..` segment or a + backslash is refused. Artifacts from other repositories keep their own pins. - The certificate names the build by `build` (`data_releases..version`, the certification's `data_build_id` and the changelog fragment name), so two builds of one release never match. - At import, a remote manifest that names another release or build than the bundle's pinned folder is refused, even when the bundle carries a - certificate. + certificate. Import doesn't read `releases.json`: a build withdrawn after it + was certified keeps loading until the bundle is certified again. Manifests published before this layout, whose `build` is an object, are certified as described above. diff --git a/src/policyengine/provenance/certification.py b/src/policyengine/provenance/certification.py index 7ecc4ce6..b81b142d 100644 --- a/src/policyengine/provenance/certification.py +++ b/src/policyengine/provenance/certification.py @@ -33,6 +33,8 @@ by its manifest's ``release`` and ``build`` strings) has no tag. Certify it from that folder at the commit that published it; its own files are pinned inside the folder at that revision, and its ``build`` names the certificate. +The repository's ``releases.json`` at ``main`` must list the build and not +have withdrawn it. ``scripts/bundle.py certify-data`` is the operator-facing wrapper. Network access is required (HF manifest fetch + PyPI wheel metadata). Countries whose From 58d43c6fdc53c5e1543560b15062d12c68f81ee7 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 23:01:58 +0400 Subject: [PATCH 09/12] Test that certification accepts only releases.json records the writer accepts The release writer's parser (microcosm-emitter#24, ReleasesRecord.parse with check_record) refuses a releases.json with missing or unknown keys, a wrong schema_version, malformed release or build entries, unsafe release or build names, a default naming an unlisted release, a current naming an unlisted or withdrawn build, or more than 4 MiB. Certification should refuse the same records, even when the build being certified is listed and not withdrawn. Adds 24 refused records, one oversized record, and 3 records the writer accepts (no default, a release without builds, a withdrawn build that is not current), which certification must keep accepting. The helper can now serve releases.json as raw bytes. The record fixture now makes the national release's current build its newest listed, non-withdrawn build. Before, the withdrawn-build and unlisted-build tests used records whose current named the withdrawn or unlisted build, which the writer refuses for that reason alone. Red: 22 failed, 46 passed (68 in the file). All 22 fail with "DID NOT RAISE CertificationError". Three refused records (missing schema_version, schema_version 2, missing releases) were already refused. Co-Authored-By: Claude Opus 5.5 --- tests/test_release_build_folders.py | 227 ++++++++++++++++++++++++++-- 1 file changed, 216 insertions(+), 11 deletions(-) diff --git a/tests/test_release_build_folders.py b/tests/test_release_build_folders.py index cee8ddf4..98a657a8 100644 --- a/tests/test_release_build_folders.py +++ b/tests/test_release_build_folders.py @@ -20,6 +20,7 @@ from __future__ import annotations import json +from collections.abc import Callable from pathlib import Path from unittest.mock import MagicMock, patch @@ -240,15 +241,19 @@ def folder_layout_dense_payload() -> dict: } -def manifest_response(payload: dict) -> MagicMock: +def file_response(content: bytes) -> MagicMock: response = MagicMock() response.status_code = 200 - response.content = json.dumps(payload).encode() - response.text = response.content.decode() + response.content = content + response.text = content.decode() response.raise_for_status.return_value = None return response +def manifest_response(payload: dict) -> MagicMock: + return file_response(json.dumps(payload).encode()) + + def missing_file_response() -> MagicMock: response = MagicMock() response.status_code = 404 @@ -268,10 +273,11 @@ def releases_record_payload( ) -> dict: """The repository's ``releases.json``, in the release writer's format. - Lists both national builds (the newer one current) and the dense build - (no current build), none withdrawn, except the ``(release, build)`` pairs - named in ``withdrawn`` or ``unlisted``. Certification never reads - ``current``, so whether the dense release has one does not matter here. + Lists both national builds and the dense build (no current build), none + withdrawn, except the ``(release, build)`` pairs named in ``withdrawn`` + or ``unlisted``. The national release's current build is its newest build + that is listed and not withdrawn, so the record stays one the release + writer accepts. """ builds_by_release = { @@ -280,9 +286,19 @@ def releases_record_payload( } releases = {} for release_name, build_names in builds_by_release.items(): + usable_builds = [ + build_name + for build_name in build_names + if (release_name, build_name) not in withdrawn | unlisted + ] + current_build = ( + usable_builds[-1] + if release_name == NATIONAL_RELEASE and usable_builds + else None + ) releases[release_name] = { "role": "national_default" if release_name == NATIONAL_RELEASE else "dense", - "current": NATIONAL_BUILD if release_name == NATIONAL_RELEASE else None, + "current": current_build, "builds": { build_name: { "published_at": "2026-10-19T08:33:23Z", @@ -321,7 +337,7 @@ def certify_uk( bundle_path: Path, manifests: list[dict], manifest_uri: str = NATIONAL_URI, - releases_record: dict | None = None, + releases_record: dict | bytes | None = None, requested_urls: list[str] | None = None, **options, ): @@ -329,8 +345,9 @@ def certify_uk( Requests for the repository's ``releases.json`` at ``main`` get ``releases_record`` (``releases_record_payload()`` when ``None``, a 404 - when empty); every other request gets the next of ``manifests``. Each - requested URL is appended to ``requested_urls`` when given. + when empty, served as they are when bytes); every other request gets the + next of ``manifests``. Each requested URL is appended to + ``requested_urls`` when given. """ record_payload = ( @@ -344,6 +361,8 @@ def fake_get(url: str, headers=None, timeout=None) -> MagicMock: if url == RELEASES_RECORD_URL: if not record_payload: return missing_file_response() + if isinstance(record_payload, bytes): + return file_response(record_payload) return manifest_response(record_payload) return manifest_response(manifests_to_serve.pop(0)) @@ -559,10 +578,196 @@ def test__given_tagged_release__then_paths_and_tag_are_as_before(self, tmp_path) assert result.build_id == tag +#: The largest ``releases.json`` the release writer accepts +#: (``microcosm_emitter_interface.releases.MAX_RECORD_BYTES``). +RELEASES_RECORD_SIZE_LIMIT = 4 * 1024 * 1024 + + +def national_release_entry(record: dict) -> dict: + return record["releases"][NATIONAL_RELEASE] + + +def dense_release_entry(record: dict) -> dict: + return record["releases"][DENSE_RELEASE] + + +def certified_build_entry(record: dict) -> dict: + return national_release_entry(record)["builds"][NATIONAL_BUILD] + + +def earlier_national_build_entry(record: dict) -> dict: + return national_release_entry(record)["builds"][EARLIER_NATIONAL_BUILD] + + +def make_withdrawn_build_current(record: dict) -> None: + earlier_national_build_entry(record)["withdrawn"] = True + national_release_entry(record)["current"] = EARLIER_NATIONAL_BUILD + + +def published_build_entry() -> dict: + return {"published_at": "2026-10-19T08:33:23Z", "withdrawn": False} + + +#: Changes that turn ``releases_record_payload()`` into a record the release +#: writer's parser (``ReleasesRecord.parse`` with ``check_record``, +#: microcosm-emitter#24) refuses. Unless the rule needs otherwise, each record +#: still lists the certified national build as not withdrawn. +RECORD_CHANGES_THE_WRITER_REFUSES = [ + pytest.param( + lambda record: record.pop("schema_version"), id="missing schema_version" + ), + pytest.param(lambda record: record.update(schema_version=2), id="schema_version 2"), + pytest.param(lambda record: record.pop("default"), id="missing default"), + pytest.param(lambda record: record.pop("releases"), id="missing releases"), + pytest.param( + lambda record: record.update(comment="edited by hand"), + id="unknown top-level key", + ), + pytest.param(lambda record: record.update(default=1), id="default not a string"), + pytest.param( + lambda record: record.update(default="microcosm-uk-2024-25-unlisted"), + id="default names an unlisted release", + ), + pytest.param( + lambda record: record["releases"].update( + {"../escape": {"role": "dense", "current": None, "builds": {}}} + ), + id="unsafe release name", + ), + pytest.param(lambda record: dense_release_entry(record).pop("role"), id="no role"), + pytest.param( + lambda record: dense_release_entry(record).update(role="Dense-Release"), + id="invalid role", + ), + pytest.param( + lambda record: dense_release_entry(record).pop("current"), id="no current" + ), + pytest.param( + lambda record: dense_release_entry(record).update(current=1), + id="current not a string", + ), + pytest.param( + lambda record: dense_release_entry(record).pop("builds"), id="no builds" + ), + pytest.param( + lambda record: dense_release_entry(record).update(builds=[]), + id="builds not an object", + ), + pytest.param( + lambda record: dense_release_entry(record).update(note="edited by hand"), + id="unknown release key", + ), + pytest.param( + lambda record: dense_release_entry(record)["builds"].update( + {"build with spaces": published_build_entry()} + ), + id="unsafe build name", + ), + pytest.param( + lambda record: certified_build_entry(record).pop("published_at"), + id="certified build without published_at", + ), + pytest.param( + lambda record: earlier_national_build_entry(record).update(published_at=""), + id="empty published_at", + ), + pytest.param( + lambda record: earlier_national_build_entry(record).update( + published_at=20261019 + ), + id="published_at not a string", + ), + pytest.param( + lambda record: earlier_national_build_entry(record).update(withdrawn="false"), + id="withdrawn not a boolean", + ), + pytest.param( + lambda record: earlier_national_build_entry(record).update(sha256="a" * 64), + id="unknown build key", + ), + pytest.param( + lambda record: national_release_entry(record)["builds"].update( + {EARLIER_NATIONAL_BUILD: "2026-10-14T21:44:43Z"} + ), + id="build entry not an object", + ), + pytest.param( + lambda record: national_release_entry(record).update( + current="uk-frs-calibration-attempt-20261020T000000Z-unlisted" + ), + id="current names an unlisted build", + ), + pytest.param(make_withdrawn_build_current, id="current names a withdrawn build"), +] + +#: Changes after which the release writer's parser still accepts the record. +RECORD_CHANGES_THE_WRITER_ACCEPTS = [ + pytest.param(lambda record: record.update(default=None), id="no default"), + pytest.param( + lambda record: record["releases"].update( + { + "microcosm-uk-2024-25-empty": { + "role": "dense", + "current": None, + "builds": {}, + } + } + ), + id="release without builds", + ), + pytest.param( + lambda record: earlier_national_build_entry(record).update(withdrawn=True), + id="withdrawn build that is not current", + ), +] + + class TestCertifyOnlyListedBuilds: """``releases.json`` at ``main`` is the record of every published build and of which ones are withdrawn; certification reads it.""" + @pytest.mark.parametrize("change_record", RECORD_CHANGES_THE_WRITER_REFUSES) + def test__given_record_the_release_writer_refuses__then_refuses( + self, tmp_path, change_record: Callable[[dict], object] + ): + record = releases_record_payload() + change_record(record) + + with pytest.raises(CertificationError, match="releases.json"): + certify_uk( + empty_bundle(tmp_path), + [folder_layout_manifest_payload()], + releases_record=record, + ) + + def test__given_record_larger_than_the_release_writer_accepts__then_refuses( + self, tmp_path + ): + record_bytes = json.dumps(releases_record_payload()).encode() + padding = b" " * (RELEASES_RECORD_SIZE_LIMIT + 1 - len(record_bytes)) + + with pytest.raises(CertificationError, match="releases.json"): + certify_uk( + empty_bundle(tmp_path), + [folder_layout_manifest_payload()], + releases_record=record_bytes + padding, + ) + + @pytest.mark.parametrize("change_record", RECORD_CHANGES_THE_WRITER_ACCEPTS) + def test__given_record_the_release_writer_accepts__then_certifies( + self, tmp_path, change_record: Callable[[dict], object] + ): + record = releases_record_payload() + change_record(record) + + _, release, _ = certify_uk( + empty_bundle(tmp_path), + [folder_layout_manifest_payload()], + releases_record=record, + ) + + assert release["version"] == NATIONAL_BUILD + def test__given_listed_build__then_reads_releases_record_at_main(self, tmp_path): requested_urls: list[str] = [] From 9030190968baa70bda10682b1ab23e8e2ad99d6a Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 23:06:57 +0400 Subject: [PATCH 10/12] Certify only from a releases.json record the release writer accepts check_build_listed now parses the whole record with the release writer's rules before it looks up the build, so .py no longer certifies from a record that the writer and its other readers refuse. The rules repeat those of microcosm-emitter's ReleasesRecord.parse and check_record (microcosm_emitter_interface.releases, microcosm-emitter#24 at 35481d4d): exact keys at every level, schema_version 1, safe release and build names, valid roles, a non-empty published_at and a boolean withdrawn on every build, a default that is null or a listed release, a current that is null or a listed build that is not withdrawn, and at most 4 MiB. .py does not depend on that package; a comment names the writer as the source of the rules. Also corrects the certification guide: a build withdrawn after it was certified keeps loading in every installed .py release whose bundle pins it, and certifying again changes only the .py releases published afterwards. That is a documentation correction, so it needs no red step. Green: 68 passed (the test file is unchanged since the previous commit). Co-Authored-By: Claude Opus 5.5 --- changelog.d/682.added.md | 2 +- docs/engineering/skills/data-certification.md | 10 +- src/policyengine/provenance/certification.py | 5 +- src/policyengine/provenance/manifest.py | 145 ++++++++++++++---- 4 files changed, 128 insertions(+), 34 deletions(-) diff --git a/changelog.d/682.added.md b/changelog.d/682.added.md index 56a58aa0..fd763093 100644 --- a/changelog.d/682.added.md +++ b/changelog.d/682.added.md @@ -1 +1 @@ -Certify Microcosm builds published side by side in their own release folders (`releases///`, with no Hugging Face tag): a release manifest that names its `release` and `build` as strings is certified from that folder, its own files are pinned inside it at the certified revision, and its `build` names the certificate, so two builds of one release never match. Certification refuses such a build unless the repository's `releases.json` at `main` lists it and has not withdrawn it, and refuses any of its own files that is not a bare file name directly inside its folder. At import, a remote manifest that names another release or build than the bundle's pinned folder is refused. Manifests published before this layout are read as before. +Certify Microcosm builds published side by side in their own release folders (`releases///`, with no Hugging Face tag): a release manifest that names its `release` and `build` as strings is certified from that folder, its own files are pinned inside it at the certified revision, and its `build` names the certificate, so two builds of one release never match. Certification refuses such a build unless the repository's `releases.json` at `main` lists it, has not withdrawn it, and is a record the release writer's parser accepts, and refuses any of its own files that is not a bare file name directly inside its folder. At import, a remote manifest that names another release or build than the bundle's pinned folder is refused. Manifests published before this layout are read as before. diff --git a/docs/engineering/skills/data-certification.md b/docs/engineering/skills/data-certification.md index ed3ed488..154c190e 100644 --- a/docs/engineering/skills/data-certification.md +++ b/docs/engineering/skills/data-certification.md @@ -56,6 +56,13 @@ python scripts/bundle.py certify-data --country uk --data-producer populace \ mark it withdrawn, or certification refuses it. A local-area release passed with `--regional-manifest-uri` is checked the same way. Which build is `current` doesn't matter: a certificate pins one build. +- The whole record must also be one the release writer's parser + (microcosm-emitter's `ReleasesRecord.parse`) accepts, or certification + refuses it. .py repeats the writer's rules: exactly the keys the writer + writes, at every level; `schema_version` 1; safe release and build names and + valid roles; a non-empty `published_at` and a boolean `withdrawn` on every + build; a `default` that is null or a listed release; a `current` that is null + or a listed build that isn't withdrawn; and at most 4 MiB. - The build's own files are pinned at `releases///` and at the URI's revision, whatever revision their manifest entries name. Each one's `path` must be a bare file name, alone or after the folder, as the @@ -67,7 +74,8 @@ python scripts/bundle.py certify-data --country uk --data-producer populace \ - At import, a remote manifest that names another release or build than the bundle's pinned folder is refused, even when the bundle carries a certificate. Import doesn't read `releases.json`: a build withdrawn after it - was certified keeps loading until the bundle is certified again. + was certified keeps loading in every installed .py release whose bundle pins + it. Certifying again changes only the .py releases published afterwards. Manifests published before this layout, whose `build` is an object, are certified as described above. diff --git a/src/policyengine/provenance/certification.py b/src/policyengine/provenance/certification.py index b81b142d..af375e5e 100644 --- a/src/policyengine/provenance/certification.py +++ b/src/policyengine/provenance/certification.py @@ -34,7 +34,7 @@ from that folder at the commit that published it; its own files are pinned inside the folder at that revision, and its ``build`` names the certificate. The repository's ``releases.json`` at ``main`` must list the build and not -have withdrawn it. +have withdrawn it, and must be a record the release writer accepts. ``scripts/bundle.py certify-data`` is the operator-facing wrapper. Network access is required (HF manifest fetch + PyPI wheel metadata). Countries whose @@ -296,7 +296,8 @@ def fetch_release_manifest( A build published in its own release folder must be read from that folder, and its own files are pinned inside it at the URI's revision (``read_build_folder``). The repository's ``releases.json`` at ``main`` - must list the build and not have withdrawn it.""" + must list the build and not have withdrawn it, and must be a record the + release writer accepts (``check_build_listed``).""" parts = parse_manifest_uri(manifest_uri) url = https_manifest_url(parts) headers = {"Authorization": f"Bearer {token}"} if token else {} diff --git a/src/policyengine/provenance/manifest.py b/src/policyengine/provenance/manifest.py index 4187597e..8f27713b 100644 --- a/src/policyengine/provenance/manifest.py +++ b/src/policyengine/provenance/manifest.py @@ -32,10 +32,19 @@ RELEASES_RECORD_NAME = "releases.json" RELEASES_RECORD_REVISION = "main" RELEASES_RECORD_SCHEMA_VERSION = 1 +#: The largest ``releases.json`` the release writer reads +#: (``microcosm_emitter_interface.releases.MAX_RECORD_BYTES``). +RELEASES_RECORD_MAX_BYTES = 4 * 1024 * 1024 #: The publisher refuses any other release or build name #: (``microcosm_emitter_interface.identifiers.SAFE_IDENTIFIER``), and both #: are interpolated into repository paths. RELEASE_FOLDER_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") +#: The publisher refuses any other release role +#: (``microcosm_emitter_interface.identifiers.ROLE_IDENTIFIER``). +RELEASE_ROLE_NAME = re.compile(r"^[a-z][a-z0-9_]{0,63}$") +_RELEASES_RECORD_KEYS = frozenset({"schema_version", "default", "releases"}) +_RELEASE_ENTRY_KEYS = frozenset({"role", "current", "builds"}) +_BUILD_ENTRY_KEYS = frozenset({"published_at", "withdrawn"}) class DataReleaseManifestUnavailableError(ValueError): @@ -417,48 +426,124 @@ def check_build_listed(record_bytes: bytes, *, release: str, build: str) -> None "published_at": ..., "withdrawn": }}}}} Which build is current does not matter here: a certificate pins one - build, current or not. + build, current or not. The whole record must still be one the release + writer accepts (``_read_releases_record``). Raises: - ReleaseBuildFolderError: If the record is unreadable, has another - schema version, does not list the build, or marks it withdrawn. + ReleaseBuildFolderError: If the release writer would refuse the + record, or the record does not list the build, or marks it + withdrawn. """ + record = _read_releases_record(record_bytes) + release_entry = record["releases"].get(release) + if release_entry is None or build not in release_entry["builds"]: + raise ReleaseBuildFolderError( + f"{RELEASES_RECORD_NAME} does not list build {build} of release {release}." + ) + if release_entry["builds"][build]["withdrawn"]: + raise ReleaseBuildFolderError( + f"Build {build} of release {release} is withdrawn in " + f"{RELEASES_RECORD_NAME}." + ) + + +def _read_releases_record(record_bytes: bytes) -> dict[str, Any]: + """Parse ``releases.json``, refusing any record the release writer refuses. + + These rules repeat those of the release writer's parser, microcosm-emitter's + ``microcosm_emitter_interface.releases.ReleasesRecord.parse`` and its + ``check_record``, so that .py certifies only from a record the writer and + its other readers accept. .py does not depend on that package; when the + writer's rules change, change these with them. + + Raises: + ReleaseBuildFolderError: If the record is larger than + ``RELEASES_RECORD_MAX_BYTES`` or not a JSON document, has other + keys or another schema version, has a malformed release or build + entry or an unsafe release or build name, or names as ``default`` + a release it does not list or as ``current`` a build it does not + list or has withdrawn. + """ + + def refuse(problem: str) -> ReleaseBuildFolderError: + return ReleaseBuildFolderError(f"{RELEASES_RECORD_NAME} {problem}") + + if len(record_bytes) > RELEASES_RECORD_MAX_BYTES: + raise refuse(f"is larger than {RELEASES_RECORD_MAX_BYTES} bytes.") try: record = json.loads(record_bytes) except (ValueError, UnicodeDecodeError, RecursionError): - raise ReleaseBuildFolderError( - f"{RELEASES_RECORD_NAME} is not a readable JSON document." - ) from None - schema_version = record.get("schema_version") if isinstance(record, dict) else None + raise refuse("is not a readable JSON document.") from None + if not isinstance(record, dict) or set(record) != _RELEASES_RECORD_KEYS: + raise refuse(f"must have exactly the keys {sorted(_RELEASES_RECORD_KEYS)}.") + schema_version = record["schema_version"] # ``True`` and ``1.0`` equal 1 in Python; only the integer is the version. if ( type(schema_version) is not int or schema_version != RELEASES_RECORD_SCHEMA_VERSION ): - raise ReleaseBuildFolderError( - f"{RELEASES_RECORD_NAME} must have schema_version " - f"{RELEASES_RECORD_SCHEMA_VERSION}." - ) - releases = record.get("releases") - release_entry = releases.get(release) if isinstance(releases, dict) else None - builds = release_entry.get("builds") if isinstance(release_entry, dict) else None - build_entry = builds.get(build) if isinstance(builds, dict) else None - if not isinstance(build_entry, dict): - raise ReleaseBuildFolderError( - f"{RELEASES_RECORD_NAME} does not list build {build} of release {release}." - ) - withdrawn = build_entry.get("withdrawn") - if not isinstance(withdrawn, bool): - raise ReleaseBuildFolderError( - f"{RELEASES_RECORD_NAME} gives build {build} of release {release} " - "no withdrawn flag." - ) - if withdrawn: - raise ReleaseBuildFolderError( - f"Build {build} of release {release} is withdrawn in " - f"{RELEASES_RECORD_NAME}." - ) + raise refuse(f"must have schema_version {RELEASES_RECORD_SCHEMA_VERSION}.") + releases = record["releases"] + if not isinstance(releases, dict): + raise refuse("must have an object as releases.") + for release_name, release_entry in releases.items(): + if not RELEASE_FOLDER_NAME.fullmatch(release_name): + raise refuse("names a release that is not a safe identifier.") + if ( + not isinstance(release_entry, dict) + or set(release_entry) != _RELEASE_ENTRY_KEYS + ): + raise refuse( + f"must give release {release_name} exactly the keys " + f"{sorted(_RELEASE_ENTRY_KEYS)}." + ) + role = release_entry["role"] + if not isinstance(role, str) or not RELEASE_ROLE_NAME.fullmatch(role): + raise refuse(f"gives release {release_name} an invalid role.") + current = release_entry["current"] + if current is not None and not isinstance(current, str): + raise refuse( + f"must give release {release_name} a build or null as current." + ) + builds = release_entry["builds"] + if not isinstance(builds, dict): + raise refuse(f"must give release {release_name} an object as builds.") + for build_name, build_entry in builds.items(): + if not RELEASE_FOLDER_NAME.fullmatch(build_name): + raise refuse( + f"names a build of release {release_name} that is not a " + "safe identifier." + ) + if ( + not isinstance(build_entry, dict) + or set(build_entry) != _BUILD_ENTRY_KEYS + or not isinstance(build_entry["published_at"], str) + or not build_entry["published_at"] + or not isinstance(build_entry["withdrawn"], bool) + ): + raise refuse( + f"must give build {build_name} of release {release_name} " + "a non-empty published_at string and a boolean withdrawn, " + "and nothing else." + ) + if current is not None: + if current not in builds: + raise refuse( + f"names unlisted build {current} as the current build of " + f"release {release_name}." + ) + if builds[current]["withdrawn"]: + raise refuse( + f"names withdrawn build {current} as the current build of " + f"release {release_name}." + ) + default = record["default"] + if default is not None and not isinstance(default, str): + raise refuse("must have a release name or null as default.") + if default is not None and default not in releases: + raise refuse(f"names unlisted release {default} as default.") + return record def build_hf_uri(repo_id: str, path_in_repo: str, revision: str) -> str: From 89c2cff38251096dfa477069ec3b803a22903a1c Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 21:51:33 +0400 Subject: [PATCH 11/12] Test the releases.json rules most likely to drift from the writer's .py repeats the release writer's parser for releases.json instead of importing it. These cases pin the rules that are easiest to lose in a later edit: schema_version must be the integer 1 (not true or 1.0), and a record that is not JSON, not text, or nested too deeply to decode is refused like any other record the writer refuses. They pass already; the fake response now decodes text the way requests does. Co-Authored-By: Claude Opus 5.5 --- tests/test_release_build_folders.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/tests/test_release_build_folders.py b/tests/test_release_build_folders.py index 98a657a8..6197bae7 100644 --- a/tests/test_release_build_folders.py +++ b/tests/test_release_build_folders.py @@ -245,7 +245,8 @@ def file_response(content: bytes) -> MagicMock: response = MagicMock() response.status_code = 200 response.content = content - response.text = content.decode() + # Like requests, which replaces bytes it cannot decode. + response.text = content.decode(errors="replace") response.raise_for_status.return_value = None return response @@ -617,6 +618,12 @@ def published_build_entry() -> dict: lambda record: record.pop("schema_version"), id="missing schema_version" ), pytest.param(lambda record: record.update(schema_version=2), id="schema_version 2"), + pytest.param( + lambda record: record.update(schema_version=True), id="schema_version true" + ), + pytest.param( + lambda record: record.update(schema_version=1.0), id="schema_version 1.0" + ), pytest.param(lambda record: record.pop("default"), id="missing default"), pytest.param(lambda record: record.pop("releases"), id="missing releases"), pytest.param( @@ -753,6 +760,24 @@ def test__given_record_larger_than_the_release_writer_accepts__then_refuses( releases_record=record_bytes + padding, ) + @pytest.mark.parametrize( + "record_bytes", + [ + pytest.param(b'{"schema_version": 1,', id="truncated JSON"), + pytest.param(b"\xff\xfe\x00", id="not text"), + pytest.param(b"[" * 100_000 + b"]" * 100_000, id="nested too deeply"), + ], + ) + def test__given_record_the_release_writer_cannot_read__then_refuses( + self, tmp_path, record_bytes: bytes + ): + with pytest.raises(CertificationError, match="releases.json"): + certify_uk( + empty_bundle(tmp_path), + [folder_layout_manifest_payload()], + releases_record=record_bytes, + ) + @pytest.mark.parametrize("change_record", RECORD_CHANGES_THE_WRITER_ACCEPTS) def test__given_record_the_release_writer_accepts__then_certifies( self, tmp_path, change_record: Callable[[dict], object] From 72347a46d446dc6614d6faea635e5fe7c541614d Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 21:51:33 +0400 Subject: [PATCH 12/12] Say that ReleaseBuildFolderError also reports releases.json refusals Certification raises it when releases.json is a record the release writer refuses, does not list the build, or marks it withdrawn, as well as for a manifest or file outside its build folder. Co-Authored-By: Claude Opus 5.5 --- src/policyengine/provenance/manifest.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/policyengine/provenance/manifest.py b/src/policyengine/provenance/manifest.py index 8f27713b..285d0967 100644 --- a/src/policyengine/provenance/manifest.py +++ b/src/policyengine/provenance/manifest.py @@ -52,10 +52,12 @@ class DataReleaseManifestUnavailableError(ValueError): class ReleaseBuildFolderError(ValueError): - """A build in its own release folder that does not match that folder. + """A build in its own release folder that cannot be certified or loaded. Its manifest names another release or build than the folder it was read - from, or one of its own files lies outside the folder. Unlike an + from, or one of its own files lies outside the folder; or, at + certification, the repository's ``releases.json`` is a record the release + writer refuses, does not list the build, or marks it withdrawn. Unlike an unavailable manifest, this is never covered by a bundled certificate. """