diff --git a/changelog.d/682.added.md b/changelog.d/682.added.md new file mode 100644 index 000000000..fd763093a --- /dev/null +++ b/changelog.d/682.added.md @@ -0,0 +1 @@ +Certify Microcosm builds published side by side in their own release folders (`releases///`, with no Hugging Face tag): a release manifest that names its `release` and `build` as strings is certified from that folder, its own files are pinned inside it at the certified revision, and its `build` names the certificate, so two builds of one release never match. Certification refuses such a build unless the repository's `releases.json` at `main` lists it, has not withdrawn it, and is a record the release writer's parser accepts, and refuses any of its own files that is not a bare file name directly inside its folder. At import, a remote manifest that names another release or build than the bundle's pinned folder is refused. Manifests published before this layout are read as before. diff --git a/docs/engineering/skills/data-certification.md b/docs/engineering/skills/data-certification.md index 42eec89be..154c190ee 100644 --- a/docs/engineering/skills/data-certification.md +++ b/docs/engineering/skills/data-certification.md @@ -34,6 +34,52 @@ uses the cut tag as the build id (`data_releases..version`, the certification's `data_build_id` and the changelog fragment name), so two cuts of one line never look identical. +### Builds in their own release folders + +Once Microcosm publishes each build in its own folder, +`releases///`, a build is never changed after publishing and +gets no Hugging Face tag. The repository's root `releases.json` records which +build of each release is current. That build's `release_manifest.json` names +its `release` and `build` as strings, for example +`microcosm-uk-2024-25-national` and +`uk-frs-calibration-attempt-20261019T083323Z-6cba5d71`. Certify the build from +its own folder, at the commit that published it: + +```bash +python scripts/bundle.py certify-data --country uk --data-producer populace \ + --manifest-uri "hf://dataset/policyengine/populace-uk-private@/releases///release_manifest.json" +``` + +- The URI must be the build's own folder: a manifest that names another + release or build than the folder it was read from is refused. +- The repository's `releases.json` at `main` must list the build and must not + mark it withdrawn, or certification refuses it. A local-area release passed + with `--regional-manifest-uri` is checked the same way. Which build is + `current` doesn't matter: a certificate pins one build. +- The whole record must also be one the release writer's parser + (microcosm-emitter's `ReleasesRecord.parse`) accepts, or certification + refuses it. .py repeats the writer's rules: exactly the keys the writer + writes, at every level; `schema_version` 1; safe release and build names and + valid roles; a non-empty `published_at` and a boolean `withdrawn` on every + build; a `default` that is null or a listed release; a `current` that is null + or a listed build that isn't withdrawn; and at most 4 MiB. +- The build's own files are pinned at `releases///` and + at the URI's revision, whatever revision their manifest entries name. Each + one's `path` must be a bare file name, alone or after the folder, as the + release writer publishes it; a nested path, a `.` or `..` segment or a + backslash is refused. Artifacts from other repositories keep their own pins. +- The certificate names the build by `build` + (`data_releases..version`, the certification's `data_build_id` and + the changelog fragment name), so two builds of one release never match. +- At import, a remote manifest that names another release or build than the + bundle's pinned folder is refused, even when the bundle carries a + certificate. Import doesn't read `releases.json`: a build withdrawn after it + was certified keeps loading in every installed .py release whose bundle pins + it. Certifying again changes only the .py releases published afterwards. + +Manifests published before this layout, whose `build` is an object, are +certified as described above. + ### UK constituency and local-authority regions UK constituency and local-authority regions row-filter the dataset a regional diff --git a/src/policyengine/provenance/certification.py b/src/policyengine/provenance/certification.py index 31327f576..af375e5e7 100644 --- a/src/policyengine/provenance/certification.py +++ b/src/policyengine/provenance/certification.py @@ -29,6 +29,13 @@ Pin the cut tag (``@``), never a branch: a release line's directory (``releases//``) is rewritten by each cut. +A build published in its own folder (``releases///``, named +by its manifest's ``release`` and ``build`` strings) has no tag. Certify it +from that folder at the commit that published it; its own files are pinned +inside the folder at that revision, and its ``build`` names the certificate. +The repository's ``releases.json`` at ``main`` must list the build and not +have withdrawn it, and must be a record the release writer accepts. + ``scripts/bundle.py certify-data`` is the operator-facing wrapper. Network access is required (HF manifest fetch + PyPI wheel metadata). Countries whose data release predates release manifests need a dedicated data-producer strategy @@ -54,11 +61,16 @@ ) from policyengine.provenance.manifest import ( HF_REQUEST_TIMEOUT_SECONDS, + RELEASES_RECORD_NAME, + RELEASES_RECORD_REVISION, DataReleaseArtifact, DataReleaseManifest, RegionalReleaseKind, + ReleaseBuildFolderError, _specifier_matches, + check_build_listed, fetch_pypi_wheel_metadata, + read_build_folder, release_build_id, ) @@ -239,12 +251,53 @@ def release_manifest_dir(parts: dict) -> str: return parts["path"].rsplit("/", 1)[0] +def https_releases_record_url(parts: dict) -> str: + """The repository's ``releases.json`` at ``main``, its current record.""" + prefix = "datasets/" if parts["repo_type"] == "dataset" else "" + return ( + f"https://huggingface.co/{prefix}{parts['repo_id']}/resolve/" + f"{RELEASES_RECORD_REVISION}/{RELEASES_RECORD_NAME}" + ) + + +def _require_listed_build( + manifest: DataReleaseManifest, parts: dict, headers: dict +) -> None: + """Refuse a build in its own folder that ``releases.json`` at ``main`` + does not list, or has withdrawn. + + The record is read at ``main``, not at the URI's revision: a build is + withdrawn after the commit that published it. + """ + release, build = manifest.release, release_build_id(manifest.build) + if release is None or build is None: + return + url = https_releases_record_url(parts) + response = requests.get(url, headers=headers, timeout=HF_REQUEST_TIMEOUT_SECONDS) + if response.status_code != 200: + raise CertificationError( + f"Could not fetch {RELEASES_RECORD_NAME} ({response.status_code}) " + f"from {url}; a build in its own release folder is certified only " + "while that record lists it." + ) + try: + check_build_listed(response.content, release=release, build=build) + except ReleaseBuildFolderError as exc: + raise CertificationError(str(exc)) from exc + + def fetch_release_manifest( manifest_uri: str, token: Optional[str] = None, ) -> tuple[DataReleaseManifest, str, dict]: """Fetch and parse the data release manifest; returns - (manifest, sha256 of the fetched bytes, parsed URI parts).""" + (manifest, sha256 of the fetched bytes, parsed URI parts). + + A build published in its own release folder must be read from that + folder, and its own files are pinned inside it at the URI's revision + (``read_build_folder``). The repository's ``releases.json`` at ``main`` + must list the build and not have withdrawn it, and must be a record the + release writer accepts (``check_build_listed``).""" parts = parse_manifest_uri(manifest_uri) url = https_manifest_url(parts) headers = {"Authorization": f"Bearer {token}"} if token else {} @@ -257,6 +310,17 @@ def fetch_release_manifest( sha256 = hashlib.sha256(response.content).hexdigest() payload = json.loads(response.content) manifest = DataReleaseManifest.model_validate(payload) + try: + manifest = read_build_folder( + manifest, + repo_id=parts["repo_id"], + manifest_path=parts["path"], + revision=parts["revision"], + ) + except ReleaseBuildFolderError as exc: + raise CertificationError(str(exc)) from exc + if manifest.build_folder is not None: + _require_listed_build(manifest, parts, headers) return manifest, sha256, parts diff --git a/src/policyengine/provenance/manifest.py b/src/policyengine/provenance/manifest.py index d43e3f74b..285d09670 100644 --- a/src/policyengine/provenance/manifest.py +++ b/src/policyengine/provenance/manifest.py @@ -1,23 +1,67 @@ import hashlib import json import os +import re from functools import lru_cache from importlib.resources import files from pathlib import Path -from typing import Literal, Optional +from typing import Any, Literal, Optional from urllib.parse import quote import requests -from pydantic import BaseModel, Field +from pydantic import ( + BaseModel, + Field, + SerializerFunctionWrapHandler, + model_serializer, + model_validator, +) HF_REQUEST_TIMEOUT_SECONDS = 30 PYPI_REQUEST_TIMEOUT_SECONDS = 30 +#: Microcosm publishes each build once, in its own folder +#: ``releases///``, and never changes it afterwards; a root +#: ``releases.json`` records which build of each release is current +#: (microcosm-emitter's ``microcosm_emitter_interface.releases``). +RELEASES_FOLDER = "releases" +RELEASE_MANIFEST_NAME = "release_manifest.json" +#: The record of every published build and of which ones are withdrawn, at +#: the repository's root. Only the release writer changes it, always on +#: ``main``, so ``main`` holds the current record. +RELEASES_RECORD_NAME = "releases.json" +RELEASES_RECORD_REVISION = "main" +RELEASES_RECORD_SCHEMA_VERSION = 1 +#: The largest ``releases.json`` the release writer reads +#: (``microcosm_emitter_interface.releases.MAX_RECORD_BYTES``). +RELEASES_RECORD_MAX_BYTES = 4 * 1024 * 1024 +#: The publisher refuses any other release or build name +#: (``microcosm_emitter_interface.identifiers.SAFE_IDENTIFIER``), and both +#: are interpolated into repository paths. +RELEASE_FOLDER_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") +#: The publisher refuses any other release role +#: (``microcosm_emitter_interface.identifiers.ROLE_IDENTIFIER``). +RELEASE_ROLE_NAME = re.compile(r"^[a-z][a-z0-9_]{0,63}$") +_RELEASES_RECORD_KEYS = frozenset({"schema_version", "default", "releases"}) +_RELEASE_ENTRY_KEYS = frozenset({"role", "current", "builds"}) +_BUILD_ENTRY_KEYS = frozenset({"published_at", "withdrawn"}) + class DataReleaseManifestUnavailableError(ValueError): """Raised when a data release manifest cannot be fetched or is absent.""" +class ReleaseBuildFolderError(ValueError): + """A build in its own release folder that cannot be certified or loaded. + + Its manifest names another release or build than the folder it was read + from, or one of its own files lies outside the folder; or, at + certification, the repository's ``releases.json`` is a record the release + writer refuses, does not list the build, or marks it withdrawn. Unlike an + unavailable manifest, this is never covered by a bundled certificate. + """ + + class PackageVersion(BaseModel): name: str version: str @@ -47,17 +91,25 @@ class BuiltWithModelPackage(PackageVersion): class DataBuildInfo(BaseModel): build_id: Optional[str] = None + """The build's name. For a build published in its own release folder + (``DataReleaseManifest.release`` set), this is the manifest's ``build`` + string, the attempt ID that names the folder.""" cut_tag: Optional[str] = None """The immutable tag of one cut of a release line. Microcosm keeps ``build_id`` equal to the line name for every cut, so the cut tag is what - tells two cuts apart (see ``release_build_id``).""" + tells two cuts apart (see ``release_build_id``). Builds published in their + own release folders have no tag.""" built_at: Optional[str] = None built_with_core_package: Optional[PackageVersion] = None built_with_model_package: Optional[BuiltWithModelPackage] = None def release_build_id(build: Optional[DataBuildInfo]) -> Optional[str]: - """Identify one data build: its cut tag when it has one, else its build id.""" + """Identify one data build: its cut tag when it has one, else its build id. + + A build published in its own release folder has no cut tag, and its build + id is the folder's build name, so two builds of one release differ. + """ if build is None: return None return build.cut_tag or build.build_id @@ -116,7 +168,10 @@ class DataReleaseArtifact(BaseModel): kind: str path: str repo_id: str - revision: str + revision: Optional[str] = None + """Required unless the manifest is a build in its own release folder, + whose own files are pinned at the revision its manifest is certified at + (``read_build_folder``).""" repo_type: Optional[Literal["model", "dataset"]] = None sha256: Optional[str] = None size_bytes: Optional[int] = None @@ -127,6 +182,8 @@ class DataReleaseArtifact(BaseModel): @property def uri(self) -> str: + if self.revision is None: + raise ValueError(f"Artifact {self.path!r} has no revision pin.") return build_hf_uri( repo_id=self.repo_id, path_in_repo=self.path, @@ -144,6 +201,12 @@ class DataReleaseManifest(BaseModel): compatible_model_packages: list[CompatiblePackage] = Field(default_factory=list) compatible_core_packages: list[CompatiblePackage] = Field(default_factory=list) default_datasets: dict[str, str] = Field(default_factory=dict) + release: Optional[str] = None + """The release of a build published in its own folder, + ``releases///``. Such a manifest names its ``release`` + and ``build`` as strings; ``build`` is read as ``DataBuildInfo`` with only + ``build_id`` set, and written back as the string. ``None`` for a manifest + published before that layout, whose ``build`` is an object.""" build: Optional[DataBuildInfo] = None artifacts: dict[str, DataReleaseArtifact] = Field(default_factory=dict) preservation_dois: list[str] = Field(default_factory=list) @@ -157,6 +220,69 @@ class DataReleaseManifest(BaseModel): source_sha256: Optional[str] = Field(default=None, exclude=True) """Byte sha256 of the fetched manifest before runtime URI rewrites.""" + @model_validator(mode="before") + @classmethod + def _read_build_folder_identity(cls, data: Any) -> Any: + """Read a string ``build`` with its ``release`` as the build folder.""" + + if not isinstance(data, dict): + return data + release = data.get("release") + build = data.get("build") + if release is None and not isinstance(build, str): + return data + if release is None: + raise ValueError( + "A release manifest that names its build as a string must " + "name its release." + ) + if not isinstance(build, str): + raise ValueError( + "A release manifest that names its release must name its " + "build as a string." + ) + for label, name in (("release", release), ("build", build)): + if not isinstance(name, str) or not RELEASE_FOLDER_NAME.fullmatch(name): + raise ValueError(f"{label} must match {RELEASE_FOLDER_NAME.pattern}.") + return {**data, "build": {"build_id": build}} + + @model_validator(mode="after") + def _require_revision_pins_outside_build_folders(self) -> "DataReleaseManifest": + if self.release is None: + unpinned = sorted( + name + for name, artifact in self.artifacts.items() + if artifact.revision is None + ) + if unpinned: + raise ValueError( + f"Artifacts {unpinned} have no revision; only a build in " + "its own release folder may leave it to certification." + ) + return self + + @model_serializer(mode="wrap") + def _write_build_folder_identity( + self, handler: SerializerFunctionWrapHandler + ) -> Any: + data = handler(self) + if ( + self.release is not None + and self.build is not None + and isinstance(data, dict) + and "build" in data + ): + data["build"] = self.build.build_id + return data + + @property + def build_folder(self) -> Optional[str]: + """``releases//`` for a build in its own folder.""" + + if self.release is None or self.build is None or self.build.build_id is None: + return None + return f"{RELEASES_FOLDER}/{self.release}/{self.build.build_id}" + class DataCertification(BaseModel): compatibility_basis: str @@ -220,6 +346,208 @@ def default_dataset_uri(self) -> str: return resolve_dataset_reference(self.country_id, self.default_dataset) +def read_build_folder( + manifest: DataReleaseManifest, + *, + repo_id: str, + manifest_path: str, + revision: str, +) -> DataReleaseManifest: + """Pin a build published in its own release folder to that folder. + + ``manifest_path`` and ``revision`` say where the manifest was read from in + ``repo_id``. The manifest must be the one in its own build folder, + ``releases///release_manifest.json``. Its own files (the + artifacts in ``repo_id``) are then pinned at + ``releases///`` and at ``revision``: the publisher + writes a build's files and its manifest in one commit and never changes + them afterwards, so any revision holding the manifest holds the same + files, while an artifact's own ``revision`` field cannot name that commit. + Artifacts in other repositories keep their own paths and pins. + + The publisher writes only bare file names, directly inside the folder, so + an own file's ``path`` must be such a name, alone or after the folder. + + A manifest published before that layout is returned unchanged. + + Raises: + ReleaseBuildFolderError: If the manifest was read from another folder, + or one of its own files is not directly inside the folder. + """ + + folder = manifest.build_folder + if folder is None: + return manifest + expected_path = f"{folder}/{RELEASE_MANIFEST_NAME}" + if manifest_path != expected_path: + raise ReleaseBuildFolderError( + f"The release manifest at {manifest_path} names release " + f"{manifest.release} and build {release_build_id(manifest.build)}, " + f"whose manifest is {expected_path}." + ) + pinned = manifest.model_copy(deep=True) + for name, artifact in pinned.artifacts.items(): + if artifact.repo_id != repo_id: + continue + if "/" not in artifact.path: + file_name = artifact.path + elif artifact.path.startswith(f"{folder}/"): + file_name = artifact.path.removeprefix(f"{folder}/") + else: + raise ReleaseBuildFolderError( + f"Artifact {name!r} ({artifact.path}) is outside its build " + f"folder {folder}/." + ) + if not _is_bare_file_name(file_name): + # HTTP clients remove dot segments, so "/..//" + # would fetch another build's file. + raise ReleaseBuildFolderError( + f"Artifact {name!r} ({artifact.path}) is not a file directly " + f"inside its build folder {folder}/." + ) + artifact.path = f"{folder}/{file_name}" + artifact.revision = revision + return pinned + + +def _is_bare_file_name(name: str) -> bool: + """One path segment that names a file: what the publisher accepts + (``microcosm_emitter_interface.identifiers.bare_file_name``).""" + + return bool(name) and name not in {".", ".."} and not set(name) & {"/", "\\"} + + +def check_build_listed(record_bytes: bytes, *, release: str, build: str) -> None: + """Refuse a build that ``releases.json`` does not list, or has withdrawn. + + ``record_bytes`` is the repository's ``releases.json`` in the release + writer's format (``microcosm_emitter_interface.releases``):: + + {"schema_version": 1, "default": ..., "releases": {: { + "role": ..., "current": ..., "builds": {: { + "published_at": ..., "withdrawn": }}}}} + + Which build is current does not matter here: a certificate pins one + build, current or not. The whole record must still be one the release + writer accepts (``_read_releases_record``). + + Raises: + ReleaseBuildFolderError: If the release writer would refuse the + record, or the record does not list the build, or marks it + withdrawn. + """ + + record = _read_releases_record(record_bytes) + release_entry = record["releases"].get(release) + if release_entry is None or build not in release_entry["builds"]: + raise ReleaseBuildFolderError( + f"{RELEASES_RECORD_NAME} does not list build {build} of release {release}." + ) + if release_entry["builds"][build]["withdrawn"]: + raise ReleaseBuildFolderError( + f"Build {build} of release {release} is withdrawn in " + f"{RELEASES_RECORD_NAME}." + ) + + +def _read_releases_record(record_bytes: bytes) -> dict[str, Any]: + """Parse ``releases.json``, refusing any record the release writer refuses. + + These rules repeat those of the release writer's parser, microcosm-emitter's + ``microcosm_emitter_interface.releases.ReleasesRecord.parse`` and its + ``check_record``, so that .py certifies only from a record the writer and + its other readers accept. .py does not depend on that package; when the + writer's rules change, change these with them. + + Raises: + ReleaseBuildFolderError: If the record is larger than + ``RELEASES_RECORD_MAX_BYTES`` or not a JSON document, has other + keys or another schema version, has a malformed release or build + entry or an unsafe release or build name, or names as ``default`` + a release it does not list or as ``current`` a build it does not + list or has withdrawn. + """ + + def refuse(problem: str) -> ReleaseBuildFolderError: + return ReleaseBuildFolderError(f"{RELEASES_RECORD_NAME} {problem}") + + if len(record_bytes) > RELEASES_RECORD_MAX_BYTES: + raise refuse(f"is larger than {RELEASES_RECORD_MAX_BYTES} bytes.") + try: + record = json.loads(record_bytes) + except (ValueError, UnicodeDecodeError, RecursionError): + raise refuse("is not a readable JSON document.") from None + if not isinstance(record, dict) or set(record) != _RELEASES_RECORD_KEYS: + raise refuse(f"must have exactly the keys {sorted(_RELEASES_RECORD_KEYS)}.") + schema_version = record["schema_version"] + # ``True`` and ``1.0`` equal 1 in Python; only the integer is the version. + if ( + type(schema_version) is not int + or schema_version != RELEASES_RECORD_SCHEMA_VERSION + ): + raise refuse(f"must have schema_version {RELEASES_RECORD_SCHEMA_VERSION}.") + releases = record["releases"] + if not isinstance(releases, dict): + raise refuse("must have an object as releases.") + for release_name, release_entry in releases.items(): + if not RELEASE_FOLDER_NAME.fullmatch(release_name): + raise refuse("names a release that is not a safe identifier.") + if ( + not isinstance(release_entry, dict) + or set(release_entry) != _RELEASE_ENTRY_KEYS + ): + raise refuse( + f"must give release {release_name} exactly the keys " + f"{sorted(_RELEASE_ENTRY_KEYS)}." + ) + role = release_entry["role"] + if not isinstance(role, str) or not RELEASE_ROLE_NAME.fullmatch(role): + raise refuse(f"gives release {release_name} an invalid role.") + current = release_entry["current"] + if current is not None and not isinstance(current, str): + raise refuse( + f"must give release {release_name} a build or null as current." + ) + builds = release_entry["builds"] + if not isinstance(builds, dict): + raise refuse(f"must give release {release_name} an object as builds.") + for build_name, build_entry in builds.items(): + if not RELEASE_FOLDER_NAME.fullmatch(build_name): + raise refuse( + f"names a build of release {release_name} that is not a " + "safe identifier." + ) + if ( + not isinstance(build_entry, dict) + or set(build_entry) != _BUILD_ENTRY_KEYS + or not isinstance(build_entry["published_at"], str) + or not build_entry["published_at"] + or not isinstance(build_entry["withdrawn"], bool) + ): + raise refuse( + f"must give build {build_name} of release {release_name} " + "a non-empty published_at string and a boolean withdrawn, " + "and nothing else." + ) + if current is not None: + if current not in builds: + raise refuse( + f"names unlisted build {current} as the current build of " + f"release {release_name}." + ) + if builds[current]["withdrawn"]: + raise refuse( + f"names withdrawn build {current} as the current build of " + f"release {release_name}." + ) + default = record["default"] + if default is not None and not isinstance(default, str): + raise refuse("must have a release name or null as default.") + if default is not None and default not in releases: + raise refuse(f"names unlisted release {default} as default.") + return record + + def build_hf_uri(repo_id: str, path_in_repo: str, revision: str) -> str: return f"hf://{repo_id}/{path_in_repo}@{revision}" @@ -399,6 +727,16 @@ def get_data_release_manifest(country_id: str) -> DataReleaseManifest: if not isinstance(source_bytes, bytes): source_bytes = response.text.encode("utf-8") data_release_manifest.source_sha256 = hashlib.sha256(source_bytes).hexdigest() + if data_release_manifest.build_folder is not None: + # A build in its own release folder must be the build the bundle + # pinned; ReleaseBuildFolderError is not an unavailable manifest, so + # no bundled certificate covers it. + return read_build_folder( + data_release_manifest, + repo_id=country_manifest.data_package.repo_id, + manifest_path=country_manifest.data_package.release_manifest_path, + revision=_artifact_revision(country_manifest.data_package), + ) release_revision = country_manifest.data_package.release_manifest_revision if release_revision is not None: for artifact in data_release_manifest.artifacts.values(): diff --git a/tests/test_release_build_folders.py b/tests/test_release_build_folders.py new file mode 100644 index 000000000..6197bae7e --- /dev/null +++ b/tests/test_release_build_folders.py @@ -0,0 +1,1014 @@ +"""Microcosm builds published side by side in their own release folders. + +In this layout each build is published once, in ``releases///``, +and never changed afterwards; a root ``releases.json`` records which build of +each release is current, and no new Hugging Face tags are made. Each build's +``release_manifest.json`` names its ``release`` and ``build`` as strings +(microcosm-emitter#24 checks that they equal the folder's names). + +The manifest shape below is assumed until Microcosm's release writer change +(H2c) defines it: top-level ``release`` and ``build`` strings, the existing +``artifacts`` entries with bare file names, and no ``cut_tag``. + +Invariants: + +- a build is named by its ``build``, so two builds of one release never match; +- a manifest published before the layout (``build`` an object, maybe with + ``cut_tag``) is read exactly as before. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from pydantic import ValidationError + +from policyengine.provenance.certification import ( + CertificationError, + certify_data_release, +) +from policyengine.provenance.manifest import ( + CountryReleaseManifest, + DataReleaseManifest, + ReleaseBuildFolderError, + certify_data_release_compatibility, + get_data_release_manifest, + release_build_id, +) + +REPOSITORY = "policyengine/populace-uk-private" +NATIONAL_RELEASE = "microcosm-uk-2024-25-national" +NATIONAL_BUILD = "uk-frs-calibration-attempt-20261019T083323Z-6cba5d71" +EARLIER_NATIONAL_BUILD = "uk-frs-calibration-attempt-20261014T214443Z-d12ff9b3" +UK_MODEL_VERSION = "2.126.12" + + +def folder_layout_manifest_payload( + release: str = NATIONAL_RELEASE, + build: str = NATIONAL_BUILD, +) -> dict: + """A national release manifest in the side-by-side layout.""" + + return { + "schema_version": 1, + "release": release, + "build": build, + "data_package": {"name": "microcosm-data", "version": "0.1.0"}, + "default_datasets": {"national": "microcosm_uk_2024_25"}, + "compatible_model_packages": [ + {"name": "policyengine-uk", "specifier": f"=={UK_MODEL_VERSION}"} + ], + "artifacts": { + "microcosm_uk_2024_25": { + "kind": "microdata", + "role": "dataset", + "path": "microcosm_uk_2024_25.h5", + "repo_id": REPOSITORY, + "revision": "main", + "sha256": "a" * 64, + }, + "calibration_diagnostics": { + "kind": "diagnostics", + "role": "calibration_diagnostics", + "path": "calibration_diagnostics.json", + "repo_id": REPOSITORY, + "revision": "main", + "sha256": "b" * 64, + }, + }, + } + + +def tagged_layout_manifest_payload() -> dict: + """A manifest published before the layout: ``build`` is an object.""" + + tag = f"{NATIONAL_RELEASE}-20261019T083323Z-6cba5d71" + payload = folder_layout_manifest_payload() + del payload["release"] + payload["build"] = { + "build_id": NATIONAL_RELEASE, + "cut_tag": tag, + "built_with_model_package": { + "name": "policyengine-uk", + "version": UK_MODEL_VERSION, + }, + } + for artifact in payload["artifacts"].values(): + artifact["revision"] = tag + return payload + + +class TestBuildFolderManifest: + def test__given_folder_layout_manifest__then_build_is_the_data_build_id(self): + manifest = DataReleaseManifest.model_validate(folder_layout_manifest_payload()) + + assert manifest.release == NATIONAL_RELEASE + assert release_build_id(manifest.build) == NATIONAL_BUILD + + def test__given_folder_layout_manifest__then_names_its_build_folder(self): + manifest = DataReleaseManifest.model_validate(folder_layout_manifest_payload()) + + assert manifest.build_folder == ( + f"releases/{NATIONAL_RELEASE}/{NATIONAL_BUILD}" + ) + + def test__given_two_builds_of_one_release__then_their_build_ids_differ(self): + newer = DataReleaseManifest.model_validate(folder_layout_manifest_payload()) + earlier = DataReleaseManifest.model_validate( + folder_layout_manifest_payload(build=EARLIER_NATIONAL_BUILD) + ) + + assert release_build_id(newer.build) != release_build_id(earlier.build) + + def test__given_folder_layout_manifest__then_records_no_build_details(self): + # The string build carries no built_with_* packages; until Microcosm + # says where they go, certification relies on compatibility claims. + manifest = DataReleaseManifest.model_validate(folder_layout_manifest_payload()) + + assert manifest.build is not None + assert manifest.build.built_with_model_package is None + assert manifest.build.cut_tag is None + + def test__given_folder_layout_manifest__then_dump_keeps_build_a_string(self): + manifest = DataReleaseManifest.model_validate(folder_layout_manifest_payload()) + + dumped = manifest.model_dump(mode="json", exclude_none=True) + + assert dumped["release"] == NATIONAL_RELEASE + assert dumped["build"] == NATIONAL_BUILD + assert DataReleaseManifest.model_validate(dumped) == manifest + + def test__given_folder_layout_artifact_without_revision__then_parses(self): + payload = folder_layout_manifest_payload() + for artifact in payload["artifacts"].values(): + del artifact["revision"] + + manifest = DataReleaseManifest.model_validate(payload) + + assert manifest.artifacts["microcosm_uk_2024_25"].revision is None + + def test__given_tagged_layout_artifact_without_revision__then_refuses(self): + payload = tagged_layout_manifest_payload() + del payload["artifacts"]["microcosm_uk_2024_25"]["revision"] + + with pytest.raises(ValidationError, match="revision"): + DataReleaseManifest.model_validate(payload) + + def test__given_string_build_without_release__then_refuses(self): + payload = folder_layout_manifest_payload() + del payload["release"] + + with pytest.raises(ValidationError, match="release"): + DataReleaseManifest.model_validate(payload) + + def test__given_release_with_object_build__then_refuses(self): + payload = tagged_layout_manifest_payload() + payload["release"] = NATIONAL_RELEASE + + with pytest.raises(ValidationError, match="build"): + DataReleaseManifest.model_validate(payload) + + @pytest.mark.parametrize( + ("field_name", "unsafe_name"), + [("release", "../microcosm-uk"), ("build", "attempt/../other")], + ) + def test__given_unsafe_release_or_build_name__then_refuses( + self, field_name, unsafe_name + ): + payload = folder_layout_manifest_payload() + payload[field_name] = unsafe_name + + with pytest.raises(ValidationError, match=field_name): + DataReleaseManifest.model_validate(payload) + + def test__given_tagged_layout_manifest__then_cut_tag_stays_the_data_build_id( + self, + ): + manifest = DataReleaseManifest.model_validate(tagged_layout_manifest_payload()) + + assert manifest.release is None + assert manifest.build_folder is None + assert release_build_id(manifest.build) == ( + f"{NATIONAL_RELEASE}-20261019T083323Z-6cba5d71" + ) + dumped = manifest.model_dump(mode="json", exclude_none=True) + assert dumped["build"]["build_id"] == NATIONAL_RELEASE + + +NATIONAL_COMMIT = "0123456789abcdef0123456789abcdef01234567" +DENSE_RELEASE = "microcosm-uk-2024-25-dense" +DENSE_BUILD = "uk-frs-calibration-attempt-20261019T000432Z-4e5f6a7b" +DENSE_COMMIT = "89abcdef0123456789abcdef0123456789abcdef" +NATIONAL_FOLDER = f"releases/{NATIONAL_RELEASE}/{NATIONAL_BUILD}" +DENSE_FOLDER = f"releases/{DENSE_RELEASE}/{DENSE_BUILD}" + + +def manifest_uri(folder: str, revision: str) -> str: + return f"hf://dataset/{REPOSITORY}@{revision}/{folder}/release_manifest.json" + + +NATIONAL_URI = manifest_uri(NATIONAL_FOLDER, NATIONAL_COMMIT) +DENSE_URI = manifest_uri(DENSE_FOLDER, DENSE_COMMIT) + + +def folder_layout_dense_payload() -> dict: + """A local-area release manifest in the side-by-side layout.""" + + return { + "schema_version": 1, + "release": DENSE_RELEASE, + "build": DENSE_BUILD, + "data_package": {"name": "microcosm-data", "version": "0.1.0"}, + "dataset_role": "non_default_local_area", + "is_default": False, + "default_datasets": {}, + "compatible_model_packages": [ + {"name": "policyengine-uk", "specifier": f"=={UK_MODEL_VERSION}"} + ], + "artifacts": { + "microcosm_uk_2024_25_dense": { + "kind": "microdata", + "role": "dataset", + "path": "microcosm_uk_2024_25_dense.h5", + "repo_id": REPOSITORY, + "sha256": "c" * 64, + }, + }, + } + + +def file_response(content: bytes) -> MagicMock: + response = MagicMock() + response.status_code = 200 + response.content = content + # Like requests, which replaces bytes it cannot decode. + response.text = content.decode(errors="replace") + response.raise_for_status.return_value = None + return response + + +def manifest_response(payload: dict) -> MagicMock: + return file_response(json.dumps(payload).encode()) + + +def missing_file_response() -> MagicMock: + response = MagicMock() + response.status_code = 404 + response.content = b"Entry not found" + response.text = response.content.decode() + return response + + +RELEASES_RECORD_URL = ( + f"https://huggingface.co/datasets/{REPOSITORY}/resolve/main/releases.json" +) + + +def releases_record_payload( + withdrawn: frozenset[tuple[str, str]] = frozenset(), + unlisted: frozenset[tuple[str, str]] = frozenset(), +) -> dict: + """The repository's ``releases.json``, in the release writer's format. + + Lists both national builds and the dense build (no current build), none + withdrawn, except the ``(release, build)`` pairs named in ``withdrawn`` + or ``unlisted``. The national release's current build is its newest build + that is listed and not withdrawn, so the record stays one the release + writer accepts. + """ + + builds_by_release = { + NATIONAL_RELEASE: [EARLIER_NATIONAL_BUILD, NATIONAL_BUILD], + DENSE_RELEASE: [DENSE_BUILD], + } + releases = {} + for release_name, build_names in builds_by_release.items(): + usable_builds = [ + build_name + for build_name in build_names + if (release_name, build_name) not in withdrawn | unlisted + ] + current_build = ( + usable_builds[-1] + if release_name == NATIONAL_RELEASE and usable_builds + else None + ) + releases[release_name] = { + "role": "national_default" if release_name == NATIONAL_RELEASE else "dense", + "current": current_build, + "builds": { + build_name: { + "published_at": "2026-10-19T08:33:23Z", + "withdrawn": (release_name, build_name) in withdrawn, + } + for build_name in build_names + if (release_name, build_name) not in unlisted + }, + } + return {"schema_version": 1, "default": NATIONAL_RELEASE, "releases": releases} + + +def empty_bundle(tmp_path: Path) -> Path: + bundle_path = tmp_path / "manifest.json" + bundle_path.write_text( + json.dumps( + { + "schema_version": 2, + "bundle_version": "9.9.9", + "policyengine_version": "9.9.9", + "packages": { + "policyengine": {"name": "policyengine", "version": "9.9.9"}, + "policyengine-uk": {"name": "policyengine-uk", "version": "2.0.0"}, + }, + "extras": {}, + "countries": {"uk": {"model_package": "policyengine-uk"}}, + "data_releases": {}, + } + ) + + "\n" + ) + return bundle_path + + +def certify_uk( + bundle_path: Path, + manifests: list[dict], + manifest_uri: str = NATIONAL_URI, + releases_record: dict | bytes | None = None, + requested_urls: list[str] | None = None, + **options, +): + """Certify with fetched files and reachability checks replaced. + + Requests for the repository's ``releases.json`` at ``main`` get + ``releases_record`` (``releases_record_payload()`` when ``None``, a 404 + when empty, served as they are when bytes); every other request gets the + next of ``manifests``. Each requested URL is appended to + ``requested_urls`` when given. + """ + + record_payload = ( + releases_record_payload() if releases_record is None else releases_record + ) + manifests_to_serve = list(manifests) + + def fake_get(url: str, headers=None, timeout=None) -> MagicMock: + if requested_urls is not None: + requested_urls.append(url) + if url == RELEASES_RECORD_URL: + if not record_payload: + return missing_file_response() + if isinstance(record_payload, bytes): + return file_response(record_payload) + return manifest_response(record_payload) + return manifest_response(manifests_to_serve.pop(0)) + + with ( + patch( + "policyengine.provenance.certification.requests.get", + side_effect=fake_get, + ), + patch( + "policyengine.provenance.certification.head_artifact", + return_value=True, + ) as head_artifact, + patch( + "policyengine.provenance.certification.head_artifact_reference", + return_value=True, + ), + patch( + "policyengine.provenance.certification.fetch_pypi_wheel_metadata", + return_value={"sha256": "d" * 64, "url": "https://example"}, + ), + patch( + "policyengine.provenance.certification.policyengine_version", + return_value="9.9.9", + ), + ): + result = certify_data_release( + country="uk", + data_producer="populace", + manifest_uri=manifest_uri, + model_version=UK_MODEL_VERSION, + bundle_path=bundle_path, + **options, + ) + release = json.loads(bundle_path.read_text())["data_releases"]["uk"] + return result, release, head_artifact + + +class TestCertifyBuildFolder: + def test__given_folder_layout_release__then_datasets_are_pinned_in_its_folder( + self, tmp_path + ): + _, release, _ = certify_uk( + empty_bundle(tmp_path), [folder_layout_manifest_payload()] + ) + + assert release["datasets"]["microcosm_uk_2024_25"] == { + "path": f"{NATIONAL_FOLDER}/microcosm_uk_2024_25.h5", + "revision": NATIONAL_COMMIT, + "repo_id": REPOSITORY, + "sha256": "a" * 64, + } + assert release["datasets"]["calibration_diagnostics"]["path"] == ( + f"{NATIONAL_FOLDER}/calibration_diagnostics.json" + ) + assert release["region_datasets"]["national"] == { + "path_template": f"{NATIONAL_FOLDER}/microcosm_uk_2024_25.h5" + } + assert release["default_dataset_uri"] == ( + f"hf://{REPOSITORY}/{NATIONAL_FOLDER}/microcosm_uk_2024_25.h5" + f"@{NATIONAL_COMMIT}" + ) + + def test__given_folder_layout_release__then_runtime_reads_its_folder_manifest( + self, tmp_path + ): + _, release, _ = certify_uk( + empty_bundle(tmp_path), [folder_layout_manifest_payload()] + ) + + assert release["data_package"]["release_manifest_path"] == ( + f"{NATIONAL_FOLDER}/release_manifest.json" + ) + assert release["data_package"]["release_manifest_revision"] == (NATIONAL_COMMIT) + + def test__given_folder_layout_release__then_its_build_names_the_certificate( + self, tmp_path + ): + result, release, _ = certify_uk( + empty_bundle(tmp_path), [folder_layout_manifest_payload()] + ) + + assert release["version"] == NATIONAL_BUILD + assert release["certification"]["data_build_id"] == NATIONAL_BUILD + assert release["certified_data_artifact"]["build_id"] == NATIONAL_BUILD + assert result.build_id == NATIONAL_BUILD + + def test__given_folder_layout_release__then_checks_the_dataset_in_its_folder( + self, tmp_path + ): + _, _, head_artifact = certify_uk( + empty_bundle(tmp_path), [folder_layout_manifest_payload()] + ) + + checked = head_artifact.call_args.args[0] + assert (checked.path, checked.revision) == ( + f"{NATIONAL_FOLDER}/microcosm_uk_2024_25.h5", + NATIONAL_COMMIT, + ) + + def test__given_artifacts_without_revision__then_certified_revision_pins_them( + self, tmp_path + ): + payload = folder_layout_manifest_payload() + for artifact in payload["artifacts"].values(): + del artifact["revision"] + + _, release, _ = certify_uk(empty_bundle(tmp_path), [payload]) + + assert {dataset["revision"] for dataset in release["datasets"].values()} == { + NATIONAL_COMMIT + } + + def test__given_manifest_read_from_another_build_folder__then_refuses( + self, tmp_path + ): + other_folder = f"releases/{NATIONAL_RELEASE}/{EARLIER_NATIONAL_BUILD}" + + with pytest.raises(CertificationError, match=NATIONAL_BUILD): + certify_uk( + empty_bundle(tmp_path), + [folder_layout_manifest_payload()], + manifest_uri=manifest_uri(other_folder, NATIONAL_COMMIT), + ) + + def test__given_artifact_outside_its_build_folder__then_refuses(self, tmp_path): + payload = folder_layout_manifest_payload() + payload["artifacts"]["microcosm_uk_2024_25"]["path"] = ( + f"releases/{NATIONAL_RELEASE}/{EARLIER_NATIONAL_BUILD}/" + "microcosm_uk_2024_25.h5" + ) + + with pytest.raises(CertificationError, match="outside"): + certify_uk(empty_bundle(tmp_path), [payload]) + + @pytest.mark.parametrize( + "artifact_path", + [ + # Starts with the folder, then leaves it: HTTP clients remove the + # dot segments, so this would fetch the earlier build's file. + f"{NATIONAL_FOLDER}/../{EARLIER_NATIONAL_BUILD}/microcosm_uk_2024_25.h5", + f"{NATIONAL_FOLDER}/nested/microcosm_uk_2024_25.h5", + f"{NATIONAL_FOLDER}/..", + "..", + ".", + "microcosm_uk_2024_25\\..\\other.h5", + ], + ) + def test__given_artifact_path_not_directly_inside_its_folder__then_refuses( + self, tmp_path, artifact_path + ): + # The release writer publishes only bare file names, directly inside + # the build folder. + payload = folder_layout_manifest_payload() + payload["artifacts"]["microcosm_uk_2024_25"]["path"] = artifact_path + + with pytest.raises(CertificationError, match="directly inside"): + certify_uk(empty_bundle(tmp_path), [payload]) + + def test__given_artifact_from_another_repository__then_keeps_its_own_pins( + self, tmp_path + ): + payload = folder_layout_manifest_payload() + payload["artifacts"]["frs_2024_25"] = { + "kind": "microdata", + "path": "frs_2024_25.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "revision": "1.58.0", + "sha256": "e" * 64, + } + + _, release, _ = certify_uk(empty_bundle(tmp_path), [payload]) + + inherited = release["datasets"]["frs_2024_25"] + assert (inherited["path"], inherited["revision"]) == ( + "frs_2024_25.h5", + "1.58.0", + ) + + def test__given_folder_layout_local_area_release__then_areas_use_its_folder( + self, tmp_path + ): + _, release, _ = certify_uk( + empty_bundle(tmp_path), + [folder_layout_manifest_payload(), folder_layout_dense_payload()], + regional_manifest_uri=DENSE_URI, + ) + + dense_path = f"{DENSE_FOLDER}/microcosm_uk_2024_25_dense.h5" + assert release["regional_release_kind"] == "local_area" + for region_type in ("constituency", "local_authority"): + assert release["region_datasets"][region_type] == { + "path_template": dense_path + } + dense = release["datasets"]["microcosm_uk_2024_25_dense"] + assert (dense["path"], dense["revision"]) == (dense_path, DENSE_COMMIT) + + def test__given_tagged_release__then_paths_and_tag_are_as_before(self, tmp_path): + tag = f"{NATIONAL_RELEASE}-20261019T083323Z-6cba5d71" + + result, release, _ = certify_uk( + empty_bundle(tmp_path), + [tagged_layout_manifest_payload()], + manifest_uri=manifest_uri(f"releases/{NATIONAL_RELEASE}", tag), + ) + + assert release["datasets"]["microcosm_uk_2024_25"]["path"] == ( + "microcosm_uk_2024_25.h5" + ) + assert release["datasets"]["microcosm_uk_2024_25"]["revision"] == tag + assert release["datasets"]["calibration_diagnostics"]["path"] == ( + f"releases/{NATIONAL_RELEASE}/calibration_diagnostics.json" + ) + assert result.build_id == tag + + +#: The largest ``releases.json`` the release writer accepts +#: (``microcosm_emitter_interface.releases.MAX_RECORD_BYTES``). +RELEASES_RECORD_SIZE_LIMIT = 4 * 1024 * 1024 + + +def national_release_entry(record: dict) -> dict: + return record["releases"][NATIONAL_RELEASE] + + +def dense_release_entry(record: dict) -> dict: + return record["releases"][DENSE_RELEASE] + + +def certified_build_entry(record: dict) -> dict: + return national_release_entry(record)["builds"][NATIONAL_BUILD] + + +def earlier_national_build_entry(record: dict) -> dict: + return national_release_entry(record)["builds"][EARLIER_NATIONAL_BUILD] + + +def make_withdrawn_build_current(record: dict) -> None: + earlier_national_build_entry(record)["withdrawn"] = True + national_release_entry(record)["current"] = EARLIER_NATIONAL_BUILD + + +def published_build_entry() -> dict: + return {"published_at": "2026-10-19T08:33:23Z", "withdrawn": False} + + +#: Changes that turn ``releases_record_payload()`` into a record the release +#: writer's parser (``ReleasesRecord.parse`` with ``check_record``, +#: microcosm-emitter#24) refuses. Unless the rule needs otherwise, each record +#: still lists the certified national build as not withdrawn. +RECORD_CHANGES_THE_WRITER_REFUSES = [ + pytest.param( + lambda record: record.pop("schema_version"), id="missing schema_version" + ), + pytest.param(lambda record: record.update(schema_version=2), id="schema_version 2"), + pytest.param( + lambda record: record.update(schema_version=True), id="schema_version true" + ), + pytest.param( + lambda record: record.update(schema_version=1.0), id="schema_version 1.0" + ), + pytest.param(lambda record: record.pop("default"), id="missing default"), + pytest.param(lambda record: record.pop("releases"), id="missing releases"), + pytest.param( + lambda record: record.update(comment="edited by hand"), + id="unknown top-level key", + ), + pytest.param(lambda record: record.update(default=1), id="default not a string"), + pytest.param( + lambda record: record.update(default="microcosm-uk-2024-25-unlisted"), + id="default names an unlisted release", + ), + pytest.param( + lambda record: record["releases"].update( + {"../escape": {"role": "dense", "current": None, "builds": {}}} + ), + id="unsafe release name", + ), + pytest.param(lambda record: dense_release_entry(record).pop("role"), id="no role"), + pytest.param( + lambda record: dense_release_entry(record).update(role="Dense-Release"), + id="invalid role", + ), + pytest.param( + lambda record: dense_release_entry(record).pop("current"), id="no current" + ), + pytest.param( + lambda record: dense_release_entry(record).update(current=1), + id="current not a string", + ), + pytest.param( + lambda record: dense_release_entry(record).pop("builds"), id="no builds" + ), + pytest.param( + lambda record: dense_release_entry(record).update(builds=[]), + id="builds not an object", + ), + pytest.param( + lambda record: dense_release_entry(record).update(note="edited by hand"), + id="unknown release key", + ), + pytest.param( + lambda record: dense_release_entry(record)["builds"].update( + {"build with spaces": published_build_entry()} + ), + id="unsafe build name", + ), + pytest.param( + lambda record: certified_build_entry(record).pop("published_at"), + id="certified build without published_at", + ), + pytest.param( + lambda record: earlier_national_build_entry(record).update(published_at=""), + id="empty published_at", + ), + pytest.param( + lambda record: earlier_national_build_entry(record).update( + published_at=20261019 + ), + id="published_at not a string", + ), + pytest.param( + lambda record: earlier_national_build_entry(record).update(withdrawn="false"), + id="withdrawn not a boolean", + ), + pytest.param( + lambda record: earlier_national_build_entry(record).update(sha256="a" * 64), + id="unknown build key", + ), + pytest.param( + lambda record: national_release_entry(record)["builds"].update( + {EARLIER_NATIONAL_BUILD: "2026-10-14T21:44:43Z"} + ), + id="build entry not an object", + ), + pytest.param( + lambda record: national_release_entry(record).update( + current="uk-frs-calibration-attempt-20261020T000000Z-unlisted" + ), + id="current names an unlisted build", + ), + pytest.param(make_withdrawn_build_current, id="current names a withdrawn build"), +] + +#: Changes after which the release writer's parser still accepts the record. +RECORD_CHANGES_THE_WRITER_ACCEPTS = [ + pytest.param(lambda record: record.update(default=None), id="no default"), + pytest.param( + lambda record: record["releases"].update( + { + "microcosm-uk-2024-25-empty": { + "role": "dense", + "current": None, + "builds": {}, + } + } + ), + id="release without builds", + ), + pytest.param( + lambda record: earlier_national_build_entry(record).update(withdrawn=True), + id="withdrawn build that is not current", + ), +] + + +class TestCertifyOnlyListedBuilds: + """``releases.json`` at ``main`` is the record of every published build + and of which ones are withdrawn; certification reads it.""" + + @pytest.mark.parametrize("change_record", RECORD_CHANGES_THE_WRITER_REFUSES) + def test__given_record_the_release_writer_refuses__then_refuses( + self, tmp_path, change_record: Callable[[dict], object] + ): + record = releases_record_payload() + change_record(record) + + with pytest.raises(CertificationError, match="releases.json"): + certify_uk( + empty_bundle(tmp_path), + [folder_layout_manifest_payload()], + releases_record=record, + ) + + def test__given_record_larger_than_the_release_writer_accepts__then_refuses( + self, tmp_path + ): + record_bytes = json.dumps(releases_record_payload()).encode() + padding = b" " * (RELEASES_RECORD_SIZE_LIMIT + 1 - len(record_bytes)) + + with pytest.raises(CertificationError, match="releases.json"): + certify_uk( + empty_bundle(tmp_path), + [folder_layout_manifest_payload()], + releases_record=record_bytes + padding, + ) + + @pytest.mark.parametrize( + "record_bytes", + [ + pytest.param(b'{"schema_version": 1,', id="truncated JSON"), + pytest.param(b"\xff\xfe\x00", id="not text"), + pytest.param(b"[" * 100_000 + b"]" * 100_000, id="nested too deeply"), + ], + ) + def test__given_record_the_release_writer_cannot_read__then_refuses( + self, tmp_path, record_bytes: bytes + ): + with pytest.raises(CertificationError, match="releases.json"): + certify_uk( + empty_bundle(tmp_path), + [folder_layout_manifest_payload()], + releases_record=record_bytes, + ) + + @pytest.mark.parametrize("change_record", RECORD_CHANGES_THE_WRITER_ACCEPTS) + def test__given_record_the_release_writer_accepts__then_certifies( + self, tmp_path, change_record: Callable[[dict], object] + ): + record = releases_record_payload() + change_record(record) + + _, release, _ = certify_uk( + empty_bundle(tmp_path), + [folder_layout_manifest_payload()], + releases_record=record, + ) + + assert release["version"] == NATIONAL_BUILD + + def test__given_listed_build__then_reads_releases_record_at_main(self, tmp_path): + requested_urls: list[str] = [] + + certify_uk( + empty_bundle(tmp_path), + [folder_layout_manifest_payload()], + requested_urls=requested_urls, + ) + + assert RELEASES_RECORD_URL in requested_urls + + def test__given_build_withdrawn_in_releases_record__then_refuses(self, tmp_path): + record = releases_record_payload( + withdrawn=frozenset({(NATIONAL_RELEASE, NATIONAL_BUILD)}) + ) + + with pytest.raises(CertificationError, match="withdrawn"): + certify_uk( + empty_bundle(tmp_path), + [folder_layout_manifest_payload()], + releases_record=record, + ) + + def test__given_build_not_listed_in_releases_record__then_refuses(self, tmp_path): + record = releases_record_payload( + unlisted=frozenset({(NATIONAL_RELEASE, NATIONAL_BUILD)}) + ) + + with pytest.raises(CertificationError, match="does not list"): + certify_uk( + empty_bundle(tmp_path), + [folder_layout_manifest_payload()], + releases_record=record, + ) + + def test__given_withdrawn_local_area_build__then_refuses(self, tmp_path): + record = releases_record_payload( + withdrawn=frozenset({(DENSE_RELEASE, DENSE_BUILD)}) + ) + + with pytest.raises(CertificationError, match=f"{DENSE_BUILD}.*withdrawn"): + certify_uk( + empty_bundle(tmp_path), + [folder_layout_manifest_payload(), folder_layout_dense_payload()], + releases_record=record, + regional_manifest_uri=DENSE_URI, + ) + + def test__given_no_releases_record__then_refuses(self, tmp_path): + with pytest.raises(CertificationError, match="releases.json"): + certify_uk( + empty_bundle(tmp_path), + [folder_layout_manifest_payload()], + releases_record={}, + ) + + def test__given_tagged_release__then_releases_record_is_not_read(self, tmp_path): + tag = f"{NATIONAL_RELEASE}-20261019T083323Z-6cba5d71" + requested_urls: list[str] = [] + + certify_uk( + empty_bundle(tmp_path), + [tagged_layout_manifest_payload()], + manifest_uri=manifest_uri(f"releases/{NATIONAL_RELEASE}", tag), + requested_urls=requested_urls, + ) + + assert RELEASES_RECORD_URL not in requested_urls + + +PAIR_ID = "a" * 64 + + +def folder_layout_country_manifest( + certification: dict | None = None, +) -> CountryReleaseManifest: + """A bundle entry certified from the national build's own folder.""" + + return CountryReleaseManifest.model_validate( + { + "country_id": "uk", + "policyengine_version": "9.9.9", + "model_package": {"name": "policyengine-uk", "version": UK_MODEL_VERSION}, + "data_package": { + "name": "microcosm-data", + "version": "0.1.0", + "repo_id": REPOSITORY, + "repo_type": "dataset", + "release_manifest_path": f"{NATIONAL_FOLDER}/release_manifest.json", + "release_manifest_revision": NATIONAL_COMMIT, + }, + "default_dataset": "microcosm_uk_2024_25", + "datasets": { + "microcosm_uk_2024_25": { + "path": f"{NATIONAL_FOLDER}/microcosm_uk_2024_25.h5", + "revision": NATIONAL_COMMIT, + "repo_id": REPOSITORY, + } + }, + **({"certification": certification} if certification else {}), + } + ) + + +class TestImportCheckBuildFolder: + def setup_method(self): + get_data_release_manifest.cache_clear() + + def teardown_method(self): + get_data_release_manifest.cache_clear() + + def _patched(self, remote_payload: dict, country_manifest=None): + return ( + patch( + "policyengine.provenance.manifest.get_release_manifest", + return_value=country_manifest or folder_layout_country_manifest(), + ), + patch( + "policyengine.provenance.manifest.requests.get", + return_value=manifest_response(remote_payload), + ), + ) + + def test__given_remote_manifest_of_its_folder__then_artifacts_resolve_in_it( + self, + ): + bundle_patch, request_patch = self._patched(folder_layout_manifest_payload()) + with bundle_patch, request_patch as get: + manifest = get_data_release_manifest("uk") + + assert get.call_args.args[0] == ( + f"https://huggingface.co/datasets/{REPOSITORY}/resolve/" + f"{NATIONAL_COMMIT}/{NATIONAL_FOLDER}/release_manifest.json" + ) + assert manifest.artifacts["microcosm_uk_2024_25"].uri == ( + f"hf://{REPOSITORY}/{NATIONAL_FOLDER}/microcosm_uk_2024_25.h5" + f"@{NATIONAL_COMMIT}" + ) + + def test__given_remote_manifest_naming_another_build__then_refuses(self): + bundle_patch, request_patch = self._patched( + folder_layout_manifest_payload(build=EARLIER_NATIONAL_BUILD) + ) + with ( + bundle_patch, + request_patch, + pytest.raises(ReleaseBuildFolderError, match=EARLIER_NATIONAL_BUILD), + ): + get_data_release_manifest("uk") + + def test__given_remote_artifact_leaving_its_folder__then_refuses(self): + remote = folder_layout_manifest_payload() + remote["artifacts"]["microcosm_uk_2024_25"]["path"] = ( + f"{NATIONAL_FOLDER}/../{EARLIER_NATIONAL_BUILD}/microcosm_uk_2024_25.h5" + ) + bundle_patch, request_patch = self._patched(remote) + with ( + bundle_patch, + request_patch, + pytest.raises(ReleaseBuildFolderError, match="directly inside"), + ): + get_data_release_manifest("uk") + + def test__given_mismatched_remote_manifest__then_bundled_certificate_is_not_used( + self, + ): + # A manifest naming another build is wrong, not unavailable: the + # fallback to the bundled certificate must not hide it. + bundled = folder_layout_country_manifest( + certification={ + "compatibility_basis": "legacy_compatible_model_package", + "certified_for_model_version": UK_MODEL_VERSION, + "data_build_id": NATIONAL_BUILD, + } + ) + bundle_patch, request_patch = self._patched( + folder_layout_manifest_payload(build=EARLIER_NATIONAL_BUILD), bundled + ) + with ( + bundle_patch, + request_patch, + pytest.raises(ValueError, match=EARLIER_NATIONAL_BUILD), + ): + certify_data_release_compatibility( + "uk", runtime_model_version=UK_MODEL_VERSION + ) + + def test__given_publisher_claim_on_folder_build__then_build_names_it(self): + bundle_patch, request_patch = self._patched(folder_layout_manifest_payload()) + with bundle_patch, request_patch: + certification = certify_data_release_compatibility( + "uk", runtime_model_version=UK_MODEL_VERSION + ) + + assert certification.compatibility_basis == "legacy_compatible_model_package" + assert certification.data_build_id == NATIONAL_BUILD + + def test__given_qualified_folder_build__then_import_check_honours_its_build( + self, + ): + qualified = { + "compatibility_basis": "qualified_exact_pair", + "certified_for_model_version": "2.130.1", + "certified_for_core_version": "3.33.0", + "data_build_id": NATIONAL_BUILD, + "qualification_pair_id": PAIR_ID, + } + remote = folder_layout_manifest_payload() + remote["compatible_model_packages"] = [] + bundle_patch, request_patch = self._patched( + remote, folder_layout_country_manifest(certification=qualified) + ) + with bundle_patch, request_patch: + certification = certify_data_release_compatibility( + "uk", runtime_model_version="2.130.1", runtime_core_version="3.33.0" + ) + + assert certification.compatibility_basis == "qualified_exact_pair" + assert certification.data_build_id == NATIONAL_BUILD