Publish the model metadata the Public API can only read through a private method

changelog.d/700.added.md

@@ -0,0 +1 @@
+Model versions publish the metadata the Public API could only read through the private `_load_system()` method: each model version's `entities` (key, plural and person flag) and `basic_inputs`; each variable's `is_input_variable`, `hidden_input`, `module_name`, `enum_values` (each value with its label, as declared) and, where its `adds` or `subtracts` names a parameter, `adds_parameter_path` or `subtracts_parameter_path`; each parameter's `economy` and `household` flags; and each bracket schedule as a parameter node with its stored label. Existing fields are unchanged: `adds` and `subtracts` still hold the list as of 2025-01-01.

src/policyengine/core/__init__.py

@@ -10,6 +10,7 @@
 from .dataset import YearData as YearData
 from .dataset import map_to_entity as map_to_entity
 from .dynamic import Dynamic as Dynamic
+from .entity import Entity as Entity
 from .output import Output as Output
 from .output import OutputCollection as OutputCollection
 from .parameter import Parameter as Parameter
@@ -31,6 +32,7 @@
 from .tax_benefit_model_version import (
     TaxBenefitModelVersion as TaxBenefitModelVersion,
 )
+from .variable import EnumValue as EnumValue
 from .variable import Variable as Variable
 
 # Rebuild models to resolve forward references

src/policyengine/core/entity.py

@@ -0,0 +1,17 @@
+from pydantic import BaseModel, Field
+
+
+class Entity(BaseModel):
+    """One entity of a country model, such as a person or a household.
+
+    Records what a household builder needs to place its inputs: the key that
+    ``Variable.entity`` names, the plural that groups a household's members of
+    this entity, and whether it is the person entity. Entity labels,
+    descriptions and roles are not recorded.
+    """
+
+    key: str = Field(description="Entity key, as named by Variable.entity")
+    plural: str = Field(
+        description="Plural key grouping a household's entities of this kind"
+    )
+    is_person: bool = Field(description="Whether this is the person entity")

src/policyengine/core/parameter.py

@@ -21,6 +21,22 @@ class Parameter(BaseModel):
     data_type: Optional[type] = None
     tax_benefit_model_version: TaxBenefitModelVersion
     unit: Optional[str] = None
+    economy: bool = Field(
+        default=True,
+        description=(
+            "Whether the parameter applies to economy-wide (population) "
+            "analysis, from the country package's 'economy' metadata. True "
+            "when the package does not say."
+        ),
+    )
+    household: bool = Field(
+        default=True,
+        description=(
+            "Whether the parameter applies to household calculations, from "
+            "the country package's 'household' metadata. True when the "
+            "package does not say."
+        ),
+    )
 
     # Lazy loading: store core param ref, build values on demand
     _core_param: Any = PrivateAttr(default=None)

src/policyengine/core/parameter_node.py

@@ -14,6 +14,10 @@ class ParameterNode(BaseModel):
     "gov.hmrc", "gov.hmrc.income_tax"). They provide structure and human-readable
     labels for navigating the parameter tree, but don't have values themselves.
 
+    A bracket schedule (e.g., "gov.hmrc.income_tax.rates.uk") is a node, and
+    so is each of its brackets ("gov.hmrc.income_tax.rates.uk[0]"), whose
+    parameters are its rate, threshold or amount.
+
     Unlike Parameter objects (which are leaf nodes with actual values),
     ParameterNode objects are purely organizational.
     """

src/policyengine/core/tax_benefit_model_version.py

@@ -13,6 +13,7 @@
 )
 from policyengine.provenance.trace import build_trace_tro_from_release_bundle
 
+from .entity import Entity
 from .tax_benefit_model import TaxBenefitModel
 
 if TYPE_CHECKING:
@@ -46,6 +47,17 @@ class TaxBenefitModelVersion(BaseModel):
     variables: list["Variable"] = Field(default_factory=list)
     parameters: list["Parameter"] = Field(default_factory=list)
     parameter_nodes: list["ParameterNode"] = Field(default_factory=list)
+    entities: list[Entity] = Field(
+        default_factory=list,
+        description="The country model's entities, in the country package's order",
+    )
+    basic_inputs: list[str] = Field(
+        default_factory=list,
+        description=(
+            "Variables a household builder shows by default, as the country "
+            "package declares them"
+        ),
+    )
 
     # Region registry for geographic simulations
     region_registry: "Optional[RegionRegistry]" = Field(

src/policyengine/core/variable.py

@@ -8,6 +8,23 @@
 from .tax_benefit_model_version import TaxBenefitModelVersion
 
 
+class EnumValue(BaseModel):
+    """One value an enum variable can take, with the label users see."""
+
+    value: str = Field(
+        description="Name stored in household data, such as 'SINGLE'",
+    )
+    # Pydantic's smart union keeps each declared scalar's own type: True stays
+    # True, 1.0 stays 1.0 and None stays None, rather than being converted.
+    label: Union[str, int, float, bool, None] = Field(
+        description=(
+            "Label shown to users, such as 'Single': the value the country "
+            "model declares for the member, unchanged, so a member declared "
+            "with an integer, number, boolean or None keeps that value"
+        ),
+    )
+
+
 class Variable(BaseModel):
     id: str
     name: str
@@ -30,6 +47,50 @@ class Variable(BaseModel):
     max_value: Optional[Union[int, float]] = None
     is_period_size_independent: Optional[bool] = None
     metadata: dict[str, Any] = Field(default_factory=dict)
+    is_input_variable: Optional[bool] = Field(
+        default=None,
+        description=(
+            "Whether users enter the variable rather than it being calculated: "
+            "it has no formula and no adds or subtracts. None when the country "
+            "model does not say."
+        ),
+    )
+    hidden_input: Optional[bool] = Field(
+        default=None,
+        description=(
+            "Whether the variable is hidden from lists of inputs. None when the "
+            "country model does not say."
+        ),
+    )
+    module_name: Optional[str] = Field(
+        default=None,
+        description="Module that defines the variable, as the country model names it",
+    )
+    adds_parameter_path: Optional[str] = Field(
+        default=None,
+        description=(
+            "Path of the parameter whose value lists the variables this one "
+            "adds, when the country model names a parameter rather than a "
+            "list. adds then holds that parameter's list as of 2025-01-01 only."
+        ),
+    )
+    subtracts_parameter_path: Optional[str] = Field(
+        default=None,
+        description=(
+            "Path of the parameter whose value lists the variables this one "
+            "subtracts, when the country model names a parameter rather than a "
+            "list. subtracts then holds that parameter's list as of 2025-01-01 "
+            "only."
+        ),
+    )
+    enum_values: Optional[list[EnumValue]] = Field(
+        default=None,
+        description=(
+            "For an enum variable, each value it can take with its label, in "
+            "the order the country model declares them. possible_values holds "
+            "the same names without labels."
+        ),
+    )
 
     @field_validator("reference", mode="before")
     @classmethod

src/policyengine/tax_benefit_models/common/model_version.py

@@ -30,6 +30,8 @@
 import pandas as pd
 
 from policyengine.core import (
+    Entity,
+    EnumValue,
     Parameter,
     ParameterNode,
     StaleOutputError,
@@ -93,6 +95,14 @@ def build_runtime_dataset_provenance(
     return provenance
 
 
+def _core_input_flag(core_variable: Any) -> Optional[bool]:
+    """Return whether users enter a core variable, or None if core cannot say."""
+    is_input_variable = getattr(core_variable, "is_input_variable", None)
+    if is_input_variable is None:
+        return None
+    return bool(is_input_variable())
+
+
 class MicrosimulationModelVersion(TaxBenefitModelVersion):
     """Shared init / save / load logic for country microsim model versions.
 
@@ -166,6 +176,7 @@ def __init__(self, **kwargs: Any) -> None:
         self.id = f"{self.model.id}@{self.version}"
 
         system = self._load_system()
+        self._populate_entities(system)
         self._populate_variables(system)
         self._populate_parameters(system)
 
@@ -189,6 +200,18 @@ def _dataset_class(self):
         raise NotImplementedError
 
     # --- Shared loading helpers ------------------------------------------
+    def _populate_entities(self, system) -> None:
+        """Copy the entities and basic inputs the country package declares."""
+        self.entities = [
+            Entity(
+                key=core_entity.key,
+                plural=core_entity.plural,
+                is_person=bool(core_entity.is_person),
+            )
+            for core_entity in system.entities
+        ]
+        self.basic_inputs = list(getattr(system, "basic_inputs", None) or [])
+
     def _populate_variables(self, system) -> None:
         from policyengine_core.enums import Enum
         from policyengine_core.parameters.operations.get_parameter import (
@@ -225,6 +248,9 @@ def _populate_variables(self, system) -> None:
                     var_obj, "is_period_size_independent", None
                 ),
                 metadata=getattr(var_obj, "metadata", None) or {},
+                is_input_variable=_core_input_flag(var_obj),
+                hidden_input=getattr(var_obj, "hidden_input", None),
+                module_name=getattr(var_obj, "module_name", None),
             )
             if (
                 hasattr(var_obj, "possible_values")
@@ -236,13 +262,19 @@ def _populate_variables(self, system) -> None:
                         var_obj.possible_values._value2member_map_.values(),
                     )
                 )
+                variable.enum_values = [
+                    EnumValue(value=member.name, label=member.value)
+                    for member in var_obj.possible_values
+                ]
             # Resolve parameter-path adds/subtracts to concrete lists so
-            # consumers always see list[str].
+            # consumers always see list[str], and record the path itself,
+            # since the parameter's list can differ in other years.
             for attr in ("adds", "subtracts"):
                 value = getattr(var_obj, attr, None)
                 if value is None:
                     continue
                 if isinstance(value, str):
+                    setattr(variable, f"{attr}_parameter_path", value)
                     try:
                         param = get_parameter(system.parameters, value)
                         setattr(variable, attr, list(param("2025-01-01")))
@@ -255,6 +287,7 @@ def _populate_variables(self, system) -> None:
     def _populate_parameters(self, system) -> None:
         from policyengine_core.parameters import Parameter as CoreParameter
         from policyengine_core.parameters import ParameterNode as CoreParameterNode
+        from policyengine_core.parameters import ParameterScale as CoreParameterScale
 
         scale_lookup = build_scale_lookup(system)
 
@@ -270,9 +303,24 @@ def _populate_parameters(self, system) -> None:
                     description=param_node.description,
                     data_type=type(param_node(2025)),
                     unit=param_node.metadata.get("unit"),
+                    economy=bool(param_node.metadata.get("economy", True)),
+                    household=bool(param_node.metadata.get("household", True)),
                     _core_param=param_node,
                 )
                 self.add_parameter(parameter)
+            elif isinstance(param_node, CoreParameterScale):
+                # A bracket schedule is not a core ParameterNode, but it groups
+                # its brackets (``name[0]``, ``name[1]``, ...) as a node does.
+                if not param_node.brackets:
+                    continue
+                schedule = ParameterNode(
+                    id=self.id + "-" + param_node.name,
+                    name=param_node.name,
+                    label=param_node.metadata.get("label"),
+                    description=param_node.description,
+                    tax_benefit_model_version=self,
+                )
+                self.add_parameter_node(schedule)
             elif isinstance(param_node, CoreParameterNode):
                 if not param_node.children:
                     continue

tests/conftest.py

@@ -3,6 +3,11 @@
 import pytest
 
 # Import fixtures from fixtures module so pytest can discover them
+from tests.fixtures.country_model_fixtures import (  # noqa: F401
+    country_model,
+    uk_country_model,
+    us_country_model,
+)
 from tests.fixtures.filtering_fixtures import (  # noqa: F401
     create_uk_test_dataset,
     create_us_test_dataset,

tests/fixtures/country_model_fixtures.py

@@ -0,0 +1,56 @@
+"""Country model versions for metadata tests, skipped when a package is missing.
+
+``policyengine.tax_benefit_models.uk`` and ``.us`` define ``uk_latest`` and
+``us_latest`` only when the country package is installed, so these fixtures
+check for the package first and import the model version afterwards.
+"""
+
+import importlib
+from types import SimpleNamespace
+
+import pytest
+
+from policyengine.core import TaxBenefitModel, TaxBenefitModelVersion
+
+
+def make_empty_model_version() -> TaxBenefitModelVersion:
+    """A model version with no country package, for populating from stubs."""
+    return TaxBenefitModelVersion(
+        id="test@1",
+        model=TaxBenefitModel(id="test", name="Test"),
+        version="1",
+    )
+
+
+def load_country_model(country_id: str) -> SimpleNamespace:
+    """Return a country's published model version and its core system.
+
+    The result has ``country_id``, ``model_version`` (``uk_latest`` or
+    ``us_latest``) and ``core_system`` (the country package's tax-benefit
+    system). Skips the calling test when the country package is not installed.
+    """
+    core_system = pytest.importorskip(f"policyengine_{country_id}.system").system
+    country_module = importlib.import_module(
+        f"policyengine.tax_benefit_models.{country_id}"
+    )
+    return SimpleNamespace(
+        country_id=country_id,
+        model_version=getattr(country_module, f"{country_id}_latest"),
+        core_system=core_system,
+    )
+
+
+@pytest.fixture(params=["uk", "us"])
+def country_model(request) -> SimpleNamespace:
+    """Each country's model version and core system, from load_country_model."""
+    return load_country_model(request.param)
+
+
+@pytest.fixture
+def uk_country_model() -> SimpleNamespace:
+    return load_country_model("uk")
+
+
+@pytest.fixture
+def us_country_model() -> SimpleNamespace:
+    return load_country_model("us")