State the conventions corpus's parts, and score them like CLDR's

tests/v2/test_cldr_agreement.py

@@ -1,19 +1,21 @@
-"""The CLDR and GND agreement ratchets, and the comparison rules they rest on.
+"""The agreement ratchets, and the comparison rules they rest on.
 
 tools/differential/cldr_agreement.py decides, for each rendering and
 each role, whether nameparser agrees with the source's stated fields
-(CLDR's, and the GND's written in CLDR's vocabulary). The rules
+(CLDR's, and the GND's, Wikidata's and the conventions table's written
+in CLDR's vocabulary). The rules
 are pinned here first, each with the wrong answer the plausible
 alternative gives recorded beside it; the ratchet test at the end then
 holds each source's corpus to its recorded disagreements in
-tools/differential/cldr_agreement.json and gnd_agreement.json.
+tools/differential/, one record per source.
 
 tools/ is not a package, so the module is loaded by path, as
 _differential_fixtures.py loads compare.py.
 """
 import importlib.util
 import json
 import sys
+from collections import Counter
 from pathlib import Path
 from types import ModuleType
 
@@ -127,6 +129,23 @@ class WithSurname2:
     assert agreement.got_roles(WithSurname2())["surname2"] == "García"  # type: ignore[arg-type]
 
 
+def test_an_honorific_is_scored_inside_the_suffix() -> None:
+    """A word of address written after the name (the conventions
+    source's `honorific`, #686) is a suffix, as nameparser reads さん."""
+    roles = agreement.expected_roles({"given": "Murat", "honorific": "Bey"}, "Murat Bey")
+    assert roles["suffix"] == "Bey"
+    assert roles["title"] == ""
+    assert roles["given_names"] == "Murat"
+    assert roles["family"] == ""
+
+
+def test_a_field_no_rule_reads_raises() -> None:
+    """A mistyped field would be expected in no role, and a row holding
+    it would pass as agreeing with an empty parse."""
+    with pytest.raises(ValueError, match="honorfic"):
+        agreement.expected_roles({"given": "Murat", "honorfic": "Bey"}, "Murat Bey")
+
+
 def test_expected_roles_compare_particles_only_where_cldr_splits_them() -> None:
     plain = agreement.expected_roles({"given": "Iris", "surname": "Falke"}, "Iris Falke")
     assert set(plain) == {"title", "given_names", "family", "suffix"}
@@ -220,15 +239,20 @@ def test_changes_reports_a_rendering_that_is_gone() -> None:
 #: exceed: a truncated file would otherwise pass both checks below, and
 #: an emptied one re-recorded would match its emptied record.
 _CORPORA = {"CLDR": ("corpus_cldr.jsonl", 1000), "GND": ("corpus_gnd.jsonl", 500),
-            "Wikidata": ("corpus_wikidata.jsonl", 500)}
+            "Wikidata": ("corpus_wikidata.jsonl", 500),
+            "Conventions": ("corpus_conventions.jsonl", 25)}
+#: Sources whose fields file states only some of its corpus's names: the
+#: conventions table leaves a row without fields where no field can state
+#: it, and says why in the row's `unstated`.
+_PARTIAL = frozenset({"Conventions"})
 
 
 def test_every_source_has_a_corpus_file() -> None:
     assert set(_CORPORA) == set(agreement.sources())
 
 
 @pytest.mark.parametrize("source", sorted(_CORPORA))
-def test_the_corpus_and_the_fields_file_hold_the_same_renderings(source: str) -> None:
+def test_the_fields_file_holds_the_corpus_renderings(source: str) -> None:
     fields, _ = agreement.sources()[source]
     corpus, least = _CORPORA[source]
     from_fields = {(row["text"], row["shape"]) for row in agreement.load_rows(fields)}
@@ -238,7 +262,10 @@ def test_the_corpus_and_the_fields_file_hold_the_same_renderings(source: str) ->
             entry = json.loads(line)
             from_corpus.add((entry, None) if isinstance(entry, str)
                             else (entry["name"], entry["shape"]))
-    assert from_corpus == from_fields
+    if source in _PARTIAL:
+        assert from_fields <= from_corpus
+    else:
+        assert from_corpus == from_fields
     assert len(from_fields) > least, f"the {source} corpus is missing or truncated"
 
 
@@ -255,6 +282,27 @@ def test_every_committed_field_occurs_in_its_rendering(source: str) -> None:
     assert not missing
 
 
+@pytest.mark.parametrize("source", sorted(set(_CORPORA) - {"CLDR"}))
+def test_the_fields_partition_their_rendering(source: str) -> None:
+    """Every word of a rendering is stated by exactly one field, so a part
+    left out is a failure here rather than an expectation of empty. CLDR
+    is exempt for the reason above; the other sources' builders claim
+    every word, and the conventions table is written to."""
+    fields, _ = agreement.sources()[source]
+    bad = []
+    for row in agreement.load_rows(fields):
+        values = list(row["fields"].values())
+        if agreement._has_space(row["text"]):
+            ok = (Counter(w for v in values for w in agreement._words(v))
+                  == Counter(agreement._words(row["text"])))
+        else:
+            ok = (sum(len(agreement._squash(v)) for v in values)
+                  == len(agreement._squash(row["text"])))
+        if not ok:
+            bad.append((row["text"], row["fields"]))
+    assert not bad
+
+
 @pytest.mark.parametrize("source", sorted(_CORPORA))
 def test_nameparser_still_agrees_with_each_source_where_it_did(source: str) -> None:
     fields, record = agreement.sources()[source]

tests/v2/test_corpora.py

@@ -43,12 +43,14 @@ def test_shapes_corpus_matches_the_case_table() -> None:
 
 
 def test_conventions_corpus_matches_its_table() -> None:
-    """corpus_conventions.jsonl is generated from conventions.py's
-    hand-written table (#686)."""
+    """corpus_conventions.jsonl and conventions_fields.jsonl are
+    generated from conventions.py's hand-written table (#686)."""
     module = load_tool("build_conventions_corpus")
-    assert _checked_in("corpus_conventions.jsonl") == module.selected(), (
-        "corpus_conventions.jsonl is stale: regenerate with "
-        "`uv run python tools/differential/build_conventions_corpus.py`")
+    for name, expected in (("corpus_conventions.jsonl", module.selected()),
+                           ("conventions_fields.jsonl", module.fields_rows())):
+        assert _checked_in(name) == expected, (
+            f"{name} is stale: regenerate with "
+            "`uv run python tools/differential/build_conventions_corpus.py`")
 
 
 def test_case_shape_ids_exist_in_the_inventory() -> None:

tools/differential/README.md

@@ -317,13 +317,11 @@ what a reader who knows the convention would take each part to be:
 uv run python tools/differential/build_conventions_corpus.py
 ```
 
-That expectation is a note, not a pin, and nothing scores against
-it: many are stated in terms no field states (a father's or
-grandfather's name, an honorific that is not a title, a married form),
-some are undecided, and a fields file must state every part, since the
-scorer reads an unstated one as an empty role (#686). A row whose
-reading is settled can graduate to `tests/v2/cases.py`. The names are
-invented, never copied from the references the conventions come
+The note is prose. Where a field can state the parts, the row
+states them too, and the builder writes those rows to
+`conventions_fields.jsonl` for the agreement ratchet (below). A row
+whose reading is settled can graduate to `tests/v2/cases.py`. The names
+are invented, never copied from the references the conventions come
 from.
 
 ## What the corpora do not cover
@@ -426,6 +424,8 @@ A second view measures the locale packs (#672). `LOCALE_FILES` in `cldr_agreemen
 
 **The Wikidata source.** `build_wikidata_corpus.py` exists for the two fields no other open source labels, and states them in `wikidata_fields.jsonl` as `surname2` (P1950, CLDR's own field name) and `patronymic` (P5056, which CLDR has no field for), beside `given` (P735) and `surname` (P734); its disagreements are in `wikidata_agreement.json`. A person's written names are token-aligned to the parts rather than composed from them: a text is kept only where every word is claimed by exactly one part, a connective (`bin`, `i`, `y`) joining the part after it, and a woman's Russian surname may match the feminine form of its item's label. ParsedName has no `surname2` or `patronymic` attribute, so `cldr_agreement.py` lists them in `PENDING_ROLES` and scores them inside `family` and `given_names` respectively -- a patronymic is not a family name, so `Vigdís Finnbogadóttir` expects the family empty -- and also scores each on its own the day ParsedName gains an attribute of that name (whether the merged role then drops it is for the 3.0 design). Until then the summary says how many renderings state each. CLDR's `surname2` is in the same position. A stratum that falls below its floor, a query that matches nothing, and a response that is not what was asked for each stop the build before anything is written; Malaysia's floor is set below its quota because only 17 Malaysian public figures with a P5056 statement matched the query on 2026-10-16, and 9 of them aligned. Raw responses are cached outside the tree, in a `wikidata/` directory under `$NAMEPARSER_TOOLS_CACHE` (default `~/.cache/nameparser-tools`); `--refresh` fetches them again.
 
+**The conventions source.** `conventions_fields.jsonl` states the parts of the hand-written conventions corpus, from the `fields` of `conventions.py`'s rows, and its disagreements are in `conventions_agreement.json`. The names are invented, so the record is a worklist of conventions rather than a measure of real data: a patronymic read as a family name (`Hassan bin Yusof`), a title read as a given name (`U Kyaw Win`). It uses the Wikidata source's `patronymic`, written with its connective (`bin Yusof`), and adds `honorific`, a word of address written after the name (`Bey`, `ssi`, `san`), scored inside `suffix` as nameparser reads `さん` (Derek, 2026-10-16). A row no field can state honestly is left out of the fields file and stays in the corpus, which the snapshot still watches: the kunya `Abu Karim`, the `-né` married form `Szabóné Kiss Éva`, and `arap`/`ole`/`wa`, whose last word is a parent's name or a family name depending on the family. Such a row says why in its `unstated`, and the builder refuses a row with neither `fields` nor `unstated`, so this fields file states a subset of its corpus without a row's fields being lost unseen. Being hand-written, it is also held to two checks the generated sources pass anyway: its fields partition each text word for word (`test_the_fields_partition_their_rendering`), and the scorer raises on a field name no rule reads, for every source.
+
 `--record` rewrites every record. `JA` needs the `ja` extra (`namedivider-python`), which only CI's `ja-extra` job installs: without it the `JA` ratchet skips, the summary says `not measured`, and `--record` keeps the recorded `ja` entries unchanged. Re-record with the extra installed (`uv sync --group dev --extra ja`) when a change moves Japanese.
 
 `_CORPUS_NAMES` in `tests/v2/_differential_fixtures.py` leaves the file out (2026-10-12): the tests reading it carry counts measured over the names the other corpora held. `corpus_gnd.jsonl` and `corpus_wikidata.jsonl` stay in it (2026-10-16): the counts those tests carry are floors and dated snapshots, which a larger corpus cannot falsify, and their invariants now walk both files' names too. A release figure counts GND's moved names in, its composed renderings included, since the names are real; AGENTS.md's rule of counting out names only `corpus_cldr.jsonl` holds stays CLDR's alone (Derek, 2026-10-16). Wikidata's moved names count in under the same rule, which singles out CLDR alone; they are real and written, not composed. `differential_corpus()` in `tests/v2/conftest.py`, which feeds the property tests, globs `corpus*.jsonl`, so it reads CLDR (since #652), GND and Wikidata, and the decomposed-twin test walks all their names.

tools/differential/build_conventions_corpus.py

@@ -1,17 +1,20 @@
-"""Regenerate corpus_conventions.jsonl from conventions.py's table (#686).
+"""Regenerate corpus_conventions.jsonl and conventions_fields.jsonl from
+conventions.py's table (#686).
 
 The table holds invented names patterned on naming conventions the other
 corpora barely cover; conventions.py says where the conventions come
 from and why each row's expected reading is a note rather than a pin.
 A row read under a non-default order is written with the shapes.py id
-for that order, so the snapshot reads it under that order.
+for that order, so the snapshot reads it under that order. A row with
+`fields` is also written to conventions_fields.jsonl, in the row format
+cldr_agreement.py compares, keyed by the name and its order.
 
 Regenerate after editing the table:
 
     uv run python tools/differential/build_conventions_corpus.py
 
-tests/v2/test_corpora.py pins the checked-in file against this module's
-selection, so a row added without regenerating fails the suite.
+tests/v2/test_corpora.py pins both checked-in files against this
+module's selection, so a row added without regenerating fails the suite.
 """
 from __future__ import annotations
 
@@ -22,6 +25,7 @@
 
 HERE = Path(__file__).resolve().parent
 OUT = HERE / "corpus_conventions.jsonl"
+FIELDS_OUT = HERE / "conventions_fields.jsonl"
 
 
 def _load(stem: str) -> ModuleType:
@@ -55,6 +59,9 @@ def selected() -> list[object]:
     seen: set[tuple[str, str | None]] = set()
     lines: list[object] = []
     for row in CONVENTIONS:
+        if (row.fields is None) == (row.unstated is None):
+            raise ValueError(f"conventions.py row {row.name!r} needs exactly one "
+                             f"of `fields` and `unstated` (why no field states it)")
         if (row.name, row.order) in seen:
             raise ValueError(f"conventions.py lists {row.name!r} under "
                              f"{row.order or 'the default order'} twice")
@@ -64,12 +71,26 @@ def selected() -> list[object]:
     return lines
 
 
-def main() -> None:
-    lines = selected()
-    with OUT.open("w", encoding="utf-8") as fh:
+def fields_rows() -> list[dict[str, object]]:
+    """The fields file's rows, for the table's rows that state fields."""
+    selected()  # the same row checks as the corpus
+    return [{"fields": row.fields, "file": "conventions", "label": row.name,
+             "order": row.order or "default",
+             "shape": None if row.order is None else _shape_for(row.order),
+             "text": row.name}
+            for row in CONVENTIONS if row.fields is not None]
+
+
+def _write(path: Path, lines: list[object]) -> None:
+    with path.open("w", encoding="utf-8") as fh:
         for line in lines:
-            fh.write(json.dumps(line, ensure_ascii=False) + "\n")
-    print(f"wrote {len(lines)} names to {OUT.name}")
+            fh.write(json.dumps(line, ensure_ascii=False, sort_keys=True) + "\n")
+    print(f"wrote {len(lines)} lines to {path.name}")
+
+
+def main() -> None:
+    _write(OUT, selected())
+    _write(FIELDS_OUT, list(fields_rows()))
 
 
 if __name__ == "__main__":