Add a conventions corpus: invented names for naming traditions the corpora miss

tests/v2/test_corpora.py

@@ -42,6 +42,15 @@ def test_shapes_corpus_matches_the_case_table() -> None:
         "`uv run python tools/differential/build_shapes_corpus.py`")
 
 
+def test_conventions_corpus_matches_its_table() -> None:
+    """corpus_conventions.jsonl is generated from conventions.py's
+    hand-written table (#686)."""
+    module = load_tool("build_conventions_corpus")
+    assert _checked_in("corpus_conventions.jsonl") == module.selected(), (
+        "corpus_conventions.jsonl is stale: regenerate with "
+        "`uv run python tools/differential/build_conventions_corpus.py`")
+
+
 def test_case_shape_ids_exist_in_the_inventory() -> None:
     """cases.py cannot import tools/, so Case.__post_init__ validates
     every shape tag against its own hand copy of the inventory's ids

tools/differential/README.md

@@ -189,6 +189,7 @@ invocation shows it.
 | `corpus_cldr.jsonl` | Unicode CLDR's person-name formatting test data at `release-48`: each fictional name's long, formal rendering in givenFirst, surnameFirst and sorting order, about 110 locales in native scripts, via `build_cldr_corpus.py` (2026-10-12); Unicode-3.0, notice in `cldr_LICENSE` | real-world name distributions: the names are written to fill every field, so full-house names are over-represented and plain two-word names under-represented |
 | `corpus_gnd.jsonl` | the German National Library's authority file (GND, via lobid), real people who are deceased and have a Wikipedia article: each preferred name as cataloged (`Gogh, Vincent van`) and, outside family-first areas, a given-first rendering composed from its fields (a regnal name is composed only, `Johann II.`, without its German descriptor), sampled by shape and area via `build_gnd_corpus.py` (2026-10-16); CC0, credit in `gnd_LICENSE` | names as their bearers write them: German library romanization and German forms for historic figures, Latin script only, titles, credentials and nicknames kept out of the preferred name (a few slip into a `personalName`: `Imam Bonjol`), and the catalog's entry rules (not the bearer's culture) decide what counts as a surname |
 | `corpus_wikidata.jsonl` | Wikidata, real public figures (deceased, or with at least 10 sitelinks) whose item states a second surname (P1950: Spain, Mexico, Colombia) or a patronymic (P5056: Russia, the Soviet Union, the Russian Empire, Iceland, Malaysia): each P1559 native-language name and own-language label whose every word aligns with exactly one stated part, via `build_wikidata_corpus.py` (2026-10-16); CC0, credit in `wikidata_LICENSE` | everything the strict alignment drops: short display forms without the second surname, nicknames, and names with an unrecorded part, so tidy names are over-represented; only those eight citizenships; no titles, credentials or suffixes |
+| `corpus_conventions.jsonl` | invented names patterned on naming conventions the other corpora barely cover -- no family name, religious names, positional patronymics, honorifics after the name -- from the hand-written table in `conventions.py`, via `build_conventions_corpus.py` (#686) | real names, and every convention nobody has written a row for: about one row per convention, so it shows that a convention moved, not how often |
 | `corpus_rules.jsonl` | every example in `docs/design/rules.md`, via `build_rules_corpus.py` (#414) | anything the rules doc has no example for |
 | `corpus_shapes.jsonl` | shape-tagged rows of `tests/v2/cases.py`, via `build_shapes_corpus.py` (#468) | anything no one has tagged a row for |
 
@@ -301,11 +302,29 @@ uv run python tools/differential/build_cjk_corpus.py
 ```
 
 Every distinct CJK-bearing `text` in `tests/v2/cases.py`, `tolerated`
-rows included. The generated corpora (`corpus_cjk`, `corpus_rules`,
-`corpus_shapes`) are each pinned by `tests/v2/test_corpora.py` against
-their generator's selection, so a case row or rules.md example added
-without regenerating fails the suite instead of silently narrowing
-what the snapshot watches.
+rows included. The generated corpora (`corpus_cjk`,
+`corpus_conventions`, `corpus_rules`, `corpus_shapes`) are each pinned
+by `tests/v2/test_corpora.py` against their generator's selection, so a
+case row, a rules.md example or a `conventions.py` row added without
+regenerating fails the suite instead of silently narrowing what the
+snapshot watches.
+
+`corpus_conventions.jsonl` is generated from `conventions.py`, a table
+whose rows record the tradition, the convention a name exercises, and
+what a reader who knows the convention would take each part to be:
+
+```
+uv run python tools/differential/build_conventions_corpus.py
+```
+
+That expectation is a note, not a pin, and nothing scores against
+it: many are stated in terms no field states (a father's or
+grandfather's name, an honorific that is not a title, a married form),
+some are undecided, and a fields file must state every part, since the
+scorer reads an unstated one as an empty role (#686). A row whose
+reading is settled can graduate to `tests/v2/cases.py`. The names are
+invented, never copied from the references the conventions come
+from.
 
 ## What the corpora do not cover
 

tools/differential/build_conventions_corpus.py

@@ -0,0 +1,76 @@
+"""Regenerate corpus_conventions.jsonl from conventions.py's table (#686).
+
+The table holds invented names patterned on naming conventions the other
+corpora barely cover; conventions.py says where the conventions come
+from and why each row's expected reading is a note rather than a pin.
+A row read under a non-default order is written with the shapes.py id
+for that order, so the snapshot reads it under that order.
+
+Regenerate after editing the table:
+
+    uv run python tools/differential/build_conventions_corpus.py
+
+tests/v2/test_corpora.py pins the checked-in file against this module's
+selection, so a row added without regenerating fails the suite.
+"""
+from __future__ import annotations
+
+import importlib.util
+import json
+from pathlib import Path
+from types import ModuleType
+
+HERE = Path(__file__).resolve().parent
+OUT = HERE / "corpus_conventions.jsonl"
+
+
+def _load(stem: str) -> ModuleType:
+    """A sibling module, loaded by path as snapshot.load_shapes does:
+    tools/ is not a package, and putting tools/differential/ on sys.path
+    would let its modules shadow bare imports for the rest of a test run."""
+    spec = importlib.util.spec_from_file_location(f"_{stem}", HERE / f"{stem}.py")
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+CONVENTIONS = _load("conventions").CONVENTIONS
+SHAPES = _load("shapes").SHAPES
+
+
+def _shape_for(order: str) -> int:
+    """The one shapes.py id that is an input shape for `order`."""
+    ids = [k for k, shape in SHAPES.items() if shape.order == order]
+    if len(ids) != 1:
+        raise ValueError(f"conventions.py names order {order!r}, which "
+                         f"{len(ids)} shapes.py shapes declare; use one of "
+                         f"{sorted({s.order for s in SHAPES.values() if s.order})}")
+    return ids[0]
+
+
+def selected() -> list[object]:
+    """The corpus lines, in table order: a bare name under the default
+    order, else the name with its order's shape."""
+    seen: set[tuple[str, str | None]] = set()
+    lines: list[object] = []
+    for row in CONVENTIONS:
+        if (row.name, row.order) in seen:
+            raise ValueError(f"conventions.py lists {row.name!r} under "
+                             f"{row.order or 'the default order'} twice")
+        seen.add((row.name, row.order))
+        lines.append(row.name if row.order is None
+                     else {"name": row.name, "shape": _shape_for(row.order)})
+    return lines
+
+
+def main() -> None:
+    lines = selected()
+    with OUT.open("w", encoding="utf-8") as fh:
+        for line in lines:
+            fh.write(json.dumps(line, ensure_ascii=False) + "\n")
+    print(f"wrote {len(lines)} names to {OUT.name}")
+
+
+if __name__ == "__main__":
+    main()

tools/differential/conventions.py

@@ -0,0 +1,157 @@
+"""Hand-written names for naming conventions the other corpora barely
+cover (#686): traditions with no family name, religious names,
+patronymics placed by position, honorifics written after the name.
+
+Every name here is INVENTED, patterned on a convention documented in
+naming-practice references (a 2006 UK government guide distributed by
+Interpol among them, which carries no licence notice), never copied from
+one. build_conventions_corpus.py writes corpus_conventions.jsonl from
+this table, and the snapshot watches those names like every other
+corpus's.
+
+`expect` is a NOTE, not a pin: what a reader who knows the convention
+would take each part to be, often in terms the model cannot express yet
+(a father's or grandfather's name, an honorific that is not a title, a
+married form), and sometimes left undecided (`H. Budi Santoso`). Nothing
+scores the parser against it (#686 decided against a fields file). Today's reading is deliberately not recorded here either:
+snapshot.jsonl holds it, and a second copy would go stale. A row whose
+reading is settled can graduate to tests/v2/cases.py, where it becomes a
+contract.
+
+`order` names an order some shapes.py shape declares, `"FAMILY_FIRST"`
+or `"FAMILY_FIRST_GIVEN_LAST"`; the default order is None, not
+`"GIVEN_FIRST"`, which no shape declares.
+"""
+from typing import NamedTuple
+
+
+class Convention(NamedTuple):
+    tradition: str
+    convention: str
+    name: str
+    order: str | None
+    expect: str
+
+
+CONVENTIONS: tuple[Convention, ...] = (
+    Convention("Somali", "given + father's + grandfather's name, no family name",
+               "Hodan Farah Warsame", None,
+               "given Hodan; Farah and Warsame are her father's and grandfather's "
+               "given names, not a family name"),
+    Convention("Sikh", "religious name between given and family name",
+               "Harpreet Kaur Dhillon", None,
+               "given Harpreet; Kaur is the religious name Sikh women traditionally "
+               "take, written as a surname when no family name is; family Dhillon"),
+    Convention("Tamil", "father's given name as a leading initial",
+               "K. Sivakumar", None,
+               "K. is the father's given name; Sivakumar is the person's own given "
+               "name; no family name"),
+    Convention("Spanish", "H. (hijo) as the Jr. equivalent",
+               "Pedro López Ruiz H.", None,
+               "given Pedro; family López Ruiz (paternal and maternal surnames); "
+               "H. a generational suffix"),
+    Convention("Spanish", "Vda. de (widow of) before the late husband's surname",
+               "Carmen Díaz Vda. de Mora", None,
+               "given Carmen; her own family name Díaz; Vda. de Mora marks her as "
+               "the widow of a Mora"),
+    Convention("Turkish", "oğlu ('son of') written apart, part of the family name",
+               "Ali Kara Oğlu", None,
+               "given Ali; family Kara Oğlu, the separated spelling of Karaoğlu"),
+    Convention("Turkish", "honorific after the given name",
+               "Murat Bey", None,
+               "given Murat; Bey an honorific, not a family name"),
+    Convention("Hungarian", "title after the name",
+               "Szabó Péter úr", "FAMILY_FIRST",
+               "family Szabó, given Péter; úr ('Mr.') a title written last"),
+    Convention("Hungarian", "married form: -né on the husband's given name",
+               "Szabó Péterné", "FAMILY_FIRST",
+               "family Szabó; Péterné ('wife of Péter') stands where her given "
+               "name would"),
+    Convention("Korean", "two-syllable given name written as two words",
+               "Kim Min Ho", "FAMILY_FIRST",
+               "family Kim; given Min Ho, one name in two words"),
+    Convention("Vietnamese", "Thi as the female middle name",
+               "Tran Thi Mai", "FAMILY_FIRST",
+               "family Tran; middle Thi; given Mai, which the convention writes "
+               "last (the order FAMILY_FIRST_GIVEN_LAST names; read under "
+               "FAMILY_FIRST as the issue measured it); Mai is also a "
+               "suffix acronym, so either order reads it as a suffix"),
+    Convention("Bulgarian", "patronymic between given and family name",
+               "Ivan Petrov Dimitrov", None,
+               "given Ivan; Petrov his patronymic, from his father Petar; family "
+               "Dimitrov"),
+    Convention("Filipino", "Ma. for María, maternal surname in the middle",
+               "Ma. Teresa Cruz Bautista", None,
+               "given Ma. Teresa (Ma. is María, not a title); Cruz the mother's "
+               "surname; family Bautista"),
+    Convention("Korean", "romanized honorific, spaced",
+               "Park Ji-hoon ssi", "FAMILY_FIRST",
+               "family Park; given Ji-hoon; ssi (씨) an honorific"),
+    Convention("Japanese", "romanized honorific, spaced",
+               "Suzuki san", "FAMILY_FIRST",
+               "family Suzuki; san an honorific; no given name"),
+    Convention("Japanese", "romanized honorific, hyphenated",
+               "Suzuki-san", "FAMILY_FIRST",
+               "family Suzuki; -san an honorific glued to it"),
+    Convention("Burmese", "no family name; honorific U for a man",
+               "U Kyaw Win", None,
+               "U an honorific; Kyaw Win the whole personal name; no family name"),
+    Convention("Burmese", "honorific Daw for a woman",
+               "Daw Thida", None,
+               "Daw an honorific; Thida the whole personal name"),
+    Convention("Thai", "Khun before the given name",
+               "Khun Malee", None,
+               "Khun a polite title; Malee the given name, which Thai address uses"),
+    Convention("Malay", "bin ('son of') patronymic, no family name",
+               "Hassan bin Yusof", None,
+               "given Hassan; bin Yusof his patronymic; no family name"),
+    Convention("Malay", "bin abbreviated B.",
+               "Hassan B. Yusof", None,
+               "as Hassan bin Yusof: B. is bin, not a middle initial"),
+    Convention("Malaysian Indian", "a/l (anak lelaki, 'son of')",
+               "Suresh a/l Muthu", None,
+               "given Suresh; a/l Muthu his patronymic; no family name"),
+    Convention("Malaysian Indian", "a/p (anak perempuan, 'daughter of')",
+               "Kavitha a/p Raman", None,
+               "given Kavitha; a/p Raman her patronymic; no family name"),
+    Convention("Indonesian", "H. (Haji), a title shaped like an initial",
+               "H. Budi Santoso", None,
+               "H. a title (one who has made the hajj); given Budi; Santoso a "
+               "second given name or a family name"),
+    Convention("Kalenjin", "arap ('son of')",
+               "David arap Kirui", None,
+               "given David; arap Kirui, used as a family name"),
+    Convention("Maasai", "ole ('son of')",
+               "Joseph ole Sankale", None,
+               "given Joseph; ole Sankale, used as a family name"),
+    Convention("Kikuyu", "wa ('of')",
+               "Wanjiru wa Kamau", None,
+               "given Wanjiru; wa Kamau names her father"),
+    Convention("Hong Kong", "English given name before the Chinese name",
+               "Michael Wong Kar Ming", None,
+               "given Michael; family Wong; Kar Ming the Chinese given name"),
+    Convention("Singapore", "English given name after the Chinese name",
+               "Lim Mei Ling Jessica", "FAMILY_FIRST",
+               "family Lim; Mei Ling the Chinese given name; Jessica the English "
+               "given name"),
+    Convention("Cantonese", "given name that is also a title modifier",
+               "Wing Lau", None,
+               "given Wing, not a title; family Lau"),
+    Convention("Hungarian", "married form on the husband's family name",
+               "Szabóné Kiss Éva", "FAMILY_FIRST",
+               "Szabóné ('wife of Szabó') a married form; family Kiss; given Éva"),
+    Convention("Afrikaans/Dutch", "capitalized particle, no given name",
+               "Van der Walt", None,
+               "family Van der Walt; no given name"),
+    Convention("Arabic", "kunya ('father of')",
+               "Abu Karim", None,
+               "a kunya, 'father of Karim', used as a name of address; not a given "
+               "name plus a family name"),
+    Convention("Iranian", "-zadeh written as a separate word",
+               "Reza Hossein Zadeh", None,
+               "given Reza; family Hossein Zadeh, the separated spelling of "
+               "Hosseinzadeh"),
+    Convention("French", "index form, elided particle after the given name",
+               "Orval, Pierre d'", None,
+               "family d'Orval; given Pierre; d' belongs to the family"),
+)

tools/differential/corpus_conventions.jsonl

@@ -0,0 +1,35 @@
+"Hodan Farah Warsame"
+"Harpreet Kaur Dhillon"
+"K. Sivakumar"
+"Pedro López Ruiz H."
+"Carmen Díaz Vda. de Mora"
+"Ali Kara Oğlu"
+"Murat Bey"
+{"name": "Szabó Péter úr", "shape": 4}
+{"name": "Szabó Péterné", "shape": 4}
+{"name": "Kim Min Ho", "shape": 4}
+{"name": "Tran Thi Mai", "shape": 4}
+"Ivan Petrov Dimitrov"
+"Ma. Teresa Cruz Bautista"
+{"name": "Park Ji-hoon ssi", "shape": 4}
+{"name": "Suzuki san", "shape": 4}
+{"name": "Suzuki-san", "shape": 4}
+"U Kyaw Win"
+"Daw Thida"
+"Khun Malee"
+"Hassan bin Yusof"
+"Hassan B. Yusof"
+"Suresh a/l Muthu"
+"Kavitha a/p Raman"
+"H. Budi Santoso"
+"David arap Kirui"
+"Joseph ole Sankale"
+"Wanjiru wa Kamau"
+"Michael Wong Kar Ming"
+{"name": "Lim Mei Ling Jessica", "shape": 4}
+"Wing Lau"
+{"name": "Szabóné Kiss Éva", "shape": 4}
+"Van der Walt"
+"Abu Karim"
+"Reza Hossein Zadeh"
+"Orval, Pierre d'"