From 5fe705d648d95d76e53be70091647b4628cfff4b Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 16 Oct 2026 19:52:06 -0700 Subject: [PATCH 1/3] State the conventions corpus's parts, and score them like CLDR's MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to #686 (#691). 30 of conventions.py's 35 rows now carry `fields` in CLDR's vocabulary, drawn from the UK naming-practices guide and Cultural Atlas research the rows were written from; build_conventions_corpus.py writes them to conventions_fields.jsonl and cldr_agreement.py scores them as a fourth source, recorded in conventions_agreement.json: 5 of 30 agree in every role. Derek's calls: a word of address written after the name (Bey, ssi, san) is a new `honorific` field scored inside `suffix`, as nameparser reads さん; Kaur is given2, Péterné the given name, H. Budi Santoso a title and one personal name, Ma.'s Cruz a surname2 and Vda. de Mora part of the surname. Five rows stay unstated, since a stated field reads an unstated part as an empty role: the kunya, the glued -san, Szabóné's married form, and arap/ole, patronymic or family name by family. The fields file is therefore a subset of its corpus, which test_cldr_agreement now allows for this source alone. The other sources' records do not move; the corpus file is unchanged. Co-Authored-By: Claude Opus 5.5 --- tests/v2/test_cldr_agreement.py | 21 +- tests/v2/test_corpora.py | 12 +- tools/differential/README.md | 12 +- .../differential/build_conventions_corpus.py | 36 +- tools/differential/cldr_agreement.py | 14 +- tools/differential/conventions.py | 111 +++++-- tools/differential/conventions_agreement.json | 309 ++++++++++++++++++ tools/differential/conventions_fields.jsonl | 30 ++ 8 files changed, 485 insertions(+), 60 deletions(-) create mode 100644 tools/differential/conventions_agreement.json create mode 100644 tools/differential/conventions_fields.jsonl diff --git a/tests/v2/test_cldr_agreement.py b/tests/v2/test_cldr_agreement.py index 5b43d457..edd4b641 100644 --- a/tests/v2/test_cldr_agreement.py +++ b/tests/v2/test_cldr_agreement.py @@ -127,6 +127,15 @@ class WithSurname2: assert agreement.got_roles(WithSurname2())["surname2"] == "García" # type: ignore[arg-type] +def test_an_honorific_is_scored_inside_the_suffix() -> None: + """A word of address written after the name (the conventions + source's `honorific`, #686) is a suffix, as nameparser reads さん.""" + roles = agreement.expected_roles({"given": "Murat", "honorific": "Bey"}, "Murat Bey") + assert roles["suffix"] == "Bey" + assert roles["given_names"] == "Murat" + assert roles["family"] == "" + + def test_expected_roles_compare_particles_only_where_cldr_splits_them() -> None: plain = agreement.expected_roles({"given": "Iris", "surname": "Falke"}, "Iris Falke") assert set(plain) == {"title", "given_names", "family", "suffix"} @@ -220,7 +229,12 @@ def test_changes_reports_a_rendering_that_is_gone() -> None: #: exceed: a truncated file would otherwise pass both checks below, and #: an emptied one re-recorded would match its emptied record. _CORPORA = {"CLDR": ("corpus_cldr.jsonl", 1000), "GND": ("corpus_gnd.jsonl", 500), - "Wikidata": ("corpus_wikidata.jsonl", 500)} + "Wikidata": ("corpus_wikidata.jsonl", 500), + "Conventions": ("corpus_conventions.jsonl", 25)} +#: Sources whose fields file states only some of its corpus's names: the +#: conventions table leaves a row without fields where no field can state +#: it (conventions.py says which and why). +_PARTIAL = frozenset({"Conventions"}) def test_every_source_has_a_corpus_file() -> None: @@ -238,7 +252,10 @@ def test_the_corpus_and_the_fields_file_hold_the_same_renderings(source: str) -> entry = json.loads(line) from_corpus.add((entry, None) if isinstance(entry, str) else (entry["name"], entry["shape"])) - assert from_corpus == from_fields + if source in _PARTIAL: + assert from_fields <= from_corpus + else: + assert from_corpus == from_fields assert len(from_fields) > least, f"the {source} corpus is missing or truncated" diff --git a/tests/v2/test_corpora.py b/tests/v2/test_corpora.py index b4472ae0..195d6af5 100644 --- a/tests/v2/test_corpora.py +++ b/tests/v2/test_corpora.py @@ -43,12 +43,14 @@ def test_shapes_corpus_matches_the_case_table() -> None: def test_conventions_corpus_matches_its_table() -> None: - """corpus_conventions.jsonl is generated from conventions.py's - hand-written table (#686).""" + """corpus_conventions.jsonl and conventions_fields.jsonl are + generated from conventions.py's hand-written table (#686).""" module = load_tool("build_conventions_corpus") - assert _checked_in("corpus_conventions.jsonl") == module.selected(), ( - "corpus_conventions.jsonl is stale: regenerate with " - "`uv run python tools/differential/build_conventions_corpus.py`") + for name, expected in (("corpus_conventions.jsonl", module.selected()), + ("conventions_fields.jsonl", module.fields_rows())): + assert _checked_in(name) == expected, ( + f"{name} is stale: regenerate with " + "`uv run python tools/differential/build_conventions_corpus.py`") def test_case_shape_ids_exist_in_the_inventory() -> None: diff --git a/tools/differential/README.md b/tools/differential/README.md index 40ec8f01..74978fec 100644 --- a/tools/differential/README.md +++ b/tools/differential/README.md @@ -317,12 +317,10 @@ what a reader who knows the convention would take each part to be: uv run python tools/differential/build_conventions_corpus.py ``` -That expectation is a note, not a pin, and nothing scores against -it: many are stated in terms no field states (a father's or -grandfather's name, an honorific that is not a title, a married form), -some are undecided, and a fields file must state every part, since the -scorer reads an unstated one as an empty role (#686). A row whose -reading is settled can graduate to `tests/v2/cases.py`. The names are +The note is prose. Where a field can state the parts, the row +states them too, and the builder writes those rows to +`conventions_fields.jsonl` for the agreement ratchet (below). A row +whose reading is settled can graduate to `tests/v2/cases.py`. The names are invented, never copied from the references the conventions come from. @@ -426,6 +424,8 @@ A second view measures the locale packs (#672). `LOCALE_FILES` in `cldr_agreemen **The Wikidata source.** `build_wikidata_corpus.py` exists for the two fields no other open source labels, and states them in `wikidata_fields.jsonl` as `surname2` (P1950, CLDR's own field name) and `patronymic` (P5056, which CLDR has no field for), beside `given` (P735) and `surname` (P734); its disagreements are in `wikidata_agreement.json`. A person's written names are token-aligned to the parts rather than composed from them: a text is kept only where every word is claimed by exactly one part, a connective (`bin`, `i`, `y`) joining the part after it, and a woman's Russian surname may match the feminine form of its item's label. ParsedName has no `surname2` or `patronymic` attribute, so `cldr_agreement.py` lists them in `PENDING_ROLES` and scores them inside `family` and `given_names` respectively -- a patronymic is not a family name, so `Vigdís Finnbogadóttir` expects the family empty -- and also scores each on its own the day ParsedName gains an attribute of that name (whether the merged role then drops it is for the 3.0 design). Until then the summary says how many renderings state each. CLDR's `surname2` is in the same position. A stratum that falls below its floor, a query that matches nothing, and a response that is not what was asked for each stop the build before anything is written; Malaysia's floor is set below its quota because only 17 Malaysian public figures with a P5056 statement matched the query on 2026-10-16, and 9 of them aligned. Raw responses are cached outside the tree, in a `wikidata/` directory under `$NAMEPARSER_TOOLS_CACHE` (default `~/.cache/nameparser-tools`); `--refresh` fetches them again. +**The conventions source.** `conventions_fields.jsonl` states the parts of the hand-written conventions corpus, from the `fields` of `conventions.py`'s rows, and its disagreements are in `conventions_agreement.json`. The names are invented, so the record is a worklist of conventions rather than a measure of real data: a patronymic read as a family name (`Hassan bin Yusof`), a title read as a given name (`U Kyaw Win`). It uses the Wikidata source's `patronymic`, written with its connective (`bin Yusof`), and adds `honorific`, a word of address written after the name (`Bey`, `ssi`, `san`), scored inside `suffix` as nameparser reads `さん` (Derek, 2026-10-16). A row no field can state honestly is left out of the fields file and stays in the corpus, which the snapshot still watches: the kunya `Abu Karim`, the glued `Suzuki-san`, the `-né` married form `Szabóné Kiss Éva`, and `arap`/`ole`, whose last word is a patronymic or a family name depending on the family (`conventions.py` says which). So this fields file states a subset of its corpus, where the other sources state all of theirs. + `--record` rewrites every record. `JA` needs the `ja` extra (`namedivider-python`), which only CI's `ja-extra` job installs: without it the `JA` ratchet skips, the summary says `not measured`, and `--record` keeps the recorded `ja` entries unchanged. Re-record with the extra installed (`uv sync --group dev --extra ja`) when a change moves Japanese. `_CORPUS_NAMES` in `tests/v2/_differential_fixtures.py` leaves the file out (2026-10-12): the tests reading it carry counts measured over the names the other corpora held. `corpus_gnd.jsonl` and `corpus_wikidata.jsonl` stay in it (2026-10-16): the counts those tests carry are floors and dated snapshots, which a larger corpus cannot falsify, and their invariants now walk both files' names too. A release figure counts GND's moved names in, its composed renderings included, since the names are real; AGENTS.md's rule of counting out names only `corpus_cldr.jsonl` holds stays CLDR's alone (Derek, 2026-10-16). Wikidata's moved names count in under the same rule, which singles out CLDR alone; they are real and written, not composed. `differential_corpus()` in `tests/v2/conftest.py`, which feeds the property tests, globs `corpus*.jsonl`, so it reads CLDR (since #652), GND and Wikidata, and the decomposed-twin test walks all their names. diff --git a/tools/differential/build_conventions_corpus.py b/tools/differential/build_conventions_corpus.py index 677e616b..f1a00af4 100644 --- a/tools/differential/build_conventions_corpus.py +++ b/tools/differential/build_conventions_corpus.py @@ -1,17 +1,20 @@ -"""Regenerate corpus_conventions.jsonl from conventions.py's table (#686). +"""Regenerate corpus_conventions.jsonl and conventions_fields.jsonl from +conventions.py's table (#686). The table holds invented names patterned on naming conventions the other corpora barely cover; conventions.py says where the conventions come from and why each row's expected reading is a note rather than a pin. A row read under a non-default order is written with the shapes.py id -for that order, so the snapshot reads it under that order. +for that order, so the snapshot reads it under that order. A row with +`fields` is also written to conventions_fields.jsonl, in the row format +cldr_agreement.py compares, keyed by the name and its order. Regenerate after editing the table: uv run python tools/differential/build_conventions_corpus.py -tests/v2/test_corpora.py pins the checked-in file against this module's -selection, so a row added without regenerating fails the suite. +tests/v2/test_corpora.py pins both checked-in files against this +module's selection, so a row added without regenerating fails the suite. """ from __future__ import annotations @@ -22,6 +25,7 @@ HERE = Path(__file__).resolve().parent OUT = HERE / "corpus_conventions.jsonl" +FIELDS_OUT = HERE / "conventions_fields.jsonl" def _load(stem: str) -> ModuleType: @@ -64,12 +68,26 @@ def selected() -> list[object]: return lines -def main() -> None: - lines = selected() - with OUT.open("w", encoding="utf-8") as fh: +def fields_rows() -> list[dict[str, object]]: + """The fields file's rows, for the table's rows that state fields.""" + selected() # the same row checks as the corpus + return [{"fields": row.fields, "file": "conventions", "label": row.name, + "order": row.order or "default", + "shape": None if row.order is None else _shape_for(row.order), + "text": row.name} + for row in CONVENTIONS if row.fields is not None] + + +def _write(path: Path, lines: list[object]) -> None: + with path.open("w", encoding="utf-8") as fh: for line in lines: - fh.write(json.dumps(line, ensure_ascii=False) + "\n") - print(f"wrote {len(lines)} names to {OUT.name}") + fh.write(json.dumps(line, ensure_ascii=False, sort_keys=True) + "\n") + print(f"wrote {len(lines)} lines to {path.name}") + + +def main() -> None: + _write(OUT, selected()) + _write(FIELDS_OUT, list(fields_rows())) if __name__ == "__main__": diff --git a/tools/differential/cldr_agreement.py b/tools/differential/cldr_agreement.py index c740ada7..72a1fc5a 100644 --- a/tools/differential/cldr_agreement.py +++ b/tools/differential/cldr_agreement.py @@ -9,8 +9,11 @@ fields in CLDR's vocabulary, so it is a second SOURCE compared by the same rules, with its record in gnd_agreement.json, and so is the Wikidata corpus (build_wikidata_corpus.py, wikidata_agreement.json), -which states the two fields CLDR's vocabulary merges or lacks; "CLDR" -below means any source wherever the rule is about fields. For each source the +which states the two fields CLDR's vocabulary merges or lacks, and the +hand-written conventions table (build_conventions_corpus.py, +conventions_agreement.json), which adds `honorific`, a word of address +written after the name, scored inside `suffix`; "CLDR" below means any +source wherever the rule is about fields. For each source the test fails on: - a role that agreed now disagrees (a regression); @@ -86,6 +89,8 @@ GND_RECORD = HERE / "gnd_agreement.json" WIKIDATA_FIELDS = HERE / "wikidata_fields.jsonl" WIKIDATA_RECORD = HERE / "wikidata_agreement.json" +CONVENTIONS_FIELDS = HERE / "conventions_fields.jsonl" +CONVENTIONS_RECORD = HERE / "conventions_agreement.json" RECORD_COMMAND = "uv run python tools/differential/cldr_agreement.py --record" #: shapes.py's FAMILY_FIRST shape, the only tag build_cldr_corpus.py writes. FAMILY_FIRST_SHAPE = 4 @@ -173,7 +178,7 @@ def pending(role: str) -> tuple[str, ...]: roles = {"title": present("title"), "given_names": present("given", "given2", *pending("given_names")), "family": present(*family, *pending("family")), - "suffix": present("generation", "credentials")} + "suffix": present("generation", "credentials", "honorific")} if "surname-core" in fields: roles["family_particles"] = present("surname-prefix") roles["family_base"] = present("surname-core", *pending("family")) @@ -317,7 +322,8 @@ def sources() -> dict[str, tuple[Path, Path]]: """Each source of stated fields: name -> (fields file, record). Read at call time, so a test that points a record elsewhere is obeyed.""" return {"CLDR": (FIELDS, RECORD), "GND": (GND_FIELDS, GND_RECORD), - "Wikidata": (WIKIDATA_FIELDS, WIKIDATA_RECORD)} + "Wikidata": (WIKIDATA_FIELDS, WIKIDATA_RECORD), + "Conventions": (CONVENTIONS_FIELDS, CONVENTIONS_RECORD)} def _summary(name: str, rows: list[dict[str, Any]], diff --git a/tools/differential/conventions.py b/tools/differential/conventions.py index 71e4bf45..0cb82536 100644 --- a/tools/differential/conventions.py +++ b/tools/differential/conventions.py @@ -12,9 +12,21 @@ `expect` is a NOTE, not a pin: what a reader who knows the convention would take each part to be, often in terms the model cannot express yet (a father's or grandfather's name, an honorific that is not a title, a -married form), and sometimes left undecided (`H. Budi Santoso`). Nothing -scores the parser against it (#686 decided against a fields file). Today's reading is deliberately not recorded here either: -snapshot.jsonl holds it, and a second copy would go stale. A row whose +married form). Nothing scores the parser against it. + +`fields` states the parts in CLDR's field vocabulary, as +cldr_agreement.py compares them, wherever a field can: `patronymic` is +scored inside the given names and `surname2` inside the family, as for +the Wikidata source, and `honorific` (a word of address written after +the name: Bey, ssi, san) inside the suffix. A row no field can state is +left without one rather than stated wrongly, since the comparison reads +an unstated part as an empty role: the kunya, the glued `-san`, the +`-né` married form on a family name, and `arap`/`ole`, whose last word +is a patronymic or a family name by family. build_conventions_corpus.py +writes conventions_fields.jsonl from these. + +Today's reading is deliberately not recorded here: snapshot.jsonl and +conventions_agreement.json hold it, and a second copy would go stale. A row whose reading is settled can graduate to tests/v2/cases.py, where it becomes a contract. @@ -31,93 +43,117 @@ class Convention(NamedTuple): name: str order: str | None expect: str + fields: dict[str, str] | None = None CONVENTIONS: tuple[Convention, ...] = ( Convention("Somali", "given + father's + grandfather's name, no family name", "Hodan Farah Warsame", None, "given Hodan; Farah and Warsame are her father's and grandfather's " - "given names, not a family name"), + "given names, not a family name", + fields={'given': 'Hodan', 'patronymic': 'Farah Warsame'}), Convention("Sikh", "religious name between given and family name", "Harpreet Kaur Dhillon", None, "given Harpreet; Kaur is the religious name Sikh women traditionally " - "take, written as a surname when no family name is; family Dhillon"), + "take, written as a surname when no family name is; family Dhillon", + fields={'given': 'Harpreet', 'given2': 'Kaur', 'surname': 'Dhillon'}), Convention("Tamil", "father's given name as a leading initial", "K. Sivakumar", None, "K. is the father's given name; Sivakumar is the person's own given " - "name; no family name"), + "name; no family name", + fields={'patronymic': 'K.', 'given': 'Sivakumar'}), Convention("Spanish", "H. (hijo) as the Jr. equivalent", "Pedro López Ruiz H.", None, "given Pedro; family López Ruiz (paternal and maternal surnames); " - "H. a generational suffix"), + "H. a generational suffix", + fields={'given': 'Pedro', 'surname': 'López', 'surname2': 'Ruiz', 'generation': 'H.'}), Convention("Spanish", "Vda. de (widow of) before the late husband's surname", "Carmen Díaz Vda. de Mora", None, "given Carmen; her own family name Díaz; Vda. de Mora marks her as " - "the widow of a Mora"), + "the widow of a Mora", + fields={'given': 'Carmen', 'surname': 'Díaz Vda. de Mora'}), Convention("Turkish", "oğlu ('son of') written apart, part of the family name", "Ali Kara Oğlu", None, - "given Ali; family Kara Oğlu, the separated spelling of Karaoğlu"), + "given Ali; family Kara Oğlu, the separated spelling of Karaoğlu", + fields={'given': 'Ali', 'surname': 'Kara Oğlu'}), Convention("Turkish", "honorific after the given name", "Murat Bey", None, - "given Murat; Bey an honorific, not a family name"), + "given Murat; Bey an honorific, not a family name", + fields={'given': 'Murat', 'honorific': 'Bey'}), Convention("Hungarian", "title after the name", "Szabó Péter úr", "FAMILY_FIRST", - "family Szabó, given Péter; úr ('Mr.') a title written last"), + "family Szabó, given Péter; úr ('Mr.') a title written last", + fields={'surname': 'Szabó', 'given': 'Péter', 'honorific': 'úr'}), Convention("Hungarian", "married form: -né on the husband's given name", "Szabó Péterné", "FAMILY_FIRST", "family Szabó; Péterné ('wife of Péter') stands where her given " - "name would"), + "name would", + fields={'surname': 'Szabó', 'given': 'Péterné'}), Convention("Korean", "two-syllable given name written as two words", "Kim Min Ho", "FAMILY_FIRST", - "family Kim; given Min Ho, one name in two words"), + "family Kim; given Min Ho, one name in two words", + fields={'surname': 'Kim', 'given': 'Min Ho'}), Convention("Vietnamese", "Thi as the female middle name", "Tran Thi Mai", "FAMILY_FIRST", "family Tran; middle Thi; given Mai, which the convention writes " "last (the order FAMILY_FIRST_GIVEN_LAST names; read under " "FAMILY_FIRST as the issue measured it); Mai is also a " - "suffix acronym, so either order reads it as a suffix"), + "suffix acronym, so either order reads it as a suffix", + fields={'surname': 'Tran', 'given2': 'Thi', 'given': 'Mai'}), Convention("Bulgarian", "patronymic between given and family name", "Ivan Petrov Dimitrov", None, "given Ivan; Petrov his patronymic, from his father Petar; family " - "Dimitrov"), + "Dimitrov", + fields={'given': 'Ivan', 'patronymic': 'Petrov', 'surname': 'Dimitrov'}), Convention("Filipino", "Ma. for María, maternal surname in the middle", "Ma. Teresa Cruz Bautista", None, "given Ma. Teresa (Ma. is María, not a title); Cruz the mother's " - "surname; family Bautista"), + "surname; family Bautista", + fields={'given': 'Ma. Teresa', 'surname2': 'Cruz', 'surname': 'Bautista'}), Convention("Korean", "romanized honorific, spaced", "Park Ji-hoon ssi", "FAMILY_FIRST", - "family Park; given Ji-hoon; ssi (씨) an honorific"), + "family Park; given Ji-hoon; ssi (씨) an honorific", + fields={'surname': 'Park', 'given': 'Ji-hoon', 'honorific': 'ssi'}), Convention("Japanese", "romanized honorific, spaced", "Suzuki san", "FAMILY_FIRST", - "family Suzuki; san an honorific; no given name"), + "family Suzuki; san an honorific; no given name", + fields={'surname': 'Suzuki', 'honorific': 'san'}), Convention("Japanese", "romanized honorific, hyphenated", "Suzuki-san", "FAMILY_FIRST", "family Suzuki; -san an honorific glued to it"), Convention("Burmese", "no family name; honorific U for a man", "U Kyaw Win", None, - "U an honorific; Kyaw Win the whole personal name; no family name"), + "U an honorific; Kyaw Win the whole personal name; no family name", + fields={'title': 'U', 'given': 'Kyaw Win'}), Convention("Burmese", "honorific Daw for a woman", "Daw Thida", None, - "Daw an honorific; Thida the whole personal name"), + "Daw an honorific; Thida the whole personal name", + fields={'title': 'Daw', 'given': 'Thida'}), Convention("Thai", "Khun before the given name", "Khun Malee", None, - "Khun a polite title; Malee the given name, which Thai address uses"), + "Khun a polite title; Malee the given name, which Thai address uses", + fields={'title': 'Khun', 'given': 'Malee'}), Convention("Malay", "bin ('son of') patronymic, no family name", "Hassan bin Yusof", None, - "given Hassan; bin Yusof his patronymic; no family name"), + "given Hassan; bin Yusof his patronymic; no family name", + fields={'given': 'Hassan', 'patronymic': 'bin Yusof'}), Convention("Malay", "bin abbreviated B.", "Hassan B. Yusof", None, - "as Hassan bin Yusof: B. is bin, not a middle initial"), + "as Hassan bin Yusof: B. is bin, not a middle initial", + fields={'given': 'Hassan', 'patronymic': 'B. Yusof'}), Convention("Malaysian Indian", "a/l (anak lelaki, 'son of')", "Suresh a/l Muthu", None, - "given Suresh; a/l Muthu his patronymic; no family name"), + "given Suresh; a/l Muthu his patronymic; no family name", + fields={'given': 'Suresh', 'patronymic': 'a/l Muthu'}), Convention("Malaysian Indian", "a/p (anak perempuan, 'daughter of')", "Kavitha a/p Raman", None, - "given Kavitha; a/p Raman her patronymic; no family name"), + "given Kavitha; a/p Raman her patronymic; no family name", + fields={'given': 'Kavitha', 'patronymic': 'a/p Raman'}), Convention("Indonesian", "H. (Haji), a title shaped like an initial", "H. Budi Santoso", None, - "H. a title (one who has made the hajj); given Budi; Santoso a " - "second given name or a family name"), + "H. a title (one who has made the hajj); given Budi Santoso, " + "the whole personal name: Indonesian law recognizes no surname", + fields={'title': 'H.', 'given': 'Budi Santoso'}), Convention("Kalenjin", "arap ('son of')", "David arap Kirui", None, "given David; arap Kirui, used as a family name"), @@ -126,23 +162,28 @@ class Convention(NamedTuple): "given Joseph; ole Sankale, used as a family name"), Convention("Kikuyu", "wa ('of')", "Wanjiru wa Kamau", None, - "given Wanjiru; wa Kamau names her father"), + "given Wanjiru; wa Kamau names her father", + fields={'given': 'Wanjiru', 'patronymic': 'wa Kamau'}), Convention("Hong Kong", "English given name before the Chinese name", "Michael Wong Kar Ming", None, - "given Michael; family Wong; Kar Ming the Chinese given name"), + "given Michael; family Wong; Kar Ming the Chinese given name", + fields={'given': 'Michael', 'surname': 'Wong', 'given2': 'Kar Ming'}), Convention("Singapore", "English given name after the Chinese name", "Lim Mei Ling Jessica", "FAMILY_FIRST", "family Lim; Mei Ling the Chinese given name; Jessica the English " - "given name"), + "given name", + fields={'surname': 'Lim', 'given': 'Mei Ling', 'given2': 'Jessica'}), Convention("Cantonese", "given name that is also a title modifier", "Wing Lau", None, - "given Wing, not a title; family Lau"), + "given Wing, not a title; family Lau", + fields={'given': 'Wing', 'surname': 'Lau'}), Convention("Hungarian", "married form on the husband's family name", "Szabóné Kiss Éva", "FAMILY_FIRST", "Szabóné ('wife of Szabó') a married form; family Kiss; given Éva"), Convention("Afrikaans/Dutch", "capitalized particle, no given name", "Van der Walt", None, - "family Van der Walt; no given name"), + "family Van der Walt; no given name", + fields={'surname-prefix': 'Van der', 'surname-core': 'Walt'}), Convention("Arabic", "kunya ('father of')", "Abu Karim", None, "a kunya, 'father of Karim', used as a name of address; not a given " @@ -150,8 +191,10 @@ class Convention(NamedTuple): Convention("Iranian", "-zadeh written as a separate word", "Reza Hossein Zadeh", None, "given Reza; family Hossein Zadeh, the separated spelling of " - "Hosseinzadeh"), + "Hosseinzadeh", + fields={'given': 'Reza', 'surname': 'Hossein Zadeh'}), Convention("French", "index form, elided particle after the given name", "Orval, Pierre d'", None, - "family d'Orval; given Pierre; d' belongs to the family"), + "family d'Orval; given Pierre; d' belongs to the family", + fields={'surname-prefix': "d'", 'surname-core': 'Orval', 'given': 'Pierre'}), ) diff --git a/tools/differential/conventions_agreement.json b/tools/differential/conventions_agreement.json new file mode 100644 index 00000000..79e2e87d --- /dev/null +++ b/tools/differential/conventions_agreement.json @@ -0,0 +1,309 @@ +{ + "conventions/Ali Kara Oğlu/default": { + "family": { + "expected": "Kara Oğlu", + "got": "Oğlu" + }, + "given_names": { + "expected": "Ali", + "got": "Ali Kara" + }, + "text": "Ali Kara Oğlu" + }, + "conventions/Carmen Díaz Vda. de Mora/default": { + "family": { + "expected": "Díaz Vda. de Mora", + "got": "de Mora" + }, + "given_names": { + "expected": "Carmen", + "got": "Carmen Díaz Vda." + }, + "text": "Carmen Díaz Vda. de Mora" + }, + "conventions/Daw Thida/default": { + "family": { + "expected": "", + "got": "Thida" + }, + "given_names": { + "expected": "Thida", + "got": "Daw" + }, + "text": "Daw Thida", + "title": { + "expected": "Daw", + "got": "" + } + }, + "conventions/H. Budi Santoso/default": { + "family": { + "expected": "", + "got": "Santoso" + }, + "given_names": { + "expected": "Budi Santoso", + "got": "H. Budi" + }, + "text": "H. Budi Santoso", + "title": { + "expected": "H.", + "got": "" + } + }, + "conventions/Hassan B. Yusof/default": { + "family": { + "expected": "", + "got": "Yusof" + }, + "given_names": { + "expected": "Hassan B. Yusof", + "got": "Hassan B." + }, + "text": "Hassan B. Yusof" + }, + "conventions/Hassan bin Yusof/default": { + "family": { + "expected": "", + "got": "bin Yusof" + }, + "given_names": { + "expected": "Hassan bin Yusof", + "got": "Hassan" + }, + "text": "Hassan bin Yusof" + }, + "conventions/Hodan Farah Warsame/default": { + "family": { + "expected": "", + "got": "Warsame" + }, + "given_names": { + "expected": "Hodan Farah Warsame", + "got": "Hodan Farah" + }, + "text": "Hodan Farah Warsame" + }, + "conventions/K. Sivakumar/default": { + "family": { + "expected": "", + "got": "Sivakumar" + }, + "given_names": { + "expected": "K. Sivakumar", + "got": "K." + }, + "text": "K. Sivakumar" + }, + "conventions/Kavitha a/p Raman/default": { + "family": { + "expected": "", + "got": "Raman" + }, + "given_names": { + "expected": "Kavitha a/p Raman", + "got": "Kavitha a/p" + }, + "text": "Kavitha a/p Raman" + }, + "conventions/Khun Malee/default": { + "family": { + "expected": "", + "got": "Malee" + }, + "given_names": { + "expected": "Malee", + "got": "Khun" + }, + "text": "Khun Malee", + "title": { + "expected": "Khun", + "got": "" + } + }, + "conventions/Ma. Teresa Cruz Bautista/default": { + "family": { + "expected": "Cruz Bautista", + "got": "Bautista" + }, + "given_names": { + "expected": "Ma. Teresa", + "got": "Teresa Cruz" + }, + "text": "Ma. Teresa Cruz Bautista", + "title": { + "expected": "", + "got": "Ma." + } + }, + "conventions/Michael Wong Kar Ming/default": { + "family": { + "expected": "Wong", + "got": "Ming" + }, + "given_names": { + "expected": "Michael Kar Ming", + "got": "Michael Wong Kar" + }, + "text": "Michael Wong Kar Ming" + }, + "conventions/Murat Bey/default": { + "family": { + "expected": "", + "got": "Bey" + }, + "suffix": { + "expected": "Bey", + "got": "" + }, + "text": "Murat Bey" + }, + "conventions/Orval, Pierre d'/default": { + "family": { + "expected": "Orval d'", + "got": "Orval" + }, + "family_particles": { + "expected": "d'", + "got": "" + }, + "given_names": { + "expected": "Pierre", + "got": "Pierre d'" + }, + "text": "Orval, Pierre d'" + }, + "conventions/Park Ji-hoon ssi/FAMILY_FIRST": { + "given_names": { + "expected": "Ji-hoon", + "got": "Ji-hoon ssi" + }, + "suffix": { + "expected": "ssi", + "got": "" + }, + "text": "Park Ji-hoon ssi" + }, + "conventions/Pedro López Ruiz H./default": { + "family": { + "expected": "López Ruiz", + "got": "H." + }, + "given_names": { + "expected": "Pedro", + "got": "Pedro López Ruiz" + }, + "suffix": { + "expected": "H.", + "got": "" + }, + "text": "Pedro López Ruiz H." + }, + "conventions/Reza Hossein Zadeh/default": { + "family": { + "expected": "Hossein Zadeh", + "got": "Zadeh" + }, + "given_names": { + "expected": "Reza", + "got": "Reza Hossein" + }, + "text": "Reza Hossein Zadeh" + }, + "conventions/Suresh a/l Muthu/default": { + "family": { + "expected": "", + "got": "Muthu" + }, + "given_names": { + "expected": "Suresh a/l Muthu", + "got": "Suresh a/l" + }, + "text": "Suresh a/l Muthu" + }, + "conventions/Suzuki san/FAMILY_FIRST": { + "given_names": { + "expected": "", + "got": "san" + }, + "suffix": { + "expected": "san", + "got": "" + }, + "text": "Suzuki san" + }, + "conventions/Szabó Péter úr/FAMILY_FIRST": { + "given_names": { + "expected": "Péter", + "got": "Péter úr" + }, + "suffix": { + "expected": "úr", + "got": "" + }, + "text": "Szabó Péter úr" + }, + "conventions/Tran Thi Mai/FAMILY_FIRST": { + "given_names": { + "expected": "Thi Mai", + "got": "Thi" + }, + "suffix": { + "expected": "", + "got": "Mai" + }, + "text": "Tran Thi Mai" + }, + "conventions/U Kyaw Win/default": { + "family": { + "expected": "", + "got": "Win" + }, + "given_names": { + "expected": "Kyaw Win", + "got": "U Kyaw" + }, + "text": "U Kyaw Win", + "title": { + "expected": "U", + "got": "" + } + }, + "conventions/Van der Walt/default": { + "family": { + "expected": "Van der Walt", + "got": "der Walt" + }, + "family_particles": { + "expected": "Van der", + "got": "der" + }, + "given_names": { + "expected": "", + "got": "Van" + }, + "text": "Van der Walt" + }, + "conventions/Wanjiru wa Kamau/default": { + "family": { + "expected": "", + "got": "Kamau" + }, + "given_names": { + "expected": "Wanjiru wa Kamau", + "got": "Wanjiru wa" + }, + "text": "Wanjiru wa Kamau" + }, + "conventions/Wing Lau/default": { + "given_names": { + "expected": "Wing", + "got": "" + }, + "text": "Wing Lau", + "title": { + "expected": "", + "got": "Wing" + } + } +} diff --git a/tools/differential/conventions_fields.jsonl b/tools/differential/conventions_fields.jsonl new file mode 100644 index 00000000..2655dcac --- /dev/null +++ b/tools/differential/conventions_fields.jsonl @@ -0,0 +1,30 @@ +{"fields": {"given": "Hodan", "patronymic": "Farah Warsame"}, "file": "conventions", "label": "Hodan Farah Warsame", "order": "default", "shape": null, "text": "Hodan Farah Warsame"} +{"fields": {"given": "Harpreet", "given2": "Kaur", "surname": "Dhillon"}, "file": "conventions", "label": "Harpreet Kaur Dhillon", "order": "default", "shape": null, "text": "Harpreet Kaur Dhillon"} +{"fields": {"given": "Sivakumar", "patronymic": "K."}, "file": "conventions", "label": "K. Sivakumar", "order": "default", "shape": null, "text": "K. Sivakumar"} +{"fields": {"generation": "H.", "given": "Pedro", "surname": "López", "surname2": "Ruiz"}, "file": "conventions", "label": "Pedro López Ruiz H.", "order": "default", "shape": null, "text": "Pedro López Ruiz H."} +{"fields": {"given": "Carmen", "surname": "Díaz Vda. de Mora"}, "file": "conventions", "label": "Carmen Díaz Vda. de Mora", "order": "default", "shape": null, "text": "Carmen Díaz Vda. de Mora"} +{"fields": {"given": "Ali", "surname": "Kara Oğlu"}, "file": "conventions", "label": "Ali Kara Oğlu", "order": "default", "shape": null, "text": "Ali Kara Oğlu"} +{"fields": {"given": "Murat", "honorific": "Bey"}, "file": "conventions", "label": "Murat Bey", "order": "default", "shape": null, "text": "Murat Bey"} +{"fields": {"given": "Péter", "honorific": "úr", "surname": "Szabó"}, "file": "conventions", "label": "Szabó Péter úr", "order": "FAMILY_FIRST", "shape": 4, "text": "Szabó Péter úr"} +{"fields": {"given": "Péterné", "surname": "Szabó"}, "file": "conventions", "label": "Szabó Péterné", "order": "FAMILY_FIRST", "shape": 4, "text": "Szabó Péterné"} +{"fields": {"given": "Min Ho", "surname": "Kim"}, "file": "conventions", "label": "Kim Min Ho", "order": "FAMILY_FIRST", "shape": 4, "text": "Kim Min Ho"} +{"fields": {"given": "Mai", "given2": "Thi", "surname": "Tran"}, "file": "conventions", "label": "Tran Thi Mai", "order": "FAMILY_FIRST", "shape": 4, "text": "Tran Thi Mai"} +{"fields": {"given": "Ivan", "patronymic": "Petrov", "surname": "Dimitrov"}, "file": "conventions", "label": "Ivan Petrov Dimitrov", "order": "default", "shape": null, "text": "Ivan Petrov Dimitrov"} +{"fields": {"given": "Ma. Teresa", "surname": "Bautista", "surname2": "Cruz"}, "file": "conventions", "label": "Ma. Teresa Cruz Bautista", "order": "default", "shape": null, "text": "Ma. Teresa Cruz Bautista"} +{"fields": {"given": "Ji-hoon", "honorific": "ssi", "surname": "Park"}, "file": "conventions", "label": "Park Ji-hoon ssi", "order": "FAMILY_FIRST", "shape": 4, "text": "Park Ji-hoon ssi"} +{"fields": {"honorific": "san", "surname": "Suzuki"}, "file": "conventions", "label": "Suzuki san", "order": "FAMILY_FIRST", "shape": 4, "text": "Suzuki san"} +{"fields": {"given": "Kyaw Win", "title": "U"}, "file": "conventions", "label": "U Kyaw Win", "order": "default", "shape": null, "text": "U Kyaw Win"} +{"fields": {"given": "Thida", "title": "Daw"}, "file": "conventions", "label": "Daw Thida", "order": "default", "shape": null, "text": "Daw Thida"} +{"fields": {"given": "Malee", "title": "Khun"}, "file": "conventions", "label": "Khun Malee", "order": "default", "shape": null, "text": "Khun Malee"} +{"fields": {"given": "Hassan", "patronymic": "bin Yusof"}, "file": "conventions", "label": "Hassan bin Yusof", "order": "default", "shape": null, "text": "Hassan bin Yusof"} +{"fields": {"given": "Hassan", "patronymic": "B. Yusof"}, "file": "conventions", "label": "Hassan B. Yusof", "order": "default", "shape": null, "text": "Hassan B. Yusof"} +{"fields": {"given": "Suresh", "patronymic": "a/l Muthu"}, "file": "conventions", "label": "Suresh a/l Muthu", "order": "default", "shape": null, "text": "Suresh a/l Muthu"} +{"fields": {"given": "Kavitha", "patronymic": "a/p Raman"}, "file": "conventions", "label": "Kavitha a/p Raman", "order": "default", "shape": null, "text": "Kavitha a/p Raman"} +{"fields": {"given": "Budi Santoso", "title": "H."}, "file": "conventions", "label": "H. Budi Santoso", "order": "default", "shape": null, "text": "H. Budi Santoso"} +{"fields": {"given": "Wanjiru", "patronymic": "wa Kamau"}, "file": "conventions", "label": "Wanjiru wa Kamau", "order": "default", "shape": null, "text": "Wanjiru wa Kamau"} +{"fields": {"given": "Michael", "given2": "Kar Ming", "surname": "Wong"}, "file": "conventions", "label": "Michael Wong Kar Ming", "order": "default", "shape": null, "text": "Michael Wong Kar Ming"} +{"fields": {"given": "Mei Ling", "given2": "Jessica", "surname": "Lim"}, "file": "conventions", "label": "Lim Mei Ling Jessica", "order": "FAMILY_FIRST", "shape": 4, "text": "Lim Mei Ling Jessica"} +{"fields": {"given": "Wing", "surname": "Lau"}, "file": "conventions", "label": "Wing Lau", "order": "default", "shape": null, "text": "Wing Lau"} +{"fields": {"surname-core": "Walt", "surname-prefix": "Van der"}, "file": "conventions", "label": "Van der Walt", "order": "default", "shape": null, "text": "Van der Walt"} +{"fields": {"given": "Reza", "surname": "Hossein Zadeh"}, "file": "conventions", "label": "Reza Hossein Zadeh", "order": "default", "shape": null, "text": "Reza Hossein Zadeh"} +{"fields": {"given": "Pierre", "surname-core": "Orval", "surname-prefix": "d'"}, "file": "conventions", "label": "Orval, Pierre d'", "order": "default", "shape": null, "text": "Orval, Pierre d'"} From 5df3318b543e65703023b172b9416bb02de6209a Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 16 Oct 2026 20:10:59 -0700 Subject: [PATCH 2/3] Guard the hand-written fields: unknown names raise, fields partition the text Review of #692: - A field name no rule reads now raises in expected_roles, for every source: a mistyped honorfic was expected in no role and passed as agreement. CLDR's locale and given-informal are the only names skipped on purpose (KNOWN_FIELDS, IGNORED_FIELDS). - test_the_fields_partition_their_rendering: every word of a non-CLDR rendering is stated by exactly one field, so a left-out part fails. GND, Wikidata and conventions already held it. - Each conventions row carries fields or an unstated reason, and the builder refuses neither or both, so a row's fields cannot drop unseen behind the subset check. - _PARSERS gains shape 5, so a FAMILY_FIRST_GIVEN_LAST row no longer crashes the scorer. - Suzuki-san is stated after all: unspaced text matches by substring, so surname Suzuki + honorific -san partition it. My earlier reason for leaving it out was wrong. 26 of 31 rows now disagree. - Stale docstrings (two sources named of four), the ur note (a word of address, as its fields say), and the record of where today's reading lives. Co-Authored-By: Claude Opus 5.5 --- tests/v2/test_cldr_agreement.py | 41 ++++++++++++++++--- tools/differential/README.md | 6 +-- .../differential/build_conventions_corpus.py | 3 ++ tools/differential/cldr_agreement.py | 34 ++++++++++++--- tools/differential/conventions.py | 31 ++++++++------ tools/differential/conventions_agreement.json | 11 +++++ tools/differential/conventions_fields.jsonl | 1 + 7 files changed, 101 insertions(+), 26 deletions(-) diff --git a/tests/v2/test_cldr_agreement.py b/tests/v2/test_cldr_agreement.py index edd4b641..0dd402ba 100644 --- a/tests/v2/test_cldr_agreement.py +++ b/tests/v2/test_cldr_agreement.py @@ -1,12 +1,13 @@ -"""The CLDR and GND agreement ratchets, and the comparison rules they rest on. +"""The agreement ratchets, and the comparison rules they rest on. tools/differential/cldr_agreement.py decides, for each rendering and each role, whether nameparser agrees with the source's stated fields -(CLDR's, and the GND's written in CLDR's vocabulary). The rules +(CLDR's, and the GND's, Wikidata's and the conventions table's written +in CLDR's vocabulary). The rules are pinned here first, each with the wrong answer the plausible alternative gives recorded beside it; the ratchet test at the end then holds each source's corpus to its recorded disagreements in -tools/differential/cldr_agreement.json and gnd_agreement.json. +tools/differential/, one record per source. tools/ is not a package, so the module is loaded by path, as _differential_fixtures.py loads compare.py. @@ -14,6 +15,7 @@ import importlib.util import json import sys +from collections import Counter from pathlib import Path from types import ModuleType @@ -132,10 +134,18 @@ def test_an_honorific_is_scored_inside_the_suffix() -> None: source's `honorific`, #686) is a suffix, as nameparser reads さん.""" roles = agreement.expected_roles({"given": "Murat", "honorific": "Bey"}, "Murat Bey") assert roles["suffix"] == "Bey" + assert roles["title"] == "" assert roles["given_names"] == "Murat" assert roles["family"] == "" +def test_a_field_no_rule_reads_raises() -> None: + """A mistyped field would be expected in no role, and a row holding + it would pass as agreeing with an empty parse.""" + with pytest.raises(ValueError, match="honorfic"): + agreement.expected_roles({"given": "Murat", "honorfic": "Bey"}, "Murat Bey") + + def test_expected_roles_compare_particles_only_where_cldr_splits_them() -> None: plain = agreement.expected_roles({"given": "Iris", "surname": "Falke"}, "Iris Falke") assert set(plain) == {"title", "given_names", "family", "suffix"} @@ -233,7 +243,7 @@ def test_changes_reports_a_rendering_that_is_gone() -> None: "Conventions": ("corpus_conventions.jsonl", 25)} #: Sources whose fields file states only some of its corpus's names: the #: conventions table leaves a row without fields where no field can state -#: it (conventions.py says which and why). +#: it, and says why in the row's `unstated`. _PARTIAL = frozenset({"Conventions"}) @@ -242,7 +252,7 @@ def test_every_source_has_a_corpus_file() -> None: @pytest.mark.parametrize("source", sorted(_CORPORA)) -def test_the_corpus_and_the_fields_file_hold_the_same_renderings(source: str) -> None: +def test_the_fields_file_holds_the_corpus_renderings(source: str) -> None: fields, _ = agreement.sources()[source] corpus, least = _CORPORA[source] from_fields = {(row["text"], row["shape"]) for row in agreement.load_rows(fields)} @@ -272,6 +282,27 @@ def test_every_committed_field_occurs_in_its_rendering(source: str) -> None: assert not missing +@pytest.mark.parametrize("source", sorted(set(_CORPORA) - {"CLDR"})) +def test_the_fields_partition_their_rendering(source: str) -> None: + """Every word of a rendering is stated by exactly one field, so a part + left out is a failure here rather than an expectation of empty. CLDR + is exempt for the reason above; the other sources' builders claim + every word, and the conventions table is written to.""" + fields, _ = agreement.sources()[source] + bad = [] + for row in agreement.load_rows(fields): + values = list(row["fields"].values()) + if agreement._has_space(row["text"]): + ok = (Counter(w for v in values for w in agreement._words(v)) + == Counter(agreement._words(row["text"]))) + else: + ok = (sum(len(agreement._squash(v)) for v in values) + == len(agreement._squash(row["text"]))) + if not ok: + bad.append((row["text"], row["fields"])) + assert not bad + + @pytest.mark.parametrize("source", sorted(_CORPORA)) def test_nameparser_still_agrees_with_each_source_where_it_did(source: str) -> None: fields, record = agreement.sources()[source] diff --git a/tools/differential/README.md b/tools/differential/README.md index 74978fec..47a1901d 100644 --- a/tools/differential/README.md +++ b/tools/differential/README.md @@ -320,8 +320,8 @@ uv run python tools/differential/build_conventions_corpus.py The note is prose. Where a field can state the parts, the row states them too, and the builder writes those rows to `conventions_fields.jsonl` for the agreement ratchet (below). A row -whose reading is settled can graduate to `tests/v2/cases.py`. The names are -invented, never copied from the references the conventions come +whose reading is settled can graduate to `tests/v2/cases.py`. The names +are invented, never copied from the references the conventions come from. ## What the corpora do not cover @@ -424,7 +424,7 @@ A second view measures the locale packs (#672). `LOCALE_FILES` in `cldr_agreemen **The Wikidata source.** `build_wikidata_corpus.py` exists for the two fields no other open source labels, and states them in `wikidata_fields.jsonl` as `surname2` (P1950, CLDR's own field name) and `patronymic` (P5056, which CLDR has no field for), beside `given` (P735) and `surname` (P734); its disagreements are in `wikidata_agreement.json`. A person's written names are token-aligned to the parts rather than composed from them: a text is kept only where every word is claimed by exactly one part, a connective (`bin`, `i`, `y`) joining the part after it, and a woman's Russian surname may match the feminine form of its item's label. ParsedName has no `surname2` or `patronymic` attribute, so `cldr_agreement.py` lists them in `PENDING_ROLES` and scores them inside `family` and `given_names` respectively -- a patronymic is not a family name, so `Vigdís Finnbogadóttir` expects the family empty -- and also scores each on its own the day ParsedName gains an attribute of that name (whether the merged role then drops it is for the 3.0 design). Until then the summary says how many renderings state each. CLDR's `surname2` is in the same position. A stratum that falls below its floor, a query that matches nothing, and a response that is not what was asked for each stop the build before anything is written; Malaysia's floor is set below its quota because only 17 Malaysian public figures with a P5056 statement matched the query on 2026-10-16, and 9 of them aligned. Raw responses are cached outside the tree, in a `wikidata/` directory under `$NAMEPARSER_TOOLS_CACHE` (default `~/.cache/nameparser-tools`); `--refresh` fetches them again. -**The conventions source.** `conventions_fields.jsonl` states the parts of the hand-written conventions corpus, from the `fields` of `conventions.py`'s rows, and its disagreements are in `conventions_agreement.json`. The names are invented, so the record is a worklist of conventions rather than a measure of real data: a patronymic read as a family name (`Hassan bin Yusof`), a title read as a given name (`U Kyaw Win`). It uses the Wikidata source's `patronymic`, written with its connective (`bin Yusof`), and adds `honorific`, a word of address written after the name (`Bey`, `ssi`, `san`), scored inside `suffix` as nameparser reads `さん` (Derek, 2026-10-16). A row no field can state honestly is left out of the fields file and stays in the corpus, which the snapshot still watches: the kunya `Abu Karim`, the glued `Suzuki-san`, the `-né` married form `Szabóné Kiss Éva`, and `arap`/`ole`, whose last word is a patronymic or a family name depending on the family (`conventions.py` says which). So this fields file states a subset of its corpus, where the other sources state all of theirs. +**The conventions source.** `conventions_fields.jsonl` states the parts of the hand-written conventions corpus, from the `fields` of `conventions.py`'s rows, and its disagreements are in `conventions_agreement.json`. The names are invented, so the record is a worklist of conventions rather than a measure of real data: a patronymic read as a family name (`Hassan bin Yusof`), a title read as a given name (`U Kyaw Win`). It uses the Wikidata source's `patronymic`, written with its connective (`bin Yusof`), and adds `honorific`, a word of address written after the name (`Bey`, `ssi`, `san`), scored inside `suffix` as nameparser reads `さん` (Derek, 2026-10-16). A row no field can state honestly is left out of the fields file and stays in the corpus, which the snapshot still watches: the kunya `Abu Karim`, the `-né` married form `Szabóné Kiss Éva`, and `arap`/`ole`, whose last word is a patronymic or a family name depending on the family. Such a row says why in its `unstated`, and the builder refuses a row with neither `fields` nor `unstated`, so this fields file states a subset of its corpus without a row's fields being lost unseen. Being hand-written, it is also held to two checks the generated sources pass anyway: its fields partition each text word for word (`test_the_fields_partition_their_rendering`), and the scorer raises on a field name no rule reads, for every source. `--record` rewrites every record. `JA` needs the `ja` extra (`namedivider-python`), which only CI's `ja-extra` job installs: without it the `JA` ratchet skips, the summary says `not measured`, and `--record` keeps the recorded `ja` entries unchanged. Re-record with the extra installed (`uv sync --group dev --extra ja`) when a change moves Japanese. diff --git a/tools/differential/build_conventions_corpus.py b/tools/differential/build_conventions_corpus.py index f1a00af4..b3fd7024 100644 --- a/tools/differential/build_conventions_corpus.py +++ b/tools/differential/build_conventions_corpus.py @@ -59,6 +59,9 @@ def selected() -> list[object]: seen: set[tuple[str, str | None]] = set() lines: list[object] = [] for row in CONVENTIONS: + if (row.fields is None) == (row.unstated is None): + raise ValueError(f"conventions.py row {row.name!r} needs exactly one " + f"of `fields` and `unstated` (why no field states it)") if (row.name, row.order) in seen: raise ValueError(f"conventions.py lists {row.name!r} under " f"{row.order or 'the default order'} twice") diff --git a/tools/differential/cldr_agreement.py b/tools/differential/cldr_agreement.py index 72a1fc5a..6b04b45a 100644 --- a/tools/differential/cldr_agreement.py +++ b/tools/differential/cldr_agreement.py @@ -1,4 +1,4 @@ -"""Where nameparser disagrees with CLDR and the GND about a name's fields -- a ratchet. +"""Where nameparser disagrees with each source's stated fields -- a ratchet. CLDR's person-name test data (see build_cldr_corpus.py) states every name's fields separately. This module parses each CLDR rendering with the @@ -27,8 +27,7 @@ uv run python tools/differential/cldr_agreement.py # summary uv run python tools/differential/cldr_agreement.py --record # re-record ---record rewrites every record, the GND one and the locale one below -included, and first prints, per source, how many of each kind of change +--record rewrites every record, each source's and the locale one below, and first prints, per source, how many of each kind of change it is accepting, so a re-record made for one source cannot slip another source's regressions in unseen. @@ -55,7 +54,14 @@ gains an attribute of that name; until then the summary says how many renderings state it. Scoring a field on its own does not end the merged scoring; only a 3.0 decision to take the field out of `family` or -`given_names` changes PENDING_ROLES. +`given_names` changes PENDING_ROLES. The conventions source's +`honorific` is not pending: it is scored inside `suffix` by design, as +nameparser reads `さん`, and ParsedName is not meant to gain it. + +A field name no rule reads raises: a mistyped `honorfic` would +otherwise be expected in no role and pass as agreement. KNOWN_FIELDS is +every name expected_roles reads, IGNORED_FIELDS what a source states +that no rendering writes (CLDR's `locale` and `given-informal`). Nothing is special-cased. A deliberately unnatural rendering, `さん` read as a suffix where CLDR calls it a title, a lone word under a family-first @@ -72,7 +78,8 @@ from pathlib import Path from typing import Any -from nameparser import FAMILY_FIRST, ParsedName, Parser, Policy, locales, parser_for +from nameparser import (FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, ParsedName, Parser, Policy, + locales, parser_for) ROLES = ("title", "given_names", "family", "suffix", "family_particles", "family_base") @@ -94,8 +101,12 @@ RECORD_COMMAND = "uv run python tools/differential/cldr_agreement.py --record" #: shapes.py's FAMILY_FIRST shape, the only tag build_cldr_corpus.py writes. FAMILY_FIRST_SHAPE = 4 +#: shapes.py's FAMILY_FIRST_GIVEN_LAST shape, which a conventions row may name. +FAMILY_FIRST_GIVEN_LAST_SHAPE = 5 _PARSERS = {None: Parser(), - FAMILY_FIRST_SHAPE: Parser(policy=Policy(name_order=FAMILY_FIRST))} + FAMILY_FIRST_SHAPE: Parser(policy=Policy(name_order=FAMILY_FIRST)), + FAMILY_FIRST_GIVEN_LAST_SHAPE: + Parser(policy=Policy(name_order=FAMILY_FIRST_GIVEN_LAST))} #: The locale view: CLDR file -> the nameparser.locales pack its #: renderings are parsed with. One pack per file, so the locale record is @@ -157,12 +168,23 @@ def occurs(value: str, text: str) -> bool: return bool(_squash(value)) and _squash(value) in _squash(text) +#: Every field name expected_roles reads, and the ones it skips on purpose. +KNOWN_FIELDS = frozenset({"title", "given", "given2", "surname", "surname-prefix", + "surname-core", "generation", "credentials", "honorific", + *PENDING_ROLES}) +IGNORED_FIELDS = frozenset({"locale", "given-informal"}) + + def expected_roles(fields: dict[str, str], text: str) -> dict[str, str]: """What the source says each role holds IN THIS RENDERING. CLDR's formats drop fields by locale and order, so a role is built only from the fields whose value occurs in the text; a role with none expects ''. The particle and base views are compared only where the source states the split, i.e. the name has a surname-core.""" + unknown = set(fields) - KNOWN_FIELDS - IGNORED_FIELDS + if unknown: + raise ValueError(f"fields {sorted(unknown)} of {text!r} are read by no " + f"rule; use one of {sorted(KNOWN_FIELDS)}") def present(*names: str) -> str: # In the order the values occur in the text, not CLDR's field # order: an unspaced rendering compares in order, and one that diff --git a/tools/differential/conventions.py b/tools/differential/conventions.py index 0cb82536..2af5f990 100644 --- a/tools/differential/conventions.py +++ b/tools/differential/conventions.py @@ -20,13 +20,14 @@ the Wikidata source, and `honorific` (a word of address written after the name: Bey, ssi, san) inside the suffix. A row no field can state is left without one rather than stated wrongly, since the comparison reads -an unstated part as an empty role: the kunya, the glued `-san`, the -`-né` married form on a family name, and `arap`/`ole`, whose last word -is a patronymic or a family name by family. build_conventions_corpus.py -writes conventions_fields.jsonl from these. +an unstated part as an empty role, and says why in `unstated`; every +row carries `fields` or `unstated`, so neither can be left out by +accident. build_conventions_corpus.py writes conventions_fields.jsonl +from the `fields`. -Today's reading is deliberately not recorded here: snapshot.jsonl and -conventions_agreement.json hold it, and a second copy would go stale. A row whose +Today's reading is deliberately not recorded here: snapshot.jsonl holds +it, and conventions_agreement.json holds it where it disagrees with +`fields`. A second copy would go stale. A row whose reading is settled can graduate to tests/v2/cases.py, where it becomes a contract. @@ -44,6 +45,7 @@ class Convention(NamedTuple): order: str | None expect: str fields: dict[str, str] | None = None + unstated: str | None = None CONVENTIONS: tuple[Convention, ...] = ( @@ -82,7 +84,7 @@ class Convention(NamedTuple): fields={'given': 'Murat', 'honorific': 'Bey'}), Convention("Hungarian", "title after the name", "Szabó Péter úr", "FAMILY_FIRST", - "family Szabó, given Péter; úr ('Mr.') a title written last", + "family Szabó, given Péter; úr ('Mr.') a word of address written last", fields={'surname': 'Szabó', 'given': 'Péter', 'honorific': 'úr'}), Convention("Hungarian", "married form: -né on the husband's given name", "Szabó Péterné", "FAMILY_FIRST", @@ -120,7 +122,8 @@ class Convention(NamedTuple): fields={'surname': 'Suzuki', 'honorific': 'san'}), Convention("Japanese", "romanized honorific, hyphenated", "Suzuki-san", "FAMILY_FIRST", - "family Suzuki; -san an honorific glued to it"), + "family Suzuki; -san an honorific glued to it", + fields={'surname': 'Suzuki', 'honorific': '-san'}), Convention("Burmese", "no family name; honorific U for a man", "U Kyaw Win", None, "U an honorific; Kyaw Win the whole personal name; no family name", @@ -156,10 +159,12 @@ class Convention(NamedTuple): fields={'title': 'H.', 'given': 'Budi Santoso'}), Convention("Kalenjin", "arap ('son of')", "David arap Kirui", None, - "given David; arap Kirui, used as a family name"), + "given David; arap Kirui, used as a family name", + unstated="arap Kirui is a patronymic or a family name by family"), Convention("Maasai", "ole ('son of')", "Joseph ole Sankale", None, - "given Joseph; ole Sankale, used as a family name"), + "given Joseph; ole Sankale, used as a family name", + unstated="ole Sankale is a patronymic or a family name by family"), Convention("Kikuyu", "wa ('of')", "Wanjiru wa Kamau", None, "given Wanjiru; wa Kamau names her father", @@ -179,7 +184,8 @@ class Convention(NamedTuple): fields={'given': 'Wing', 'surname': 'Lau'}), Convention("Hungarian", "married form on the husband's family name", "Szabóné Kiss Éva", "FAMILY_FIRST", - "Szabóné ('wife of Szabó') a married form; family Kiss; given Éva"), + "Szabóné ('wife of Szabó') a married form; family Kiss; given Éva", + unstated="no field states a married form on the husband's family name"), Convention("Afrikaans/Dutch", "capitalized particle, no given name", "Van der Walt", None, "family Van der Walt; no given name", @@ -187,7 +193,8 @@ class Convention(NamedTuple): Convention("Arabic", "kunya ('father of')", "Abu Karim", None, "a kunya, 'father of Karim', used as a name of address; not a given " - "name plus a family name"), + "name plus a family name", + unstated="no field states a kunya"), Convention("Iranian", "-zadeh written as a separate word", "Reza Hossein Zadeh", None, "given Reza; family Hossein Zadeh, the separated spelling of " diff --git a/tools/differential/conventions_agreement.json b/tools/differential/conventions_agreement.json index 79e2e87d..8787d6dd 100644 --- a/tools/differential/conventions_agreement.json +++ b/tools/differential/conventions_agreement.json @@ -232,6 +232,17 @@ }, "text": "Suzuki san" }, + "conventions/Suzuki-san/FAMILY_FIRST": { + "family": { + "expected": "Suzuki", + "got": "Suzuki-san" + }, + "suffix": { + "expected": "-san", + "got": "" + }, + "text": "Suzuki-san" + }, "conventions/Szabó Péter úr/FAMILY_FIRST": { "given_names": { "expected": "Péter", diff --git a/tools/differential/conventions_fields.jsonl b/tools/differential/conventions_fields.jsonl index 2655dcac..be5b69e6 100644 --- a/tools/differential/conventions_fields.jsonl +++ b/tools/differential/conventions_fields.jsonl @@ -13,6 +13,7 @@ {"fields": {"given": "Ma. Teresa", "surname": "Bautista", "surname2": "Cruz"}, "file": "conventions", "label": "Ma. Teresa Cruz Bautista", "order": "default", "shape": null, "text": "Ma. Teresa Cruz Bautista"} {"fields": {"given": "Ji-hoon", "honorific": "ssi", "surname": "Park"}, "file": "conventions", "label": "Park Ji-hoon ssi", "order": "FAMILY_FIRST", "shape": 4, "text": "Park Ji-hoon ssi"} {"fields": {"honorific": "san", "surname": "Suzuki"}, "file": "conventions", "label": "Suzuki san", "order": "FAMILY_FIRST", "shape": 4, "text": "Suzuki san"} +{"fields": {"honorific": "-san", "surname": "Suzuki"}, "file": "conventions", "label": "Suzuki-san", "order": "FAMILY_FIRST", "shape": 4, "text": "Suzuki-san"} {"fields": {"given": "Kyaw Win", "title": "U"}, "file": "conventions", "label": "U Kyaw Win", "order": "default", "shape": null, "text": "U Kyaw Win"} {"fields": {"given": "Thida", "title": "Daw"}, "file": "conventions", "label": "Daw Thida", "order": "default", "shape": null, "text": "Daw Thida"} {"fields": {"given": "Malee", "title": "Khun"}, "file": "conventions", "label": "Khun Malee", "order": "default", "shape": null, "text": "Khun Malee"} From 48763183dafe327a15db3f36e2511b7a36b45872 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Fri, 16 Oct 2026 20:14:31 -0700 Subject: [PATCH 3/3] Leave wa unstated beside arap and ole Derek's call: the research groups the three as declining Kenyan parent-name markers whose last word may now be a family name, and its wa example names a mother, so wa is no surer a patronymic than the others. Its row stays in the corpus; 30 rows are stated, 25 disagree. Co-Authored-By: Claude Opus 5.5 --- tools/differential/README.md | 2 +- tools/differential/conventions.py | 5 +++-- tools/differential/conventions_agreement.json | 11 ----------- tools/differential/conventions_fields.jsonl | 1 - 4 files changed, 4 insertions(+), 15 deletions(-) diff --git a/tools/differential/README.md b/tools/differential/README.md index 47a1901d..d8f17a3a 100644 --- a/tools/differential/README.md +++ b/tools/differential/README.md @@ -424,7 +424,7 @@ A second view measures the locale packs (#672). `LOCALE_FILES` in `cldr_agreemen **The Wikidata source.** `build_wikidata_corpus.py` exists for the two fields no other open source labels, and states them in `wikidata_fields.jsonl` as `surname2` (P1950, CLDR's own field name) and `patronymic` (P5056, which CLDR has no field for), beside `given` (P735) and `surname` (P734); its disagreements are in `wikidata_agreement.json`. A person's written names are token-aligned to the parts rather than composed from them: a text is kept only where every word is claimed by exactly one part, a connective (`bin`, `i`, `y`) joining the part after it, and a woman's Russian surname may match the feminine form of its item's label. ParsedName has no `surname2` or `patronymic` attribute, so `cldr_agreement.py` lists them in `PENDING_ROLES` and scores them inside `family` and `given_names` respectively -- a patronymic is not a family name, so `Vigdís Finnbogadóttir` expects the family empty -- and also scores each on its own the day ParsedName gains an attribute of that name (whether the merged role then drops it is for the 3.0 design). Until then the summary says how many renderings state each. CLDR's `surname2` is in the same position. A stratum that falls below its floor, a query that matches nothing, and a response that is not what was asked for each stop the build before anything is written; Malaysia's floor is set below its quota because only 17 Malaysian public figures with a P5056 statement matched the query on 2026-10-16, and 9 of them aligned. Raw responses are cached outside the tree, in a `wikidata/` directory under `$NAMEPARSER_TOOLS_CACHE` (default `~/.cache/nameparser-tools`); `--refresh` fetches them again. -**The conventions source.** `conventions_fields.jsonl` states the parts of the hand-written conventions corpus, from the `fields` of `conventions.py`'s rows, and its disagreements are in `conventions_agreement.json`. The names are invented, so the record is a worklist of conventions rather than a measure of real data: a patronymic read as a family name (`Hassan bin Yusof`), a title read as a given name (`U Kyaw Win`). It uses the Wikidata source's `patronymic`, written with its connective (`bin Yusof`), and adds `honorific`, a word of address written after the name (`Bey`, `ssi`, `san`), scored inside `suffix` as nameparser reads `さん` (Derek, 2026-10-16). A row no field can state honestly is left out of the fields file and stays in the corpus, which the snapshot still watches: the kunya `Abu Karim`, the `-né` married form `Szabóné Kiss Éva`, and `arap`/`ole`, whose last word is a patronymic or a family name depending on the family. Such a row says why in its `unstated`, and the builder refuses a row with neither `fields` nor `unstated`, so this fields file states a subset of its corpus without a row's fields being lost unseen. Being hand-written, it is also held to two checks the generated sources pass anyway: its fields partition each text word for word (`test_the_fields_partition_their_rendering`), and the scorer raises on a field name no rule reads, for every source. +**The conventions source.** `conventions_fields.jsonl` states the parts of the hand-written conventions corpus, from the `fields` of `conventions.py`'s rows, and its disagreements are in `conventions_agreement.json`. The names are invented, so the record is a worklist of conventions rather than a measure of real data: a patronymic read as a family name (`Hassan bin Yusof`), a title read as a given name (`U Kyaw Win`). It uses the Wikidata source's `patronymic`, written with its connective (`bin Yusof`), and adds `honorific`, a word of address written after the name (`Bey`, `ssi`, `san`), scored inside `suffix` as nameparser reads `さん` (Derek, 2026-10-16). A row no field can state honestly is left out of the fields file and stays in the corpus, which the snapshot still watches: the kunya `Abu Karim`, the `-né` married form `Szabóné Kiss Éva`, and `arap`/`ole`/`wa`, whose last word is a parent's name or a family name depending on the family. Such a row says why in its `unstated`, and the builder refuses a row with neither `fields` nor `unstated`, so this fields file states a subset of its corpus without a row's fields being lost unseen. Being hand-written, it is also held to two checks the generated sources pass anyway: its fields partition each text word for word (`test_the_fields_partition_their_rendering`), and the scorer raises on a field name no rule reads, for every source. `--record` rewrites every record. `JA` needs the `ja` extra (`namedivider-python`), which only CI's `ja-extra` job installs: without it the `JA` ratchet skips, the summary says `not measured`, and `--record` keeps the recorded `ja` entries unchanged. Re-record with the extra installed (`uv sync --group dev --extra ja`) when a change moves Japanese. diff --git a/tools/differential/conventions.py b/tools/differential/conventions.py index 2af5f990..3e3b850c 100644 --- a/tools/differential/conventions.py +++ b/tools/differential/conventions.py @@ -167,8 +167,9 @@ class Convention(NamedTuple): unstated="ole Sankale is a patronymic or a family name by family"), Convention("Kikuyu", "wa ('of')", "Wanjiru wa Kamau", None, - "given Wanjiru; wa Kamau names her father", - fields={'given': 'Wanjiru', 'patronymic': 'wa Kamau'}), + "given Wanjiru; wa Kamau names a parent, her father or her mother", + unstated="wa Kamau is a patronymic, a matronymic or a family name " + "by family"), Convention("Hong Kong", "English given name before the Chinese name", "Michael Wong Kar Ming", None, "given Michael; family Wong; Kar Ming the Chinese given name", diff --git a/tools/differential/conventions_agreement.json b/tools/differential/conventions_agreement.json index 8787d6dd..3a668106 100644 --- a/tools/differential/conventions_agreement.json +++ b/tools/differential/conventions_agreement.json @@ -295,17 +295,6 @@ }, "text": "Van der Walt" }, - "conventions/Wanjiru wa Kamau/default": { - "family": { - "expected": "", - "got": "Kamau" - }, - "given_names": { - "expected": "Wanjiru wa Kamau", - "got": "Wanjiru wa" - }, - "text": "Wanjiru wa Kamau" - }, "conventions/Wing Lau/default": { "given_names": { "expected": "Wing", diff --git a/tools/differential/conventions_fields.jsonl b/tools/differential/conventions_fields.jsonl index be5b69e6..3ab3ea75 100644 --- a/tools/differential/conventions_fields.jsonl +++ b/tools/differential/conventions_fields.jsonl @@ -22,7 +22,6 @@ {"fields": {"given": "Suresh", "patronymic": "a/l Muthu"}, "file": "conventions", "label": "Suresh a/l Muthu", "order": "default", "shape": null, "text": "Suresh a/l Muthu"} {"fields": {"given": "Kavitha", "patronymic": "a/p Raman"}, "file": "conventions", "label": "Kavitha a/p Raman", "order": "default", "shape": null, "text": "Kavitha a/p Raman"} {"fields": {"given": "Budi Santoso", "title": "H."}, "file": "conventions", "label": "H. Budi Santoso", "order": "default", "shape": null, "text": "H. Budi Santoso"} -{"fields": {"given": "Wanjiru", "patronymic": "wa Kamau"}, "file": "conventions", "label": "Wanjiru wa Kamau", "order": "default", "shape": null, "text": "Wanjiru wa Kamau"} {"fields": {"given": "Michael", "given2": "Kar Ming", "surname": "Wong"}, "file": "conventions", "label": "Michael Wong Kar Ming", "order": "default", "shape": null, "text": "Michael Wong Kar Ming"} {"fields": {"given": "Mei Ling", "given2": "Jessica", "surname": "Lim"}, "file": "conventions", "label": "Lim Mei Ling Jessica", "order": "FAMILY_FIRST", "shape": 4, "text": "Lim Mei Ling Jessica"} {"fields": {"given": "Wing", "surname": "Lau"}, "file": "conventions", "label": "Wing Lau", "order": "default", "shape": null, "text": "Wing Lau"}