Share the corpus builders' cache and file helpers

tests/v2/_differential_fixtures.py

@@ -48,8 +48,9 @@ def load_tool(stem: str) -> ModuleType:
 
     `tools/` is outside testpaths, and adding it would run
     --doctest-modules over the corpus builders, so these are imported
-    this way rather than made importable. None has import-time side
-    effects: each main() is behind a __name__ guard."""
+    this way rather than made importable. Each main() is behind a
+    __name__ guard; the one import-time effect is that a corpus builder
+    appends its own directory to sys.path to find corpus_build.py."""
     spec = importlib.util.spec_from_file_location(
         f"differential_{stem}", _TOOLS / f"{stem}.py")
     assert spec is not None and spec.loader is not None, stem

tests/v2/test_corpus_build.py

@@ -0,0 +1,114 @@
+"""Offline checks on tools/differential/corpus_build.py, the helpers the
+labeled-corpus builders share.
+
+The builders' own tests reach these helpers through one builder at a
+time; what is pinned here is each helper's contract, and that every
+builder can find the module at all. tools/ is not a package, so modules
+are loaded by path, as _differential_fixtures.py loads compare.py.
+"""
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+from ._differential_fixtures import _TOOLS, load_tool
+
+corpus_build = load_tool("corpus_build")
+
+_BUILDERS = ("build_cldr_corpus", "build_gnd_corpus", "build_wikidata_corpus")
+
+
+@pytest.mark.parametrize("stem", _BUILDERS)
+def test_each_builder_finds_the_shared_module_on_its_own(
+        stem: str, tmp_path: Path) -> None:
+    # In a fresh interpreter, from an unrelated cwd and with safe-path on:
+    # in the test process an earlier builder's sys.path entry, or a cached
+    # import, would hide a builder that forgot its own.
+    loader = ("import importlib.util as u, sys; "
+              f"s = u.spec_from_file_location('b', {str(_TOOLS / (stem + '.py'))!r}); "
+              "m = u.module_from_spec(s); s.loader.exec_module(m); "
+              "print(m.corpus_lines is sys.modules['corpus_build'].corpus_lines)")
+    result = subprocess.run([sys.executable, "-P", "-c", loader], cwd=tmp_path,
+                            capture_output=True, text=True, check=False)
+    assert result.returncode == 0, result.stderr
+    assert result.stdout.strip() == "True"
+
+
+def test_a_shape_must_be_an_int_or_none() -> None:
+    with pytest.raises(TypeError, match="shape must be an int or None, not '4'"):
+        corpus_build.corpus_lines([{"text": "A", "shape": "4"}])
+
+
+def test_an_untagged_line_sorts_before_the_tagged_ones_of_its_text() -> None:
+    # Shape 0 is the case to watch: a key that ties it with None leaves
+    # the order to set iteration, which a single text can pass by luck.
+    texts = [f"name {i}" for i in range(40)]
+    rows = [{"text": t, "shape": shape} for t in texts for shape in (0, None, 4)]
+    assert corpus_build.corpus_lines(rows) == [
+        line for t in sorted(texts)
+        for line in (json.dumps(t), json.dumps({"name": t, "shape": 0}),
+                     json.dumps({"name": t, "shape": 4}))]
+
+
+def _row(text: str, order: str = "givenFirst") -> dict[str, object]:
+    return {"file": "Q1", "label": "x", "order": order, "text": text,
+            "shape": None, "fields": {"given": text}}
+
+
+def test_write_corpus_writes_the_corpus_and_the_fields_file(tmp_path: Path) -> None:
+    corpus, fields = tmp_path / "corpus.jsonl", tmp_path / "fields.jsonl"
+    corpus_build.write_corpus([_row("B", "sorting"), _row("A")], corpus, fields)
+    assert corpus.read_text(encoding="utf-8") == '"A"\n"B"\n'
+    assert [json.loads(line)["order"] for line in
+            fields.read_text(encoding="utf-8").splitlines()] == ["givenFirst", "sorting"]
+
+
+def test_a_refused_row_leaves_both_files_as_they_were(tmp_path: Path) -> None:
+    corpus, fields = tmp_path / "corpus.jsonl", tmp_path / "fields.jsonl"
+    corpus.write_text("old corpus\n", encoding="utf-8")
+    fields.write_text("old fields\n", encoding="utf-8")
+    with pytest.raises(ValueError, match="duplicate rendering Q1/x/givenFirst"):
+        corpus_build.write_corpus([_row("A"), _row("B")], corpus, fields)
+    assert corpus.read_text(encoding="utf-8") == "old corpus\n"
+    assert fields.read_text(encoding="utf-8") == "old fields\n"
+
+
+def test_a_failed_write_leaves_both_files_as_they_were(
+        tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    corpus, fields = tmp_path / "corpus.jsonl", tmp_path / "fields.jsonl"
+    corpus.write_text("old corpus\n", encoding="utf-8")
+    fields.write_text("old fields\n", encoding="utf-8")
+    real = Path.write_text
+
+    def full_disk(self: Path, text: str, encoding: str | None = None) -> int:
+        if self.name.startswith("fields"):
+            raise OSError("No space left on device")
+        return real(self, text, encoding=encoding)
+    monkeypatch.setattr(Path, "write_text", full_disk)
+    with pytest.raises(OSError, match="No space"):
+        corpus_build.write_corpus([_row("A")], corpus, fields)
+    monkeypatch.undo()
+    assert corpus.read_text(encoding="utf-8") == "old corpus\n"
+    assert fields.read_text(encoding="utf-8") == "old fields\n"
+    assert sorted(p.name for p in tmp_path.iterdir()) == ["corpus.jsonl", "fields.jsonl"]
+
+
+def _check(body: bytes | str, source: str) -> dict[str, object]:
+    data = corpus_build.parse_json(body, source)
+    assert isinstance(data, dict)
+    return data
+
+
+def test_an_interrupted_cache_write_leaves_no_cache_file(
+        tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    # The body goes to a .part file and is renamed into place; if the
+    # rename never happens, the next run must not find a cached answer.
+    def interrupted(src: object, dst: object) -> None:
+        raise OSError("interrupted")
+    monkeypatch.setattr(corpus_build.os, "replace", interrupted)
+    with pytest.raises(OSError, match="interrupted"):
+        corpus_build.cached_json(tmp_path, "k", "label", lambda: b'{"a": 1}',
+                                 _check, refresh=False)
+    assert not list(tmp_path.glob("*.json"))

tests/v2/test_gnd_corpus.py

@@ -147,7 +147,8 @@ def test_a_record_its_fields_cannot_account_for_is_left_out(
 
 
 def test_the_corpus_holds_each_text_once() -> None:
-    rows = [{"text": "Björk"}, {"text": "Björk"}, {"text": "Abe, Kōbō"}]
+    rows = [{"text": "Björk", "shape": None}, {"text": "Björk", "shape": None},
+            {"text": "Abe, Kōbō", "shape": None}]
     assert build.corpus_lines(rows) == ['"Abe, Kōbō"', '"Björk"']
 
 

tools/differential/README.md

@@ -403,9 +403,9 @@ A second view measures the locale packs (#672). `LOCALE_FILES` in `cldr_agreemen
 
 **The GND source.** `build_gnd_corpus.py` writes the GND's name parts in CLDR's field vocabulary (`forename` → `given`, `prefix` → `surname-prefix` with a one-word `surname` as `surname-core`, any other family → `surname` whole, `counting` → `generation`, and a `personalName` -- a person entered under the whole name, with no surname -- as `given` alone, so the family is expected empty), so `cldr_agreement.py` compares `gnd_fields.jsonl` by the same rules and records its disagreements in `gnd_agreement.json`, ratcheted by the same test. The locale view reads CLDR only. A composed rendering says so (`composed: true`); where a particle is glued to the surname (`d'Arenberg`, `al-Ǧaʿfarī`) the composed line states the glued word as a whole `surname`, since the comparison matches fields word by word, and the index form, where the particle is a word of its own, keeps the split. A family with no `prefix`, or with a surname of several words, is stated whole for the same kind of reason: GND does not split a particle inside a surname (`Vélez de Guevara`, `Zuylen van Nijevelt`), so what it leaves unsplit means "not stated", not "no particle", and only `family` is compared there. Raw lobid responses are cached outside the tree, in a `gnd/` directory under `$NAMEPARSER_TOOLS_CACHE` (default `~/.cache/nameparser-tools`); `--refresh` fetches them again. The build stops before writing anything when a stratum falls below its floor, a query matches nothing, or a pool pages short, so a lobid change fails it rather than shrinking the corpus; `--record` prints, per source, how many changes of each kind it accepts.
 
-**Adding a source.** Map only what the source states. The comparison reads a missing field as a statement that the role is empty, so a field the source leaves unstated must be mapped so that its role is not compared at all: #688's first build wrote a prefixless GND surname as `surname-core`, and 19 renderings recorded the parser's `de` in `Vélez de Guevara` as a disagreement no fix could clear; its fix then missed the same gap inside a several-word core after a prefix (`Zuylen van Nijevelt, Julius van`), so test each way a field can be left unstated, not the one that was reported. A field must also occur in its rendering's text, word by word (`test_every_committed_field_occurs_in_its_rendering`), or it is expected empty too. A build must fail rather than shrink: give every stratum a floor, check a response's shape before caching it (an API error arrives as HTTP 200), and raise on a query that matches nothing. But tell a broken fetch from a fact about the data: #689's first fix raised on any item Wikidata reported missing, so one deleted name-part item anywhere in a pool would have stopped every rebuild, with `--refresh` advice that could not help; a missing part now only refuses the person stating it.
+**Adding a source.** Map only what the source states. The comparison reads a missing field as a statement that the role is empty, so a field the source leaves unstated must be mapped so that its role is not compared at all: #688's first build wrote a prefixless GND surname as `surname-core`, and 19 renderings recorded the parser's `de` in `Vélez de Guevara` as a disagreement no fix could clear; its fix then missed the same gap inside a several-word core after a prefix (`Zuylen van Nijevelt, Julius van`), so test each way a field can be left unstated, not the one that was reported. A field must also occur in its rendering's text, word by word (`test_every_committed_field_occurs_in_its_rendering`), or it is expected empty too. A build must fail rather than shrink: give every stratum a floor, check a response's shape before caching it (an API error arrives as HTTP 200), and raise on a query that matches nothing. But tell a broken fetch from a fact about the data: #689's first fix raised on any item Wikidata reported missing, so one deleted name-part item anywhere in a pool would have stopped every rebuild, with `--refresh` advice that could not help; a missing part now only refuses the person stating it. The cache (`cache_dir`, `cached_json`, which checks a response before caching it and writes atomically) and the two output files (`corpus_lines`, `fields_lines`, `write_corpus`) are shared in `corpus_build.py`; a new builder imports them rather than copying another builder's.
 
-**The Wikidata source.** `build_wikidata_corpus.py` exists for the two fields no other open source labels, and states them in `wikidata_fields.jsonl` as `surname2` (P1950, CLDR's own field name) and `patronymic` (P5056, which CLDR has no field for), beside `given` (P735) and `surname` (P734); its disagreements are in `wikidata_agreement.json`. A person's written names are token-aligned to the parts rather than composed from them: a text is kept only where every word is claimed by exactly one part, a connective (`bin`, `i`, `y`) joining the part after it, and a woman's Russian surname may match the feminine form of its item's label. ParsedName has no `surname2` or `patronymic` attribute, so `cldr_agreement.py` lists them in `PENDING_ROLES` and scores them inside `family` and `given_names` respectively -- a patronymic is not a family name, so `Vigdís Finnbogadóttir` expects the family empty -- and also scores each on its own the day ParsedName gains an attribute of that name (whether the merged role then drops it is for the 3.0 design). Until then the summary says how many renderings state each. CLDR's `surname2` is in the same position. A stratum that falls below its floor, a query that matches nothing, and a response that is not what was asked for each stop the build before anything is written; Malaysia's floor is set below its quota because only 17 Malaysian public figures with a P5056 statement matched the query on 2026-10-16, and 9 of them aligned. Raw responses are cached outside the tree (`$NAMEPARSER_TOOLS_CACHE` or `~/.cache/nameparser-tools/wikidata`); `--refresh` fetches them again.
+**The Wikidata source.** `build_wikidata_corpus.py` exists for the two fields no other open source labels, and states them in `wikidata_fields.jsonl` as `surname2` (P1950, CLDR's own field name) and `patronymic` (P5056, which CLDR has no field for), beside `given` (P735) and `surname` (P734); its disagreements are in `wikidata_agreement.json`. A person's written names are token-aligned to the parts rather than composed from them: a text is kept only where every word is claimed by exactly one part, a connective (`bin`, `i`, `y`) joining the part after it, and a woman's Russian surname may match the feminine form of its item's label. ParsedName has no `surname2` or `patronymic` attribute, so `cldr_agreement.py` lists them in `PENDING_ROLES` and scores them inside `family` and `given_names` respectively -- a patronymic is not a family name, so `Vigdís Finnbogadóttir` expects the family empty -- and also scores each on its own the day ParsedName gains an attribute of that name (whether the merged role then drops it is for the 3.0 design). Until then the summary says how many renderings state each. CLDR's `surname2` is in the same position. A stratum that falls below its floor, a query that matches nothing, and a response that is not what was asked for each stop the build before anything is written; Malaysia's floor is set below its quota because only 17 Malaysian public figures with a P5056 statement matched the query on 2026-10-16, and 9 of them aligned. Raw responses are cached outside the tree, in a `wikidata/` directory under `$NAMEPARSER_TOOLS_CACHE` (default `~/.cache/nameparser-tools`); `--refresh` fetches them again.
 
 `--record` rewrites every record. `JA` needs the `ja` extra (`namedivider-python`), which only CI's `ja-extra` job installs: without it the `JA` ratchet skips, the summary says `not measured`, and `--record` keeps the recorded `ja` entries unchanged. Re-record with the extra installed (`uv sync --group dev --extra ja`) when a change moves Japanese.
 

tools/differential/build_cldr_corpus.py

@@ -48,9 +48,18 @@
 
 import argparse
 import json
+import sys
 import urllib.request
 from pathlib import Path
 
+# The shared cache and file helpers (corpus_build.py), found by putting
+# this directory on sys.path: see corpus_build's docstring for why.
+_HERE = str(Path(__file__).resolve().parent)
+if _HERE not in sys.path:
+    sys.path.append(_HERE)
+
+from corpus_build import corpus_lines, fields_lines, write_corpus  # noqa: E402,F401
+
 HERE = Path(__file__).resolve().parent
 CLDR_TAG = "release-48"
 _REPO = "unicode-org/cldr"
@@ -128,32 +137,6 @@ def parse_test_file(text: str, stem: str) -> list[dict[str, object]]:
     return rows
 
 
-def corpus_lines(rows: list[dict[str, object]]) -> list[str]:
-    """One harness line per distinct (text, shape), sorted for stable diffs."""
-    pairs: set[tuple[str, int | None]] = set()
-    for row in rows:
-        shape = row["shape"]
-        if shape is not None and not isinstance(shape, int):
-            raise TypeError(f"shape must be an int or None, not {shape!r}")
-        pairs.add((str(row["text"]), shape))
-    return [json.dumps(text, ensure_ascii=False) if shape is None
-            else json.dumps({"name": text, "shape": shape}, ensure_ascii=False)
-            for text, shape in sorted(pairs, key=lambda p: (p[0], p[1] or 0))]
-
-
-def fields_lines(rows: list[dict[str, object]]) -> list[str]:
-    """One line per rendering, sorted by its key. A key is unique per
-    rendering; a duplicate means the format was misread."""
-    keyed: dict[str, dict[str, object]] = {}
-    for row in rows:
-        key = f"{row['file']}/{row['label']}/{row['order']}"
-        if key in keyed:
-            raise ValueError(f"duplicate rendering {key}")
-        keyed[key] = row
-    return [json.dumps(keyed[k], ensure_ascii=False, sort_keys=True)
-            for k in sorted(keyed)]
-
-
 def _get(url: str) -> bytes:
     request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
     with urllib.request.urlopen(request, timeout=60) as response:
@@ -170,8 +153,7 @@ def main() -> None:
     for name in names:
         text = _get(f"{RAW}/{_DIR}/{name}").decode("utf-8")
         rows.extend(parse_test_file(text, name[:-len(".txt")]))
-    CORPUS.write_text("\n".join(corpus_lines(rows)) + "\n", encoding="utf-8")
-    FIELDS.write_text("\n".join(fields_lines(rows)) + "\n", encoding="utf-8")
+    write_corpus(rows, CORPUS, FIELDS)
     LICENSE.write_bytes(_get(f"{RAW}/LICENSE"))
     print(f"CLDR {CLDR_TAG}: {len(names)} files, {len(rows)} renderings; "
           f"wrote {CORPUS.name}, {FIELDS.name}, {LICENSE.name}")