Use the hf auth login token for dataset downloads

changelog.d/hf-cli-login-token.fixed.md

@@ -0,0 +1 @@
+Dataset downloads now use the token saved by `hf auth login` when no token environment variable is set, and a 401/403 says whether any token was sent and how to request UK data access.

src/policyengine/provenance/dataset_materialization.py

@@ -20,6 +20,7 @@
     dataset_logical_name,
     get_release_manifest,
     https_dataset_uri,
+    hugging_face_auth_failure,
     hugging_face_auth_headers,
     resolve_managed_dataset_reference,
 )
@@ -258,10 +259,7 @@ def _download_hugging_face_dataset(
             ) as response:
                 if response.status_code in {401, 403}:
                     raise DatasetMaterializationError(
-                        "Could not download explicit dataset "
-                        f"{dataset_uri!r}: Hugging Face rejected the configured "
-                        "credentials. Set HUGGING_FACE_TOKEN to a token with "
-                        "access to the repository."
+                        hugging_face_auth_failure(f"explicit dataset {dataset_uri!r}")
                     )
                 if response.status_code == 404:
                     if repo_type == "model":
@@ -336,9 +334,7 @@ def _reuse_or_download_bundle_files(
             ) as response:
                 if response.status_code in {401, 403}:
                     raise DatasetMaterializationError(
-                        f"Could not download {description}: Hugging Face rejected "
-                        "the configured credentials. Set HUGGING_FACE_TOKEN to a "
-                        "token with access to the repository."
+                        hugging_face_auth_failure(description)
                     )
                 if response.status_code == 404:
                     raise DatasetMaterializationError(

src/policyengine/provenance/manifest.py

@@ -252,17 +252,55 @@ def https_dataset_uri(
     )
 
 
+def _cached_hugging_face_token() -> Optional[str]:
+    """Return the token saved by ``hf auth login``, if there is one."""
+
+    try:
+        from huggingface_hub import get_token
+    except ImportError:
+        return None
+    try:
+        return get_token()
+    except Exception:
+        return None
+
+
 def hugging_face_auth_headers() -> dict[str, str]:
-    """Return authentication headers for Hugging Face requests."""
+    """Return authentication headers for Hugging Face requests.
+
+    The environment variables take precedence. Without them, the token saved
+    by ``hf auth login`` (``huggingface-cli login``) is used, as
+    ``huggingface_hub`` itself does.
+    """
 
     token = (
         os.environ.get("HUGGING_FACE_TOKEN")
         or os.environ.get("HF_TOKEN")
         or os.environ.get("HUGGINGFACE_HUB_TOKEN")
+        or _cached_hugging_face_token()
     )
     return {"Authorization": f"Bearer {token}"} if token else {}
 
 
+HUGGING_FACE_ACCESS_HINT = (
+    "Set HUGGING_FACE_TOKEN, or run `hf auth login`, with a token from a "
+    "Hugging Face account that has been granted access to the repository. "
+    "UK datasets are private because they derive from licensed UK Data "
+    "Service surveys. To request access, email contact@policyengine.org with "
+    "your Hugging Face username and evidence of your UK Data Service access."
+)
+
+
+def hugging_face_auth_failure(description: str) -> str:
+    """Explain a 401/403 from Hugging Face, saying whether a token was sent."""
+
+    if hugging_face_auth_headers():
+        reason = "Hugging Face rejected the configured credentials"
+    else:
+        reason = "no Hugging Face credentials were found"
+    return f"Could not download {description}: {reason}. {HUGGING_FACE_ACCESS_HINT}"
+
+
 def _artifact_revision(data_package: "DataPackageVersion") -> str:
     return data_package.release_manifest_revision or data_package.version
 

tests/test_dataset_materialization.py

@@ -240,6 +240,75 @@ def test_download_passes_hugging_face_token(monkeypatch, tmp_path):
     assert session.calls[0][1]["headers"] == {"Authorization": "Bearer secret-token"}
 
 
+def _clear_token_env(monkeypatch):
+    for name in ("HUGGING_FACE_TOKEN", "HF_TOKEN", "HUGGINGFACE_HUB_TOKEN"):
+        monkeypatch.delenv(name, raising=False)
+
+
+def test_download_uses_hugging_face_cli_login(monkeypatch, tmp_path):
+    import policyengine.provenance.manifest as manifest
+
+    _clear_token_env(monkeypatch)
+    monkeypatch.setattr(manifest, "_cached_hugging_face_token", lambda: "cli-token")
+    payload = b"certified"
+    session = _Session(_Response(payload))
+
+    _download(_manifest_with_hash(_sha256(payload)), tmp_path, session)
+
+    assert session.calls[0][1]["headers"] == {"Authorization": "Bearer cli-token"}
+
+
+def test_environment_token_takes_precedence_over_cli_login(monkeypatch, tmp_path):
+    import policyengine.provenance.manifest as manifest
+
+    _clear_token_env(monkeypatch)
+    monkeypatch.setenv("HF_TOKEN", "env-token")
+    monkeypatch.setattr(manifest, "_cached_hugging_face_token", lambda: "cli-token")
+    payload = b"certified"
+    session = _Session(_Response(payload))
+
+    _download(_manifest_with_hash(_sha256(payload)), tmp_path, session)
+
+    assert session.calls[0][1]["headers"] == {"Authorization": "Bearer env-token"}
+
+
+def test_authentication_failure_without_token_says_none_was_found(
+    monkeypatch, tmp_path
+):
+    import policyengine.provenance.manifest as manifest
+
+    _clear_token_env(monkeypatch)
+    monkeypatch.setattr(manifest, "_cached_hugging_face_token", lambda: None)
+
+    with pytest.raises(DatasetMaterializationError) as error:
+        _download(
+            _manifest_with_hash(_sha256(b"certified")),
+            tmp_path,
+            _Session(_Response(status_code=401)),
+        )
+
+    message = str(error.value)
+    assert "no Hugging Face credentials were found" in message
+    assert "rejected" not in message
+    assert "hf auth login" in message
+    assert "contact@policyengine.org" in message
+
+
+def test_authentication_failure_with_token_says_it_was_rejected(monkeypatch, tmp_path):
+    _clear_token_env(monkeypatch)
+    monkeypatch.setenv("HUGGING_FACE_TOKEN", "secret-token")
+
+    with pytest.raises(DatasetMaterializationError) as error:
+        _download(
+            _manifest_with_hash(_sha256(b"certified")),
+            tmp_path,
+            _Session(_Response(status_code=403)),
+        )
+
+    assert "rejected the configured credentials" in str(error.value)
+    assert "secret-token" not in str(error.value)
+
+
 @pytest.mark.parametrize("status_code", [401, 403])
 def test_authentication_failure_does_not_retry_repository_type(tmp_path, status_code):
     session = _Session(_Response(status_code=status_code))