diff --git a/changelog.d/hf-cli-login-token.fixed.md b/changelog.d/hf-cli-login-token.fixed.md new file mode 100644 index 000000000..713493f78 --- /dev/null +++ b/changelog.d/hf-cli-login-token.fixed.md @@ -0,0 +1 @@ +Dataset downloads now use the token saved by `hf auth login` when no token environment variable is set, and a 401/403 says whether any token was sent and how to request UK data access. diff --git a/src/policyengine/provenance/dataset_materialization.py b/src/policyengine/provenance/dataset_materialization.py index 28e8f60b9..0f12fafa5 100644 --- a/src/policyengine/provenance/dataset_materialization.py +++ b/src/policyengine/provenance/dataset_materialization.py @@ -20,6 +20,7 @@ dataset_logical_name, get_release_manifest, https_dataset_uri, + hugging_face_auth_failure, hugging_face_auth_headers, resolve_managed_dataset_reference, ) @@ -258,10 +259,7 @@ def _download_hugging_face_dataset( ) as response: if response.status_code in {401, 403}: raise DatasetMaterializationError( - "Could not download explicit dataset " - f"{dataset_uri!r}: Hugging Face rejected the configured " - "credentials. Set HUGGING_FACE_TOKEN to a token with " - "access to the repository." + hugging_face_auth_failure(f"explicit dataset {dataset_uri!r}") ) if response.status_code == 404: if repo_type == "model": @@ -336,9 +334,7 @@ def _reuse_or_download_bundle_files( ) as response: if response.status_code in {401, 403}: raise DatasetMaterializationError( - f"Could not download {description}: Hugging Face rejected " - "the configured credentials. Set HUGGING_FACE_TOKEN to a " - "token with access to the repository." + hugging_face_auth_failure(description) ) if response.status_code == 404: raise DatasetMaterializationError( diff --git a/src/policyengine/provenance/manifest.py b/src/policyengine/provenance/manifest.py index f3b15fb5d..6cf09bdf4 100644 --- a/src/policyengine/provenance/manifest.py +++ b/src/policyengine/provenance/manifest.py @@ -252,17 +252,55 @@ def https_dataset_uri( ) +def _cached_hugging_face_token() -> Optional[str]: + """Return the token saved by ``hf auth login``, if there is one.""" + + try: + from huggingface_hub import get_token + except ImportError: + return None + try: + return get_token() + except Exception: + return None + + def hugging_face_auth_headers() -> dict[str, str]: - """Return authentication headers for Hugging Face requests.""" + """Return authentication headers for Hugging Face requests. + + The environment variables take precedence. Without them, the token saved + by ``hf auth login`` (``huggingface-cli login``) is used, as + ``huggingface_hub`` itself does. + """ token = ( os.environ.get("HUGGING_FACE_TOKEN") or os.environ.get("HF_TOKEN") or os.environ.get("HUGGINGFACE_HUB_TOKEN") + or _cached_hugging_face_token() ) return {"Authorization": f"Bearer {token}"} if token else {} +HUGGING_FACE_ACCESS_HINT = ( + "Set HUGGING_FACE_TOKEN, or run `hf auth login`, with a token from a " + "Hugging Face account that has been granted access to the repository. " + "UK datasets are private because they derive from licensed UK Data " + "Service surveys. To request access, email contact@policyengine.org with " + "your Hugging Face username and evidence of your UK Data Service access." +) + + +def hugging_face_auth_failure(description: str) -> str: + """Explain a 401/403 from Hugging Face, saying whether a token was sent.""" + + if hugging_face_auth_headers(): + reason = "Hugging Face rejected the configured credentials" + else: + reason = "no Hugging Face credentials were found" + return f"Could not download {description}: {reason}. {HUGGING_FACE_ACCESS_HINT}" + + def _artifact_revision(data_package: "DataPackageVersion") -> str: return data_package.release_manifest_revision or data_package.version diff --git a/tests/test_dataset_materialization.py b/tests/test_dataset_materialization.py index 5180ebd88..b7d2965d2 100644 --- a/tests/test_dataset_materialization.py +++ b/tests/test_dataset_materialization.py @@ -240,6 +240,75 @@ def test_download_passes_hugging_face_token(monkeypatch, tmp_path): assert session.calls[0][1]["headers"] == {"Authorization": "Bearer secret-token"} +def _clear_token_env(monkeypatch): + for name in ("HUGGING_FACE_TOKEN", "HF_TOKEN", "HUGGINGFACE_HUB_TOKEN"): + monkeypatch.delenv(name, raising=False) + + +def test_download_uses_hugging_face_cli_login(monkeypatch, tmp_path): + import policyengine.provenance.manifest as manifest + + _clear_token_env(monkeypatch) + monkeypatch.setattr(manifest, "_cached_hugging_face_token", lambda: "cli-token") + payload = b"certified" + session = _Session(_Response(payload)) + + _download(_manifest_with_hash(_sha256(payload)), tmp_path, session) + + assert session.calls[0][1]["headers"] == {"Authorization": "Bearer cli-token"} + + +def test_environment_token_takes_precedence_over_cli_login(monkeypatch, tmp_path): + import policyengine.provenance.manifest as manifest + + _clear_token_env(monkeypatch) + monkeypatch.setenv("HF_TOKEN", "env-token") + monkeypatch.setattr(manifest, "_cached_hugging_face_token", lambda: "cli-token") + payload = b"certified" + session = _Session(_Response(payload)) + + _download(_manifest_with_hash(_sha256(payload)), tmp_path, session) + + assert session.calls[0][1]["headers"] == {"Authorization": "Bearer env-token"} + + +def test_authentication_failure_without_token_says_none_was_found( + monkeypatch, tmp_path +): + import policyengine.provenance.manifest as manifest + + _clear_token_env(monkeypatch) + monkeypatch.setattr(manifest, "_cached_hugging_face_token", lambda: None) + + with pytest.raises(DatasetMaterializationError) as error: + _download( + _manifest_with_hash(_sha256(b"certified")), + tmp_path, + _Session(_Response(status_code=401)), + ) + + message = str(error.value) + assert "no Hugging Face credentials were found" in message + assert "rejected" not in message + assert "hf auth login" in message + assert "contact@policyengine.org" in message + + +def test_authentication_failure_with_token_says_it_was_rejected(monkeypatch, tmp_path): + _clear_token_env(monkeypatch) + monkeypatch.setenv("HUGGING_FACE_TOKEN", "secret-token") + + with pytest.raises(DatasetMaterializationError) as error: + _download( + _manifest_with_hash(_sha256(b"certified")), + tmp_path, + _Session(_Response(status_code=403)), + ) + + assert "rejected the configured credentials" in str(error.value) + assert "secret-token" not in str(error.value) + + @pytest.mark.parametrize("status_code", [401, 403]) def test_authentication_failure_does_not_retry_repository_type(tmp_path, status_code): session = _Session(_Response(status_code=status_code))