From 43ec04df8ec5daf1d372c457eb4c9da376a204f6 Mon Sep 17 00:00:00 2001 From: quinn-galileo Date: Wed, 14 Oct 2026 13:14:48 -0400 Subject: [PATCH 1/8] feat: scope evaluators to the current project on create Add project_ids and is_global to the evaluator create paths and default the scope to the current project, so non-admins can create evaluators when metric RBAC is on. Surface is_global and scope_projects on Evaluator. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 18 +++ src/splunk_ao/evaluator.py | 41 +++++- src/splunk_ao/evaluators.py | 107 ++++++++++++++- tests/test_evaluator.py | 188 +++++++++++++++++++++++++- tests/test_evaluators.py | 257 +++++++++++++++++++++++++++++++++++- 5 files changed, 607 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3d357e5..27895612 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- `LlmEvaluator`, `CodeEvaluator`, `Evaluators.create_custom_llm_evaluator()` and the module-level + `create_custom_llm_evaluator()` accept `project_ids` and `is_global` to choose the scope of a new evaluator. + Passing both, or an empty `project_ids`, raises `ValidationError` before any request is sent. +- `Evaluator.is_global` and `Evaluator.scope_projects` report the scope the API returned, after + `create()`, `get()`, `list()` and `refresh()`. + +### Changed + +- New evaluators are scoped to the current project by default: the decorator context project, then + `SPLUNK_AO_PROJECT_ID`, then `SPLUNK_AO_PROJECT`. If that project cannot be found, creation raises + `NotFoundError` rather than falling back to a global evaluator. When no current project is set and no scope is + passed, no scope is sent and the server default (global, org admins only) applies. Org admins who relied on creating + global evaluators with a project set must now pass `is_global=True`. +- Creating project-scoped evaluators requires `splunk-ao` 0.6.0 or later. Earlier versions send no scope, so on servers + with metric RBAC enabled, non-admin users get a 403 when creating an evaluator. + ## [0.5.0] ### Added diff --git a/src/splunk_ao/evaluator.py b/src/splunk_ao/evaluator.py index 587b7cfa..b85bc640 100644 --- a/src/splunk_ao/evaluator.py +++ b/src/splunk_ao/evaluator.py @@ -19,7 +19,7 @@ from galileo_core.schemas.shared.metric import MetricValueType from splunk_ao.config import SplunkAOConfig from splunk_ao.configuration import Configuration -from splunk_ao.evaluators import Evaluators +from splunk_ao.evaluators import Evaluators, _resolve_create_scope from splunk_ao.resources.api.data import ( create_code_scorer_version_scorers_scorer_id_version_code_post, create_scorers_post, @@ -33,6 +33,7 @@ CreateScorerRequest, HTTPValidationError, OutputTypeEnum, + ScorerScopeProjectRef, ScorerTypes, TaskResultStatus, UpdateScorerRequest, @@ -107,6 +108,10 @@ class Evaluator(StateManagementMixin, ABC): created_at (datetime | None): When the metric was created. updated_at (datetime | None): When the metric was last updated. version (int | None): Evaluator version number. + is_global (bool | None): Whether the evaluator is global, as reported by the API. + ``None`` until populated from the API. + scope_projects (list[ScorerScopeProjectRef]): Projects the evaluator is scoped to, as + reported by the API. Empty until populated, and for global evaluators. Class Attributes ---------------- @@ -163,6 +168,10 @@ def my_scorer(trace_or_span): judges: int | None cot_enabled: bool | None + # Access scope reported by the API + is_global: bool | None + scope_projects: list[ScorerScopeProjectRef] + def __init__( self, name: str, *, description: str = "", tags: list[str] | None = None, version: int | None = None ) -> None: @@ -190,6 +199,9 @@ def __init__( self.judges = None self.cot_enabled = None + self.is_global = None + self.scope_projects = [] + self._set_state(SyncState.LOCAL_ONLY) @staticmethod @@ -396,6 +408,9 @@ def _populate_from_scorer_response(self, scorer_response: Any) -> None: judges = None cot_enabled = None + is_global = None if isinstance(scorer_response.is_global, Unset) else scorer_response.is_global + scope_projects = [] if isinstance(scorer_response.scope_projects, Unset) else scorer_response.scope_projects + self._sync_attrs( id=scorer_response.id, name=scorer_response.name, @@ -408,6 +423,8 @@ def _populate_from_scorer_response(self, scorer_response: Any) -> None: model=model, judges=judges, cot_enabled=cot_enabled, + is_global=is_global, + scope_projects=list(scope_projects), ) # LLM-specific attributes (only set if this is an LlmEvaluator) @@ -708,6 +725,9 @@ def __init__( cot_enabled: bool | None = None, output_type: str | OutputTypeEnum | None = None, ground_truth: bool = False, + # Access scope + project_ids: list[str] | None = None, + is_global: bool | None = None, # Common parameters description: str = "", tags: list[str] | None = None, @@ -730,6 +750,9 @@ def __init__( output_type: Output type ("percentage", "boolean", etc.). ground_truth: Whether the scorer requires ground truth (``reference_output``) from the dataset. When True, the judge LLM receives the row's ground-truth value in its prompt. + project_ids: Projects to scope the metric to when it is created. Defaults to the current + project (decorator context, SPLUNK_AO_PROJECT_ID, or SPLUNK_AO_PROJECT). + is_global: Create a global metric (org admins only). Cannot be combined with ``project_ids``. description: Description of the metric. tags: Tags associated with the metric. version: Specific version to reference (for existing metrics). @@ -775,6 +798,8 @@ def __init__( self.output_type = output_type or OutputTypeEnum.BOOLEAN self.ground_truth = ground_truth + self.project_ids = project_ids + self.is_global = is_global self.scorer_type = ScorerTypes.LLM @@ -817,6 +842,8 @@ def create(self) -> LlmEvaluator: if isinstance(self.output_type, OutputTypeEnum) else OutputTypeEnum.BOOLEAN, ground_truth=self.ground_truth, + project_ids=self.project_ids, + is_global=self.is_global, ) # Update attributes from response without triggering dirty-tracking @@ -893,6 +920,8 @@ def __init__( node_level: StepType | None = None, output_type: str | OutputTypeEnum | None = None, required_metrics: list[str] | None = None, + project_ids: list[str] | None = None, + is_global: bool | None = None, description: str = "", tags: list[str] | None = None, version: int | None = None, @@ -907,6 +936,9 @@ def __init__( output_type: Output type for the metric ("percentage", "boolean", "categorical", "count", "discrete"). Accepts string or OutputTypeEnum. required_metrics: List of metric names that this code metric depends on. + project_ids: Projects to scope the metric to when it is created. Defaults to the current + project (decorator context, SPLUNK_AO_PROJECT_ID, or SPLUNK_AO_PROJECT). + is_global: Create a global metric (org admins only). Cannot be combined with ``project_ids``. description: Description of the metric. tags: Tags associated with the metric. version: Specific version to reference (for existing metrics). @@ -916,6 +948,8 @@ def __init__( self.code = code self.node_level = node_level or StepType.llm self.required_metrics = required_metrics + self.project_ids = project_ids + self.is_global = is_global self.scorer_type = ScorerTypes.CODE self.output_type = Evaluator._parse_output_type(output_type) @@ -1099,6 +1133,9 @@ def create(self) -> CodeEvaluator: # Ensure node_level is set (should always be set in __init__, but checking for type safety) assert self.node_level is not None + # Resolve scope first so a bad scope fails before any API call + scope_is_global, scope_project_ids = _resolve_create_scope(self.project_ids, self.is_global) + # Step 1: Validate the code and get validation result logger.debug(f"CodeEvaluator.create: name='{self.name}' - validating code") validation_result = self._validate_code(config) @@ -1113,6 +1150,8 @@ def create(self) -> CodeEvaluator: scoreable_node_types=[self.node_level.value], output_type=self.output_type, required_scorers=self.required_metrics, + is_global=scope_is_global, + project_ids=scope_project_ids, ) scorer_response = create_scorers_post.sync(client=config.api_client, body=scorer_request) diff --git a/src/splunk_ao/evaluators.py b/src/splunk_ao/evaluators.py index 2e53932f..e1b0805f 100644 --- a/src/splunk_ao/evaluators.py +++ b/src/splunk_ao/evaluators.py @@ -20,12 +20,73 @@ from splunk_ao.resources.models.create_scorer_request import CreateScorerRequest from splunk_ao.resources.models.output_type_enum import OutputTypeEnum from splunk_ao.resources.models.scorer_defaults import ScorerDefaults +from splunk_ao.resources.types import UNSET, Unset from splunk_ao.scorers import Scorers from splunk_ao.search import FilterType +from splunk_ao.shared.exceptions import ValidationError, _resolve_project_identifiers +from splunk_ao.shared.project_resolver import _resolve_project _logger = logging.getLogger(__name__) +def _resolve_create_scope( + project_ids: list[str] | None, is_global: bool | None +) -> tuple[bool | Unset, list[str] | Unset]: + """ + Resolve the ``(is_global, project_ids)`` pair to send when creating an evaluator. + + Explicit arguments win. Otherwise the scope defaults to the current project, taken from + the decorator context, then ``SPLUNK_AO_PROJECT_ID``, then ``SPLUNK_AO_PROJECT``. When + none of those is set and nothing was passed, no scope is sent and the server default + (global, org admins only) applies. + + Parameters + ---------- + project_ids + Projects to scope the evaluator to. + is_global + Whether the evaluator is global. Cannot be combined with ``project_ids``. + + Returns + ------- + tuple[bool | Unset, list[str] | Unset] + The values for ``CreateScorerRequest.is_global`` and ``CreateScorerRequest.project_ids``. + + Raises + ------ + ValidationError + If the arguments conflict, ``project_ids`` is empty, or ``is_global=False`` is passed + with no project to scope to. + NotFoundError + If a current project is set but cannot be found. + """ + if is_global is True: + if project_ids: + raise ValidationError("An evaluator cannot be both global and scoped: pass is_global=True or project_ids.") + return True, UNSET + + if project_ids is not None: + if not project_ids: + raise ValidationError("project_ids must not be empty. Pass is_global=True for a global evaluator.") + return False, list(dict.fromkeys(project_ids)) + + # Imported here because decorator.py imports modules that depend on this one. + from splunk_ao.decorator import splunk_ao_context + + context_project = splunk_ao_context.get_current_project() + if context_project: + project_id, project_name = None, context_project + else: + project_id, project_name = _resolve_project_identifiers(None, None) + + if project_id or project_name: + return False, [_resolve_project(project_id, project_name).id] + + if is_global is False: + raise ValidationError("is_global=False needs project_ids or a current project.") + return UNSET, UNSET + + class Evaluators: config: SplunkAOConfig @@ -57,6 +118,8 @@ def create_custom_llm_evaluator( tags: list[str] | None = None, output_type: OutputTypeEnum = OutputTypeEnum.BOOLEAN, ground_truth: bool = False, + project_ids: list[str] | None = None, + is_global: bool | None = None, ) -> BaseScorerVersionResponse: """ Create a custom LLM metric. @@ -84,14 +147,28 @@ def create_custom_llm_evaluator( ground_truth: bool Whether the scorer requires ground truth (``reference_output``) from the dataset. When True, the judge LLM receives the row's ground-truth value in its prompt. + project_ids: List[str] + Projects to scope the metric to. Defaults to the current project (decorator context, + ``SPLUNK_AO_PROJECT_ID``, or ``SPLUNK_AO_PROJECT``) when one is set. + is_global: bool + Create a global metric, visible to every project. Requires the org admin role and + cannot be combined with ``project_ids``. Returns ------- BaseScorerVersionResponse Response containing the created metric details. + + Raises + ------ + ValidationError + If the scope arguments conflict or ``project_ids`` is empty. + NotFoundError + If the current project is set but cannot be found. """ if tags is None: tags = [] + scope_is_global, scope_project_ids = _resolve_create_scope(project_ids, is_global) create_scorer_request = CreateScorerRequest( name=name, scorer_type=ScorerTypes.LLM, @@ -101,6 +178,8 @@ def create_custom_llm_evaluator( scoreable_node_types=[node_level], output_type=output_type, ground_truth=ground_truth, + is_global=scope_is_global, + project_ids=scope_project_ids, ) scorer = create_scorers_post.sync(body=create_scorer_request, client=self.config.api_client) @@ -159,6 +238,8 @@ def create_custom_llm_evaluator( tags: list[str] | None = None, output_type: OutputTypeEnum = OutputTypeEnum.BOOLEAN, ground_truth: bool = False, + project_ids: list[str] | None = None, + is_global: bool | None = None, ) -> BaseScorerVersionResponse: """ Create a custom LLM metric. @@ -186,16 +267,40 @@ def create_custom_llm_evaluator( ground_truth: bool Whether the scorer requires ground truth (``reference_output``) from the dataset. When True, the judge LLM receives the row's ground-truth value in its prompt. + project_ids: List[str] + Projects to scope the metric to. Defaults to the current project (decorator context, + ``SPLUNK_AO_PROJECT_ID``, or ``SPLUNK_AO_PROJECT``) when one is set. + is_global: bool + Create a global metric, visible to every project. Requires the org admin role and + cannot be combined with ``project_ids``. Returns ------- BaseScorerVersionResponse Response containing the created metric details. + + Raises + ------ + ValidationError + If the scope arguments conflict or ``project_ids`` is empty. + NotFoundError + If the current project is set but cannot be found. """ if tags is None: tags = [] return Evaluators().create_custom_llm_evaluator( - name, user_prompt, node_level, cot_enabled, model_name, num_judges, description, tags, output_type, ground_truth + name, + user_prompt, + node_level, + cot_enabled, + model_name, + num_judges, + description, + tags, + output_type, + ground_truth, + project_ids=project_ids, + is_global=is_global, ) diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py index 82eb42f8..cc1b2e00 100644 --- a/tests/test_evaluator.py +++ b/tests/test_evaluator.py @@ -6,16 +6,50 @@ import pytest from galileo_core.schemas.logging.step import StepType +from splunk_ao.decorator import _project_context from splunk_ao.evaluator import CodeEvaluator, LlmEvaluator, LocalEvaluator, Evaluator -from splunk_ao.resources.models import HTTPValidationError, OutputTypeEnum, ScorerTypes +from splunk_ao.resources.models import HTTPValidationError, OutputTypeEnum, ScorerScopeProjectRef, ScorerTypes from splunk_ao.resources.models.invalid_result import InvalidResult from splunk_ao.resources.models.task_result_status import TaskResultStatus +from splunk_ao.resources.types import UNSET from splunk_ao.shared.base import SyncState from splunk_ao.shared.exceptions import APIError, ValidationError # Test fixtures and helper functions +@pytest.fixture(autouse=True) +def no_current_project(monkeypatch: pytest.MonkeyPatch): + """Start every test with no current project in the environment or the decorator context.""" + monkeypatch.delenv("SPLUNK_AO_PROJECT", raising=False) + monkeypatch.delenv("SPLUNK_AO_PROJECT_ID", raising=False) + token = _project_context.set(None) + yield + _project_context.reset(token) + + +def _scope_scorer(scorer_id: str, *, scorer_type: ScorerTypes = ScorerTypes.LLM, is_global=False, scope_projects=UNSET): + """Build a scorer response carrying scope fields.""" + scorer = MagicMock() + scorer.id = scorer_id + scorer.name = "Scoped Evaluator" + scorer.scorer_type = scorer_type + scorer.tags = [] + scorer.description = "" + scorer.created_at = MagicMock() + scorer.updated_at = MagicMock() + scorer.output_type = OutputTypeEnum.BOOLEAN + scorer.user_prompt = "Is it accurate?" + scorer.defaults = MagicMock() + scorer.defaults.model_name = "gpt-4.1-mini" + scorer.defaults.num_judges = 3 + scorer.defaults.cot_enabled = True + scorer.scoreable_node_types = ["llm"] + scorer.is_global = is_global + scorer.scope_projects = scope_projects + return scorer + + @pytest.fixture def mock_api_client(): """Create a mock API client.""" @@ -2025,3 +2059,155 @@ def test_configuration_from_env_vars(self, reset_configuration: None, monkeypatc assert Configuration.code_validation_initial_delay == 3.0 assert Configuration.code_validation_max_delay == 45.0 assert Configuration.code_validation_backoff_multiplier == 1.8 + + +class TestEvaluatorScope: + """Scope on evaluator construction, creation and population from the API.""" + + def test_fresh_evaluator_has_no_scope(self, reset_configuration: None) -> None: + # Given a local evaluator that has never been synced + evaluator = LocalEvaluator(name="local", scorer_fn=lambda x: 1.0) + + # Then it reports no server scope + assert evaluator.is_global is None + assert evaluator.scope_projects == [] + + @pytest.mark.parametrize( + ("is_global", "scope_projects", "expected_global", "expected_projects"), + [ + (False, [ScorerScopeProjectRef(id="p1", name="One")], False, [ScorerScopeProjectRef(id="p1", name="One")]), + (True, [], True, []), + (UNSET, UNSET, None, []), + ], + ) + @patch("splunk_ao.evaluator.Scorers") + def test_get_list_and_refresh_populate_scope( + self, mock_scorers_class, is_global, scope_projects, expected_global, expected_projects, reset_configuration + ) -> None: + # Given a scorer response with scope fields + scorer_id = str(uuid4()) + scorer = _scope_scorer(scorer_id, is_global=is_global, scope_projects=scope_projects) + mock_scorers_class.return_value.list.return_value = [scorer] + + # When read through get(), list() and refresh() + from_get = Evaluator.get(name="Scoped Evaluator") + from_list = Evaluator.list()[0] + refreshed = LlmEvaluator(name="Scoped Evaluator", prompt="x") + refreshed.id = scorer_id + refreshed.refresh() + + # Then each reflects the response + for evaluator in (from_get, from_list, refreshed): + assert evaluator.is_global is expected_global + assert evaluator.scope_projects == expected_projects + + @patch("splunk_ao.evaluator.Evaluators") + @patch("splunk_ao.evaluator.Scorers") + def test_llm_evaluator_forwards_scope_and_populates_after_create( + self, mock_scorers_class, mock_evaluators_class, reset_configuration: None + ) -> None: + # Given an LLM evaluator built with an explicit scope + scorer_id = str(uuid4()) + mock_version = MagicMock() + mock_version.scorer_id = scorer_id + mock_evaluators_class.return_value.create_custom_llm_evaluator.return_value = mock_version + projects = [ScorerScopeProjectRef(id="a", name="A"), ScorerScopeProjectRef(id="b", name="B")] + mock_scorers_class.return_value.list.return_value = [ + _scope_scorer(scorer_id, is_global=False, scope_projects=projects) + ] + + # When created + evaluator = LlmEvaluator(name="Scoped Evaluator", prompt="Is it accurate?", project_ids=["a", "b"]).create() + + # Then the scope reaches the service and the server scope is surfaced + kwargs = mock_evaluators_class.return_value.create_custom_llm_evaluator.call_args.kwargs + assert kwargs["project_ids"] == ["a", "b"] + assert kwargs["is_global"] is None + assert evaluator.is_global is False + assert evaluator.scope_projects == projects + + @patch("splunk_ao.evaluator.Evaluators") + @patch("splunk_ao.evaluator.Scorers") + def test_llm_evaluator_forwards_is_global( + self, mock_scorers_class, mock_evaluators_class, reset_configuration: None + ) -> None: + scorer_id = str(uuid4()) + mock_evaluators_class.return_value.create_custom_llm_evaluator.return_value = MagicMock(scorer_id=scorer_id) + mock_scorers_class.return_value.list.return_value = [_scope_scorer(scorer_id, is_global=True)] + + evaluator = LlmEvaluator(name="Scoped Evaluator", prompt="x", is_global=True).create() + + kwargs = mock_evaluators_class.return_value.create_custom_llm_evaluator.call_args.kwargs + assert kwargs["is_global"] is True + assert kwargs["project_ids"] is None + assert evaluator.is_global is True + + @patch("splunk_ao.evaluator.SplunkAOConfig.get") + @patch("splunk_ao.evaluator.get_validate_code_scorer_task_result_scorers_code_validate_task_id_get") + @patch("splunk_ao.evaluator.validate_code_scorer_scorers_code_validate_post") + @patch("splunk_ao.evaluator.create_code_scorer_version_scorers_scorer_id_version_code_post") + @patch("splunk_ao.evaluator.create_scorers_post") + @patch("splunk_ao.evaluator.Scorers") + def test_code_evaluator_sends_scope_and_populates_after_create( + self, + mock_scorers_class, + mock_create_scorers, + mock_create_version, + mock_validate_post, + mock_validate_get, + mock_config, + reset_configuration, + mock_validation_response, + mock_validation_task_result, + mock_version_response, + mock_scorer_response, + ) -> None: + # Given a code evaluator with explicit scope and a working API + scorer_id = str(uuid4()) + mock_validate_post.sync.return_value = mock_validation_response() + mock_validate_get.sync.return_value = mock_validation_task_result(TaskResultStatus.COMPLETED) + mock_create_scorers.sync.return_value = mock_scorer_response(scorer_id, "Scoped Code") + mock_create_version.sync.return_value = mock_version_response(scorer_id) + projects = [ScorerScopeProjectRef(id="a", name="A")] + mock_scorers_class.return_value.list.return_value = [ + _scope_scorer(scorer_id, scorer_type=ScorerTypes.CODE, scope_projects=projects) + ] + + # When created + evaluator = CodeEvaluator(name="Scoped Code", code="def score(t): return 1.0", project_ids=["a", "a"]).create() + + # Then the request carries the deduplicated scope and the server scope is surfaced + body = mock_create_scorers.sync.call_args.kwargs["body"] + assert body.is_global is False + assert body.project_ids == ["a"] + assert evaluator.scope_projects == projects + + @patch("splunk_ao.evaluator.SplunkAOConfig.get") + @patch("splunk_ao.evaluator.create_scorers_post") + def test_code_evaluator_global_sends_no_project_ids( + self, mock_create_scorers, mock_config, reset_configuration + ) -> None: + mock_create_scorers.sync.return_value = None + + with patch.object(CodeEvaluator, "_validate_code", return_value="{}"): + with pytest.raises(ValueError, match="No response"): + CodeEvaluator(name="G", code="x", is_global=True).create() + + body = mock_create_scorers.sync.call_args.kwargs["body"] + assert body.is_global is True + assert "project_ids" not in body.to_dict() + + @pytest.mark.parametrize( + "kwargs", [{"is_global": True, "project_ids": ["a"]}, {"project_ids": []}, {"is_global": False}] + ) + @patch("splunk_ao.evaluator.SplunkAOConfig.get") + @patch("splunk_ao.evaluator.validate_code_scorer_scorers_code_validate_post") + @patch("splunk_ao.evaluator.create_scorers_post") + def test_code_evaluator_resolves_scope_before_validating_code( + self, mock_create_scorers, mock_validate_post, mock_config, kwargs, reset_configuration + ) -> None: + with pytest.raises(ValidationError): + CodeEvaluator(name="Bad", code="x", **kwargs).create() + + mock_validate_post.sync.assert_not_called() + mock_create_scorers.sync.assert_not_called() diff --git a/tests/test_evaluators.py b/tests/test_evaluators.py index b8c1c3b0..15c517a9 100644 --- a/tests/test_evaluators.py +++ b/tests/test_evaluators.py @@ -6,7 +6,14 @@ import pytest from galileo_core.schemas.logging.step import StepType -from splunk_ao.evaluators import Evaluators, create_custom_llm_evaluator, delete_evaluator, get_evaluators +from splunk_ao.decorator import _project_context +from splunk_ao.evaluators import ( + Evaluators, + _resolve_create_scope, + create_custom_llm_evaluator, + delete_evaluator, + get_evaluators, +) from splunk_ao.resources.models import ( BucketedMetrics, HTTPValidationError, @@ -22,10 +29,33 @@ from splunk_ao.resources.models.scorer_defaults import ScorerDefaults from splunk_ao.resources.models.scorer_response import ScorerResponse from splunk_ao.resources.models.scorer_types import ScorerTypes +from splunk_ao.resources.types import UNSET +from splunk_ao.shared.exceptions import NotFoundError +from splunk_ao.shared.exceptions import ValidationError as SplunkAOValidationError FIXED_PROJECT_ID = str(uuid4()) +@pytest.fixture(autouse=True) +def no_current_project(monkeypatch: pytest.MonkeyPatch): + """Start every test with no current project in the environment or the decorator context.""" + monkeypatch.delenv("SPLUNK_AO_PROJECT", raising=False) + monkeypatch.delenv("SPLUNK_AO_PROJECT_ID", raising=False) + token = _project_context.set(None) + yield + _project_context.reset(token) + + +@pytest.fixture +def mock_projects(): + """Patch the project lookup used to resolve the current project; the found project has id ``project-id-1``.""" + with patch("splunk_ao.shared.project_resolver.Projects") as projects_class: + project = MagicMock() + project.id = "project-id-1" + projects_class.return_value.get_with_env_fallbacks.return_value = project + yield projects_class.return_value + + @pytest.fixture def mock_scorer_response(): """Mock scorer response for testing.""" @@ -299,6 +329,8 @@ def test_create_custom_llm_metric_function(self, mock_metrics_class) -> None: ["test"], OutputTypeEnum.CATEGORICAL, True, + project_ids=None, + is_global=None, ) # Verify the result is returned @@ -328,6 +360,8 @@ def test_create_custom_llm_metric_function_default_parameters(self, mock_metrics [], # default OutputTypeEnum.BOOLEAN, # default False, # default ground_truth + project_ids=None, + is_global=None, ) # Verify the result is returned @@ -501,3 +535,224 @@ def test_passes_all_parameters_correctly(self, mock_api_call): assert body.filters == filters assert body.group_by == group_by assert body.interval == interval + + +class TestResolveCreateScope: + """Scope resolution for evaluator creation.""" + + def test_project_id_env_defaults_to_that_project(self, monkeypatch, mock_projects) -> None: + # Given only SPLUNK_AO_PROJECT_ID is set + monkeypatch.setenv("SPLUNK_AO_PROJECT_ID", "env-id") + + # When resolving with no explicit scope + result = _resolve_create_scope(None, None) + + # Then it is scoped to the project looked up by id + assert result == (False, ["project-id-1"]) + mock_projects.get_with_env_fallbacks.assert_called_once_with(id="env-id", name=None) + + def test_project_name_env_is_resolved_to_id(self, monkeypatch, mock_projects) -> None: + # Given only SPLUNK_AO_PROJECT is set + monkeypatch.setenv("SPLUNK_AO_PROJECT", "my-project") + + # When resolving with no explicit scope + result = _resolve_create_scope(None, None) + + # Then the name is looked up and its id is used + assert result == (False, ["project-id-1"]) + mock_projects.get_with_env_fallbacks.assert_called_once_with(id=None, name="my-project") + + def test_context_project_wins_over_env(self, monkeypatch, mock_projects) -> None: + # Given a context project and both env vars + monkeypatch.setenv("SPLUNK_AO_PROJECT_ID", "env-id") + monkeypatch.setenv("SPLUNK_AO_PROJECT", "env-name") + _project_context.set("context-project") + + # When resolving with no explicit scope + result = _resolve_create_scope(None, None) + + # Then the context project name is the one looked up + assert result == (False, ["project-id-1"]) + mock_projects.get_with_env_fallbacks.assert_called_once_with(id=None, name="context-project") + + def test_project_id_env_wins_over_project_name_env(self, monkeypatch, mock_projects) -> None: + # Given both env vars + monkeypatch.setenv("SPLUNK_AO_PROJECT_ID", "env-id") + monkeypatch.setenv("SPLUNK_AO_PROJECT", "env-name") + + # When resolving with no explicit scope + _resolve_create_scope(None, None) + + # Then the id is the one looked up + mock_projects.get_with_env_fallbacks.assert_called_once_with(id="env-id", name=None) + + def test_explicit_project_ids_override_context_and_env(self, monkeypatch, mock_projects) -> None: + # Given a context project and env vars + monkeypatch.setenv("SPLUNK_AO_PROJECT_ID", "env-id") + _project_context.set("context-project") + + # When explicit project ids are passed, with a duplicate + result = _resolve_create_scope(["a", "b", "a"], None) + + # Then they are used as-is (deduplicated, order kept) and no lookup happens + assert result == (False, ["a", "b"]) + mock_projects.get_with_env_fallbacks.assert_not_called() + + def test_explicit_project_ids_with_is_global_false(self, mock_projects) -> None: + assert _resolve_create_scope(["a"], False) == (False, ["a"]) + + def test_is_global_true_sends_no_project_ids_even_with_current_project(self, monkeypatch, mock_projects) -> None: + # Given a current project + monkeypatch.setenv("SPLUNK_AO_PROJECT_ID", "env-id") + + # When is_global=True is passed + result = _resolve_create_scope(None, True) + + # Then the evaluator is global and no lookup happens + assert result == (True, UNSET) + mock_projects.get_with_env_fallbacks.assert_not_called() + + def test_nothing_set_sends_no_scope(self, mock_projects) -> None: + assert _resolve_create_scope(None, None) == (UNSET, UNSET) + mock_projects.get_with_env_fallbacks.assert_not_called() + + def test_whitespace_env_counts_as_unset(self, monkeypatch, mock_projects) -> None: + monkeypatch.setenv("SPLUNK_AO_PROJECT", " ") + assert _resolve_create_scope(None, None) == (UNSET, UNSET) + + def test_global_and_project_ids_conflict(self, mock_projects) -> None: + with pytest.raises(SplunkAOValidationError, match="both global and scoped"): + _resolve_create_scope(["a"], True) + + def test_empty_project_ids_rejected(self, monkeypatch, mock_projects) -> None: + # Given a current project, which must not paper over an explicit empty list + monkeypatch.setenv("SPLUNK_AO_PROJECT_ID", "env-id") + + with pytest.raises(SplunkAOValidationError, match="must not be empty"): + _resolve_create_scope([], None) + + def test_is_global_false_without_any_project_rejected(self, mock_projects) -> None: + with pytest.raises(SplunkAOValidationError, match="is_global=False needs project_ids"): + _resolve_create_scope(None, False) + + def test_is_global_false_uses_current_project(self, monkeypatch, mock_projects) -> None: + monkeypatch.setenv("SPLUNK_AO_PROJECT_ID", "env-id") + assert _resolve_create_scope(None, False) == (False, ["project-id-1"]) + + def test_current_project_not_found_raises_not_found(self, monkeypatch, mock_projects) -> None: + # Given a current project that does not exist + monkeypatch.setenv("SPLUNK_AO_PROJECT", "missing") + mock_projects.get_with_env_fallbacks.return_value = None + + # Then there is no silent fallback to global + with pytest.raises(NotFoundError, match="missing"): + _resolve_create_scope(None, None) + + +class TestCreateCustomLlmEvaluatorScope: + """The scope reaches the CreateScorerRequest on both public create paths.""" + + @staticmethod + def _create(use_function: bool, **kwargs): + if use_function: + return create_custom_llm_evaluator("m", "prompt", **kwargs) + return Evaluators().create_custom_llm_evaluator(name="m", user_prompt="prompt", **kwargs) + + @pytest.mark.parametrize("use_function", [False, True], ids=["method", "module_function"]) + @patch("splunk_ao.evaluators.create_llm_scorer_version_scorers_scorer_id_version_llm_post") + @patch("splunk_ao.evaluators.create_scorers_post") + def test_defaults_to_current_project( + self, mock_create_scorer, mock_create_version, use_function, monkeypatch, mock_projects, mock_scorer_response + ) -> None: + # Given a current project from the environment + monkeypatch.setenv("SPLUNK_AO_PROJECT_ID", "env-id") + mock_create_scorer.sync.return_value = mock_scorer_response + + # When creating with no explicit scope + self._create(use_function) + + # Then the request is scoped to that project + body = mock_create_scorer.sync.call_args.kwargs["body"] + assert body.is_global is False + assert body.project_ids == ["project-id-1"] + + @pytest.mark.parametrize("use_function", [False, True], ids=["method", "module_function"]) + @patch("splunk_ao.evaluators.create_llm_scorer_version_scorers_scorer_id_version_llm_post") + @patch("splunk_ao.evaluators.create_scorers_post") + def test_explicit_project_ids( + self, mock_create_scorer, mock_create_version, use_function, monkeypatch, mock_projects, mock_scorer_response + ) -> None: + monkeypatch.setenv("SPLUNK_AO_PROJECT_ID", "env-id") + mock_create_scorer.sync.return_value = mock_scorer_response + + self._create(use_function, project_ids=["a", "b"]) + + body = mock_create_scorer.sync.call_args.kwargs["body"] + assert body.is_global is False + assert body.project_ids == ["a", "b"] + + @pytest.mark.parametrize("use_function", [False, True], ids=["method", "module_function"]) + @patch("splunk_ao.evaluators.create_llm_scorer_version_scorers_scorer_id_version_llm_post") + @patch("splunk_ao.evaluators.create_scorers_post") + def test_explicit_global_overrides_current_project( + self, mock_create_scorer, mock_create_version, use_function, monkeypatch, mock_projects, mock_scorer_response + ) -> None: + monkeypatch.setenv("SPLUNK_AO_PROJECT_ID", "env-id") + mock_create_scorer.sync.return_value = mock_scorer_response + + self._create(use_function, is_global=True) + + body = mock_create_scorer.sync.call_args.kwargs["body"] + assert body.is_global is True + assert "project_ids" not in body.to_dict() + + @pytest.mark.parametrize("use_function", [False, True], ids=["method", "module_function"]) + @patch("splunk_ao.evaluators.create_llm_scorer_version_scorers_scorer_id_version_llm_post") + @patch("splunk_ao.evaluators.create_scorers_post") + def test_no_current_project_sends_no_scope( + self, mock_create_scorer, mock_create_version, use_function, mock_projects, mock_scorer_response + ) -> None: + mock_create_scorer.sync.return_value = mock_scorer_response + + self._create(use_function) + + body_dict = mock_create_scorer.sync.call_args.kwargs["body"].to_dict() + assert "is_global" not in body_dict + assert "project_ids" not in body_dict + + @pytest.mark.parametrize("use_function", [False, True], ids=["method", "module_function"]) + @pytest.mark.parametrize( + "kwargs", [{"is_global": True, "project_ids": ["a"]}, {"project_ids": []}, {"is_global": False}] + ) + @patch("splunk_ao.evaluators.create_scorers_post") + def test_invalid_scope_fails_before_any_http_call( + self, mock_create_scorer, kwargs, use_function, mock_projects + ) -> None: + with pytest.raises(SplunkAOValidationError): + self._create(use_function, **kwargs) + + mock_create_scorer.sync.assert_not_called() + + @pytest.mark.parametrize("use_function", [False, True], ids=["method", "module_function"]) + @patch("splunk_ao.evaluators.create_scorers_post") + def test_current_project_not_found_sends_no_create_request( + self, mock_create_scorer, use_function, monkeypatch, mock_projects + ) -> None: + monkeypatch.setenv("SPLUNK_AO_PROJECT", "missing") + mock_projects.get_with_env_fallbacks.return_value = None + + with pytest.raises(NotFoundError): + self._create(use_function) + + mock_create_scorer.sync.assert_not_called() + + @patch("splunk_ao.evaluators.Evaluators") + def test_module_function_keeps_positional_order_and_forwards_scope_by_keyword(self, mock_evaluators_class) -> None: + create_custom_llm_evaluator( + "m", "prompt", StepType.llm, True, "gpt-4o", 5, "desc", ["t"], OutputTypeEnum.PERCENTAGE, True, ["a"], False + ) + + mock_evaluators_class.return_value.create_custom_llm_evaluator.assert_called_once_with( + "m", "prompt", StepType.llm, True, "gpt-4o", 5, "desc", ["t"], OutputTypeEnum.PERCENTAGE, True, + project_ids=["a"], is_global=False, + ) # fmt: skip From bd613de527b414080305d9b1f749e67346a72ab4 Mon Sep 17 00:00:00 2001 From: quinn-galileo Date: Wed, 14 Oct 2026 13:18:48 -0400 Subject: [PATCH 2/8] fix: default project_ids on all evaluators and share the no-current-project fixture Co-Authored-By: Claude Opus 5.5 (1M context) --- src/splunk_ao/evaluator.py | 10 +++++++--- tests/conftest.py | 12 +++++++++++- tests/test_evaluator.py | 12 ++---------- tests/test_evaluators.py | 9 +-------- 4 files changed, 21 insertions(+), 22 deletions(-) diff --git a/src/splunk_ao/evaluator.py b/src/splunk_ao/evaluator.py index b85bc640..5ab96649 100644 --- a/src/splunk_ao/evaluator.py +++ b/src/splunk_ao/evaluator.py @@ -108,8 +108,9 @@ class Evaluator(StateManagementMixin, ABC): created_at (datetime | None): When the metric was created. updated_at (datetime | None): When the metric was last updated. version (int | None): Evaluator version number. - is_global (bool | None): Whether the evaluator is global, as reported by the API. - ``None`` until populated from the API. + is_global (bool | None): Whether the evaluator is global. ``None`` on a fresh evaluator. On + ``LlmEvaluator`` and ``CodeEvaluator`` it holds the ``is_global`` passed to the constructor until + ``create()`` or ``refresh()`` replaces it with the value the API reports. scope_projects (list[ScorerScopeProjectRef]): Projects the evaluator is scoped to, as reported by the API. Empty until populated, and for global evaluators. @@ -172,6 +173,9 @@ def my_scorer(trace_or_span): is_global: bool | None scope_projects: list[ScorerScopeProjectRef] + # Requested scope used by create(); a class default because get() and list() skip __init__ + project_ids: list[str] | None = None + def __init__( self, name: str, *, description: str = "", tags: list[str] | None = None, version: int | None = None ) -> None: @@ -408,7 +412,7 @@ def _populate_from_scorer_response(self, scorer_response: Any) -> None: judges = None cot_enabled = None - is_global = None if isinstance(scorer_response.is_global, Unset) else scorer_response.is_global + is_global = False if isinstance(scorer_response.is_global, Unset) else scorer_response.is_global scope_projects = [] if isinstance(scorer_response.scope_projects, Unset) else scorer_response.scope_projects self._sync_attrs( diff --git a/tests/conftest.py b/tests/conftest.py index 3f046681..84906cae 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -62,7 +62,7 @@ from splunk_ao.collaborator import CollaboratorRole from splunk_ao.config import SplunkAOConfig from splunk_ao.configuration import _CONFIGURATION_KEYS, Configuration -from splunk_ao.decorator import _mode_context, _session_id_context +from splunk_ao.decorator import _mode_context, _project_context, _session_id_context from splunk_ao.resources.models import DatasetContent, DatasetRow, DatasetRowValuesDict from splunk_ao.resources.models.messages_list_item import MessagesListItem from splunk_ao.utils.singleton import SplunkAOLoggerSingleton @@ -602,3 +602,13 @@ def mock_collaborator() -> MagicMock: mock_collab.last_name = "Collaborator" mock_collab.permissions = [] return mock_collab + + +@pytest.fixture +def no_current_project(monkeypatch: pytest.MonkeyPatch) -> Generator[None, None, None]: + """Clear the current project from the environment and the decorator context for one test.""" + monkeypatch.delenv("SPLUNK_AO_PROJECT", raising=False) + monkeypatch.delenv("SPLUNK_AO_PROJECT_ID", raising=False) + token = _project_context.set(None) + yield + _project_context.reset(token) diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py index cc1b2e00..0f21141f 100644 --- a/tests/test_evaluator.py +++ b/tests/test_evaluator.py @@ -6,7 +6,6 @@ import pytest from galileo_core.schemas.logging.step import StepType -from splunk_ao.decorator import _project_context from splunk_ao.evaluator import CodeEvaluator, LlmEvaluator, LocalEvaluator, Evaluator from splunk_ao.resources.models import HTTPValidationError, OutputTypeEnum, ScorerScopeProjectRef, ScorerTypes from splunk_ao.resources.models.invalid_result import InvalidResult @@ -18,14 +17,7 @@ # Test fixtures and helper functions -@pytest.fixture(autouse=True) -def no_current_project(monkeypatch: pytest.MonkeyPatch): - """Start every test with no current project in the environment or the decorator context.""" - monkeypatch.delenv("SPLUNK_AO_PROJECT", raising=False) - monkeypatch.delenv("SPLUNK_AO_PROJECT_ID", raising=False) - token = _project_context.set(None) - yield - _project_context.reset(token) +pytestmark = pytest.mark.usefixtures("no_current_project") def _scope_scorer(scorer_id: str, *, scorer_type: ScorerTypes = ScorerTypes.LLM, is_global=False, scope_projects=UNSET): @@ -2077,7 +2069,7 @@ def test_fresh_evaluator_has_no_scope(self, reset_configuration: None) -> None: [ (False, [ScorerScopeProjectRef(id="p1", name="One")], False, [ScorerScopeProjectRef(id="p1", name="One")]), (True, [], True, []), - (UNSET, UNSET, None, []), + (UNSET, UNSET, False, []), ], ) @patch("splunk_ao.evaluator.Scorers") diff --git a/tests/test_evaluators.py b/tests/test_evaluators.py index 15c517a9..3b5e087b 100644 --- a/tests/test_evaluators.py +++ b/tests/test_evaluators.py @@ -36,14 +36,7 @@ FIXED_PROJECT_ID = str(uuid4()) -@pytest.fixture(autouse=True) -def no_current_project(monkeypatch: pytest.MonkeyPatch): - """Start every test with no current project in the environment or the decorator context.""" - monkeypatch.delenv("SPLUNK_AO_PROJECT", raising=False) - monkeypatch.delenv("SPLUNK_AO_PROJECT_ID", raising=False) - token = _project_context.set(None) - yield - _project_context.reset(token) +pytestmark = pytest.mark.usefixtures("no_current_project") @pytest.fixture From c5db4da1eadbe14e783bf5632bc04a90a89fcfd8 Mon Sep 17 00:00:00 2001 From: quinn-galileo Date: Wed, 14 Oct 2026 13:19:28 -0400 Subject: [PATCH 3/8] fix: cover fetched evaluators and tidy scope wording Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 2 +- src/splunk_ao/evaluator.py | 2 +- tests/test_evaluator.py | 8 ++++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 27895612..13a782e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `LlmEvaluator`, `CodeEvaluator`, `Evaluators.create_custom_llm_evaluator()` and the module-level `create_custom_llm_evaluator()` accept `project_ids` and `is_global` to choose the scope of a new evaluator. - Passing both, or an empty `project_ids`, raises `ValidationError` before any request is sent. + Passing both, or an empty `project_ids` without `is_global=True`, raises `ValidationError` before any request is sent. - `Evaluator.is_global` and `Evaluator.scope_projects` report the scope the API returned, after `create()`, `get()`, `list()` and `refresh()`. diff --git a/src/splunk_ao/evaluator.py b/src/splunk_ao/evaluator.py index 5ab96649..10cab95e 100644 --- a/src/splunk_ao/evaluator.py +++ b/src/splunk_ao/evaluator.py @@ -1137,7 +1137,7 @@ def create(self) -> CodeEvaluator: # Ensure node_level is set (should always be set in __init__, but checking for type safety) assert self.node_level is not None - # Resolve scope first so a bad scope fails before any API call + # Resolve scope before code validation or any write, so a bad scope fails fast scope_is_global, scope_project_ids = _resolve_create_scope(self.project_ids, self.is_global) # Step 1: Validate the code and get validation result diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py index 0f21141f..54f77393 100644 --- a/tests/test_evaluator.py +++ b/tests/test_evaluator.py @@ -2093,6 +2093,14 @@ def test_get_list_and_refresh_populate_scope( assert evaluator.is_global is expected_global assert evaluator.scope_projects == expected_projects + @patch("splunk_ao.evaluator.Scorers") + def test_fetched_evaluator_has_no_requested_project_ids(self, mock_scorers_class, reset_configuration) -> None: + # Given an evaluator built by get(), which skips __init__ + mock_scorers_class.return_value.list.return_value = [_scope_scorer(str(uuid4()))] + + # Then the requested-scope attribute still exists + assert Evaluator.get(name="Scoped Evaluator").project_ids is None + @patch("splunk_ao.evaluator.Evaluators") @patch("splunk_ao.evaluator.Scorers") def test_llm_evaluator_forwards_scope_and_populates_after_create( From bd119d560d1a41d09f9c2a5615fca8cf67b56ebf Mon Sep 17 00:00:00 2001 From: quinn-galileo Date: Wed, 14 Oct 2026 13:20:53 -0400 Subject: [PATCH 4/8] chore: tidy comments Co-Authored-By: Claude Opus 5.5 (1M context) --- src/splunk_ao/evaluators.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/splunk_ao/evaluators.py b/src/splunk_ao/evaluators.py index e1b0805f..26391fb8 100644 --- a/src/splunk_ao/evaluators.py +++ b/src/splunk_ao/evaluators.py @@ -162,7 +162,8 @@ def create_custom_llm_evaluator( Raises ------ ValidationError - If the scope arguments conflict or ``project_ids`` is empty. + If the scope arguments conflict, ``project_ids`` is empty, or ``is_global=False`` is + passed with no project to scope to. NotFoundError If the current project is set but cannot be found. """ @@ -282,7 +283,8 @@ def create_custom_llm_evaluator( Raises ------ ValidationError - If the scope arguments conflict or ``project_ids`` is empty. + If the scope arguments conflict, ``project_ids`` is empty, or ``is_global=False`` is + passed with no project to scope to. NotFoundError If the current project is set but cannot be found. """ From 49bef1461b98a0ca112ad09b66723a4748b7eaaf Mon Sep 17 00:00:00 2001 From: quinn-galileo Date: Mon, 19 Oct 2026 11:44:58 -0400 Subject: [PATCH 5/8] fix: remove committed conflict markers and tighten scope validation - Resolve the CHANGELOG merge conflict, keeping main's entries and ours. - Reject a bare string for project_ids instead of splitting it into characters. - DecisionEvaluator.create no longer marks the evaluator failed on a scope ValidationError, matching the LLM and code paths. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 17 +++++++---------- src/splunk_ao/evaluator.py | 4 +++- src/splunk_ao/evaluators.py | 3 +++ tests/test_evaluator.py | 12 ++++++++++++ tests/test_evaluators.py | 4 ++++ 5 files changed, 29 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7710cd13..aec0a829 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -<<<<<<< HEAD +- `create_decision_evaluator()` (also `Evaluators.create_decision_evaluator`) and `DecisionEvaluator` create and + read decision-model evaluators: one typed question per evaluator, answered with probabilities. Supports + `boolean_probability`, `categorical` and `discrete` output types; when `output_type` is omitted it is inferred + from `answer_options`. If the version request is clearly rejected (400, 403, 404, 422) the new scorer is + deleted; on a timeout, network error or 5xx it is kept and the raised error names its id. +- Scorer requests now send `X-Galileo-Scorer-Capabilities: decision`, and `ScorerTypes` and `ModelType` include + `decision`, so decision scorers appear in list and get responses and decode. - `LlmEvaluator`, `CodeEvaluator`, `DecisionEvaluator`, `Evaluators.create_custom_llm_evaluator()`, `Evaluators.create_decision_evaluator()` and the module-level `create_custom_llm_evaluator()` and `create_decision_evaluator()` accept `project_ids` and `is_global` to choose the scope of a new evaluator. @@ -26,14 +32,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 global evaluators with a project set must now pass `is_global=True`. - Creating project-scoped evaluators requires `splunk-ao` 0.6.0 or later. Earlier versions send no scope, so on servers with metric RBAC enabled, non-admin users get a 403 when creating an evaluator. -======= -- `create_decision_evaluator()` (also `Evaluators.create_decision_evaluator`) and `DecisionEvaluator` create and - read decision-model evaluators: one typed question per evaluator, answered with probabilities. Supports - `boolean_probability`, `categorical` and `discrete` output types; when `output_type` is omitted it is inferred - from `answer_options`. If the version request is clearly rejected (400, 403, 404, 422) the new scorer is - deleted; on a timeout, network error or 5xx it is kept and the raised error names its id. -- Scorer requests now send `X-Galileo-Scorer-Capabilities: decision`, and `ScorerTypes` and `ModelType` include - `decision`, so decision scorers appear in list and get responses and decode. ### Fixed @@ -46,7 +44,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Image and file parts in OpenAI Agents user messages, on both the Responses and Chat Completions paths, are now recorded as content blocks instead of a JSON dump of the whole message, including through `SplunkAOLogger(ingestion_hook=...)`. ->>>>>>> origin/main ## [0.5.0] diff --git a/src/splunk_ao/evaluator.py b/src/splunk_ao/evaluator.py index 2c636818..f4889106 100644 --- a/src/splunk_ao/evaluator.py +++ b/src/splunk_ao/evaluator.py @@ -122,7 +122,7 @@ class Evaluator(StateManagementMixin, ABC): updated_at (datetime | None): When the metric was last updated. version (int | None): Evaluator version number. is_global (bool | None): Whether the evaluator is global. ``None`` on a fresh evaluator. On - ``LlmEvaluator`` and ``CodeEvaluator`` it holds the ``is_global`` passed to the constructor until + ``LlmEvaluator``, ``CodeEvaluator`` and ``DecisionEvaluator`` it holds the ``is_global`` passed to the constructor until ``create()`` or ``refresh()`` replaces it with the value the API reports. scope_projects (list[ScorerScopeProjectRef]): Projects the evaluator is scoped to, as reported by the API. Empty until populated, and for global evaluators. @@ -1045,6 +1045,8 @@ def create(self) -> DecisionEvaluator: logger.info(f"DecisionEvaluator.create: id='{self.id}' - completed") return self + except ValidationError: + raise except Exception as e: if isinstance(e, DecisionVersionOutcomeUnknownError): self._sync_attrs(id=e.scorer_id) diff --git a/src/splunk_ao/evaluators.py b/src/splunk_ao/evaluators.py index 77875900..5f9228fa 100644 --- a/src/splunk_ao/evaluators.py +++ b/src/splunk_ao/evaluators.py @@ -219,6 +219,9 @@ def _resolve_create_scope( return True, UNSET if project_ids is not None: + requested: object = project_ids + if isinstance(requested, str): + raise ValidationError("project_ids must be a list of project ids, not a single string.") if not project_ids: raise ValidationError("project_ids must not be empty. Pass is_global=True for a global evaluator.") return False, list(dict.fromkeys(project_ids)) diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py index 88dbaca0..ffacd0eb 100644 --- a/tests/test_evaluator.py +++ b/tests/test_evaluator.py @@ -2511,6 +2511,18 @@ def test_invalid_scope_makes_no_create_request( mock_create_scorers.sync.assert_not_called() + @patch("splunk_ao.evaluators.SplunkAOConfig.get") + def test_invalid_scope_does_not_mark_evaluator_failed(self, mock_config, reset_configuration) -> None: + # Given a decision evaluator with a conflicting scope + evaluator = DecisionEvaluator(name="tone", question="Which tone?", is_global=True, project_ids=["a"]) + + # When created + with pytest.raises(ValidationError): + evaluator.create() + + # Then its state is untouched, as for LlmEvaluator and CodeEvaluator + assert evaluator.sync_state == SyncState.LOCAL_ONLY + @patch("splunk_ao.evaluator.Scorers") def test_get_populates_scope_for_fetched_decision_evaluator( self, mock_scorers_class: MagicMock, reset_configuration: None diff --git a/tests/test_evaluators.py b/tests/test_evaluators.py index 2a62c7dd..400f54fb 100644 --- a/tests/test_evaluators.py +++ b/tests/test_evaluators.py @@ -638,6 +638,10 @@ def test_empty_project_ids_rejected(self, monkeypatch, mock_projects) -> None: with pytest.raises(SplunkAOValidationError, match="must not be empty"): _resolve_create_scope([], None) + def test_string_project_ids_rejected(self, mock_projects) -> None: + with pytest.raises(SplunkAOValidationError, match="not a single string"): + _resolve_create_scope("3f2a", None) # type: ignore[arg-type] + def test_is_global_false_without_any_project_rejected(self, mock_projects) -> None: with pytest.raises(SplunkAOValidationError, match="is_global=False needs project_ids"): _resolve_create_scope(None, False) From 9fcea53ede3545388ecea872e530290ebd4c9664 Mon Sep 17 00:00:00 2001 From: quinn-galileo Date: Mon, 19 Oct 2026 12:14:09 -0400 Subject: [PATCH 6/8] fix: keep requested scope private and clarify scope errors - Store constructor project_ids/is_global privately; public is_global and scope_projects now only reflect the API response (None/[] until populated). - An Unset is_global in the response maps to None. - Explain a missing default project in the NotFoundError message, normalise a whitespace-only context project, and document scope errors on create(). - Add regression tests for create() on a fetched evaluator and tighten tests. - Correct the CHANGELOG validation wording and note the project must exist. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 5 ++- src/splunk_ao/evaluator.py | 76 +++++++++++++++++++++++------------- src/splunk_ao/evaluators.py | 15 ++++++-- tests/test_evaluator.py | 58 ++++++++++++++++++++++++---- tests/test_evaluators.py | 77 ++++++++++++++++++++++++++++++++----- 5 files changed, 183 insertions(+), 48 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aec0a829..d3702c9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `LlmEvaluator`, `CodeEvaluator`, `DecisionEvaluator`, `Evaluators.create_custom_llm_evaluator()`, `Evaluators.create_decision_evaluator()` and the module-level `create_custom_llm_evaluator()` and `create_decision_evaluator()` accept `project_ids` and `is_global` to choose the scope of a new evaluator. - Passing both, or an empty `project_ids` without `is_global=True`, raises `ValidationError` before any request is sent. + Passing `is_global=True` together with `project_ids`, an empty `project_ids`, or `is_global=False` with no + `project_ids` and no current project raises `ValidationError` before any request is sent. - `Evaluator.is_global` and `Evaluator.scope_projects` report the scope the API returned, after `create()`, `get()`, `list()` and `refresh()`. @@ -30,6 +31,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `NotFoundError` rather than falling back to a global evaluator. When no current project is set and no scope is passed, no scope is sent and the server default (global, org admins only) applies. Org admins who relied on creating global evaluators with a project set must now pass `is_global=True`. + The current project must already exist: unlike logging, creating an evaluator does not create it. Create it first, + or pass `project_ids` or `is_global=True`. - Creating project-scoped evaluators requires `splunk-ao` 0.6.0 or later. Earlier versions send no scope, so on servers with metric RBAC enabled, non-admin users get a 403 when creating an evaluator. diff --git a/src/splunk_ao/evaluator.py b/src/splunk_ao/evaluator.py index f4889106..181b1fe9 100644 --- a/src/splunk_ao/evaluator.py +++ b/src/splunk_ao/evaluator.py @@ -121,11 +121,10 @@ class Evaluator(StateManagementMixin, ABC): created_at (datetime | None): When the metric was created. updated_at (datetime | None): When the metric was last updated. version (int | None): Evaluator version number. - is_global (bool | None): Whether the evaluator is global. ``None`` on a fresh evaluator. On - ``LlmEvaluator``, ``CodeEvaluator`` and ``DecisionEvaluator`` it holds the ``is_global`` passed to the constructor until - ``create()`` or ``refresh()`` replaces it with the value the API reports. - scope_projects (list[ScorerScopeProjectRef]): Projects the evaluator is scoped to, as - reported by the API. Empty until populated, and for global evaluators. + is_global (bool | None): Whether the evaluator is global, as reported by the API. ``None`` until the + evaluator is populated from the API (after ``create()``, ``get()``, ``list()`` or ``refresh()``). + scope_projects (list[ScorerScopeProjectRef]): Projects the evaluator is scoped to, as reported by the + API. Empty until populated, and for global evaluators. Class Attributes ---------------- @@ -186,8 +185,9 @@ def my_scorer(trace_or_span): is_global: bool | None scope_projects: list[ScorerScopeProjectRef] - # Requested scope used by create(); a class default because get() and list() skip __init__ - project_ids: list[str] | None = None + # Scope requested for create(). Class defaults because get() and list() build instances without __init__. + _create_project_ids: list[str] | None = None + _create_is_global: bool | None = None def __init__( self, name: str, *, description: str = "", tags: list[str] | None = None, version: int | None = None @@ -431,7 +431,7 @@ def _populate_from_scorer_response(self, scorer_response: Any) -> None: # Decision scorers carry no LLM defaults; keep the model the evaluator already has. model = getattr(self, "model", None) - is_global = False if isinstance(scorer_response.is_global, Unset) else scorer_response.is_global + is_global = None if isinstance(scorer_response.is_global, Unset) else scorer_response.is_global scope_projects = [] if isinstance(scorer_response.scope_projects, Unset) else scorer_response.scope_projects self._sync_attrs( @@ -777,9 +777,13 @@ def __init__( output_type: Output type ("percentage", "boolean", etc.). ground_truth: Whether the scorer requires ground truth (``reference_output``) from the dataset. When True, the judge LLM receives the row's ground-truth value in its prompt. - project_ids: Projects to scope the metric to when it is created. Defaults to the current - project (decorator context, SPLUNK_AO_PROJECT_ID, or SPLUNK_AO_PROJECT). + project_ids: Projects to scope the metric to when it is created. Defaults to the current project + (decorator context, SPLUNK_AO_PROJECT_ID, or SPLUNK_AO_PROJECT), which must already exist. If no + current project is set, no scope is sent and the server creates a global metric (org admins only). + The decorator context is per thread and task, so pass ``project_ids`` explicitly when creating from + worker threads. is_global: Create a global metric (org admins only). Cannot be combined with ``project_ids``. + ``False`` requires ``project_ids`` or a current project. description: Description of the metric. tags: Tags associated with the metric. version: Specific version to reference (for existing metrics). @@ -825,8 +829,8 @@ def __init__( self.output_type = output_type or OutputTypeEnum.BOOLEAN self.ground_truth = ground_truth - self.project_ids = project_ids - self.is_global = is_global + self._create_project_ids = project_ids + self._create_is_global = is_global self.scorer_type = ScorerTypes.LLM @@ -840,7 +844,10 @@ def create(self) -> LlmEvaluator: Raises ------ - ValidationError: If configuration is invalid. + ValidationError: If configuration is invalid, or the scope is invalid (``is_global=True`` with + ``project_ids``, an empty ``project_ids``, or ``is_global=False`` with no project). Scope errors + are ``ValidationError``, not ``ValueError``. + NotFoundError: If the current project is set but cannot be found. Exception: If the API call fails. Examples @@ -869,8 +876,8 @@ def create(self) -> LlmEvaluator: if isinstance(self.output_type, OutputTypeEnum) else OutputTypeEnum.BOOLEAN, ground_truth=self.ground_truth, - project_ids=self.project_ids, - is_global=self.is_global, + project_ids=self._create_project_ids, + is_global=self._create_is_global, ) # Update attributes from response without triggering dirty-tracking @@ -963,9 +970,13 @@ def __init__( ``DecisionLevel`` / ``{level, label, description}`` dicts for discrete. input_type: What the model sees. None lets the server choose a default for the node type. ground_truth: Whether the scorer requires ground truth (``reference_output``) from the dataset. - project_ids: Projects to scope the evaluator to when it is created. Defaults to the current - project (decorator context, SPLUNK_AO_PROJECT_ID, or SPLUNK_AO_PROJECT). + project_ids: Projects to scope the evaluator to when it is created. Defaults to the current project + (decorator context, SPLUNK_AO_PROJECT_ID, or SPLUNK_AO_PROJECT), which must already exist. If no + current project is set, no scope is sent and the server creates a global evaluator (org admins only). + The decorator context is per thread and task, so pass ``project_ids`` explicitly when creating from + worker threads. is_global: Create a global evaluator (org admins only). Cannot be combined with ``project_ids``. + ``False`` requires ``project_ids`` or a current project. description: Description of the evaluator. tags: Tags associated with the evaluator. version: Specific version to reference (for existing evaluators). @@ -992,8 +1003,8 @@ def __init__( self.answer_options = normalized_options self.input_type = input_type self.ground_truth = ground_truth - self.project_ids = project_ids - self.is_global = is_global + self._create_project_ids = project_ids + self._create_is_global = is_global self.scorer_type = ScorerTypes.DECISION def create(self) -> DecisionEvaluator: @@ -1006,6 +1017,10 @@ def create(self) -> DecisionEvaluator: Raises ------ + ValidationError: If the scope is invalid (``is_global=True`` with ``project_ids``, an empty + ``project_ids``, or ``is_global=False`` with no project). Scope errors are ``ValidationError``, + not ``ValueError``. + NotFoundError: If the current project is set but cannot be found. ValueError: If an argument is invalid or the API answers with a validation error. SplunkAOAPIError: For HTTP errors. A scorer created before a clear rejection is deleted. DecisionVersionOutcomeUnknownError: When the outcome of the version request is unknown (timeout, @@ -1030,8 +1045,8 @@ def create(self) -> DecisionEvaluator: description=self.description, tags=self.tags, ground_truth=self.ground_truth, - project_ids=self.project_ids, - is_global=self.is_global, + project_ids=self._create_project_ids, + is_global=self._create_is_global, ) self._sync_attrs( @@ -1176,9 +1191,13 @@ def __init__( output_type: Output type for the metric ("percentage", "boolean", "categorical", "count", "discrete"). Accepts string or OutputTypeEnum. required_metrics: List of metric names that this code metric depends on. - project_ids: Projects to scope the metric to when it is created. Defaults to the current - project (decorator context, SPLUNK_AO_PROJECT_ID, or SPLUNK_AO_PROJECT). + project_ids: Projects to scope the metric to when it is created. Defaults to the current project + (decorator context, SPLUNK_AO_PROJECT_ID, or SPLUNK_AO_PROJECT), which must already exist. If no + current project is set, no scope is sent and the server creates a global metric (org admins only). + The decorator context is per thread and task, so pass ``project_ids`` explicitly when creating from + worker threads. is_global: Create a global metric (org admins only). Cannot be combined with ``project_ids``. + ``False`` requires ``project_ids`` or a current project. description: Description of the metric. tags: Tags associated with the metric. version: Specific version to reference (for existing metrics). @@ -1188,8 +1207,8 @@ def __init__( self.code = code self.node_level = node_level or StepType.llm self.required_metrics = required_metrics - self.project_ids = project_ids - self.is_global = is_global + self._create_project_ids = project_ids + self._create_is_global = is_global self.scorer_type = ScorerTypes.CODE self.output_type = Evaluator._parse_output_type(output_type) @@ -1339,7 +1358,10 @@ def create(self) -> CodeEvaluator: Raises ------ - ValidationError: If code is not set, validation fails, or configuration is invalid. + ValidationError: If code is not set, validation fails, or configuration is invalid, or the scope is + invalid (``is_global=True`` with ``project_ids``, an empty ``project_ids``, or ``is_global=False`` + with no project). Scope errors are ``ValidationError``, not ``ValueError``. + NotFoundError: If the current project is set but cannot be found. Exception: If the API call fails. Examples @@ -1374,7 +1396,7 @@ def create(self) -> CodeEvaluator: assert self.node_level is not None # Resolve scope before code validation or any write, so a bad scope fails fast - scope_is_global, scope_project_ids = _resolve_create_scope(self.project_ids, self.is_global) + scope_is_global, scope_project_ids = _resolve_create_scope(self._create_project_ids, self._create_is_global) # Step 1: Validate the code and get validation result logger.debug(f"CodeEvaluator.create: name='{self.name}' - validating code") diff --git a/src/splunk_ao/evaluators.py b/src/splunk_ao/evaluators.py index 5f9228fa..512da5eb 100644 --- a/src/splunk_ao/evaluators.py +++ b/src/splunk_ao/evaluators.py @@ -30,7 +30,7 @@ from splunk_ao.resources.types import UNSET, Unset from splunk_ao.scorers import Scorers from splunk_ao.search import FilterType -from splunk_ao.shared.exceptions import ValidationError, _resolve_project_identifiers +from splunk_ao.shared.exceptions import NotFoundError, ValidationError, _resolve_project_identifiers from splunk_ao.shared.project_resolver import _resolve_project _logger = logging.getLogger(__name__) @@ -219,6 +219,7 @@ def _resolve_create_scope( return True, UNSET if project_ids is not None: + # Typed as object so the str guard is not flagged unreachable for a list[str] parameter. requested: object = project_ids if isinstance(requested, str): raise ValidationError("project_ids must be a list of project ids, not a single string.") @@ -229,14 +230,22 @@ def _resolve_create_scope( # Imported here because decorator.py imports modules that depend on this one. from splunk_ao.decorator import splunk_ao_context - context_project = splunk_ao_context.get_current_project() + context_project = (splunk_ao_context.get_current_project() or "").strip() or None + project_id: str | None + project_name: str | None if context_project: project_id, project_name = None, context_project else: project_id, project_name = _resolve_project_identifiers(None, None) if project_id or project_name: - return False, [_resolve_project(project_id, project_name).id] + try: + return False, [_resolve_project(project_id, project_name).id] + except NotFoundError as e: + raise type(e)( + "Evaluator scope defaults to the current project (decorator context, SPLUNK_AO_PROJECT_ID or " + f"SPLUNK_AO_PROJECT), which was not found. Pass project_ids=[...] or is_global=True. {e}" + ) from e if is_global is False: raise ValidationError("is_global=False needs project_ids or a current project.") diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py index ffacd0eb..49301811 100644 --- a/tests/test_evaluator.py +++ b/tests/test_evaluator.py @@ -2070,7 +2070,7 @@ def test_fresh_evaluator_has_no_scope(self, reset_configuration: None) -> None: [ (False, [ScorerScopeProjectRef(id="p1", name="One")], False, [ScorerScopeProjectRef(id="p1", name="One")]), (True, [], True, []), - (UNSET, UNSET, False, []), + (UNSET, UNSET, None, []), ], ) @patch("splunk_ao.evaluator.Scorers") @@ -2094,13 +2094,45 @@ def test_get_list_and_refresh_populate_scope( assert evaluator.is_global is expected_global assert evaluator.scope_projects == expected_projects + @pytest.mark.parametrize( + ("is_global", "scope_projects"), [(False, [ScorerScopeProjectRef(id="a", name="A")]), (True, [])] + ) + @patch("splunk_ao.shared.project_resolver.Projects") + @patch("splunk_ao.evaluators.create_llm_scorer_version_scorers_scorer_id_version_llm_post") + @patch("splunk_ao.evaluators.create_scorers_post") + @patch("splunk_ao.evaluators.SplunkAOConfig.get") @patch("splunk_ao.evaluator.Scorers") - def test_fetched_evaluator_has_no_requested_project_ids(self, mock_scorers_class, reset_configuration) -> None: - # Given an evaluator built by get(), which skips __init__ - mock_scorers_class.return_value.list.return_value = [_scope_scorer(str(uuid4()))] + def test_create_on_fetched_evaluator_ignores_the_fetched_scope( + self, + mock_scorers_class, + mock_config, + mock_create_scorers, + mock_create_version, + mock_projects_class, + is_global, + scope_projects, + monkeypatch, + reset_configuration, + ) -> None: + # Given an evaluator fetched with a server scope, and a different current project in the environment + scorer_id = str(uuid4()) + mock_scorers_class.return_value.list.return_value = [ + _scope_scorer(scorer_id, is_global=is_global, scope_projects=scope_projects) + ] + fetched = Evaluator.get(name="Scoped Evaluator") + assert fetched.is_global is is_global + monkeypatch.setenv("SPLUNK_AO_PROJECT_ID", "env-id") + mock_projects_class.return_value.get_with_env_fallbacks.return_value = MagicMock(id="env-project-id") + mock_create_scorers.sync.return_value = MagicMock(id=scorer_id) + mock_create_version.sync.return_value = MagicMock(scorer_id=scorer_id) + + # When create() is called on it + fetched.create() - # Then the requested-scope attribute still exists - assert Evaluator.get(name="Scoped Evaluator").project_ids is None + # Then the scope comes from the current project, not from the fetched server values + body = mock_create_scorers.sync.call_args.kwargs["body"] + assert body.is_global is False + assert body.project_ids == ["env-project-id"] @patch("splunk_ao.evaluator.Evaluators") @patch("splunk_ao.evaluator.Scorers") @@ -2132,12 +2164,15 @@ def test_llm_evaluator_forwards_scope_and_populates_after_create( def test_llm_evaluator_forwards_is_global( self, mock_scorers_class, mock_evaluators_class, reset_configuration: None ) -> None: + # Given an LLM evaluator built with is_global=True scorer_id = str(uuid4()) mock_evaluators_class.return_value.create_custom_llm_evaluator.return_value = MagicMock(scorer_id=scorer_id) mock_scorers_class.return_value.list.return_value = [_scope_scorer(scorer_id, is_global=True)] + # When created evaluator = LlmEvaluator(name="Scoped Evaluator", prompt="x", is_global=True).create() + # Then the request is global and the server value is surfaced kwargs = mock_evaluators_class.return_value.create_custom_llm_evaluator.call_args.kwargs assert kwargs["is_global"] is True assert kwargs["project_ids"] is None @@ -2188,12 +2223,15 @@ def test_code_evaluator_sends_scope_and_populates_after_create( def test_code_evaluator_global_sends_no_project_ids( self, mock_create_scorers, mock_config, reset_configuration ) -> None: + # Given a global code evaluator and a create endpoint that returns nothing; we only inspect the request mock_create_scorers.sync.return_value = None + # When created, the missing response raises after the request is built with patch.object(CodeEvaluator, "_validate_code", return_value="{}"): with pytest.raises(ValueError, match="No response"): CodeEvaluator(name="G", code="x", is_global=True).create() + # Then the request was global with no project ids body = mock_create_scorers.sync.call_args.kwargs["body"] assert body.is_global is True assert "project_ids" not in body.to_dict() @@ -2207,9 +2245,12 @@ def test_code_evaluator_global_sends_no_project_ids( def test_code_evaluator_resolves_scope_before_validating_code( self, mock_create_scorers, mock_validate_post, mock_config, kwargs, reset_configuration ) -> None: + # Given a code evaluator with an invalid scope + # When created with pytest.raises(ValidationError): CodeEvaluator(name="Bad", code="x", **kwargs).create() + # Then neither validation nor creation was attempted mock_validate_post.sync.assert_not_called() mock_create_scorers.sync.assert_not_called() @@ -2487,13 +2528,16 @@ def test_forwards_scope_and_populates_after_create( def test_forwards_is_global( self, mock_scorers_class: MagicMock, mock_evaluators_class: MagicMock, reset_configuration: None ) -> None: + # Given a decision evaluator built with is_global=True version = MagicMock() version.scorer_id = "s1" mock_evaluators_class.return_value.create_decision_evaluator.return_value = version mock_scorers_class.return_value.list.return_value = [_decision_scorer("boolean_probability", None)] + # When created DecisionEvaluator(name="tone", question="Which tone?", is_global=True).create() + # Then the request is global kwargs = mock_evaluators_class.return_value.create_decision_evaluator.call_args.kwargs assert kwargs["is_global"] is True assert kwargs["project_ids"] is None @@ -2540,4 +2584,4 @@ def test_get_populates_scope_for_fetched_decision_evaluator( assert isinstance(evaluator, DecisionEvaluator) assert evaluator.is_global is False assert [(p.id, p.name) for p in evaluator.scope_projects] == [("a", "A")] - assert evaluator.project_ids is None + assert evaluator._create_project_ids is None diff --git a/tests/test_evaluators.py b/tests/test_evaluators.py index 400f54fb..d7cb067f 100644 --- a/tests/test_evaluators.py +++ b/tests/test_evaluators.py @@ -44,7 +44,6 @@ from splunk_ao.resources.models.scorer_response import ScorerResponse from splunk_ao.resources.models.scorer_types import ScorerTypes from splunk_ao.resources.types import UNSET -from splunk_ao.shared.exceptions import NotFoundError as ProjectLookupNotFoundError from splunk_ao.shared.exceptions import ValidationError as SplunkAOValidationError FIXED_PROJECT_ID = str(uuid4()) @@ -606,7 +605,20 @@ def test_explicit_project_ids_override_context_and_env(self, monkeypatch, mock_p mock_projects.get_with_env_fallbacks.assert_not_called() def test_explicit_project_ids_with_is_global_false(self, mock_projects) -> None: - assert _resolve_create_scope(["a"], False) == (False, ["a"]) + # Given explicit ids and is_global=False + # When resolving + result = _resolve_create_scope(["a"], False) + + # Then the ids are used + assert result == (False, ["a"]) + + def test_empty_project_ids_with_is_global_true_is_global(self, mock_projects) -> None: + # Given is_global=True and an empty list, which does not conflict + # When resolving + result = _resolve_create_scope([], True) + + # Then the evaluator is global + assert result == (True, UNSET) def test_is_global_true_sends_no_project_ids_even_with_current_project(self, monkeypatch, mock_projects) -> None: # Given a current project @@ -620,14 +632,37 @@ def test_is_global_true_sends_no_project_ids_even_with_current_project(self, mon mock_projects.get_with_env_fallbacks.assert_not_called() def test_nothing_set_sends_no_scope(self, mock_projects) -> None: - assert _resolve_create_scope(None, None) == (UNSET, UNSET) + # Given no arguments, context project or env vars + # When resolving + result = _resolve_create_scope(None, None) + + # Then no scope is sent and no lookup happens + assert result == (UNSET, UNSET) mock_projects.get_with_env_fallbacks.assert_not_called() + def test_whitespace_context_project_counts_as_unset(self, mock_projects) -> None: + # Given a whitespace-only context project and nothing else + _project_context.set(" ") + + # When resolving + result = _resolve_create_scope(None, None) + + # Then it is treated as no current project + assert result == (UNSET, UNSET) + def test_whitespace_env_counts_as_unset(self, monkeypatch, mock_projects) -> None: + # Given a whitespace-only SPLUNK_AO_PROJECT monkeypatch.setenv("SPLUNK_AO_PROJECT", " ") - assert _resolve_create_scope(None, None) == (UNSET, UNSET) + + # When resolving + result = _resolve_create_scope(None, None) + + # Then it is treated as no current project + assert result == (UNSET, UNSET) def test_global_and_project_ids_conflict(self, mock_projects) -> None: + # Given is_global=True together with project ids + # Then resolving fails with pytest.raises(SplunkAOValidationError, match="both global and scoped"): _resolve_create_scope(["a"], True) @@ -643,21 +678,31 @@ def test_string_project_ids_rejected(self, mock_projects) -> None: _resolve_create_scope("3f2a", None) # type: ignore[arg-type] def test_is_global_false_without_any_project_rejected(self, mock_projects) -> None: + # Given is_global=False and no project anywhere + # Then resolving fails with pytest.raises(SplunkAOValidationError, match="is_global=False needs project_ids"): _resolve_create_scope(None, False) def test_is_global_false_uses_current_project(self, monkeypatch, mock_projects) -> None: + # Given a current project monkeypatch.setenv("SPLUNK_AO_PROJECT_ID", "env-id") - assert _resolve_create_scope(None, False) == (False, ["project-id-1"]) + + # When is_global=False is passed with no ids + result = _resolve_create_scope(None, False) + + # Then the current project is used + assert result == (False, ["project-id-1"]) def test_current_project_not_found_raises_not_found(self, monkeypatch, mock_projects) -> None: # Given a current project that does not exist monkeypatch.setenv("SPLUNK_AO_PROJECT", "missing") mock_projects.get_with_env_fallbacks.return_value = None - # Then there is no silent fallback to global - with pytest.raises(ProjectLookupNotFoundError, match="missing"): + # Then there is no silent fallback to global, and the error says why a project was looked up + with pytest.raises(NotFoundError, match="missing") as exc_info: _resolve_create_scope(None, None) + assert "defaults to the current project" in str(exc_info.value) + assert "is_global=True" in str(exc_info.value) class TestCreateCustomLlmEvaluatorScope: @@ -690,14 +735,17 @@ def test_defaults_to_current_project( @pytest.mark.parametrize("use_function", [False, True], ids=["method", "module_function"]) @patch("splunk_ao.evaluators.create_llm_scorer_version_scorers_scorer_id_version_llm_post") @patch("splunk_ao.evaluators.create_scorers_post") - def test_explicit_project_ids( + def test_explicit_project_ids_override_current_project( self, mock_create_scorer, mock_create_version, use_function, monkeypatch, mock_projects, mock_scorer_response ) -> None: + # Given a current project from the environment monkeypatch.setenv("SPLUNK_AO_PROJECT_ID", "env-id") mock_create_scorer.sync.return_value = mock_scorer_response + # When creating with explicit project ids self._create(use_function, project_ids=["a", "b"]) + # Then the explicit ids are sent body = mock_create_scorer.sync.call_args.kwargs["body"] assert body.is_global is False assert body.project_ids == ["a", "b"] @@ -708,11 +756,14 @@ def test_explicit_project_ids( def test_explicit_global_overrides_current_project( self, mock_create_scorer, mock_create_version, use_function, monkeypatch, mock_projects, mock_scorer_response ) -> None: + # Given a current project from the environment monkeypatch.setenv("SPLUNK_AO_PROJECT_ID", "env-id") mock_create_scorer.sync.return_value = mock_scorer_response + # When creating with is_global=True self._create(use_function, is_global=True) + # Then the request is global with no project ids body = mock_create_scorer.sync.call_args.kwargs["body"] assert body.is_global is True assert "project_ids" not in body.to_dict() @@ -723,10 +774,13 @@ def test_explicit_global_overrides_current_project( def test_no_current_project_sends_no_scope( self, mock_create_scorer, mock_create_version, use_function, mock_projects, mock_scorer_response ) -> None: + # Given no current project mock_create_scorer.sync.return_value = mock_scorer_response + # When creating with no scope self._create(use_function) + # Then no scope fields are sent body_dict = mock_create_scorer.sync.call_args.kwargs["body"].to_dict() assert "is_global" not in body_dict assert "project_ids" not in body_dict @@ -739,9 +793,12 @@ def test_no_current_project_sends_no_scope( def test_invalid_scope_fails_before_any_http_call( self, mock_create_scorer, kwargs, use_function, mock_projects ) -> None: + # Given an invalid scope + # When creating with pytest.raises(SplunkAOValidationError): self._create(use_function, **kwargs) + # Then nothing is sent mock_create_scorer.sync.assert_not_called() @pytest.mark.parametrize("use_function", [False, True], ids=["method", "module_function"]) @@ -752,7 +809,7 @@ def test_current_project_not_found_sends_no_create_request( monkeypatch.setenv("SPLUNK_AO_PROJECT", "missing") mock_projects.get_with_env_fallbacks.return_value = None - with pytest.raises(ProjectLookupNotFoundError): + with pytest.raises(NotFoundError): self._create(use_function) mock_create_scorer.sync.assert_not_called() @@ -1200,7 +1257,7 @@ def test_current_project_not_found_sends_no_create_request( monkeypatch.setenv("SPLUNK_AO_PROJECT", "missing") mock_projects.get_with_env_fallbacks.return_value = None - with pytest.raises(ProjectLookupNotFoundError): + with pytest.raises(NotFoundError): self._create(use_function) mock_create_scorer.sync.assert_not_called() From 15f00ceadf4d9b7a8f17fbc47755c6fd4e14532d Mon Sep 17 00:00:00 2001 From: quinn-galileo Date: Tue, 20 Oct 2026 11:41:41 -0400 Subject: [PATCH 7/8] chore: fix duplicate Changed heading in the changelog Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 708989d9..223c9ade 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,9 +35,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 or pass `project_ids` or `is_global=True`. - Creating project-scoped evaluators requires `splunk-ao` 0.6.0 or later. Earlier versions send no scope, so on servers with metric RBAC enabled, non-admin users get a 403 when creating an evaluator. - -### Changed - - **Breaking:** the low-level API client (`splunk_ao.resources`) is regenerated from the current Client API spec. Code that imports generated symbols directly may need updating: - `ScorerConfig` is now `RuntimeScorerConfig`. It keeps the old fields, adds new ones and makes `scorer_type` From 29b5453c5e9b1c1ad17fde9129f1591ccb4be2c6 Mon Sep 17 00:00:00 2001 From: quinn-galileo Date: Tue, 20 Oct 2026 13:01:48 -0400 Subject: [PATCH 8/8] chore: correct the scope validation wording in the changelog Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 223c9ade..775e3528 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,8 +19,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `LlmEvaluator`, `CodeEvaluator`, `DecisionEvaluator`, `Evaluators.create_custom_llm_evaluator()`, `Evaluators.create_decision_evaluator()` and the module-level `create_custom_llm_evaluator()` and `create_decision_evaluator()` accept `project_ids` and `is_global` to choose the scope of a new evaluator. - Passing `is_global=True` together with `project_ids`, an empty `project_ids`, or `is_global=False` with no - `project_ids` and no current project raises `ValidationError` before any request is sent. + Passing `is_global=True` together with `project_ids`, an empty `project_ids` without `is_global=True`, or + `is_global=False` with no `project_ids` and no current project raises `ValidationError` before any request is + sent. - `Evaluator.is_global` and `Evaluator.scope_projects` report the scope the API returned, after `create()`, `get()`, `list()` and `refresh()`.