diff --git a/CHANGELOG.md b/CHANGELOG.md index c1b9024f..775e3528 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,9 +16,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 deleted; on a timeout, network error or 5xx it is kept and the raised error names its id. - Scorer requests now send `X-Galileo-Scorer-Capabilities: decision`, and `ScorerTypes` and `ModelType` include `decision`, so decision scorers appear in list and get responses and decode. +- `LlmEvaluator`, `CodeEvaluator`, `DecisionEvaluator`, `Evaluators.create_custom_llm_evaluator()`, + `Evaluators.create_decision_evaluator()` and the module-level `create_custom_llm_evaluator()` and + `create_decision_evaluator()` accept `project_ids` and `is_global` to choose the scope of a new evaluator. + Passing `is_global=True` together with `project_ids`, an empty `project_ids` without `is_global=True`, or + `is_global=False` with no `project_ids` and no current project raises `ValidationError` before any request is + sent. +- `Evaluator.is_global` and `Evaluator.scope_projects` report the scope the API returned, after + `create()`, `get()`, `list()` and `refresh()`. ### Changed +- New evaluators are scoped to the current project by default: the decorator context project, then + `SPLUNK_AO_PROJECT_ID`, then `SPLUNK_AO_PROJECT`. If that project cannot be found, creation raises + `NotFoundError` rather than falling back to a global evaluator. When no current project is set and no scope is + passed, no scope is sent and the server default (global, org admins only) applies. Org admins who relied on creating + global evaluators with a project set must now pass `is_global=True`. + The current project must already exist: unlike logging, creating an evaluator does not create it. Create it first, + or pass `project_ids` or `is_global=True`. +- Creating project-scoped evaluators requires `splunk-ao` 0.6.0 or later. Earlier versions send no scope, so on servers + with metric RBAC enabled, non-admin users get a 403 when creating an evaluator. - **Breaking:** the low-level API client (`splunk_ao.resources`) is regenerated from the current Client API spec. Code that imports generated symbols directly may need updating: - `ScorerConfig` is now `RuntimeScorerConfig`. It keeps the old fields, adds new ones and makes `scorer_type` diff --git a/src/splunk_ao/evaluator.py b/src/splunk_ao/evaluator.py index 06133eea..181b1fe9 100644 --- a/src/splunk_ao/evaluator.py +++ b/src/splunk_ao/evaluator.py @@ -23,6 +23,7 @@ DecisionAnswerOptions, DecisionVersionOutcomeUnknownError, Evaluators, + _resolve_create_scope, normalize_decision_answer_options, resolve_decision_output_type, validate_decision_request, @@ -44,6 +45,7 @@ InputTypeEnum, OutputTypeEnum, ScorerResponse, + ScorerScopeProjectRef, ScorerTypes, TaskResultStatus, UpdateScorerRequest, @@ -119,6 +121,10 @@ class Evaluator(StateManagementMixin, ABC): created_at (datetime | None): When the metric was created. updated_at (datetime | None): When the metric was last updated. version (int | None): Evaluator version number. + is_global (bool | None): Whether the evaluator is global, as reported by the API. ``None`` until the + evaluator is populated from the API (after ``create()``, ``get()``, ``list()`` or ``refresh()``). + scope_projects (list[ScorerScopeProjectRef]): Projects the evaluator is scoped to, as reported by the + API. Empty until populated, and for global evaluators. Class Attributes ---------------- @@ -175,6 +181,14 @@ def my_scorer(trace_or_span): judges: int | None cot_enabled: bool | None + # Access scope reported by the API + is_global: bool | None + scope_projects: list[ScorerScopeProjectRef] + + # Scope requested for create(). Class defaults because get() and list() build instances without __init__. + _create_project_ids: list[str] | None = None + _create_is_global: bool | None = None + def __init__( self, name: str, *, description: str = "", tags: list[str] | None = None, version: int | None = None ) -> None: @@ -202,6 +216,9 @@ def __init__( self.judges = None self.cot_enabled = None + self.is_global = None + self.scope_projects = [] + self._set_state(SyncState.LOCAL_ONLY) @staticmethod @@ -414,6 +431,9 @@ def _populate_from_scorer_response(self, scorer_response: Any) -> None: # Decision scorers carry no LLM defaults; keep the model the evaluator already has. model = getattr(self, "model", None) + is_global = None if isinstance(scorer_response.is_global, Unset) else scorer_response.is_global + scope_projects = [] if isinstance(scorer_response.scope_projects, Unset) else scorer_response.scope_projects + self._sync_attrs( id=scorer_response.id, name=scorer_response.name, @@ -426,6 +446,8 @@ def _populate_from_scorer_response(self, scorer_response: Any) -> None: model=model, judges=judges, cot_enabled=cot_enabled, + is_global=is_global, + scope_projects=list(scope_projects), ) # LLM-specific attributes (only set if this is an LlmEvaluator) @@ -730,6 +752,9 @@ def __init__( cot_enabled: bool | None = None, output_type: str | OutputTypeEnum | None = None, ground_truth: bool = False, + # Access scope + project_ids: list[str] | None = None, + is_global: bool | None = None, # Common parameters description: str = "", tags: list[str] | None = None, @@ -752,6 +777,13 @@ def __init__( output_type: Output type ("percentage", "boolean", etc.). ground_truth: Whether the scorer requires ground truth (``reference_output``) from the dataset. When True, the judge LLM receives the row's ground-truth value in its prompt. + project_ids: Projects to scope the metric to when it is created. Defaults to the current project + (decorator context, SPLUNK_AO_PROJECT_ID, or SPLUNK_AO_PROJECT), which must already exist. If no + current project is set, no scope is sent and the server creates a global metric (org admins only). + The decorator context is per thread and task, so pass ``project_ids`` explicitly when creating from + worker threads. + is_global: Create a global metric (org admins only). Cannot be combined with ``project_ids``. + ``False`` requires ``project_ids`` or a current project. description: Description of the metric. tags: Tags associated with the metric. version: Specific version to reference (for existing metrics). @@ -797,6 +829,8 @@ def __init__( self.output_type = output_type or OutputTypeEnum.BOOLEAN self.ground_truth = ground_truth + self._create_project_ids = project_ids + self._create_is_global = is_global self.scorer_type = ScorerTypes.LLM @@ -810,7 +844,10 @@ def create(self) -> LlmEvaluator: Raises ------ - ValidationError: If configuration is invalid. + ValidationError: If configuration is invalid, or the scope is invalid (``is_global=True`` with + ``project_ids``, an empty ``project_ids``, or ``is_global=False`` with no project). Scope errors + are ``ValidationError``, not ``ValueError``. + NotFoundError: If the current project is set but cannot be found. Exception: If the API call fails. Examples @@ -839,6 +876,8 @@ def create(self) -> LlmEvaluator: if isinstance(self.output_type, OutputTypeEnum) else OutputTypeEnum.BOOLEAN, ground_truth=self.ground_truth, + project_ids=self._create_project_ids, + is_global=self._create_is_global, ) # Update attributes from response without triggering dirty-tracking @@ -911,6 +950,8 @@ def __init__( answer_options: DecisionAnswerOptions | None = None, input_type: InputTypeEnum | None = None, ground_truth: bool = False, + project_ids: list[str] | None = None, + is_global: bool | None = None, description: str = "", tags: list[str] | None = None, version: int | None = None, @@ -929,6 +970,13 @@ def __init__( ``DecisionLevel`` / ``{level, label, description}`` dicts for discrete. input_type: What the model sees. None lets the server choose a default for the node type. ground_truth: Whether the scorer requires ground truth (``reference_output``) from the dataset. + project_ids: Projects to scope the evaluator to when it is created. Defaults to the current project + (decorator context, SPLUNK_AO_PROJECT_ID, or SPLUNK_AO_PROJECT), which must already exist. If no + current project is set, no scope is sent and the server creates a global evaluator (org admins only). + The decorator context is per thread and task, so pass ``project_ids`` explicitly when creating from + worker threads. + is_global: Create a global evaluator (org admins only). Cannot be combined with ``project_ids``. + ``False`` requires ``project_ids`` or a current project. description: Description of the evaluator. tags: Tags associated with the evaluator. version: Specific version to reference (for existing evaluators). @@ -955,6 +1003,8 @@ def __init__( self.answer_options = normalized_options self.input_type = input_type self.ground_truth = ground_truth + self._create_project_ids = project_ids + self._create_is_global = is_global self.scorer_type = ScorerTypes.DECISION def create(self) -> DecisionEvaluator: @@ -967,6 +1017,10 @@ def create(self) -> DecisionEvaluator: Raises ------ + ValidationError: If the scope is invalid (``is_global=True`` with ``project_ids``, an empty + ``project_ids``, or ``is_global=False`` with no project). Scope errors are ``ValidationError``, + not ``ValueError``. + NotFoundError: If the current project is set but cannot be found. ValueError: If an argument is invalid or the API answers with a validation error. SplunkAOAPIError: For HTTP errors. A scorer created before a clear rejection is deleted. DecisionVersionOutcomeUnknownError: When the outcome of the version request is unknown (timeout, @@ -991,6 +1045,8 @@ def create(self) -> DecisionEvaluator: description=self.description, tags=self.tags, ground_truth=self.ground_truth, + project_ids=self._create_project_ids, + is_global=self._create_is_global, ) self._sync_attrs( @@ -1004,6 +1060,8 @@ def create(self) -> DecisionEvaluator: logger.info(f"DecisionEvaluator.create: id='{self.id}' - completed") return self + except ValidationError: + raise except Exception as e: if isinstance(e, DecisionVersionOutcomeUnknownError): self._sync_attrs(id=e.scorer_id) @@ -1117,6 +1175,8 @@ def __init__( node_level: StepType | None = None, output_type: str | OutputTypeEnum | None = None, required_metrics: list[str] | None = None, + project_ids: list[str] | None = None, + is_global: bool | None = None, description: str = "", tags: list[str] | None = None, version: int | None = None, @@ -1131,6 +1191,13 @@ def __init__( output_type: Output type for the metric ("percentage", "boolean", "categorical", "count", "discrete"). Accepts string or OutputTypeEnum. required_metrics: List of metric names that this code metric depends on. + project_ids: Projects to scope the metric to when it is created. Defaults to the current project + (decorator context, SPLUNK_AO_PROJECT_ID, or SPLUNK_AO_PROJECT), which must already exist. If no + current project is set, no scope is sent and the server creates a global metric (org admins only). + The decorator context is per thread and task, so pass ``project_ids`` explicitly when creating from + worker threads. + is_global: Create a global metric (org admins only). Cannot be combined with ``project_ids``. + ``False`` requires ``project_ids`` or a current project. description: Description of the metric. tags: Tags associated with the metric. version: Specific version to reference (for existing metrics). @@ -1140,6 +1207,8 @@ def __init__( self.code = code self.node_level = node_level or StepType.llm self.required_metrics = required_metrics + self._create_project_ids = project_ids + self._create_is_global = is_global self.scorer_type = ScorerTypes.CODE self.output_type = Evaluator._parse_output_type(output_type) @@ -1289,7 +1358,10 @@ def create(self) -> CodeEvaluator: Raises ------ - ValidationError: If code is not set, validation fails, or configuration is invalid. + ValidationError: If code is not set, validation fails, or configuration is invalid, or the scope is + invalid (``is_global=True`` with ``project_ids``, an empty ``project_ids``, or ``is_global=False`` + with no project). Scope errors are ``ValidationError``, not ``ValueError``. + NotFoundError: If the current project is set but cannot be found. Exception: If the API call fails. Examples @@ -1323,6 +1395,9 @@ def create(self) -> CodeEvaluator: # Ensure node_level is set (should always be set in __init__, but checking for type safety) assert self.node_level is not None + # Resolve scope before code validation or any write, so a bad scope fails fast + scope_is_global, scope_project_ids = _resolve_create_scope(self._create_project_ids, self._create_is_global) + # Step 1: Validate the code and get validation result logger.debug(f"CodeEvaluator.create: name='{self.name}' - validating code") validation_result = self._validate_code(config) @@ -1337,6 +1412,8 @@ def create(self) -> CodeEvaluator: scoreable_node_types=[self.node_level.value], output_type=self.output_type, required_scorers=self.required_metrics, + is_global=scope_is_global, + project_ids=scope_project_ids, ) scorer_response = create_scorers_post.sync(client=config.api_client, body=scorer_request) diff --git a/src/splunk_ao/evaluators.py b/src/splunk_ao/evaluators.py index dbaac659..512da5eb 100644 --- a/src/splunk_ao/evaluators.py +++ b/src/splunk_ao/evaluators.py @@ -30,6 +30,8 @@ from splunk_ao.resources.types import UNSET, Unset from splunk_ao.scorers import Scorers from splunk_ao.search import FilterType +from splunk_ao.shared.exceptions import NotFoundError, ValidationError, _resolve_project_identifiers +from splunk_ao.shared.project_resolver import _resolve_project _logger = logging.getLogger(__name__) @@ -180,6 +182,76 @@ def _build_decision_version_request( return CreateDecisionScorerVersionRequest(model_name=model_name, question=question, answer_options=answer_options) +def _resolve_create_scope( + project_ids: list[str] | None, is_global: bool | None +) -> tuple[bool | Unset, list[str] | Unset]: + """ + Resolve the ``(is_global, project_ids)`` pair to send when creating an evaluator. + + Explicit arguments win. Otherwise the scope defaults to the current project, taken from + the decorator context, then ``SPLUNK_AO_PROJECT_ID``, then ``SPLUNK_AO_PROJECT``. When + none of those is set and nothing was passed, no scope is sent and the server default + (global, org admins only) applies. + + Parameters + ---------- + project_ids + Projects to scope the evaluator to. + is_global + Whether the evaluator is global. Cannot be combined with ``project_ids``. + + Returns + ------- + tuple[bool | Unset, list[str] | Unset] + The values for ``CreateScorerRequest.is_global`` and ``CreateScorerRequest.project_ids``. + + Raises + ------ + ValidationError + If the arguments conflict, ``project_ids`` is empty, or ``is_global=False`` is passed + with no project to scope to. + NotFoundError + If a current project is set but cannot be found. + """ + if is_global is True: + if project_ids: + raise ValidationError("An evaluator cannot be both global and scoped: pass is_global=True or project_ids.") + return True, UNSET + + if project_ids is not None: + # Typed as object so the str guard is not flagged unreachable for a list[str] parameter. + requested: object = project_ids + if isinstance(requested, str): + raise ValidationError("project_ids must be a list of project ids, not a single string.") + if not project_ids: + raise ValidationError("project_ids must not be empty. Pass is_global=True for a global evaluator.") + return False, list(dict.fromkeys(project_ids)) + + # Imported here because decorator.py imports modules that depend on this one. + from splunk_ao.decorator import splunk_ao_context + + context_project = (splunk_ao_context.get_current_project() or "").strip() or None + project_id: str | None + project_name: str | None + if context_project: + project_id, project_name = None, context_project + else: + project_id, project_name = _resolve_project_identifiers(None, None) + + if project_id or project_name: + try: + return False, [_resolve_project(project_id, project_name).id] + except NotFoundError as e: + raise type(e)( + "Evaluator scope defaults to the current project (decorator context, SPLUNK_AO_PROJECT_ID or " + f"SPLUNK_AO_PROJECT), which was not found. Pass project_ids=[...] or is_global=True. {e}" + ) from e + + if is_global is False: + raise ValidationError("is_global=False needs project_ids or a current project.") + return UNSET, UNSET + + class Evaluators: config: SplunkAOConfig @@ -211,6 +283,8 @@ def create_custom_llm_evaluator( tags: list[str] | None = None, output_type: OutputTypeEnum = OutputTypeEnum.BOOLEAN, ground_truth: bool = False, + project_ids: list[str] | None = None, + is_global: bool | None = None, ) -> BaseScorerVersionResponse: """ Create a custom LLM metric. @@ -238,14 +312,29 @@ def create_custom_llm_evaluator( ground_truth: bool Whether the scorer requires ground truth (``reference_output``) from the dataset. When True, the judge LLM receives the row's ground-truth value in its prompt. + project_ids: List[str] + Projects to scope the metric to. Defaults to the current project (decorator context, + ``SPLUNK_AO_PROJECT_ID``, or ``SPLUNK_AO_PROJECT``) when one is set. + is_global: bool + Create a global metric, visible to every project. Requires the org admin role and + cannot be combined with ``project_ids``. Returns ------- BaseScorerVersionResponse Response containing the created metric details. + + Raises + ------ + ValidationError + If the scope arguments conflict, ``project_ids`` is empty, or ``is_global=False`` is + passed with no project to scope to. + NotFoundError + If the current project is set but cannot be found. """ if tags is None: tags = [] + scope_is_global, scope_project_ids = _resolve_create_scope(project_ids, is_global) create_scorer_request = CreateScorerRequest( name=name, scorer_type=ScorerTypes.LLM, @@ -255,6 +344,8 @@ def create_custom_llm_evaluator( scoreable_node_types=[node_level], output_type=output_type, ground_truth=ground_truth, + is_global=scope_is_global, + project_ids=scope_project_ids, ) scorer = create_scorers_post.sync(body=create_scorer_request, client=self.config.api_client) @@ -280,6 +371,8 @@ def create_decision_evaluator( description: str = "", tags: list[str] | None = None, ground_truth: bool = False, + project_ids: list[str] | None = None, + is_global: bool | None = None, ) -> BaseScorerVersionResponse: """ Create a decision evaluator. @@ -310,6 +403,12 @@ def create_decision_evaluator( Tags associated with the evaluator. ground_truth: bool Whether the scorer requires ground truth (``reference_output``) from the dataset. + project_ids: List[str] + Projects to scope the evaluator to. Defaults to the current project (decorator context, + ``SPLUNK_AO_PROJECT_ID``, or ``SPLUNK_AO_PROJECT``) when one is set. + is_global: bool + Create a global evaluator, visible to every project. Requires the org admin role and + cannot be combined with ``project_ids``. Returns ------- @@ -318,6 +417,11 @@ def create_decision_evaluator( Raises ------ + ValidationError + If the scope arguments conflict, ``project_ids`` is empty, or ``is_global=False`` is + passed with no project to scope to. + NotFoundError + If the current project is set but cannot be found. ValueError If an argument is invalid or the API answers with a validation error. If the scorer was created but its version was rejected, the scorer is deleted before the error is raised. @@ -334,6 +438,7 @@ def create_decision_evaluator( version_req = _build_decision_version_request( model_name=model_name, question=question, answer_options=normalized_options ) + scope_is_global, scope_project_ids = _resolve_create_scope(project_ids, is_global) create_scorer_request = CreateScorerRequest( name=name, scorer_type=ScorerTypes.DECISION, @@ -343,6 +448,8 @@ def create_decision_evaluator( output_type=output_type, input_type=input_type if input_type is not None else UNSET, ground_truth=ground_truth, + is_global=scope_is_global, + project_ids=scope_project_ids, ) scorer = create_scorers_post.sync(body=create_scorer_request, client=self.config.api_client) @@ -446,6 +553,8 @@ def create_custom_llm_evaluator( tags: list[str] | None = None, output_type: OutputTypeEnum = OutputTypeEnum.BOOLEAN, ground_truth: bool = False, + project_ids: list[str] | None = None, + is_global: bool | None = None, ) -> BaseScorerVersionResponse: """ Create a custom LLM metric. @@ -473,16 +582,41 @@ def create_custom_llm_evaluator( ground_truth: bool Whether the scorer requires ground truth (``reference_output``) from the dataset. When True, the judge LLM receives the row's ground-truth value in its prompt. + project_ids: List[str] + Projects to scope the metric to. Defaults to the current project (decorator context, + ``SPLUNK_AO_PROJECT_ID``, or ``SPLUNK_AO_PROJECT``) when one is set. + is_global: bool + Create a global metric, visible to every project. Requires the org admin role and + cannot be combined with ``project_ids``. Returns ------- BaseScorerVersionResponse Response containing the created metric details. + + Raises + ------ + ValidationError + If the scope arguments conflict, ``project_ids`` is empty, or ``is_global=False`` is + passed with no project to scope to. + NotFoundError + If the current project is set but cannot be found. """ if tags is None: tags = [] return Evaluators().create_custom_llm_evaluator( - name, user_prompt, node_level, cot_enabled, model_name, num_judges, description, tags, output_type, ground_truth + name, + user_prompt, + node_level, + cot_enabled, + model_name, + num_judges, + description, + tags, + output_type, + ground_truth, + project_ids=project_ids, + is_global=is_global, ) @@ -497,6 +631,8 @@ def create_decision_evaluator( description: str = "", tags: list[str] | None = None, ground_truth: bool = False, + project_ids: list[str] | None = None, + is_global: bool | None = None, ) -> BaseScorerVersionResponse: """ Create a decision evaluator. @@ -530,6 +666,12 @@ def create_decision_evaluator( Tags associated with the evaluator. ground_truth: bool Whether the scorer requires ground truth (``reference_output``) from the dataset. + project_ids: List[str] + Projects to scope the evaluator to. Defaults to the current project (decorator context, + ``SPLUNK_AO_PROJECT_ID``, or ``SPLUNK_AO_PROJECT``) when one is set. + is_global: bool + Create a global evaluator, visible to every project. Requires the org admin role and + cannot be combined with ``project_ids``. Returns ------- @@ -538,6 +680,11 @@ def create_decision_evaluator( Raises ------ + ValidationError + If the scope arguments conflict, ``project_ids`` is empty, or ``is_global=False`` is + passed with no project to scope to. + NotFoundError + If the current project is set but cannot be found. ValueError If an argument is invalid or the API answers with a validation error. If the scorer was created but its version was rejected, the scorer is deleted before the error is raised. @@ -571,6 +718,8 @@ def create_decision_evaluator( description=description, tags=tags, ground_truth=ground_truth, + project_ids=project_ids, + is_global=is_global, ) diff --git a/tests/conftest.py b/tests/conftest.py index 3f046681..84906cae 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -62,7 +62,7 @@ from splunk_ao.collaborator import CollaboratorRole from splunk_ao.config import SplunkAOConfig from splunk_ao.configuration import _CONFIGURATION_KEYS, Configuration -from splunk_ao.decorator import _mode_context, _session_id_context +from splunk_ao.decorator import _mode_context, _project_context, _session_id_context from splunk_ao.resources.models import DatasetContent, DatasetRow, DatasetRowValuesDict from splunk_ao.resources.models.messages_list_item import MessagesListItem from splunk_ao.utils.singleton import SplunkAOLoggerSingleton @@ -602,3 +602,13 @@ def mock_collaborator() -> MagicMock: mock_collab.last_name = "Collaborator" mock_collab.permissions = [] return mock_collab + + +@pytest.fixture +def no_current_project(monkeypatch: pytest.MonkeyPatch) -> Generator[None, None, None]: + """Clear the current project from the environment and the decorator context for one test.""" + monkeypatch.delenv("SPLUNK_AO_PROJECT", raising=False) + monkeypatch.delenv("SPLUNK_AO_PROJECT_ID", raising=False) + token = _project_context.set(None) + yield + _project_context.reset(token) diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py index b6bf6855..49301811 100644 --- a/tests/test_evaluator.py +++ b/tests/test_evaluator.py @@ -8,15 +8,41 @@ from galileo_core.schemas.logging.step import StepType from splunk_ao.evaluators import DecisionVersionOutcomeUnknownError from splunk_ao.evaluator import CodeEvaluator, DecisionEvaluator, LlmEvaluator, LocalEvaluator, Evaluator -from splunk_ao.resources.models import HTTPValidationError, OutputTypeEnum, ScorerResponse, ScorerTypes +from splunk_ao.resources.models import HTTPValidationError, OutputTypeEnum, ScorerResponse, ScorerScopeProjectRef, ScorerTypes from splunk_ao.resources.models.invalid_result import InvalidResult from splunk_ao.resources.models.task_result_status import TaskResultStatus +from splunk_ao.resources.types import UNSET from splunk_ao.shared.base import SyncState from splunk_ao.shared.exceptions import APIError, ValidationError # Test fixtures and helper functions +pytestmark = pytest.mark.usefixtures("no_current_project") + + +def _scope_scorer(scorer_id: str, *, scorer_type: ScorerTypes = ScorerTypes.LLM, is_global=False, scope_projects=UNSET): + """Build a scorer response carrying scope fields.""" + scorer = MagicMock() + scorer.id = scorer_id + scorer.name = "Scoped Evaluator" + scorer.scorer_type = scorer_type + scorer.tags = [] + scorer.description = "" + scorer.created_at = MagicMock() + scorer.updated_at = MagicMock() + scorer.output_type = OutputTypeEnum.BOOLEAN + scorer.user_prompt = "Is it accurate?" + scorer.defaults = MagicMock() + scorer.defaults.model_name = "gpt-4.1-mini" + scorer.defaults.num_judges = 3 + scorer.defaults.cot_enabled = True + scorer.scoreable_node_types = ["llm"] + scorer.is_global = is_global + scorer.scope_projects = scope_projects + return scorer + + @pytest.fixture def mock_api_client(): """Create a mock API client.""" @@ -2028,6 +2054,207 @@ def test_configuration_from_env_vars(self, reset_configuration: None, monkeypatc assert Configuration.code_validation_backoff_multiplier == 1.8 +class TestEvaluatorScope: + """Scope on evaluator construction, creation and population from the API.""" + + def test_fresh_evaluator_has_no_scope(self, reset_configuration: None) -> None: + # Given a local evaluator that has never been synced + evaluator = LocalEvaluator(name="local", scorer_fn=lambda x: 1.0) + + # Then it reports no server scope + assert evaluator.is_global is None + assert evaluator.scope_projects == [] + + @pytest.mark.parametrize( + ("is_global", "scope_projects", "expected_global", "expected_projects"), + [ + (False, [ScorerScopeProjectRef(id="p1", name="One")], False, [ScorerScopeProjectRef(id="p1", name="One")]), + (True, [], True, []), + (UNSET, UNSET, None, []), + ], + ) + @patch("splunk_ao.evaluator.Scorers") + def test_get_list_and_refresh_populate_scope( + self, mock_scorers_class, is_global, scope_projects, expected_global, expected_projects, reset_configuration + ) -> None: + # Given a scorer response with scope fields + scorer_id = str(uuid4()) + scorer = _scope_scorer(scorer_id, is_global=is_global, scope_projects=scope_projects) + mock_scorers_class.return_value.list.return_value = [scorer] + + # When read through get(), list() and refresh() + from_get = Evaluator.get(name="Scoped Evaluator") + from_list = Evaluator.list()[0] + refreshed = LlmEvaluator(name="Scoped Evaluator", prompt="x") + refreshed.id = scorer_id + refreshed.refresh() + + # Then each reflects the response + for evaluator in (from_get, from_list, refreshed): + assert evaluator.is_global is expected_global + assert evaluator.scope_projects == expected_projects + + @pytest.mark.parametrize( + ("is_global", "scope_projects"), [(False, [ScorerScopeProjectRef(id="a", name="A")]), (True, [])] + ) + @patch("splunk_ao.shared.project_resolver.Projects") + @patch("splunk_ao.evaluators.create_llm_scorer_version_scorers_scorer_id_version_llm_post") + @patch("splunk_ao.evaluators.create_scorers_post") + @patch("splunk_ao.evaluators.SplunkAOConfig.get") + @patch("splunk_ao.evaluator.Scorers") + def test_create_on_fetched_evaluator_ignores_the_fetched_scope( + self, + mock_scorers_class, + mock_config, + mock_create_scorers, + mock_create_version, + mock_projects_class, + is_global, + scope_projects, + monkeypatch, + reset_configuration, + ) -> None: + # Given an evaluator fetched with a server scope, and a different current project in the environment + scorer_id = str(uuid4()) + mock_scorers_class.return_value.list.return_value = [ + _scope_scorer(scorer_id, is_global=is_global, scope_projects=scope_projects) + ] + fetched = Evaluator.get(name="Scoped Evaluator") + assert fetched.is_global is is_global + monkeypatch.setenv("SPLUNK_AO_PROJECT_ID", "env-id") + mock_projects_class.return_value.get_with_env_fallbacks.return_value = MagicMock(id="env-project-id") + mock_create_scorers.sync.return_value = MagicMock(id=scorer_id) + mock_create_version.sync.return_value = MagicMock(scorer_id=scorer_id) + + # When create() is called on it + fetched.create() + + # Then the scope comes from the current project, not from the fetched server values + body = mock_create_scorers.sync.call_args.kwargs["body"] + assert body.is_global is False + assert body.project_ids == ["env-project-id"] + + @patch("splunk_ao.evaluator.Evaluators") + @patch("splunk_ao.evaluator.Scorers") + def test_llm_evaluator_forwards_scope_and_populates_after_create( + self, mock_scorers_class, mock_evaluators_class, reset_configuration: None + ) -> None: + # Given an LLM evaluator built with an explicit scope + scorer_id = str(uuid4()) + mock_version = MagicMock() + mock_version.scorer_id = scorer_id + mock_evaluators_class.return_value.create_custom_llm_evaluator.return_value = mock_version + projects = [ScorerScopeProjectRef(id="a", name="A"), ScorerScopeProjectRef(id="b", name="B")] + mock_scorers_class.return_value.list.return_value = [ + _scope_scorer(scorer_id, is_global=False, scope_projects=projects) + ] + + # When created + evaluator = LlmEvaluator(name="Scoped Evaluator", prompt="Is it accurate?", project_ids=["a", "b"]).create() + + # Then the scope reaches the service and the server scope is surfaced + kwargs = mock_evaluators_class.return_value.create_custom_llm_evaluator.call_args.kwargs + assert kwargs["project_ids"] == ["a", "b"] + assert kwargs["is_global"] is None + assert evaluator.is_global is False + assert evaluator.scope_projects == projects + + @patch("splunk_ao.evaluator.Evaluators") + @patch("splunk_ao.evaluator.Scorers") + def test_llm_evaluator_forwards_is_global( + self, mock_scorers_class, mock_evaluators_class, reset_configuration: None + ) -> None: + # Given an LLM evaluator built with is_global=True + scorer_id = str(uuid4()) + mock_evaluators_class.return_value.create_custom_llm_evaluator.return_value = MagicMock(scorer_id=scorer_id) + mock_scorers_class.return_value.list.return_value = [_scope_scorer(scorer_id, is_global=True)] + + # When created + evaluator = LlmEvaluator(name="Scoped Evaluator", prompt="x", is_global=True).create() + + # Then the request is global and the server value is surfaced + kwargs = mock_evaluators_class.return_value.create_custom_llm_evaluator.call_args.kwargs + assert kwargs["is_global"] is True + assert kwargs["project_ids"] is None + assert evaluator.is_global is True + + @patch("splunk_ao.evaluator.SplunkAOConfig.get") + @patch("splunk_ao.evaluator.get_validate_code_scorer_task_result_scorers_code_validate_task_id_get") + @patch("splunk_ao.evaluator.validate_code_scorer_scorers_code_validate_post") + @patch("splunk_ao.evaluator.create_code_scorer_version_scorers_scorer_id_version_code_post") + @patch("splunk_ao.evaluator.create_scorers_post") + @patch("splunk_ao.evaluator.Scorers") + def test_code_evaluator_sends_scope_and_populates_after_create( + self, + mock_scorers_class, + mock_create_scorers, + mock_create_version, + mock_validate_post, + mock_validate_get, + mock_config, + reset_configuration, + mock_validation_response, + mock_validation_task_result, + mock_version_response, + mock_scorer_response, + ) -> None: + # Given a code evaluator with explicit scope and a working API + scorer_id = str(uuid4()) + mock_validate_post.sync.return_value = mock_validation_response() + mock_validate_get.sync.return_value = mock_validation_task_result(TaskResultStatus.COMPLETED) + mock_create_scorers.sync.return_value = mock_scorer_response(scorer_id, "Scoped Code") + mock_create_version.sync.return_value = mock_version_response(scorer_id) + projects = [ScorerScopeProjectRef(id="a", name="A")] + mock_scorers_class.return_value.list.return_value = [ + _scope_scorer(scorer_id, scorer_type=ScorerTypes.CODE, scope_projects=projects) + ] + + # When created + evaluator = CodeEvaluator(name="Scoped Code", code="def score(t): return 1.0", project_ids=["a", "a"]).create() + + # Then the request carries the deduplicated scope and the server scope is surfaced + body = mock_create_scorers.sync.call_args.kwargs["body"] + assert body.is_global is False + assert body.project_ids == ["a"] + assert evaluator.scope_projects == projects + + @patch("splunk_ao.evaluator.SplunkAOConfig.get") + @patch("splunk_ao.evaluator.create_scorers_post") + def test_code_evaluator_global_sends_no_project_ids( + self, mock_create_scorers, mock_config, reset_configuration + ) -> None: + # Given a global code evaluator and a create endpoint that returns nothing; we only inspect the request + mock_create_scorers.sync.return_value = None + + # When created, the missing response raises after the request is built + with patch.object(CodeEvaluator, "_validate_code", return_value="{}"): + with pytest.raises(ValueError, match="No response"): + CodeEvaluator(name="G", code="x", is_global=True).create() + + # Then the request was global with no project ids + body = mock_create_scorers.sync.call_args.kwargs["body"] + assert body.is_global is True + assert "project_ids" not in body.to_dict() + + @pytest.mark.parametrize( + "kwargs", [{"is_global": True, "project_ids": ["a"]}, {"project_ids": []}, {"is_global": False}] + ) + @patch("splunk_ao.evaluator.SplunkAOConfig.get") + @patch("splunk_ao.evaluator.validate_code_scorer_scorers_code_validate_post") + @patch("splunk_ao.evaluator.create_scorers_post") + def test_code_evaluator_resolves_scope_before_validating_code( + self, mock_create_scorers, mock_validate_post, mock_config, kwargs, reset_configuration + ) -> None: + # Given a code evaluator with an invalid scope + # When created + with pytest.raises(ValidationError): + CodeEvaluator(name="Bad", code="x", **kwargs).create() + + # Then neither validation nor creation was attempted + mock_validate_post.sync.assert_not_called() + mock_create_scorers.sync.assert_not_called() + + def _decision_scorer(output_type: str, answer_options, *, with_version: bool = True) -> ScorerResponse: """Decode a scorer the way the API client does, so answer_options arrive in their wire shape.""" row: dict = { @@ -2267,3 +2494,94 @@ def test_model_is_kept_when_the_response_has_no_decision_version( # Then: the model survives assert evaluator.model == "jev-1.12.0" + + +class TestDecisionEvaluatorScope: + """Scope on DecisionEvaluator construction, creation and population from the API.""" + + @patch("splunk_ao.evaluator.Evaluators") + @patch("splunk_ao.evaluator.Scorers") + def test_forwards_scope_and_populates_after_create( + self, mock_scorers_class: MagicMock, mock_evaluators_class: MagicMock, reset_configuration: None + ) -> None: + # Given a decision evaluator built with an explicit scope, and a server that reports it + version = MagicMock() + version.scorer_id = "s1" + mock_evaluators_class.return_value.create_decision_evaluator.return_value = version + row = _decision_scorer("boolean_probability", None).to_dict() + row["is_global"] = False + row["scope_projects"] = [{"id": "a", "name": "A"}] + mock_scorers_class.return_value.list.return_value = [ScorerResponse.from_dict(row)] + + # When created + evaluator = DecisionEvaluator(name="tone", question="Which tone?", project_ids=["a"]).create() + + # Then the scope reaches the service and the server scope is surfaced + kwargs = mock_evaluators_class.return_value.create_decision_evaluator.call_args.kwargs + assert kwargs["project_ids"] == ["a"] + assert kwargs["is_global"] is None + assert evaluator.is_global is False + assert [(p.id, p.name) for p in evaluator.scope_projects] == [("a", "A")] + + @patch("splunk_ao.evaluator.Evaluators") + @patch("splunk_ao.evaluator.Scorers") + def test_forwards_is_global( + self, mock_scorers_class: MagicMock, mock_evaluators_class: MagicMock, reset_configuration: None + ) -> None: + # Given a decision evaluator built with is_global=True + version = MagicMock() + version.scorer_id = "s1" + mock_evaluators_class.return_value.create_decision_evaluator.return_value = version + mock_scorers_class.return_value.list.return_value = [_decision_scorer("boolean_probability", None)] + + # When created + DecisionEvaluator(name="tone", question="Which tone?", is_global=True).create() + + # Then the request is global + kwargs = mock_evaluators_class.return_value.create_decision_evaluator.call_args.kwargs + assert kwargs["is_global"] is True + assert kwargs["project_ids"] is None + + @pytest.mark.parametrize( + "kwargs", [{"is_global": True, "project_ids": ["a"]}, {"project_ids": []}, {"is_global": False}] + ) + @patch("splunk_ao.evaluators.SplunkAOConfig.get") + @patch("splunk_ao.evaluators.create_scorers_post") + def test_invalid_scope_makes_no_create_request( + self, mock_create_scorers, mock_config, kwargs, reset_configuration + ) -> None: + with pytest.raises(ValidationError): + DecisionEvaluator(name="tone", question="Which tone?", **kwargs).create() + + mock_create_scorers.sync.assert_not_called() + + @patch("splunk_ao.evaluators.SplunkAOConfig.get") + def test_invalid_scope_does_not_mark_evaluator_failed(self, mock_config, reset_configuration) -> None: + # Given a decision evaluator with a conflicting scope + evaluator = DecisionEvaluator(name="tone", question="Which tone?", is_global=True, project_ids=["a"]) + + # When created + with pytest.raises(ValidationError): + evaluator.create() + + # Then its state is untouched, as for LlmEvaluator and CodeEvaluator + assert evaluator.sync_state == SyncState.LOCAL_ONLY + + @patch("splunk_ao.evaluator.Scorers") + def test_get_populates_scope_for_fetched_decision_evaluator( + self, mock_scorers_class: MagicMock, reset_configuration: None + ) -> None: + # Given a decision scorer scoped to a project + row = _decision_scorer("boolean_probability", None).to_dict() + row["is_global"] = False + row["scope_projects"] = [{"id": "a", "name": "A"}] + mock_scorers_class.return_value.list.return_value = [ScorerResponse.from_dict(row)] + + # When fetched + evaluator = Evaluator.get(name="tone") + + # Then the base-class population fills the scope + assert isinstance(evaluator, DecisionEvaluator) + assert evaluator.is_global is False + assert [(p.id, p.name) for p in evaluator.scope_projects] == [("a", "A")] + assert evaluator._create_project_ids is None diff --git a/tests/test_evaluators.py b/tests/test_evaluators.py index 663eeee6..d7cb067f 100644 --- a/tests/test_evaluators.py +++ b/tests/test_evaluators.py @@ -6,9 +6,11 @@ import pytest from galileo_core.schemas.logging.step import StepType +from splunk_ao.decorator import _project_context from splunk_ao.evaluators import ( DecisionVersionOutcomeUnknownError, Evaluators, + _resolve_create_scope, create_custom_llm_evaluator, create_decision_evaluator, delete_evaluator, @@ -41,10 +43,25 @@ from splunk_ao.resources.models.scorer_defaults import ScorerDefaults from splunk_ao.resources.models.scorer_response import ScorerResponse from splunk_ao.resources.models.scorer_types import ScorerTypes +from splunk_ao.resources.types import UNSET +from splunk_ao.shared.exceptions import ValidationError as SplunkAOValidationError FIXED_PROJECT_ID = str(uuid4()) +pytestmark = pytest.mark.usefixtures("no_current_project") + + +@pytest.fixture +def mock_projects(): + """Patch the project lookup used to resolve the current project; the found project has id ``project-id-1``.""" + with patch("splunk_ao.shared.project_resolver.Projects") as projects_class: + project = MagicMock() + project.id = "project-id-1" + projects_class.return_value.get_with_env_fallbacks.return_value = project + yield projects_class.return_value + + @pytest.fixture def mock_scorer_response(): """Mock scorer response for testing.""" @@ -318,6 +335,8 @@ def test_create_custom_llm_metric_function(self, mock_metrics_class) -> None: ["test"], OutputTypeEnum.CATEGORICAL, True, + project_ids=None, + is_global=None, ) # Verify the result is returned @@ -347,6 +366,8 @@ def test_create_custom_llm_metric_function_default_parameters(self, mock_metrics [], # default OutputTypeEnum.BOOLEAN, # default False, # default ground_truth + project_ids=None, + is_global=None, ) # Verify the result is returned @@ -522,6 +543,289 @@ def test_passes_all_parameters_correctly(self, mock_api_call): assert body.interval == interval +class TestResolveCreateScope: + """Scope resolution for evaluator creation.""" + + def test_project_id_env_defaults_to_that_project(self, monkeypatch, mock_projects) -> None: + # Given only SPLUNK_AO_PROJECT_ID is set + monkeypatch.setenv("SPLUNK_AO_PROJECT_ID", "env-id") + + # When resolving with no explicit scope + result = _resolve_create_scope(None, None) + + # Then it is scoped to the project looked up by id + assert result == (False, ["project-id-1"]) + mock_projects.get_with_env_fallbacks.assert_called_once_with(id="env-id", name=None) + + def test_project_name_env_is_resolved_to_id(self, monkeypatch, mock_projects) -> None: + # Given only SPLUNK_AO_PROJECT is set + monkeypatch.setenv("SPLUNK_AO_PROJECT", "my-project") + + # When resolving with no explicit scope + result = _resolve_create_scope(None, None) + + # Then the name is looked up and its id is used + assert result == (False, ["project-id-1"]) + mock_projects.get_with_env_fallbacks.assert_called_once_with(id=None, name="my-project") + + def test_context_project_wins_over_env(self, monkeypatch, mock_projects) -> None: + # Given a context project and both env vars + monkeypatch.setenv("SPLUNK_AO_PROJECT_ID", "env-id") + monkeypatch.setenv("SPLUNK_AO_PROJECT", "env-name") + _project_context.set("context-project") + + # When resolving with no explicit scope + result = _resolve_create_scope(None, None) + + # Then the context project name is the one looked up + assert result == (False, ["project-id-1"]) + mock_projects.get_with_env_fallbacks.assert_called_once_with(id=None, name="context-project") + + def test_project_id_env_wins_over_project_name_env(self, monkeypatch, mock_projects) -> None: + # Given both env vars + monkeypatch.setenv("SPLUNK_AO_PROJECT_ID", "env-id") + monkeypatch.setenv("SPLUNK_AO_PROJECT", "env-name") + + # When resolving with no explicit scope + _resolve_create_scope(None, None) + + # Then the id is the one looked up + mock_projects.get_with_env_fallbacks.assert_called_once_with(id="env-id", name=None) + + def test_explicit_project_ids_override_context_and_env(self, monkeypatch, mock_projects) -> None: + # Given a context project and env vars + monkeypatch.setenv("SPLUNK_AO_PROJECT_ID", "env-id") + _project_context.set("context-project") + + # When explicit project ids are passed, with a duplicate + result = _resolve_create_scope(["a", "b", "a"], None) + + # Then they are used as-is (deduplicated, order kept) and no lookup happens + assert result == (False, ["a", "b"]) + mock_projects.get_with_env_fallbacks.assert_not_called() + + def test_explicit_project_ids_with_is_global_false(self, mock_projects) -> None: + # Given explicit ids and is_global=False + # When resolving + result = _resolve_create_scope(["a"], False) + + # Then the ids are used + assert result == (False, ["a"]) + + def test_empty_project_ids_with_is_global_true_is_global(self, mock_projects) -> None: + # Given is_global=True and an empty list, which does not conflict + # When resolving + result = _resolve_create_scope([], True) + + # Then the evaluator is global + assert result == (True, UNSET) + + def test_is_global_true_sends_no_project_ids_even_with_current_project(self, monkeypatch, mock_projects) -> None: + # Given a current project + monkeypatch.setenv("SPLUNK_AO_PROJECT_ID", "env-id") + + # When is_global=True is passed + result = _resolve_create_scope(None, True) + + # Then the evaluator is global and no lookup happens + assert result == (True, UNSET) + mock_projects.get_with_env_fallbacks.assert_not_called() + + def test_nothing_set_sends_no_scope(self, mock_projects) -> None: + # Given no arguments, context project or env vars + # When resolving + result = _resolve_create_scope(None, None) + + # Then no scope is sent and no lookup happens + assert result == (UNSET, UNSET) + mock_projects.get_with_env_fallbacks.assert_not_called() + + def test_whitespace_context_project_counts_as_unset(self, mock_projects) -> None: + # Given a whitespace-only context project and nothing else + _project_context.set(" ") + + # When resolving + result = _resolve_create_scope(None, None) + + # Then it is treated as no current project + assert result == (UNSET, UNSET) + + def test_whitespace_env_counts_as_unset(self, monkeypatch, mock_projects) -> None: + # Given a whitespace-only SPLUNK_AO_PROJECT + monkeypatch.setenv("SPLUNK_AO_PROJECT", " ") + + # When resolving + result = _resolve_create_scope(None, None) + + # Then it is treated as no current project + assert result == (UNSET, UNSET) + + def test_global_and_project_ids_conflict(self, mock_projects) -> None: + # Given is_global=True together with project ids + # Then resolving fails + with pytest.raises(SplunkAOValidationError, match="both global and scoped"): + _resolve_create_scope(["a"], True) + + def test_empty_project_ids_rejected(self, monkeypatch, mock_projects) -> None: + # Given a current project, which must not paper over an explicit empty list + monkeypatch.setenv("SPLUNK_AO_PROJECT_ID", "env-id") + + with pytest.raises(SplunkAOValidationError, match="must not be empty"): + _resolve_create_scope([], None) + + def test_string_project_ids_rejected(self, mock_projects) -> None: + with pytest.raises(SplunkAOValidationError, match="not a single string"): + _resolve_create_scope("3f2a", None) # type: ignore[arg-type] + + def test_is_global_false_without_any_project_rejected(self, mock_projects) -> None: + # Given is_global=False and no project anywhere + # Then resolving fails + with pytest.raises(SplunkAOValidationError, match="is_global=False needs project_ids"): + _resolve_create_scope(None, False) + + def test_is_global_false_uses_current_project(self, monkeypatch, mock_projects) -> None: + # Given a current project + monkeypatch.setenv("SPLUNK_AO_PROJECT_ID", "env-id") + + # When is_global=False is passed with no ids + result = _resolve_create_scope(None, False) + + # Then the current project is used + assert result == (False, ["project-id-1"]) + + def test_current_project_not_found_raises_not_found(self, monkeypatch, mock_projects) -> None: + # Given a current project that does not exist + monkeypatch.setenv("SPLUNK_AO_PROJECT", "missing") + mock_projects.get_with_env_fallbacks.return_value = None + + # Then there is no silent fallback to global, and the error says why a project was looked up + with pytest.raises(NotFoundError, match="missing") as exc_info: + _resolve_create_scope(None, None) + assert "defaults to the current project" in str(exc_info.value) + assert "is_global=True" in str(exc_info.value) + + +class TestCreateCustomLlmEvaluatorScope: + """The scope reaches the CreateScorerRequest on both public create paths.""" + + @staticmethod + def _create(use_function: bool, **kwargs): + if use_function: + return create_custom_llm_evaluator("m", "prompt", **kwargs) + return Evaluators().create_custom_llm_evaluator(name="m", user_prompt="prompt", **kwargs) + + @pytest.mark.parametrize("use_function", [False, True], ids=["method", "module_function"]) + @patch("splunk_ao.evaluators.create_llm_scorer_version_scorers_scorer_id_version_llm_post") + @patch("splunk_ao.evaluators.create_scorers_post") + def test_defaults_to_current_project( + self, mock_create_scorer, mock_create_version, use_function, monkeypatch, mock_projects, mock_scorer_response + ) -> None: + # Given a current project from the environment + monkeypatch.setenv("SPLUNK_AO_PROJECT_ID", "env-id") + mock_create_scorer.sync.return_value = mock_scorer_response + + # When creating with no explicit scope + self._create(use_function) + + # Then the request is scoped to that project + body = mock_create_scorer.sync.call_args.kwargs["body"] + assert body.is_global is False + assert body.project_ids == ["project-id-1"] + + @pytest.mark.parametrize("use_function", [False, True], ids=["method", "module_function"]) + @patch("splunk_ao.evaluators.create_llm_scorer_version_scorers_scorer_id_version_llm_post") + @patch("splunk_ao.evaluators.create_scorers_post") + def test_explicit_project_ids_override_current_project( + self, mock_create_scorer, mock_create_version, use_function, monkeypatch, mock_projects, mock_scorer_response + ) -> None: + # Given a current project from the environment + monkeypatch.setenv("SPLUNK_AO_PROJECT_ID", "env-id") + mock_create_scorer.sync.return_value = mock_scorer_response + + # When creating with explicit project ids + self._create(use_function, project_ids=["a", "b"]) + + # Then the explicit ids are sent + body = mock_create_scorer.sync.call_args.kwargs["body"] + assert body.is_global is False + assert body.project_ids == ["a", "b"] + + @pytest.mark.parametrize("use_function", [False, True], ids=["method", "module_function"]) + @patch("splunk_ao.evaluators.create_llm_scorer_version_scorers_scorer_id_version_llm_post") + @patch("splunk_ao.evaluators.create_scorers_post") + def test_explicit_global_overrides_current_project( + self, mock_create_scorer, mock_create_version, use_function, monkeypatch, mock_projects, mock_scorer_response + ) -> None: + # Given a current project from the environment + monkeypatch.setenv("SPLUNK_AO_PROJECT_ID", "env-id") + mock_create_scorer.sync.return_value = mock_scorer_response + + # When creating with is_global=True + self._create(use_function, is_global=True) + + # Then the request is global with no project ids + body = mock_create_scorer.sync.call_args.kwargs["body"] + assert body.is_global is True + assert "project_ids" not in body.to_dict() + + @pytest.mark.parametrize("use_function", [False, True], ids=["method", "module_function"]) + @patch("splunk_ao.evaluators.create_llm_scorer_version_scorers_scorer_id_version_llm_post") + @patch("splunk_ao.evaluators.create_scorers_post") + def test_no_current_project_sends_no_scope( + self, mock_create_scorer, mock_create_version, use_function, mock_projects, mock_scorer_response + ) -> None: + # Given no current project + mock_create_scorer.sync.return_value = mock_scorer_response + + # When creating with no scope + self._create(use_function) + + # Then no scope fields are sent + body_dict = mock_create_scorer.sync.call_args.kwargs["body"].to_dict() + assert "is_global" not in body_dict + assert "project_ids" not in body_dict + + @pytest.mark.parametrize("use_function", [False, True], ids=["method", "module_function"]) + @pytest.mark.parametrize( + "kwargs", [{"is_global": True, "project_ids": ["a"]}, {"project_ids": []}, {"is_global": False}] + ) + @patch("splunk_ao.evaluators.create_scorers_post") + def test_invalid_scope_fails_before_any_http_call( + self, mock_create_scorer, kwargs, use_function, mock_projects + ) -> None: + # Given an invalid scope + # When creating + with pytest.raises(SplunkAOValidationError): + self._create(use_function, **kwargs) + + # Then nothing is sent + mock_create_scorer.sync.assert_not_called() + + @pytest.mark.parametrize("use_function", [False, True], ids=["method", "module_function"]) + @patch("splunk_ao.evaluators.create_scorers_post") + def test_current_project_not_found_sends_no_create_request( + self, mock_create_scorer, use_function, monkeypatch, mock_projects + ) -> None: + monkeypatch.setenv("SPLUNK_AO_PROJECT", "missing") + mock_projects.get_with_env_fallbacks.return_value = None + + with pytest.raises(NotFoundError): + self._create(use_function) + + mock_create_scorer.sync.assert_not_called() + + @patch("splunk_ao.evaluators.Evaluators") + def test_module_function_keeps_positional_order_and_forwards_scope_by_keyword(self, mock_evaluators_class) -> None: + create_custom_llm_evaluator( + "m", "prompt", StepType.llm, True, "gpt-4o", 5, "desc", ["t"], OutputTypeEnum.PERCENTAGE, True, ["a"], False + ) + + mock_evaluators_class.return_value.create_custom_llm_evaluator.assert_called_once_with( + "m", "prompt", StepType.llm, True, "gpt-4o", 5, "desc", ["t"], OutputTypeEnum.PERCENTAGE, True, + project_ids=["a"], is_global=False, + ) # fmt: skip + + @pytest.fixture def decision_scorer_response(): return ScorerResponse( @@ -860,3 +1164,100 @@ def test_server_limits_are_checked_before_any_request( with pytest.raises(ValueError): create_decision_evaluator(name="q", question=question, answer_options=answer_options) mock_create_scorer.sync.assert_not_called() + + +@patch("splunk_ao.evaluators.create_decision_scorer_version_scorers_scorer_id_version_decision_post") +@patch("splunk_ao.evaluators.create_scorers_post") +class TestCreateDecisionEvaluatorScope: + """The scope reaches the CreateScorerRequest on both decision create paths.""" + + @staticmethod + def _create(use_function: bool, **kwargs): + if use_function: + return create_decision_evaluator("tone", "Is the tone professional?", **kwargs) + return Evaluators().create_decision_evaluator(name="tone", question="Is the tone professional?", **kwargs) + + @pytest.mark.parametrize("use_function", [False, True], ids=["method", "module_function"]) + def test_defaults_to_current_project( + self, mock_create_scorer, mock_create_version, use_function, monkeypatch, mock_projects, decision_scorer_response + ) -> None: + # Given a current project from the environment + monkeypatch.setenv("SPLUNK_AO_PROJECT_ID", "env-id") + mock_create_scorer.sync.return_value = decision_scorer_response + + # When creating with no explicit scope + self._create(use_function) + + # Then the request is scoped to that project + body = mock_create_scorer.sync.call_args.kwargs["body"] + assert body.is_global is False + assert body.project_ids == ["project-id-1"] + + @pytest.mark.parametrize("use_function", [False, True], ids=["method", "module_function"]) + def test_explicit_project_ids_override_current_project( + self, mock_create_scorer, mock_create_version, use_function, monkeypatch, mock_projects, decision_scorer_response + ) -> None: + # Given a current project and explicit ids with a duplicate + monkeypatch.setenv("SPLUNK_AO_PROJECT_ID", "env-id") + mock_create_scorer.sync.return_value = decision_scorer_response + + # When creating + self._create(use_function, project_ids=["a", "b", "a"]) + + # Then the explicit ids win, deduplicated + body = mock_create_scorer.sync.call_args.kwargs["body"] + assert body.is_global is False + assert body.project_ids == ["a", "b"] + + @pytest.mark.parametrize("use_function", [False, True], ids=["method", "module_function"]) + def test_explicit_global_overrides_current_project( + self, mock_create_scorer, mock_create_version, use_function, monkeypatch, mock_projects, decision_scorer_response + ) -> None: + # Given a current project + monkeypatch.setenv("SPLUNK_AO_PROJECT_ID", "env-id") + mock_create_scorer.sync.return_value = decision_scorer_response + + # When is_global=True is passed + self._create(use_function, is_global=True) + + # Then the request is global with no project_ids + body = mock_create_scorer.sync.call_args.kwargs["body"] + assert body.is_global is True + assert "project_ids" not in body.to_dict() + + @pytest.mark.parametrize("use_function", [False, True], ids=["method", "module_function"]) + def test_no_current_project_sends_no_scope( + self, mock_create_scorer, mock_create_version, use_function, mock_projects, decision_scorer_response + ) -> None: + mock_create_scorer.sync.return_value = decision_scorer_response + + self._create(use_function) + + body_dict = mock_create_scorer.sync.call_args.kwargs["body"].to_dict() + assert "is_global" not in body_dict + assert "project_ids" not in body_dict + + @pytest.mark.parametrize("use_function", [False, True], ids=["method", "module_function"]) + @pytest.mark.parametrize( + "kwargs", [{"is_global": True, "project_ids": ["a"]}, {"project_ids": []}, {"is_global": False}] + ) + def test_invalid_scope_fails_before_any_http_call( + self, mock_create_scorer, mock_create_version, kwargs, use_function, mock_projects + ) -> None: + with pytest.raises(SplunkAOValidationError): + self._create(use_function, **kwargs) + + mock_create_scorer.sync.assert_not_called() + mock_create_version.sync.assert_not_called() + + @pytest.mark.parametrize("use_function", [False, True], ids=["method", "module_function"]) + def test_current_project_not_found_sends_no_create_request( + self, mock_create_scorer, mock_create_version, use_function, monkeypatch, mock_projects + ) -> None: + monkeypatch.setenv("SPLUNK_AO_PROJECT", "missing") + mock_projects.get_with_env_fallbacks.return_value = None + + with pytest.raises(NotFoundError): + self._create(use_function) + + mock_create_scorer.sync.assert_not_called()