From bf4a6c00bd123026b56f0a9ce5876aaa47453235 Mon Sep 17 00:00:00 2001 From: John Weiler Date: Fri, 16 Oct 2026 09:36:36 -0400 Subject: [PATCH 1/7] feat(sdk): decode decision scorers and request them from the API (SAO-18652) Send X-Galileo-Scorer-Capabilities on /scorers*, */scorer-settings and */metric_settings endpoints so the API includes decision scorers in responses, and add decision to the ScorerTypes and ModelType enums. Co-Authored-By: Claude Sonnet 5.5 --- codegen_templates/endpoint_module.py.jinja | 9 ++++++++- openapi.yaml | 2 ++ .../data/autogen_llm_scorer_scorers_llm_autogen_post.py | 3 ++- ...scorer_version_scorers_scorer_id_version_code_post.py | 3 ++- ..._scorer_version_scorers_scorer_id_version_llm_post.py | 3 ++- ...scorer_version_scorers_scorer_id_version_luna_post.py | 3 ++- ...orer_version_scorers_scorer_id_version_preset_post.py | 3 ++- src/splunk_ao/resources/api/data/create_scorers_post.py | 3 ++- .../api/data/delete_scorer_scorers_scorer_id_delete.py | 3 ++- ..._health_scores_scorers_scorer_id_health_scores_get.py | 3 ++- .../api/data/get_scorer_scorers_scorer_id_get.py | 3 ++- ...er_version_code_scorers_scorer_id_version_code_get.py | 3 ++- ...er_version_or_latest_scorers_scorer_id_version_get.py | 3 ++- ...orer_task_result_scorers_code_validate_task_id_get.py | 3 ++- ...versions_for_scorer_scorers_scorer_id_versions_get.py | 3 ++- ...ts_for_scorer_route_scorers_scorer_id_projects_get.py | 3 ++- ...te_scorers_versions_scorer_version_id_projects_get.py | 3 ++- .../data/list_scorers_with_filters_scorers_list_post.py | 3 ++- .../resources/api/data/list_tags_scorers_tags_get.py | 3 ++- ...date_multipart_scorers_llm_validate_multipart_post.py | 3 ++- .../manual_llm_validate_scorers_llm_validate_post.py | 3 ++- ...ers_scorer_id_versions_version_number_restore_post.py | 3 ++- .../data/set_scorer_scope_scorers_scorer_id_scope_put.py | 3 ++- .../resources/api/data/update_scorers_scorer_id_patch.py | 3 ++- ..._scorer_dataset_scorers_code_validate_dataset_post.py | 3 ++- ...r_log_record_scorers_code_validate_log_record_post.py | 3 ++- .../validate_code_scorer_scorers_code_validate_post.py | 3 ++- ...m_scorer_dataset_scorers_llm_validate_dataset_post.py | 3 ++- ...er_log_record_scorers_llm_validate_log_record_post.py | 3 ++- ...orer_id_versions_version_number_health_scores_post.py | 3 ++- ...t_id_experiments_experiment_id_metric_settings_get.py | 3 ++- ...id_experiments_experiment_id_metric_settings_patch.py | 3 ++- ...t_id_log_streams_log_stream_id_metric_settings_get.py | 3 ++- ...id_log_streams_log_stream_id_metric_settings_patch.py | 3 ++- ...rojects_project_id_runs_run_id_scorer_settings_get.py | 3 ++- ...jects_project_id_runs_run_id_scorer_settings_patch.py | 3 ++- ...ojects_project_id_runs_run_id_scorer_settings_post.py | 3 ++- src/splunk_ao/resources/models/model_type.py | 1 + src/splunk_ao/resources/models/scorer_types.py | 1 + src/splunk_ao/utils/headers_data.py | 5 +++++ 40 files changed, 87 insertions(+), 36 deletions(-) diff --git a/codegen_templates/endpoint_module.py.jinja b/codegen_templates/endpoint_module.py.jinja index 0638211b..22f1ad3a 100644 --- a/codegen_templates/endpoint_module.py.jinja +++ b/codegen_templates/endpoint_module.py.jinja @@ -12,7 +12,11 @@ from splunk_ao.exceptions import ( RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +{# Endpoints whose responses carry scorer_type; the API leaves decision scorers out unless the header is sent. + The blank line after the import is needed: trim_blocks drops the newline that follows {% endif %}. #} +{% set scorer_discovery = endpoint.path.startswith("/scorers") or endpoint.path.endswith("/scorer-settings") or endpoint.path.endswith("/metric_settings") %} +from splunk_ao.utils.headers_data import get_sdk_header{% if scorer_discovery %}, get_scorer_capabilities_header{% endif %} + from galileo_core.constants.request_method import RequestMethod from galileo_core.helpers.api_client import ApiClient from ...types import Response, UNSET @@ -73,6 +77,9 @@ def _get_kwargs( {% endif %} headers["Splunk-AO-SDK"] = get_sdk_header() +{% if scorer_discovery %} + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() +{% endif %} _kwargs["content_headers"] = headers return _kwargs diff --git a/openapi.yaml b/openapi.yaml index 3fcc4797..be1707c5 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -35838,6 +35838,7 @@ components: - slm - llm - code + - decision title: ModelType MultiModalModelIntegrationConfig: properties: @@ -42460,6 +42461,7 @@ components: - code - luna - preset + - decision title: ScorerTypes ScorerUpdatedAtFilter: properties: diff --git a/src/splunk_ao/resources/api/data/autogen_llm_scorer_scorers_llm_autogen_post.py b/src/splunk_ao/resources/api/data/autogen_llm_scorer_scorers_llm_autogen_post.py index 006d7971..3123d4f9 100644 --- a/src/splunk_ao/resources/api/data/autogen_llm_scorer_scorers_llm_autogen_post.py +++ b/src/splunk_ao/resources/api/data/autogen_llm_scorer_scorers_llm_autogen_post.py @@ -14,7 +14,7 @@ RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header from ... import errors from ...models.create_llm_scorer_autogen_request import CreateLLMScorerAutogenRequest @@ -37,6 +37,7 @@ def _get_kwargs(*, body: CreateLLMScorerAutogenRequest) -> dict[str, Any]: headers["Content-Type"] = "application/json" headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() _kwargs["content_headers"] = headers return _kwargs diff --git a/src/splunk_ao/resources/api/data/create_code_scorer_version_scorers_scorer_id_version_code_post.py b/src/splunk_ao/resources/api/data/create_code_scorer_version_scorers_scorer_id_version_code_post.py index 0a18b4c0..54a73016 100644 --- a/src/splunk_ao/resources/api/data/create_code_scorer_version_scorers_scorer_id_version_code_post.py +++ b/src/splunk_ao/resources/api/data/create_code_scorer_version_scorers_scorer_id_version_code_post.py @@ -14,7 +14,7 @@ RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header from ... import errors from ...models.base_scorer_version_response import BaseScorerVersionResponse @@ -37,6 +37,7 @@ def _get_kwargs(scorer_id: str, *, body: BodyCreateCodeScorerVersionScorersScore _kwargs["files"] = body.to_multipart() headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() _kwargs["content_headers"] = headers return _kwargs diff --git a/src/splunk_ao/resources/api/data/create_llm_scorer_version_scorers_scorer_id_version_llm_post.py b/src/splunk_ao/resources/api/data/create_llm_scorer_version_scorers_scorer_id_version_llm_post.py index 1d3bcf57..b966e179 100644 --- a/src/splunk_ao/resources/api/data/create_llm_scorer_version_scorers_scorer_id_version_llm_post.py +++ b/src/splunk_ao/resources/api/data/create_llm_scorer_version_scorers_scorer_id_version_llm_post.py @@ -14,7 +14,7 @@ RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header from ... import errors from ...models.base_scorer_version_response import BaseScorerVersionResponse @@ -37,6 +37,7 @@ def _get_kwargs(scorer_id: str, *, body: CreateLLMScorerVersionRequest) -> dict[ headers["Content-Type"] = "application/json" headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() _kwargs["content_headers"] = headers return _kwargs diff --git a/src/splunk_ao/resources/api/data/create_luna_scorer_version_scorers_scorer_id_version_luna_post.py b/src/splunk_ao/resources/api/data/create_luna_scorer_version_scorers_scorer_id_version_luna_post.py index 3b8f22bd..abfe7385 100644 --- a/src/splunk_ao/resources/api/data/create_luna_scorer_version_scorers_scorer_id_version_luna_post.py +++ b/src/splunk_ao/resources/api/data/create_luna_scorer_version_scorers_scorer_id_version_luna_post.py @@ -14,7 +14,7 @@ RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header from ... import errors from ...models.base_scorer_version_response import BaseScorerVersionResponse @@ -37,6 +37,7 @@ def _get_kwargs(scorer_id: str, *, body: CreateCustomLunaScorerVersionRequest) - headers["Content-Type"] = "application/json" headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() _kwargs["content_headers"] = headers return _kwargs diff --git a/src/splunk_ao/resources/api/data/create_preset_scorer_version_scorers_scorer_id_version_preset_post.py b/src/splunk_ao/resources/api/data/create_preset_scorer_version_scorers_scorer_id_version_preset_post.py index 5a2616be..38c7123d 100644 --- a/src/splunk_ao/resources/api/data/create_preset_scorer_version_scorers_scorer_id_version_preset_post.py +++ b/src/splunk_ao/resources/api/data/create_preset_scorer_version_scorers_scorer_id_version_preset_post.py @@ -14,7 +14,7 @@ RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header from ... import errors from ...models.base_scorer_version_response import BaseScorerVersionResponse @@ -37,6 +37,7 @@ def _get_kwargs(scorer_id: str, *, body: CreateScorerVersionRequest) -> dict[str headers["Content-Type"] = "application/json" headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() _kwargs["content_headers"] = headers return _kwargs diff --git a/src/splunk_ao/resources/api/data/create_scorers_post.py b/src/splunk_ao/resources/api/data/create_scorers_post.py index 6da41630..41d63d7c 100644 --- a/src/splunk_ao/resources/api/data/create_scorers_post.py +++ b/src/splunk_ao/resources/api/data/create_scorers_post.py @@ -14,7 +14,7 @@ RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header from ... import errors from ...models.create_scorer_request import CreateScorerRequest @@ -33,6 +33,7 @@ def _get_kwargs(*, body: CreateScorerRequest) -> dict[str, Any]: headers["Content-Type"] = "application/json" headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() _kwargs["content_headers"] = headers return _kwargs diff --git a/src/splunk_ao/resources/api/data/delete_scorer_scorers_scorer_id_delete.py b/src/splunk_ao/resources/api/data/delete_scorer_scorers_scorer_id_delete.py index 2e247865..236176d0 100644 --- a/src/splunk_ao/resources/api/data/delete_scorer_scorers_scorer_id_delete.py +++ b/src/splunk_ao/resources/api/data/delete_scorer_scorers_scorer_id_delete.py @@ -14,7 +14,7 @@ RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header from ... import errors from ...models.delete_scorer_response import DeleteScorerResponse @@ -32,6 +32,7 @@ def _get_kwargs(scorer_id: str) -> dict[str, Any]: } headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() _kwargs["content_headers"] = headers return _kwargs diff --git a/src/splunk_ao/resources/api/data/get_scorer_health_scores_scorers_scorer_id_health_scores_get.py b/src/splunk_ao/resources/api/data/get_scorer_health_scores_scorers_scorer_id_health_scores_get.py index 32f5a58c..94717b90 100644 --- a/src/splunk_ao/resources/api/data/get_scorer_health_scores_scorers_scorer_id_health_scores_get.py +++ b/src/splunk_ao/resources/api/data/get_scorer_health_scores_scorers_scorer_id_health_scores_get.py @@ -14,7 +14,7 @@ RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header from ... import errors from ...models.http_validation_error import HTTPValidationError @@ -39,6 +39,7 @@ def _get_kwargs(scorer_id: str, *, dataset_id: str) -> dict[str, Any]: } headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() _kwargs["content_headers"] = headers return _kwargs diff --git a/src/splunk_ao/resources/api/data/get_scorer_scorers_scorer_id_get.py b/src/splunk_ao/resources/api/data/get_scorer_scorers_scorer_id_get.py index f4c6c816..fffeae85 100644 --- a/src/splunk_ao/resources/api/data/get_scorer_scorers_scorer_id_get.py +++ b/src/splunk_ao/resources/api/data/get_scorer_scorers_scorer_id_get.py @@ -14,7 +14,7 @@ RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header from ... import errors from ...models.http_validation_error import HTTPValidationError @@ -47,6 +47,7 @@ def _get_kwargs(scorer_id: str, *, actions: list[ScorerAction] | Unset = UNSET) } headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() _kwargs["content_headers"] = headers return _kwargs diff --git a/src/splunk_ao/resources/api/data/get_scorer_version_code_scorers_scorer_id_version_code_get.py b/src/splunk_ao/resources/api/data/get_scorer_version_code_scorers_scorer_id_version_code_get.py index 640001cc..23e1e049 100644 --- a/src/splunk_ao/resources/api/data/get_scorer_version_code_scorers_scorer_id_version_code_get.py +++ b/src/splunk_ao/resources/api/data/get_scorer_version_code_scorers_scorer_id_version_code_get.py @@ -14,7 +14,7 @@ RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header from ... import errors from ...models.http_validation_error import HTTPValidationError @@ -43,6 +43,7 @@ def _get_kwargs(scorer_id: str, *, version: int | None | Unset = UNSET) -> dict[ } headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() _kwargs["content_headers"] = headers return _kwargs diff --git a/src/splunk_ao/resources/api/data/get_scorer_version_or_latest_scorers_scorer_id_version_get.py b/src/splunk_ao/resources/api/data/get_scorer_version_or_latest_scorers_scorer_id_version_get.py index 7199d701..8c411f67 100644 --- a/src/splunk_ao/resources/api/data/get_scorer_version_or_latest_scorers_scorer_id_version_get.py +++ b/src/splunk_ao/resources/api/data/get_scorer_version_or_latest_scorers_scorer_id_version_get.py @@ -14,7 +14,7 @@ RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header from ... import errors from ...models.base_scorer_version_response import BaseScorerVersionResponse @@ -39,6 +39,7 @@ def _get_kwargs(scorer_id: str, *, version: int | Unset = UNSET) -> dict[str, An } headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() _kwargs["content_headers"] = headers return _kwargs diff --git a/src/splunk_ao/resources/api/data/get_validate_code_scorer_task_result_scorers_code_validate_task_id_get.py b/src/splunk_ao/resources/api/data/get_validate_code_scorer_task_result_scorers_code_validate_task_id_get.py index 4376c7d8..95b30d8f 100644 --- a/src/splunk_ao/resources/api/data/get_validate_code_scorer_task_result_scorers_code_validate_task_id_get.py +++ b/src/splunk_ao/resources/api/data/get_validate_code_scorer_task_result_scorers_code_validate_task_id_get.py @@ -14,7 +14,7 @@ RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header from ... import errors from ...models.http_validation_error import HTTPValidationError @@ -32,6 +32,7 @@ def _get_kwargs(task_id: str) -> dict[str, Any]: } headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() _kwargs["content_headers"] = headers return _kwargs diff --git a/src/splunk_ao/resources/api/data/list_all_versions_for_scorer_scorers_scorer_id_versions_get.py b/src/splunk_ao/resources/api/data/list_all_versions_for_scorer_scorers_scorer_id_versions_get.py index 07c182e8..65d78151 100644 --- a/src/splunk_ao/resources/api/data/list_all_versions_for_scorer_scorers_scorer_id_versions_get.py +++ b/src/splunk_ao/resources/api/data/list_all_versions_for_scorer_scorers_scorer_id_versions_get.py @@ -14,7 +14,7 @@ RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header from ... import errors from ...models.http_validation_error import HTTPValidationError @@ -50,6 +50,7 @@ def _get_kwargs( } headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() _kwargs["content_headers"] = headers return _kwargs diff --git a/src/splunk_ao/resources/api/data/list_projects_for_scorer_route_scorers_scorer_id_projects_get.py b/src/splunk_ao/resources/api/data/list_projects_for_scorer_route_scorers_scorer_id_projects_get.py index 319230ee..b585e8ae 100644 --- a/src/splunk_ao/resources/api/data/list_projects_for_scorer_route_scorers_scorer_id_projects_get.py +++ b/src/splunk_ao/resources/api/data/list_projects_for_scorer_route_scorers_scorer_id_projects_get.py @@ -14,7 +14,7 @@ RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header from ... import errors from ...models.get_projects_paginated_response_v2 import GetProjectsPaginatedResponseV2 @@ -41,6 +41,7 @@ def _get_kwargs(scorer_id: str, *, starting_token: int | Unset = 0, limit: int | } headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() _kwargs["content_headers"] = headers return _kwargs diff --git a/src/splunk_ao/resources/api/data/list_projects_for_scorer_version_route_scorers_versions_scorer_version_id_projects_get.py b/src/splunk_ao/resources/api/data/list_projects_for_scorer_version_route_scorers_versions_scorer_version_id_projects_get.py index e7ac7ec9..553b608c 100644 --- a/src/splunk_ao/resources/api/data/list_projects_for_scorer_version_route_scorers_versions_scorer_version_id_projects_get.py +++ b/src/splunk_ao/resources/api/data/list_projects_for_scorer_version_route_scorers_versions_scorer_version_id_projects_get.py @@ -14,7 +14,7 @@ RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header from ... import errors from ...models.get_projects_paginated_response_v2 import GetProjectsPaginatedResponseV2 @@ -41,6 +41,7 @@ def _get_kwargs(scorer_version_id: str, *, starting_token: int | Unset = 0, limi } headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() _kwargs["content_headers"] = headers return _kwargs diff --git a/src/splunk_ao/resources/api/data/list_scorers_with_filters_scorers_list_post.py b/src/splunk_ao/resources/api/data/list_scorers_with_filters_scorers_list_post.py index bb5164c0..ca39ac5c 100644 --- a/src/splunk_ao/resources/api/data/list_scorers_with_filters_scorers_list_post.py +++ b/src/splunk_ao/resources/api/data/list_scorers_with_filters_scorers_list_post.py @@ -14,7 +14,7 @@ RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header from ... import errors from ...models.http_validation_error import HTTPValidationError @@ -62,6 +62,7 @@ def _get_kwargs( headers["Content-Type"] = "application/json" headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() _kwargs["content_headers"] = headers return _kwargs diff --git a/src/splunk_ao/resources/api/data/list_tags_scorers_tags_get.py b/src/splunk_ao/resources/api/data/list_tags_scorers_tags_get.py index b2f1adde..f301eb60 100644 --- a/src/splunk_ao/resources/api/data/list_tags_scorers_tags_get.py +++ b/src/splunk_ao/resources/api/data/list_tags_scorers_tags_get.py @@ -14,7 +14,7 @@ RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header from ... import errors from ...types import Response @@ -26,6 +26,7 @@ def _get_kwargs() -> dict[str, Any]: _kwargs: dict[str, Any] = {"method": RequestMethod.GET, "return_raw_response": True, "path": "/scorers/tags"} headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() _kwargs["content_headers"] = headers return _kwargs diff --git a/src/splunk_ao/resources/api/data/manual_llm_validate_multipart_scorers_llm_validate_multipart_post.py b/src/splunk_ao/resources/api/data/manual_llm_validate_multipart_scorers_llm_validate_multipart_post.py index 1a054c8d..946d4700 100644 --- a/src/splunk_ao/resources/api/data/manual_llm_validate_multipart_scorers_llm_validate_multipart_post.py +++ b/src/splunk_ao/resources/api/data/manual_llm_validate_multipart_scorers_llm_validate_multipart_post.py @@ -14,7 +14,7 @@ RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header from ... import errors from ...models.body_manual_llm_validate_multipart_scorers_llm_validate_multipart_post import ( @@ -37,6 +37,7 @@ def _get_kwargs(*, body: BodyManualLlmValidateMultipartScorersLlmValidateMultipa _kwargs["files"] = body.to_multipart() headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() _kwargs["content_headers"] = headers return _kwargs diff --git a/src/splunk_ao/resources/api/data/manual_llm_validate_scorers_llm_validate_post.py b/src/splunk_ao/resources/api/data/manual_llm_validate_scorers_llm_validate_post.py index 60aa2930..d91588b2 100644 --- a/src/splunk_ao/resources/api/data/manual_llm_validate_scorers_llm_validate_post.py +++ b/src/splunk_ao/resources/api/data/manual_llm_validate_scorers_llm_validate_post.py @@ -14,7 +14,7 @@ RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header from ... import errors from ...models.generated_scorer_validation_response import GeneratedScorerValidationResponse @@ -31,6 +31,7 @@ def _get_kwargs() -> dict[str, Any]: } headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() _kwargs["content_headers"] = headers return _kwargs diff --git a/src/splunk_ao/resources/api/data/restore_scorer_version_scorers_scorer_id_versions_version_number_restore_post.py b/src/splunk_ao/resources/api/data/restore_scorer_version_scorers_scorer_id_versions_version_number_restore_post.py index 106e1401..64b69410 100644 --- a/src/splunk_ao/resources/api/data/restore_scorer_version_scorers_scorer_id_versions_version_number_restore_post.py +++ b/src/splunk_ao/resources/api/data/restore_scorer_version_scorers_scorer_id_versions_version_number_restore_post.py @@ -14,7 +14,7 @@ RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header from ... import errors from ...models.base_scorer_version_response import BaseScorerVersionResponse @@ -34,6 +34,7 @@ def _get_kwargs(scorer_id: str, version_number: int) -> dict[str, Any]: } headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() _kwargs["content_headers"] = headers return _kwargs diff --git a/src/splunk_ao/resources/api/data/set_scorer_scope_scorers_scorer_id_scope_put.py b/src/splunk_ao/resources/api/data/set_scorer_scope_scorers_scorer_id_scope_put.py index 3d6205a4..ef47d0b8 100644 --- a/src/splunk_ao/resources/api/data/set_scorer_scope_scorers_scorer_id_scope_put.py +++ b/src/splunk_ao/resources/api/data/set_scorer_scope_scorers_scorer_id_scope_put.py @@ -14,7 +14,7 @@ RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header from ... import errors from ...models.http_validation_error import HTTPValidationError @@ -37,6 +37,7 @@ def _get_kwargs(scorer_id: str, *, body: UpdateScorerScopeRequest) -> dict[str, headers["Content-Type"] = "application/json" headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() _kwargs["content_headers"] = headers return _kwargs diff --git a/src/splunk_ao/resources/api/data/update_scorers_scorer_id_patch.py b/src/splunk_ao/resources/api/data/update_scorers_scorer_id_patch.py index bef42f0b..417a1316 100644 --- a/src/splunk_ao/resources/api/data/update_scorers_scorer_id_patch.py +++ b/src/splunk_ao/resources/api/data/update_scorers_scorer_id_patch.py @@ -14,7 +14,7 @@ RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header from ... import errors from ...models.http_validation_error import HTTPValidationError @@ -37,6 +37,7 @@ def _get_kwargs(scorer_id: str, *, body: UpdateScorerRequest) -> dict[str, Any]: headers["Content-Type"] = "application/json" headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() _kwargs["content_headers"] = headers return _kwargs diff --git a/src/splunk_ao/resources/api/data/validate_code_scorer_dataset_scorers_code_validate_dataset_post.py b/src/splunk_ao/resources/api/data/validate_code_scorer_dataset_scorers_code_validate_dataset_post.py index c185e4b3..02565306 100644 --- a/src/splunk_ao/resources/api/data/validate_code_scorer_dataset_scorers_code_validate_dataset_post.py +++ b/src/splunk_ao/resources/api/data/validate_code_scorer_dataset_scorers_code_validate_dataset_post.py @@ -14,7 +14,7 @@ RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header from ... import errors from ...models.body_validate_code_scorer_dataset_scorers_code_validate_dataset_post import ( @@ -37,6 +37,7 @@ def _get_kwargs(*, body: BodyValidateCodeScorerDatasetScorersCodeValidateDataset _kwargs["files"] = body.to_multipart() headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() _kwargs["content_headers"] = headers return _kwargs diff --git a/src/splunk_ao/resources/api/data/validate_code_scorer_log_record_scorers_code_validate_log_record_post.py b/src/splunk_ao/resources/api/data/validate_code_scorer_log_record_scorers_code_validate_log_record_post.py index 57c3a208..8771b973 100644 --- a/src/splunk_ao/resources/api/data/validate_code_scorer_log_record_scorers_code_validate_log_record_post.py +++ b/src/splunk_ao/resources/api/data/validate_code_scorer_log_record_scorers_code_validate_log_record_post.py @@ -14,7 +14,7 @@ RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header from ... import errors from ...models.body_validate_code_scorer_log_record_scorers_code_validate_log_record_post import ( @@ -37,6 +37,7 @@ def _get_kwargs(*, body: BodyValidateCodeScorerLogRecordScorersCodeValidateLogRe _kwargs["files"] = body.to_multipart() headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() _kwargs["content_headers"] = headers return _kwargs diff --git a/src/splunk_ao/resources/api/data/validate_code_scorer_scorers_code_validate_post.py b/src/splunk_ao/resources/api/data/validate_code_scorer_scorers_code_validate_post.py index 2ce074a7..c91a5e0e 100644 --- a/src/splunk_ao/resources/api/data/validate_code_scorer_scorers_code_validate_post.py +++ b/src/splunk_ao/resources/api/data/validate_code_scorer_scorers_code_validate_post.py @@ -14,7 +14,7 @@ RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header from ... import errors from ...models.body_validate_code_scorer_scorers_code_validate_post import BodyValidateCodeScorerScorersCodeValidatePost @@ -35,6 +35,7 @@ def _get_kwargs(*, body: BodyValidateCodeScorerScorersCodeValidatePost) -> dict[ _kwargs["files"] = body.to_multipart() headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() _kwargs["content_headers"] = headers return _kwargs diff --git a/src/splunk_ao/resources/api/data/validate_llm_scorer_dataset_scorers_llm_validate_dataset_post.py b/src/splunk_ao/resources/api/data/validate_llm_scorer_dataset_scorers_llm_validate_dataset_post.py index 42c29a50..bcbd55ea 100644 --- a/src/splunk_ao/resources/api/data/validate_llm_scorer_dataset_scorers_llm_validate_dataset_post.py +++ b/src/splunk_ao/resources/api/data/validate_llm_scorer_dataset_scorers_llm_validate_dataset_post.py @@ -14,7 +14,7 @@ RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header from ... import errors from ...models.http_validation_error import HTTPValidationError @@ -37,6 +37,7 @@ def _get_kwargs(*, body: ValidateLLMScorerDatasetRequest) -> dict[str, Any]: headers["Content-Type"] = "application/json" headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() _kwargs["content_headers"] = headers return _kwargs diff --git a/src/splunk_ao/resources/api/data/validate_llm_scorer_log_record_scorers_llm_validate_log_record_post.py b/src/splunk_ao/resources/api/data/validate_llm_scorer_log_record_scorers_llm_validate_log_record_post.py index 137725cc..54dc50e5 100644 --- a/src/splunk_ao/resources/api/data/validate_llm_scorer_log_record_scorers_llm_validate_log_record_post.py +++ b/src/splunk_ao/resources/api/data/validate_llm_scorer_log_record_scorers_llm_validate_log_record_post.py @@ -14,7 +14,7 @@ RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header from ... import errors from ...models.http_validation_error import HTTPValidationError @@ -37,6 +37,7 @@ def _get_kwargs(*, body: ValidateLLMScorerLogRecordRequest) -> dict[str, Any]: headers["Content-Type"] = "application/json" headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() _kwargs["content_headers"] = headers return _kwargs diff --git a/src/splunk_ao/resources/api/data/write_scorer_version_health_score_scorers_scorer_id_versions_version_number_health_scores_post.py b/src/splunk_ao/resources/api/data/write_scorer_version_health_score_scorers_scorer_id_versions_version_number_health_scores_post.py index f7a91403..6b2e2362 100644 --- a/src/splunk_ao/resources/api/data/write_scorer_version_health_score_scorers_scorer_id_versions_version_number_health_scores_post.py +++ b/src/splunk_ao/resources/api/data/write_scorer_version_health_score_scorers_scorer_id_versions_version_number_health_scores_post.py @@ -14,7 +14,7 @@ RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header from ... import errors from ...models.http_validation_error import HTTPValidationError @@ -39,6 +39,7 @@ def _get_kwargs(scorer_id: str, version_number: int, *, body: WriteHealthScoreRe headers["Content-Type"] = "application/json" headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() _kwargs["content_headers"] = headers return _kwargs diff --git a/src/splunk_ao/resources/api/experiment/get_metric_settings_projects_project_id_experiments_experiment_id_metric_settings_get.py b/src/splunk_ao/resources/api/experiment/get_metric_settings_projects_project_id_experiments_experiment_id_metric_settings_get.py index d025a890..14caf204 100644 --- a/src/splunk_ao/resources/api/experiment/get_metric_settings_projects_project_id_experiments_experiment_id_metric_settings_get.py +++ b/src/splunk_ao/resources/api/experiment/get_metric_settings_projects_project_id_experiments_experiment_id_metric_settings_get.py @@ -14,7 +14,7 @@ RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header from ... import errors from ...models.http_validation_error import HTTPValidationError @@ -34,6 +34,7 @@ def _get_kwargs(project_id: str, experiment_id: str) -> dict[str, Any]: } headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() _kwargs["content_headers"] = headers return _kwargs diff --git a/src/splunk_ao/resources/api/experiment/update_metric_settings_projects_project_id_experiments_experiment_id_metric_settings_patch.py b/src/splunk_ao/resources/api/experiment/update_metric_settings_projects_project_id_experiments_experiment_id_metric_settings_patch.py index 8f27e5e1..939516a9 100644 --- a/src/splunk_ao/resources/api/experiment/update_metric_settings_projects_project_id_experiments_experiment_id_metric_settings_patch.py +++ b/src/splunk_ao/resources/api/experiment/update_metric_settings_projects_project_id_experiments_experiment_id_metric_settings_patch.py @@ -14,7 +14,7 @@ RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header from ... import errors from ...models.http_validation_error import HTTPValidationError @@ -39,6 +39,7 @@ def _get_kwargs(project_id: str, experiment_id: str, *, body: MetricSettingsRequ headers["Content-Type"] = "application/json" headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() _kwargs["content_headers"] = headers return _kwargs diff --git a/src/splunk_ao/resources/api/log_stream/get_metric_settings_projects_project_id_log_streams_log_stream_id_metric_settings_get.py b/src/splunk_ao/resources/api/log_stream/get_metric_settings_projects_project_id_log_streams_log_stream_id_metric_settings_get.py index 5577675a..84590997 100644 --- a/src/splunk_ao/resources/api/log_stream/get_metric_settings_projects_project_id_log_streams_log_stream_id_metric_settings_get.py +++ b/src/splunk_ao/resources/api/log_stream/get_metric_settings_projects_project_id_log_streams_log_stream_id_metric_settings_get.py @@ -14,7 +14,7 @@ RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header from ... import errors from ...models.http_validation_error import HTTPValidationError @@ -34,6 +34,7 @@ def _get_kwargs(project_id: str, log_stream_id: str) -> dict[str, Any]: } headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() _kwargs["content_headers"] = headers return _kwargs diff --git a/src/splunk_ao/resources/api/log_stream/update_metric_settings_projects_project_id_log_streams_log_stream_id_metric_settings_patch.py b/src/splunk_ao/resources/api/log_stream/update_metric_settings_projects_project_id_log_streams_log_stream_id_metric_settings_patch.py index 79a83ca8..8bc32a20 100644 --- a/src/splunk_ao/resources/api/log_stream/update_metric_settings_projects_project_id_log_streams_log_stream_id_metric_settings_patch.py +++ b/src/splunk_ao/resources/api/log_stream/update_metric_settings_projects_project_id_log_streams_log_stream_id_metric_settings_patch.py @@ -14,7 +14,7 @@ RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header from ... import errors from ...models.http_validation_error import HTTPValidationError @@ -39,6 +39,7 @@ def _get_kwargs(project_id: str, log_stream_id: str, *, body: MetricSettingsRequ headers["Content-Type"] = "application/json" headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() _kwargs["content_headers"] = headers return _kwargs diff --git a/src/splunk_ao/resources/api/run_scorer_settings/get_settings_projects_project_id_runs_run_id_scorer_settings_get.py b/src/splunk_ao/resources/api/run_scorer_settings/get_settings_projects_project_id_runs_run_id_scorer_settings_get.py index 357a62b6..422120b3 100644 --- a/src/splunk_ao/resources/api/run_scorer_settings/get_settings_projects_project_id_runs_run_id_scorer_settings_get.py +++ b/src/splunk_ao/resources/api/run_scorer_settings/get_settings_projects_project_id_runs_run_id_scorer_settings_get.py @@ -14,7 +14,7 @@ RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header from ... import errors from ...models.http_validation_error import HTTPValidationError @@ -32,6 +32,7 @@ def _get_kwargs(project_id: str, run_id: str) -> dict[str, Any]: } headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() _kwargs["content_headers"] = headers return _kwargs diff --git a/src/splunk_ao/resources/api/run_scorer_settings/upsert_scorers_config_projects_project_id_runs_run_id_scorer_settings_patch.py b/src/splunk_ao/resources/api/run_scorer_settings/upsert_scorers_config_projects_project_id_runs_run_id_scorer_settings_patch.py index ee87a62f..7f62b334 100644 --- a/src/splunk_ao/resources/api/run_scorer_settings/upsert_scorers_config_projects_project_id_runs_run_id_scorer_settings_patch.py +++ b/src/splunk_ao/resources/api/run_scorer_settings/upsert_scorers_config_projects_project_id_runs_run_id_scorer_settings_patch.py @@ -14,7 +14,7 @@ RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header from ... import errors from ...models.http_validation_error import HTTPValidationError @@ -37,6 +37,7 @@ def _get_kwargs(project_id: str, run_id: str, *, body: RunScorerSettingsPatchReq headers["Content-Type"] = "application/json" headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() _kwargs["content_headers"] = headers return _kwargs diff --git a/src/splunk_ao/resources/api/run_scorer_settings/upsert_scorers_config_projects_project_id_runs_run_id_scorer_settings_post.py b/src/splunk_ao/resources/api/run_scorer_settings/upsert_scorers_config_projects_project_id_runs_run_id_scorer_settings_post.py index 8e55fe89..787b866d 100644 --- a/src/splunk_ao/resources/api/run_scorer_settings/upsert_scorers_config_projects_project_id_runs_run_id_scorer_settings_post.py +++ b/src/splunk_ao/resources/api/run_scorer_settings/upsert_scorers_config_projects_project_id_runs_run_id_scorer_settings_post.py @@ -14,7 +14,7 @@ RateLimitError, ServerError, ) -from splunk_ao.utils.headers_data import get_sdk_header +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header from ... import errors from ...models.http_validation_error import HTTPValidationError @@ -37,6 +37,7 @@ def _get_kwargs(project_id: str, run_id: str, *, body: RunScorerSettingsPatchReq headers["Content-Type"] = "application/json" headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() _kwargs["content_headers"] = headers return _kwargs diff --git a/src/splunk_ao/resources/models/model_type.py b/src/splunk_ao/resources/models/model_type.py index 10a5358d..2fc441ec 100644 --- a/src/splunk_ao/resources/models/model_type.py +++ b/src/splunk_ao/resources/models/model_type.py @@ -3,6 +3,7 @@ class ModelType(str, Enum): CODE = "code" + DECISION = "decision" LLM = "llm" SLM = "slm" diff --git a/src/splunk_ao/resources/models/scorer_types.py b/src/splunk_ao/resources/models/scorer_types.py index 4e1ad51e..eaf585a6 100644 --- a/src/splunk_ao/resources/models/scorer_types.py +++ b/src/splunk_ao/resources/models/scorer_types.py @@ -3,6 +3,7 @@ class ScorerTypes(str, Enum): CODE = "code" + DECISION = "decision" LLM = "llm" LUNA = "luna" PRESET = "preset" diff --git a/src/splunk_ao/utils/headers_data.py b/src/splunk_ao/utils/headers_data.py index 56c568ee..0483e79b 100644 --- a/src/splunk_ao/utils/headers_data.py +++ b/src/splunk_ao/utils/headers_data.py @@ -67,3 +67,8 @@ def get_sdk_header() -> str: sdk_header = f"{sdk_header} {method_name}" return sdk_header + + +def get_scorer_capabilities_header() -> str: + """Identify the scorer types supported by scorer response decoding.""" + return "decision" From 2e1d1009e0b0a60dc954c4b44a42c900aee1da9a Mon Sep 17 00:00:00 2001 From: John Weiler Date: Fri, 16 Oct 2026 09:36:46 -0400 Subject: [PATCH 2/7] feat(sdk): add decision scorer version endpoint and schemas to the generated client (SAO-17973) Splice the decision path, CreateDecisionScorerVersionRequest, DecisionLevel, DecisionScorerDB, boolean_probability and decision_scorer into openapi.yaml from the prod client spec (full regeneration yields an unrelated ~45k-line diff) and regenerate. Co-Authored-By: Claude Sonnet 5.5 --- openapi.yaml | 180 ++++++++++++ ...scorers_scorer_id_version_decision_post.py | 199 ++++++++++++++ src/splunk_ao/resources/models/__init__.py | 6 + .../models/base_scorer_version_db.py | 33 +++ .../models/base_scorer_version_response.py | 33 +++ .../create_decision_scorer_version_request.py | 260 ++++++++++++++++++ .../resources/models/decision_level.py | 83 ++++++ .../resources/models/decision_scorer_db.py | 165 +++++++++++ .../resources/models/output_type_enum.py | 1 + 9 files changed, 960 insertions(+) create mode 100644 src/splunk_ao/resources/api/data/create_decision_scorer_version_scorers_scorer_id_version_decision_post.py create mode 100644 src/splunk_ao/resources/models/create_decision_scorer_version_request.py create mode 100644 src/splunk_ao/resources/models/decision_level.py create mode 100644 src/splunk_ao/resources/models/decision_scorer_db.py diff --git a/openapi.yaml b/openapi.yaml index be1707c5..ae1cfae5 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -6877,6 +6877,46 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + /scorers/{scorer_id}/version/decision: + post: + tags: + - data + - prompts + - rows + summary: Create Decision Scorer Version + description: Create a decision evaluator version that holds its question, answer options and model. + operationId: create_decision_scorer_version_scorers__scorer_id__version_decision_post + security: + - APIKeyHeader: [] + - OAuth2PasswordBearer: [] + - HTTPBasic: [] + parameters: + - name: scorer_id + in: path + required: true + schema: + type: string + format: uuid4 + title: Scorer Id + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/CreateDecisionScorerVersionRequest' + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/BaseScorerVersionResponse' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' /scorers/code/validate: post: tags: @@ -13824,6 +13864,10 @@ components: anyOf: - $ref: '#/components/schemas/BaseFinetunedScorerDB' - type: 'null' + decision_scorer: + anyOf: + - $ref: '#/components/schemas/DecisionScorerDB' + - type: 'null' model_name: anyOf: - type: string @@ -13894,6 +13938,10 @@ components: anyOf: - $ref: '#/components/schemas/FineTunedScorerResponse' - type: 'null' + decision_scorer: + anyOf: + - $ref: '#/components/schemas/DecisionScorerDB' + - type: 'null' model_name: anyOf: - type: string @@ -15530,6 +15578,66 @@ components: - lora_task_id - prompt title: CreateCustomLunaScorerVersionRequest + CreateDecisionScorerVersionRequest: + properties: + model_name: + type: string + maxLength: 255 + minLength: 1 + title: Model Name + num_judges: + anyOf: + - type: integer + - type: 'null' + title: Num Judges + scoreable_node_types: + anyOf: + - items: + type: string + type: array + - type: 'null' + title: Scoreable Node Types + cot_enabled: + anyOf: + - type: boolean + - type: 'null' + title: Cot Enabled + output_type: + anyOf: + - $ref: '#/components/schemas/OutputTypeEnum' + - type: 'null' + input_type: + anyOf: + - $ref: '#/components/schemas/InputTypeEnum' + - type: 'null' + question: + type: string + maxLength: 2000 + minLength: 1 + title: Question + answer_options: + anyOf: + - items: + type: string + type: array + - items: + $ref: '#/components/schemas/DecisionLevel' + type: array + - type: 'null' + title: Answer Options + type: object + required: + - model_name + - question + title: CreateDecisionScorerVersionRequest + description: 'The question and model for a new decision scorer version. + + + The inherited scoreable_node_types, output_type and input_type are optional, but each must + + match the scorer when sent. The service checks these fields, and it checks answer_options + + against the stored output_type.' CreateJobRequest: properties: resource_limits: @@ -22724,6 +22832,77 @@ components: default: column type: object title: DatasetVersionIndexSort + DecisionLevel: + properties: + level: + type: integer + maximum: 9.0 + minimum: 0.0 + title: Level + label: + type: string + title: Label + description: + anyOf: + - type: string + - type: 'null' + title: Description + type: object + required: + - level + - label + title: DecisionLevel + description: One level of a score question. A question's levels run 0, 1, 2 and so on, with no gaps. + DecisionScorerDB: + properties: + id: + type: string + format: uuid4 + title: Id + organization_id: + type: string + format: uuid4 + title: Organization Id + created_by: + anyOf: + - type: string + format: uuid4 + - type: 'null' + title: Created By + question: + type: string + title: Question + answer_options: + anyOf: + - items: + type: string + type: array + - items: + $ref: '#/components/schemas/DecisionLevel' + type: array + - type: 'null' + title: Answer Options + created_at: + type: string + format: date-time + title: Created At + type: object + required: + - id + - organization_id + - question + - created_at + title: DecisionScorerDB + description: 'A row of the decision_scorers table. + + + answer_options depends on the output type of the scorer: + + - categorical: a list of labels (list[str]). + + - discrete: a list of DecisionLevel. + + - boolean_probability: None.' DeletePromptResponse: properties: message: @@ -36306,6 +36485,7 @@ components: - multilabel - retrieved_chunk_list_boolean - boolean_multilabel + - boolean_probability title: OutputTypeEnum description: Enumeration of output types. OverrideAction: diff --git a/src/splunk_ao/resources/api/data/create_decision_scorer_version_scorers_scorer_id_version_decision_post.py b/src/splunk_ao/resources/api/data/create_decision_scorer_version_scorers_scorer_id_version_decision_post.py new file mode 100644 index 00000000..c96bd812 --- /dev/null +++ b/src/splunk_ao/resources/api/data/create_decision_scorer_version_scorers_scorer_id_version_decision_post.py @@ -0,0 +1,199 @@ +from http import HTTPStatus +from typing import Any, Optional + +import httpx + +from galileo_core.constants.request_method import RequestMethod +from galileo_core.helpers.api_client import ApiClient +from splunk_ao.exceptions import ( + AuthenticationError, + BadRequestError, + ConflictError, + ForbiddenError, + NotFoundError, + RateLimitError, + ServerError, +) +from splunk_ao.utils.headers_data import get_scorer_capabilities_header, get_sdk_header + +from ... import errors +from ...models.base_scorer_version_response import BaseScorerVersionResponse +from ...models.create_decision_scorer_version_request import CreateDecisionScorerVersionRequest +from ...models.http_validation_error import HTTPValidationError +from ...types import Response + + +def _get_kwargs(scorer_id: str, *, body: CreateDecisionScorerVersionRequest) -> dict[str, Any]: + headers: dict[str, Any] = {} + + _kwargs: dict[str, Any] = { + "method": RequestMethod.POST, + "return_raw_response": True, + "path": "/scorers/{scorer_id}/version/decision".format(scorer_id=scorer_id), + } + + _kwargs["json"] = body.to_dict() + + headers["Content-Type"] = "application/json" + + headers["Splunk-AO-SDK"] = get_sdk_header() + headers["X-Galileo-Scorer-Capabilities"] = get_scorer_capabilities_header() + + _kwargs["content_headers"] = headers + return _kwargs + + +def _parse_response(*, client: ApiClient, response: httpx.Response) -> BaseScorerVersionResponse | HTTPValidationError: + if response.status_code == 200: + response_200 = BaseScorerVersionResponse.from_dict(response.json()) + + return response_200 + + if response.status_code == 422: + response_422 = HTTPValidationError.from_dict(response.json()) + + return response_422 + + # Handle common HTTP errors with actionable messages + if response.status_code == 400: + raise BadRequestError(response.status_code, response.content) + if response.status_code == 401: + raise AuthenticationError(response.status_code, response.content) + if response.status_code == 403: + raise ForbiddenError(response.status_code, response.content) + if response.status_code == 404: + raise NotFoundError(response.status_code, response.content) + if response.status_code == 409: + raise ConflictError(response.status_code, response.content) + if response.status_code == 429: + raise RateLimitError(response.status_code, response.content) + if response.status_code >= 500: + raise ServerError(response.status_code, response.content) + raise errors.UnexpectedStatus(response.status_code, response.content) + + +def _build_response( + *, client: ApiClient, response: httpx.Response +) -> Response[BaseScorerVersionResponse | HTTPValidationError]: + return Response( + status_code=HTTPStatus(response.status_code), + content=response.content, + headers=response.headers, + parsed=_parse_response(client=client, response=response), + ) + + +def sync_detailed( + scorer_id: str, *, client: ApiClient, body: CreateDecisionScorerVersionRequest +) -> Response[BaseScorerVersionResponse | HTTPValidationError]: + """Create Decision Scorer Version + + Create a decision evaluator version that holds its question, answer options and model. + + Args: + scorer_id (str): + body (CreateDecisionScorerVersionRequest): The question and model for a new decision + scorer version. + + The inherited scoreable_node_types, output_type and input_type are optional, but each must + match the scorer when sent. The service checks these fields, and it checks answer_options + against the stored output_type. + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[BaseScorerVersionResponse | HTTPValidationError] + """ + + kwargs = _get_kwargs(scorer_id=scorer_id, body=body) + + response = client.request(**kwargs) + + return _build_response(client=client, response=response) + + +def sync( + scorer_id: str, *, client: ApiClient, body: CreateDecisionScorerVersionRequest +) -> Optional[BaseScorerVersionResponse | HTTPValidationError]: + """Create Decision Scorer Version + + Create a decision evaluator version that holds its question, answer options and model. + + Args: + scorer_id (str): + body (CreateDecisionScorerVersionRequest): The question and model for a new decision + scorer version. + + The inherited scoreable_node_types, output_type and input_type are optional, but each must + match the scorer when sent. The service checks these fields, and it checks answer_options + against the stored output_type. + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + BaseScorerVersionResponse | HTTPValidationError + """ + + return sync_detailed(scorer_id=scorer_id, client=client, body=body).parsed + + +async def asyncio_detailed( + scorer_id: str, *, client: ApiClient, body: CreateDecisionScorerVersionRequest +) -> Response[BaseScorerVersionResponse | HTTPValidationError]: + """Create Decision Scorer Version + + Create a decision evaluator version that holds its question, answer options and model. + + Args: + scorer_id (str): + body (CreateDecisionScorerVersionRequest): The question and model for a new decision + scorer version. + + The inherited scoreable_node_types, output_type and input_type are optional, but each must + match the scorer when sent. The service checks these fields, and it checks answer_options + against the stored output_type. + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[BaseScorerVersionResponse | HTTPValidationError] + """ + + kwargs = _get_kwargs(scorer_id=scorer_id, body=body) + + response = await client.arequest(**kwargs) + + return _build_response(client=client, response=response) + + +async def asyncio( + scorer_id: str, *, client: ApiClient, body: CreateDecisionScorerVersionRequest +) -> Optional[BaseScorerVersionResponse | HTTPValidationError]: + """Create Decision Scorer Version + + Create a decision evaluator version that holds its question, answer options and model. + + Args: + scorer_id (str): + body (CreateDecisionScorerVersionRequest): The question and model for a new decision + scorer version. + + The inherited scoreable_node_types, output_type and input_type are optional, but each must + match the scorer when sent. The service checks these fields, and it checks answer_options + against the stored output_type. + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + BaseScorerVersionResponse | HTTPValidationError + """ + + return (await asyncio_detailed(scorer_id=scorer_id, client=client, body=body)).parsed diff --git a/src/splunk_ao/resources/models/__init__.py b/src/splunk_ao/resources/models/__init__.py index d43d4175..37b5643c 100644 --- a/src/splunk_ao/resources/models/__init__.py +++ b/src/splunk_ao/resources/models/__init__.py @@ -225,6 +225,7 @@ from .create_code_metric_generation_request import CreateCodeMetricGenerationRequest from .create_code_metric_generation_response import CreateCodeMetricGenerationResponse from .create_custom_luna_scorer_version_request import CreateCustomLunaScorerVersionRequest +from .create_decision_scorer_version_request import CreateDecisionScorerVersionRequest from .create_job_request import CreateJobRequest from .create_job_request_validation_config_type_0 import CreateJobRequestValidationConfigType0 from .create_job_response import CreateJobResponse @@ -439,6 +440,8 @@ from .dataset_used_in_project_filter import DatasetUsedInProjectFilter from .dataset_version_db import DatasetVersionDB from .dataset_version_index_sort import DatasetVersionIndexSort +from .decision_level import DecisionLevel +from .decision_scorer_db import DecisionScorerDB from .delete_prompt_response import DeletePromptResponse from .delete_run_response import DeleteRunResponse from .delete_scorer_response import DeleteScorerResponse @@ -1549,6 +1552,7 @@ "CreateCodeMetricGenerationRequest", "CreateCodeMetricGenerationResponse", "CreateCustomLunaScorerVersionRequest", + "CreateDecisionScorerVersionRequest", "CreateJobRequest", "CreateJobRequestValidationConfigType0", "CreateJobResponse", @@ -1689,6 +1693,8 @@ "DataType", "DataTypeOptions", "DataUnit", + "DecisionLevel", + "DecisionScorerDB", "DeletePromptResponse", "DeleteRunResponse", "DeleteScorerResponse", diff --git a/src/splunk_ao/resources/models/base_scorer_version_db.py b/src/splunk_ao/resources/models/base_scorer_version_db.py index 9ab20ebe..6cd812e1 100644 --- a/src/splunk_ao/resources/models/base_scorer_version_db.py +++ b/src/splunk_ao/resources/models/base_scorer_version_db.py @@ -14,6 +14,7 @@ from ..models.base_finetuned_scorer_db import BaseFinetunedScorerDB from ..models.base_generated_scorer_db import BaseGeneratedScorerDB from ..models.base_registered_scorer_db import BaseRegisteredScorerDB + from ..models.decision_scorer_db import DecisionScorerDB T = TypeVar("T", bound="BaseScorerVersionDB") @@ -30,6 +31,7 @@ class BaseScorerVersionDB: generated_scorer (BaseGeneratedScorerDB | None | Unset): registered_scorer (BaseRegisteredScorerDB | None | Unset): finetuned_scorer (BaseFinetunedScorerDB | None | Unset): + decision_scorer (DecisionScorerDB | None | Unset): model_name (None | str | Unset): num_judges (int | None | Unset): scoreable_node_types (list[str] | None | Unset): List of node types that can be scored by this scorer. Defaults @@ -48,6 +50,7 @@ class BaseScorerVersionDB: generated_scorer: BaseGeneratedScorerDB | None | Unset = UNSET registered_scorer: BaseRegisteredScorerDB | None | Unset = UNSET finetuned_scorer: BaseFinetunedScorerDB | None | Unset = UNSET + decision_scorer: DecisionScorerDB | None | Unset = UNSET model_name: None | str | Unset = UNSET num_judges: int | None | Unset = UNSET scoreable_node_types: list[str] | None | Unset = UNSET @@ -60,6 +63,7 @@ def to_dict(self) -> dict[str, Any]: from ..models.base_finetuned_scorer_db import BaseFinetunedScorerDB from ..models.base_generated_scorer_db import BaseGeneratedScorerDB from ..models.base_registered_scorer_db import BaseRegisteredScorerDB + from ..models.decision_scorer_db import DecisionScorerDB id = self.id @@ -91,6 +95,14 @@ def to_dict(self) -> dict[str, Any]: else: finetuned_scorer = self.finetuned_scorer + decision_scorer: dict[str, Any] | None | Unset + if isinstance(self.decision_scorer, Unset): + decision_scorer = UNSET + elif isinstance(self.decision_scorer, DecisionScorerDB): + decision_scorer = self.decision_scorer.to_dict() + else: + decision_scorer = self.decision_scorer + model_name: None | str | Unset if isinstance(self.model_name, Unset): model_name = UNSET @@ -143,6 +155,8 @@ def to_dict(self) -> dict[str, Any]: field_dict["registered_scorer"] = registered_scorer if finetuned_scorer is not UNSET: field_dict["finetuned_scorer"] = finetuned_scorer + if decision_scorer is not UNSET: + field_dict["decision_scorer"] = decision_scorer if model_name is not UNSET: field_dict["model_name"] = model_name if num_judges is not UNSET: @@ -163,6 +177,7 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: from ..models.base_finetuned_scorer_db import BaseFinetunedScorerDB from ..models.base_generated_scorer_db import BaseGeneratedScorerDB from ..models.base_registered_scorer_db import BaseRegisteredScorerDB + from ..models.decision_scorer_db import DecisionScorerDB d = dict(src_dict) id = d.pop("id") @@ -222,6 +237,23 @@ def _parse_finetuned_scorer(data: object) -> BaseFinetunedScorerDB | None | Unse finetuned_scorer = _parse_finetuned_scorer(d.pop("finetuned_scorer", UNSET)) + def _parse_decision_scorer(data: object) -> DecisionScorerDB | None | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + try: + if not isinstance(data, dict): + raise TypeError() + decision_scorer_type_0 = DecisionScorerDB.from_dict(data) + + return decision_scorer_type_0 + except: # noqa: E722 + pass + return cast(DecisionScorerDB | None | Unset, data) + + decision_scorer = _parse_decision_scorer(d.pop("decision_scorer", UNSET)) + def _parse_model_name(data: object) -> None | str | Unset: if data is None: return data @@ -307,6 +339,7 @@ def _parse_input_type(data: object) -> InputTypeEnum | None | Unset: generated_scorer=generated_scorer, registered_scorer=registered_scorer, finetuned_scorer=finetuned_scorer, + decision_scorer=decision_scorer, model_name=model_name, num_judges=num_judges, scoreable_node_types=scoreable_node_types, diff --git a/src/splunk_ao/resources/models/base_scorer_version_response.py b/src/splunk_ao/resources/models/base_scorer_version_response.py index b36eb344..2dcb1ea6 100644 --- a/src/splunk_ao/resources/models/base_scorer_version_response.py +++ b/src/splunk_ao/resources/models/base_scorer_version_response.py @@ -14,6 +14,7 @@ if TYPE_CHECKING: from ..models.chain_poll_template import ChainPollTemplate from ..models.create_update_registered_scorer_response import CreateUpdateRegisteredScorerResponse + from ..models.decision_scorer_db import DecisionScorerDB from ..models.fine_tuned_scorer_response import FineTunedScorerResponse from ..models.generated_scorer_response import GeneratedScorerResponse @@ -33,6 +34,7 @@ class BaseScorerVersionResponse: generated_scorer (GeneratedScorerResponse | None | Unset): registered_scorer (CreateUpdateRegisteredScorerResponse | None | Unset): finetuned_scorer (FineTunedScorerResponse | None | Unset): + decision_scorer (DecisionScorerDB | None | Unset): model_name (None | str | Unset): num_judges (int | None | Unset): scoreable_node_types (list[str] | None | Unset): @@ -53,6 +55,7 @@ class BaseScorerVersionResponse: generated_scorer: GeneratedScorerResponse | None | Unset = UNSET registered_scorer: CreateUpdateRegisteredScorerResponse | None | Unset = UNSET finetuned_scorer: FineTunedScorerResponse | None | Unset = UNSET + decision_scorer: DecisionScorerDB | None | Unset = UNSET model_name: None | str | Unset = UNSET num_judges: int | None | Unset = UNSET scoreable_node_types: list[str] | None | Unset = UNSET @@ -67,6 +70,7 @@ class BaseScorerVersionResponse: def to_dict(self) -> dict[str, Any]: from ..models.chain_poll_template import ChainPollTemplate from ..models.create_update_registered_scorer_response import CreateUpdateRegisteredScorerResponse + from ..models.decision_scorer_db import DecisionScorerDB from ..models.fine_tuned_scorer_response import FineTunedScorerResponse from ..models.generated_scorer_response import GeneratedScorerResponse @@ -104,6 +108,14 @@ def to_dict(self) -> dict[str, Any]: else: finetuned_scorer = self.finetuned_scorer + decision_scorer: dict[str, Any] | None | Unset + if isinstance(self.decision_scorer, Unset): + decision_scorer = UNSET + elif isinstance(self.decision_scorer, DecisionScorerDB): + decision_scorer = self.decision_scorer.to_dict() + else: + decision_scorer = self.decision_scorer + model_name: None | str | Unset if isinstance(self.model_name, Unset): model_name = UNSET @@ -178,6 +190,8 @@ def to_dict(self) -> dict[str, Any]: field_dict["registered_scorer"] = registered_scorer if finetuned_scorer is not UNSET: field_dict["finetuned_scorer"] = finetuned_scorer + if decision_scorer is not UNSET: + field_dict["decision_scorer"] = decision_scorer if model_name is not UNSET: field_dict["model_name"] = model_name if num_judges is not UNSET: @@ -203,6 +217,7 @@ def to_dict(self) -> dict[str, Any]: def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: from ..models.chain_poll_template import ChainPollTemplate from ..models.create_update_registered_scorer_response import CreateUpdateRegisteredScorerResponse + from ..models.decision_scorer_db import DecisionScorerDB from ..models.fine_tuned_scorer_response import FineTunedScorerResponse from ..models.generated_scorer_response import GeneratedScorerResponse @@ -268,6 +283,23 @@ def _parse_finetuned_scorer(data: object) -> FineTunedScorerResponse | None | Un finetuned_scorer = _parse_finetuned_scorer(d.pop("finetuned_scorer", UNSET)) + def _parse_decision_scorer(data: object) -> DecisionScorerDB | None | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + try: + if not isinstance(data, dict): + raise TypeError() + decision_scorer_type_0 = DecisionScorerDB.from_dict(data) + + return decision_scorer_type_0 + except: # noqa: E722 + pass + return cast(DecisionScorerDB | None | Unset, data) + + decision_scorer = _parse_decision_scorer(d.pop("decision_scorer", UNSET)) + def _parse_model_name(data: object) -> None | str | Unset: if data is None: return data @@ -390,6 +422,7 @@ def _parse_created_by(data: object) -> None | str | Unset: generated_scorer=generated_scorer, registered_scorer=registered_scorer, finetuned_scorer=finetuned_scorer, + decision_scorer=decision_scorer, model_name=model_name, num_judges=num_judges, scoreable_node_types=scoreable_node_types, diff --git a/src/splunk_ao/resources/models/create_decision_scorer_version_request.py b/src/splunk_ao/resources/models/create_decision_scorer_version_request.py new file mode 100644 index 00000000..4bc7aea2 --- /dev/null +++ b/src/splunk_ao/resources/models/create_decision_scorer_version_request.py @@ -0,0 +1,260 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, TypeVar, cast + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..models.input_type_enum import InputTypeEnum +from ..models.output_type_enum import OutputTypeEnum +from ..types import UNSET, Unset + +if TYPE_CHECKING: + from ..models.decision_level import DecisionLevel + + +T = TypeVar("T", bound="CreateDecisionScorerVersionRequest") + + +@_attrs_define +class CreateDecisionScorerVersionRequest: + """The question and model for a new decision scorer version. + + The inherited scoreable_node_types, output_type and input_type are optional, but each must + match the scorer when sent. The service checks these fields, and it checks answer_options + against the stored output_type. + + Attributes: + model_name (str): + question (str): + num_judges (int | None | Unset): + scoreable_node_types (list[str] | None | Unset): + cot_enabled (bool | None | Unset): + output_type (None | OutputTypeEnum | Unset): + input_type (InputTypeEnum | None | Unset): + answer_options (list[DecisionLevel] | list[str] | None | Unset): + """ + + model_name: str + question: str + num_judges: int | None | Unset = UNSET + scoreable_node_types: list[str] | None | Unset = UNSET + cot_enabled: bool | None | Unset = UNSET + output_type: None | OutputTypeEnum | Unset = UNSET + input_type: InputTypeEnum | None | Unset = UNSET + answer_options: list[DecisionLevel] | list[str] | None | Unset = UNSET + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + model_name = self.model_name + + question = self.question + + num_judges: int | None | Unset + if isinstance(self.num_judges, Unset): + num_judges = UNSET + else: + num_judges = self.num_judges + + scoreable_node_types: list[str] | None | Unset + if isinstance(self.scoreable_node_types, Unset): + scoreable_node_types = UNSET + elif isinstance(self.scoreable_node_types, list): + scoreable_node_types = self.scoreable_node_types + + else: + scoreable_node_types = self.scoreable_node_types + + cot_enabled: bool | None | Unset + if isinstance(self.cot_enabled, Unset): + cot_enabled = UNSET + else: + cot_enabled = self.cot_enabled + + output_type: None | str | Unset + if isinstance(self.output_type, Unset): + output_type = UNSET + elif isinstance(self.output_type, OutputTypeEnum): + output_type = self.output_type.value + else: + output_type = self.output_type + + input_type: None | str | Unset + if isinstance(self.input_type, Unset): + input_type = UNSET + elif isinstance(self.input_type, InputTypeEnum): + input_type = self.input_type.value + else: + input_type = self.input_type + + answer_options: list[dict[str, Any]] | list[str] | None | Unset + if isinstance(self.answer_options, Unset): + answer_options = UNSET + elif isinstance(self.answer_options, list): + answer_options = self.answer_options + + elif isinstance(self.answer_options, list): + answer_options = [] + for answer_options_type_1_item_data in self.answer_options: + answer_options_type_1_item = answer_options_type_1_item_data.to_dict() + answer_options.append(answer_options_type_1_item) + + else: + answer_options = self.answer_options + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update({"model_name": model_name, "question": question}) + if num_judges is not UNSET: + field_dict["num_judges"] = num_judges + if scoreable_node_types is not UNSET: + field_dict["scoreable_node_types"] = scoreable_node_types + if cot_enabled is not UNSET: + field_dict["cot_enabled"] = cot_enabled + if output_type is not UNSET: + field_dict["output_type"] = output_type + if input_type is not UNSET: + field_dict["input_type"] = input_type + if answer_options is not UNSET: + field_dict["answer_options"] = answer_options + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + from ..models.decision_level import DecisionLevel + + d = dict(src_dict) + model_name = d.pop("model_name") + + question = d.pop("question") + + def _parse_num_judges(data: object) -> int | None | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(int | None | Unset, data) + + num_judges = _parse_num_judges(d.pop("num_judges", UNSET)) + + def _parse_scoreable_node_types(data: object) -> list[str] | None | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + try: + if not isinstance(data, list): + raise TypeError() + scoreable_node_types_type_0 = cast(list[str], data) + + return scoreable_node_types_type_0 + except: # noqa: E722 + pass + return cast(list[str] | None | Unset, data) + + scoreable_node_types = _parse_scoreable_node_types(d.pop("scoreable_node_types", UNSET)) + + def _parse_cot_enabled(data: object) -> bool | None | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(bool | None | Unset, data) + + cot_enabled = _parse_cot_enabled(d.pop("cot_enabled", UNSET)) + + def _parse_output_type(data: object) -> None | OutputTypeEnum | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + try: + if not isinstance(data, str): + raise TypeError() + output_type_type_0 = OutputTypeEnum(data) + + return output_type_type_0 + except: # noqa: E722 + pass + return cast(None | OutputTypeEnum | Unset, data) + + output_type = _parse_output_type(d.pop("output_type", UNSET)) + + def _parse_input_type(data: object) -> InputTypeEnum | None | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + try: + if not isinstance(data, str): + raise TypeError() + input_type_type_0 = InputTypeEnum(data) + + return input_type_type_0 + except: # noqa: E722 + pass + return cast(InputTypeEnum | None | Unset, data) + + input_type = _parse_input_type(d.pop("input_type", UNSET)) + + def _parse_answer_options(data: object) -> list[DecisionLevel] | list[str] | None | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + try: + if not isinstance(data, list): + raise TypeError() + answer_options_type_0 = cast(list[str], data) + + return answer_options_type_0 + except: # noqa: E722 + pass + try: + if not isinstance(data, list): + raise TypeError() + answer_options_type_1 = [] + _answer_options_type_1 = data + for answer_options_type_1_item_data in _answer_options_type_1: + answer_options_type_1_item = DecisionLevel.from_dict(answer_options_type_1_item_data) + + answer_options_type_1.append(answer_options_type_1_item) + + return answer_options_type_1 + except: # noqa: E722 + pass + return cast(list[DecisionLevel] | list[str] | None | Unset, data) + + answer_options = _parse_answer_options(d.pop("answer_options", UNSET)) + + create_decision_scorer_version_request = cls( + model_name=model_name, + question=question, + num_judges=num_judges, + scoreable_node_types=scoreable_node_types, + cot_enabled=cot_enabled, + output_type=output_type, + input_type=input_type, + answer_options=answer_options, + ) + + create_decision_scorer_version_request.additional_properties = d + return create_decision_scorer_version_request + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/src/splunk_ao/resources/models/decision_level.py b/src/splunk_ao/resources/models/decision_level.py new file mode 100644 index 00000000..fb295461 --- /dev/null +++ b/src/splunk_ao/resources/models/decision_level.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar, cast + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..types import UNSET, Unset + +T = TypeVar("T", bound="DecisionLevel") + + +@_attrs_define +class DecisionLevel: + """One level of a score question. A question's levels run 0, 1, 2 and so on, with no gaps. + + Attributes: + level (int): + label (str): + description (None | str | Unset): + """ + + level: int + label: str + description: None | str | Unset = UNSET + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + level = self.level + + label = self.label + + description: None | str | Unset + if isinstance(self.description, Unset): + description = UNSET + else: + description = self.description + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update({"level": level, "label": label}) + if description is not UNSET: + field_dict["description"] = description + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + level = d.pop("level") + + label = d.pop("label") + + def _parse_description(data: object) -> None | str | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(None | str | Unset, data) + + description = _parse_description(d.pop("description", UNSET)) + + decision_level = cls(level=level, label=label, description=description) + + decision_level.additional_properties = d + return decision_level + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/src/splunk_ao/resources/models/decision_scorer_db.py b/src/splunk_ao/resources/models/decision_scorer_db.py new file mode 100644 index 00000000..6ab50066 --- /dev/null +++ b/src/splunk_ao/resources/models/decision_scorer_db.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +import datetime +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, TypeVar, cast + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..types import UNSET, Unset + +if TYPE_CHECKING: + from ..models.decision_level import DecisionLevel + + +T = TypeVar("T", bound="DecisionScorerDB") + + +@_attrs_define +class DecisionScorerDB: + """A row of the decision_scorers table. + + answer_options depends on the output type of the scorer: + - categorical: a list of labels (list[str]). + - discrete: a list of DecisionLevel. + - boolean_probability: None. + + Attributes: + id (str): + organization_id (str): + question (str): + created_at (datetime.datetime): + created_by (None | str | Unset): + answer_options (list[DecisionLevel] | list[str] | None | Unset): + """ + + id: str + organization_id: str + question: str + created_at: datetime.datetime + created_by: None | str | Unset = UNSET + answer_options: list[DecisionLevel] | list[str] | None | Unset = UNSET + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + id = self.id + + organization_id = self.organization_id + + question = self.question + + created_at = self.created_at.isoformat() + + created_by: None | str | Unset + if isinstance(self.created_by, Unset): + created_by = UNSET + else: + created_by = self.created_by + + answer_options: list[dict[str, Any]] | list[str] | None | Unset + if isinstance(self.answer_options, Unset): + answer_options = UNSET + elif isinstance(self.answer_options, list): + answer_options = self.answer_options + + elif isinstance(self.answer_options, list): + answer_options = [] + for answer_options_type_1_item_data in self.answer_options: + answer_options_type_1_item = answer_options_type_1_item_data.to_dict() + answer_options.append(answer_options_type_1_item) + + else: + answer_options = self.answer_options + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + {"id": id, "organization_id": organization_id, "question": question, "created_at": created_at} + ) + if created_by is not UNSET: + field_dict["created_by"] = created_by + if answer_options is not UNSET: + field_dict["answer_options"] = answer_options + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + from ..models.decision_level import DecisionLevel + + d = dict(src_dict) + id = d.pop("id") + + organization_id = d.pop("organization_id") + + question = d.pop("question") + + created_at = datetime.datetime.fromisoformat(d.pop("created_at")) + + def _parse_created_by(data: object) -> None | str | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(None | str | Unset, data) + + created_by = _parse_created_by(d.pop("created_by", UNSET)) + + def _parse_answer_options(data: object) -> list[DecisionLevel] | list[str] | None | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + try: + if not isinstance(data, list): + raise TypeError() + answer_options_type_0 = cast(list[str], data) + + return answer_options_type_0 + except: # noqa: E722 + pass + try: + if not isinstance(data, list): + raise TypeError() + answer_options_type_1 = [] + _answer_options_type_1 = data + for answer_options_type_1_item_data in _answer_options_type_1: + answer_options_type_1_item = DecisionLevel.from_dict(answer_options_type_1_item_data) + + answer_options_type_1.append(answer_options_type_1_item) + + return answer_options_type_1 + except: # noqa: E722 + pass + return cast(list[DecisionLevel] | list[str] | None | Unset, data) + + answer_options = _parse_answer_options(d.pop("answer_options", UNSET)) + + decision_scorer_db = cls( + id=id, + organization_id=organization_id, + question=question, + created_at=created_at, + created_by=created_by, + answer_options=answer_options, + ) + + decision_scorer_db.additional_properties = d + return decision_scorer_db + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/src/splunk_ao/resources/models/output_type_enum.py b/src/splunk_ao/resources/models/output_type_enum.py index 076bbb2d..61b54dab 100644 --- a/src/splunk_ao/resources/models/output_type_enum.py +++ b/src/splunk_ao/resources/models/output_type_enum.py @@ -4,6 +4,7 @@ class OutputTypeEnum(str, Enum): BOOLEAN = "boolean" BOOLEAN_MULTILABEL = "boolean_multilabel" + BOOLEAN_PROBABILITY = "boolean_probability" CATEGORICAL = "categorical" COUNT = "count" DISCRETE = "discrete" From 9b1c43ec0144ecb58c6d42897c14a1706cfaafb0 Mon Sep 17 00:00:00 2001 From: John Weiler Date: Fri, 16 Oct 2026 09:41:21 -0400 Subject: [PATCH 3/7] feat(sdk): add create_decision_evaluator and DecisionEvaluator (SAO-17973) Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 10 + src/splunk_ao/__future__/__init__.py | 11 +- src/splunk_ao/__init__.py | 10 +- src/splunk_ao/evaluator.py | 218 +++++++++++++++- src/splunk_ao/evaluators.py | 361 +++++++++++++++++++++++++++ tests/test_decision_scorers.py | 147 +++++++++++ tests/test_evaluator.py | 189 +++++++++++++- tests/test_evaluator_types.py | 93 ++++++- tests/test_evaluators.py | 340 ++++++++++++++++++++++++- 9 files changed, 1370 insertions(+), 9 deletions(-) create mode 100644 tests/test_decision_scorers.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 963fa582..72556781 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- `create_decision_evaluator()` (also `Evaluators.create_decision_evaluator`) and `DecisionEvaluator` create and + read decision-model evaluators: one typed question per evaluator, answered with probabilities. Supports + `boolean_probability`, `categorical` and `discrete` output types; when `output_type` is omitted it is inferred + from `answer_options`. If the version request is clearly rejected (400, 403, 404, 422) the new scorer is + deleted; on a timeout, network error or 5xx it is kept and the raised error names its id. +- Scorer requests now send `X-Galileo-Scorer-Capabilities: decision`, and `ScorerTypes` and `ModelType` include + `decision`, so decision scorers appear in list and get responses and decode. + ### Fixed - `SplunkAOTracingProcessor` (OpenAI Agents) now records Responses API LLM spans, the Agents SDK diff --git a/src/splunk_ao/__future__/__init__.py b/src/splunk_ao/__future__/__init__.py index efe7bae5..225649df 100644 --- a/src/splunk_ao/__future__/__init__.py +++ b/src/splunk_ao/__future__/__init__.py @@ -8,7 +8,15 @@ from splunk_ao.experiment import Experiment from splunk_ao.integration import Integration from splunk_ao.agent_stream import AgentStream -from splunk_ao.evaluator import BuiltInEvaluators, CodeEvaluator, Evaluator, LlmEvaluator, LocalEvaluator, SplunkAOEvaluator +from splunk_ao.evaluator import ( + BuiltInEvaluators, + CodeEvaluator, + DecisionEvaluator, + Evaluator, + LlmEvaluator, + LocalEvaluator, + SplunkAOEvaluator, +) from splunk_ao.model import Model from splunk_ao.project import Project from splunk_ao.prompt import Prompt @@ -34,6 +42,7 @@ "Configuration", "ConfigurationError", "Dataset", + "DecisionEvaluator", "Evaluator", "Experiment", "Integration", diff --git a/src/splunk_ao/__init__.py b/src/splunk_ao/__init__.py index e2f92824..4b2b0b8f 100644 --- a/src/splunk_ao/__init__.py +++ b/src/splunk_ao/__init__.py @@ -44,7 +44,14 @@ from splunk_ao.configuration import Configuration from splunk_ao.dataset import Dataset from splunk_ao.decorator import SplunkAODecorator, log, splunk_ao_context, start_session -from splunk_ao.evaluator import CodeEvaluator, Evaluator, LlmEvaluator, LocalEvaluator, SplunkAOEvaluator +from splunk_ao.evaluator import ( + CodeEvaluator, + DecisionEvaluator, + Evaluator, + LlmEvaluator, + LocalEvaluator, + SplunkAOEvaluator, +) from splunk_ao.exceptions import ( AuthenticationError, BadRequestError, @@ -117,6 +124,7 @@ "ControlResult", "ControlSpan", "Dataset", + "DecisionEvaluator", "Document", "Evaluator", "Experiment", diff --git a/src/splunk_ao/evaluator.py b/src/splunk_ao/evaluator.py index 587b7cfa..9a6348bc 100644 --- a/src/splunk_ao/evaluator.py +++ b/src/splunk_ao/evaluator.py @@ -19,7 +19,13 @@ from galileo_core.schemas.shared.metric import MetricValueType from splunk_ao.config import SplunkAOConfig from splunk_ao.configuration import Configuration -from splunk_ao.evaluators import Evaluators +from splunk_ao.evaluators import ( + DEFAULT_DECISION_MODEL_NAME, + DecisionAnswerOptions, + Evaluators, + normalize_decision_answer_options, + resolve_decision_output_type, +) from splunk_ao.resources.api.data import ( create_code_scorer_version_scorers_scorer_id_version_code_post, create_scorers_post, @@ -31,7 +37,9 @@ BodyCreateCodeScorerVersionScorersScorerIdVersionCodePost, BodyValidateCodeScorerScorersCodeValidatePost, CreateScorerRequest, + DecisionLevel, HTTPValidationError, + InputTypeEnum, OutputTypeEnum, ScorerTypes, TaskResultStatus, @@ -94,6 +102,7 @@ class Evaluator(StateManagementMixin, ABC): - **SplunkAOEvaluator**: Built-in Splunk AO evaluators (access via Evaluator.metrics) - **LlmEvaluator**: Custom LLM-based metrics with prompt templates + - **DecisionEvaluator**: Decision model evaluators (one typed question per evaluator) - **LocalEvaluator**: Local function-based metrics - **CodeEvaluator**: Code-based metrics (future support) @@ -226,7 +235,7 @@ def _create_metric_from_type(cls, scorer_type: ScorerTypes) -> Evaluator: Returns ------- Evaluator: An uninitialized instance of the appropriate subclass - (LlmEvaluator, CodeEvaluator, or SplunkAOEvaluator). + (LlmEvaluator, CodeEvaluator, DecisionEvaluator, or SplunkAOEvaluator). Examples -------- @@ -237,6 +246,8 @@ def _create_metric_from_type(cls, scorer_type: ScorerTypes) -> Evaluator: return LlmEvaluator.__new__(LlmEvaluator) if scorer_type == ScorerTypes.CODE: return CodeEvaluator.__new__(CodeEvaluator) + if scorer_type == ScorerTypes.DECISION: + return DecisionEvaluator.__new__(DecisionEvaluator) # Default to SplunkAOEvaluator for built-in scorers (LUNA, PRESET, etc.) return SplunkAOEvaluator.__new__(SplunkAOEvaluator) @@ -253,7 +264,8 @@ def get(cls, *, id: str | None = None, name: str | None = None) -> Evaluator | N Returns ------- - Optional[Evaluator]: The metric if found (SplunkAOEvaluator, LlmEvaluator, or CodeEvaluator), None otherwise. + Optional[Evaluator]: The metric if found (SplunkAOEvaluator, LlmEvaluator, CodeEvaluator, or DecisionEvaluator), + None otherwise. Raises ------ @@ -432,6 +444,10 @@ def _populate_from_scorer_response(self, scorer_response: Any) -> None: self._sync_attrs(output_type=output_type, prompt=prompt, node_level=node_level, ground_truth=ground_truth) + # Decision-specific attributes (only set if this is a DecisionEvaluator) + if isinstance(self, DecisionEvaluator): + self._populate_decision_attrs(scorer_response) + # Code-specific attributes (only set if this is a CodeEvaluator) if isinstance(self, CodeEvaluator): # Extract scoreable node types @@ -843,6 +859,202 @@ def __repr__(self) -> str: return f"LlmEvaluator(name='{self.name}', id='{self.id}', model='{self.model}', judges={self.judges})" +class DecisionEvaluator(Evaluator): + """ + Decision model evaluator. + + A decision evaluator asks a decision model (for example TypeSafe's Jev) one typed question and scores + the node with the probabilities of the answers. + + Attributes + ---------- + question (str | None): The question the model answers. + model (str | None): Decision model registry version, for example ``jev-1.13.0``. + node_level (StepType | None): The single node type the evaluator scores. + output_type (OutputTypeEnum | None): ``BOOLEAN_PROBABILITY``, ``CATEGORICAL`` or ``DISCRETE``. + answer_options (list[str] | list[DecisionLevel] | None): Labels (categorical), levels (discrete) or None. + input_type (InputTypeEnum | None): What the model sees. None lets the server choose. + ground_truth (bool): Whether the scorer requires ground truth from the dataset. + + Examples + -------- + evaluator = DecisionEvaluator( + name="tone", + question="Is the tone of this response professional?", + output_type=OutputTypeEnum.CATEGORICAL, + answer_options=["professional", "casual", "rude"], + ).create() + """ + + question: str | None + model: str | None + node_level: StepType | None + output_type: OutputTypeEnum | None + answer_options: list[str] | list[DecisionLevel] | None + input_type: InputTypeEnum | None + ground_truth: bool + + def __init__( + self, + name: str, + *, + question: str | None = None, + model: str | None = None, + node_level: StepType | None = None, + output_type: str | OutputTypeEnum | None = None, + answer_options: DecisionAnswerOptions | None = None, + input_type: InputTypeEnum | None = None, + ground_truth: bool = False, + description: str = "", + tags: list[str] | None = None, + version: int | None = None, + ) -> None: + """ + Initialize a decision evaluator. + + Args: + name: The name of the evaluator. + question: The question the decision model answers (1-2,000 characters). + model: Decision model registry version. Defaults to ``jev-1.13.0``. + node_level: The single node type the evaluator scores. Defaults to StepType.llm. + output_type: ``boolean_probability``, ``categorical`` or ``discrete``. Inferred from ``answer_options`` when None. + answer_options: None for boolean_probability, a list of labels for categorical, or a list of + ``DecisionLevel`` / ``{level, label, description}`` dicts for discrete. + input_type: What the model sees. None lets the server choose a default for the node type. + ground_truth: Whether the scorer requires ground truth (``reference_output``) from the dataset. + description: Description of the evaluator. + tags: Tags associated with the evaluator. + version: Specific version to reference (for existing evaluators). + + Raises + ------ + ValidationError: If question is not provided. + """ + super().__init__(name=name, description=description, tags=tags, version=version) + + if not question: + raise ValidationError("'question' must be provided for decision evaluators.") + + self.question = question + self.model = model or DEFAULT_DECISION_MODEL_NAME + self.node_level = node_level or StepType.llm + normalized_options = normalize_decision_answer_options(answer_options) + if isinstance(output_type, str) and not isinstance(output_type, OutputTypeEnum): + output_type = output_type.lower() + try: + self.output_type = resolve_decision_output_type(output_type, normalized_options) + except ValueError as error: + raise ValidationError(str(error)) from error + self.answer_options = normalized_options + self.input_type = input_type + self.ground_truth = ground_truth + self.scorer_type = ScorerTypes.DECISION + + def create(self) -> DecisionEvaluator: + """ + Persist this decision evaluator to the API. + + Returns + ------- + DecisionEvaluator: This evaluator instance with updated attributes from the API. + + Raises + ------ + ValueError: If an argument is invalid or the API answers with a validation error. + SplunkAOAPIError: For HTTP errors. A scorer created before a clear rejection is deleted. + + Examples + -------- + evaluator = DecisionEvaluator(name="tone", question="Is the tone professional?").create() + assert evaluator.is_synced() + """ + try: + logger.info(f"DecisionEvaluator.create: name='{self.name}' - started") + + created_version = Evaluators().create_decision_evaluator( + name=self.name, + question=self.question or "", + node_level=self.node_level if self.node_level is not None else StepType.llm, + model_name=self.model if self.model is not None else DEFAULT_DECISION_MODEL_NAME, + output_type=self.output_type + if isinstance(self.output_type, OutputTypeEnum) + else OutputTypeEnum.BOOLEAN_PROBABILITY, + answer_options=self.answer_options, + input_type=self.input_type, + description=self.description, + tags=self.tags, + ground_truth=self.ground_truth, + ) + + self._sync_attrs( + id=str(created_version.scorer_id), + created_at=created_version.created_at, + updated_at=created_version.updated_at, + ) + + # Refresh to get full scorer details + self.refresh() + + logger.info(f"DecisionEvaluator.create: id='{self.id}' - completed") + return self + except Exception as e: + self._set_state(SyncState.FAILED_SYNC, error=e) + logger.error(f"DecisionEvaluator.create: name='{self.name}' - failed: {e}") + raise + + def _populate_decision_attrs(self, scorer_response: Any) -> None: + """Fill decision fields from a scorer response. + + The question and answer options live on the scorer's version. When the response carries no version, + existing values are kept and instances built by ``Evaluator.get`` start as None. + """ + if not hasattr(self, "question"): + self._sync_attrs(question=None, answer_options=None) + output_type = None if isinstance(scorer_response.output_type, Unset) else scorer_response.output_type + input_type = None if isinstance(scorer_response.input_type, Unset) else scorer_response.input_type + scoreable = scorer_response.scoreable_node_types + node_level: StepType | None = None + if not isinstance(scoreable, Unset) and scoreable: + try: + node_level = StepType(scoreable[0]) + except ValueError: + node_level = None + ground_truth = ( + False + if isinstance(scorer_response.ground_truth, Unset) or scorer_response.ground_truth is None + else scorer_response.ground_truth + ) + self._sync_attrs( + output_type=output_type, input_type=input_type, node_level=node_level, ground_truth=ground_truth + ) + + for version in (scorer_response.default_version, scorer_response.latest_version): + if version is None or isinstance(version, Unset): + continue + decision = version.decision_scorer + if decision is None or isinstance(decision, Unset): + continue + self._sync_attrs(question=decision.question, answer_options=self._read_answer_options(decision)) + if not isinstance(version.model_name, Unset) and version.model_name: + self._sync_attrs(model=version.model_name) + break + + @staticmethod + def _read_answer_options(decision: Any) -> Any: + """Convert server answer options leniently: a read must never raise on unexpected data.""" + try: + return normalize_decision_answer_options(decision.answer_options) + except ValueError as error: + logger.warning(f"DecisionEvaluator: keeping raw answer_options for decision_scorer {decision.id}: {error}") + return decision.answer_options + + def __repr__(self) -> str: + """Detailed string representation of the evaluator.""" + return ( + f"DecisionEvaluator(name='{self.name}', id='{self.id}', model='{self.model}', question='{self.question}')" + ) + + class CodeEvaluator(Evaluator): r""" Code-based metric. diff --git a/src/splunk_ao/evaluators.py b/src/splunk_ao/evaluators.py index 2e53932f..7990ee26 100644 --- a/src/splunk_ao/evaluators.py +++ b/src/splunk_ao/evaluators.py @@ -1,9 +1,13 @@ import datetime import logging +from collections.abc import Mapping, Sequence +from typing import Any from galileo_core.schemas.logging.step import StepType from splunk_ao.config import SplunkAOConfig +from splunk_ao.exceptions import SplunkAOAPIError from splunk_ao.resources.api.data import ( + create_decision_scorer_version_scorers_scorer_id_version_decision_post, create_llm_scorer_version_scorers_scorer_id_version_llm_post, create_scorers_post, delete_scorer_scorers_scorer_id_delete, @@ -16,15 +20,151 @@ ScorerTypes, ) from splunk_ao.resources.models.base_scorer_version_response import BaseScorerVersionResponse +from splunk_ao.resources.models.create_decision_scorer_version_request import CreateDecisionScorerVersionRequest from splunk_ao.resources.models.create_llm_scorer_version_request import CreateLLMScorerVersionRequest from splunk_ao.resources.models.create_scorer_request import CreateScorerRequest +from splunk_ao.resources.models.decision_level import DecisionLevel +from splunk_ao.resources.models.input_type_enum import InputTypeEnum from splunk_ao.resources.models.output_type_enum import OutputTypeEnum from splunk_ao.resources.models.scorer_defaults import ScorerDefaults +from splunk_ao.resources.types import UNSET, Unset from splunk_ao.scorers import Scorers from splunk_ao.search import FilterType _logger = logging.getLogger(__name__) +DEFAULT_DECISION_MODEL_NAME = "jev-1.13.0" + +DecisionAnswerOptions = Sequence[str] | Sequence[DecisionLevel | Mapping[str, Any]] + + +def normalize_decision_answer_options( + answer_options: DecisionAnswerOptions | Unset | None, +) -> list[str] | list[DecisionLevel] | None: + """Convert user-facing answer options into the request shape. + + Strings pass through (categorical). ``DecisionLevel`` objects and ``{level, label, description}`` dicts + become ``DecisionLevel`` (discrete). ``None`` stays ``None`` (boolean_probability). + """ + if answer_options is None or isinstance(answer_options, Unset): + return None + if isinstance(answer_options, str | bytes | Mapping): + raise ValueError("answer_options must be a list, not a single string or dict.") + options = list(answer_options) + if not options: + raise ValueError("answer_options must not be empty; pass None for boolean_probability.") + if all(isinstance(option, str) for option in options): + return [str(option) for option in options] + levels: list[DecisionLevel] = [] + for index, option in enumerate(options): + if isinstance(option, DecisionLevel): + levels.append(option) + elif isinstance(option, Mapping): + missing = [key for key in ("level", "label") if key not in option] + if missing: + raise ValueError(f"answer_options[{index}] is missing required key(s): {', '.join(missing)}.") + levels.append( + DecisionLevel( + level=option["level"], label=option["label"], description=option.get("description") or UNSET + ) + ) + else: + raise ValueError( + "answer_options must be a list of strings (categorical) or a list of DecisionLevel / " + f"{{level, label, description}} dicts (discrete); got {type(option).__name__} at index {index}." + ) + return levels + + +class DecisionVersionOutcomeUnknownError(RuntimeError): + """The version request failed without a definitive rejection, so the scorer was kept. + + Attributes + ---------- + scorer_id (str): The scorer that may or may not have a version. + """ + + def __init__(self, scorer_id: str, cause: Exception) -> None: + self.scorer_id = scorer_id + super().__init__( + f"Creating the decision evaluator version failed with an unknown outcome; scorer_id={scorer_id} " + f"was kept and may have a version: {cause}" + ) + + +REJECTION_STATUS_CODES = frozenset({400, 403, 404, 422}) + + +class _VersionRejected(ValueError): + """The server answered the version request with a validation error or no body, so no version exists.""" + + +DECISION_OUTPUT_TYPES = (OutputTypeEnum.BOOLEAN_PROBABILITY, OutputTypeEnum.CATEGORICAL, OutputTypeEnum.DISCRETE) + + +def resolve_decision_output_type( + output_type: OutputTypeEnum | str | None, answer_options: list[str] | list[DecisionLevel] | None +) -> OutputTypeEnum: + """Return the explicit output type, or infer it from the normalized answer options. + + ``None`` options mean ``boolean_probability``, strings mean ``categorical`` and levels mean ``discrete``. + An explicit value wins, and the request validation then rejects a mismatch with the options. + """ + if output_type is not None: + try: + resolved = OutputTypeEnum(output_type) + except ValueError: + resolved = None + if resolved not in DECISION_OUTPUT_TYPES: + raise ValueError( + f"output_type must be one of {[t.value for t in DECISION_OUTPUT_TYPES]} for decision evaluators; " + f"got {output_type!r}." + ) + return resolved + if answer_options is None: + return OutputTypeEnum.BOOLEAN_PROBABILITY + if all(isinstance(option, str) for option in answer_options): + return OutputTypeEnum.CATEGORICAL + return OutputTypeEnum.DISCRETE + + +def _validate_decision_request( + output_type: OutputTypeEnum, answer_options: list[str] | list[DecisionLevel] | None +) -> None: + """Reject requests the server would refuse, before anything is created.""" + if output_type not in DECISION_OUTPUT_TYPES: + raise ValueError( + f"output_type must be one of {[t.value for t in DECISION_OUTPUT_TYPES]} for decision evaluators; " + f"got {output_type!r}." + ) + if output_type == OutputTypeEnum.BOOLEAN_PROBABILITY: + if answer_options is not None: + raise ValueError("answer_options must be None for boolean_probability.") + elif output_type == OutputTypeEnum.CATEGORICAL: + if not answer_options or not all(isinstance(option, str) for option in answer_options): + raise ValueError("answer_options must be a list of label strings for categorical.") + elif not answer_options or not all(isinstance(option, DecisionLevel) for option in answer_options): + raise ValueError("answer_options must be a list of levels (DecisionLevel or dict) for discrete.") + + +def _build_decision_version_request( + model_name: str, question: str, answer_options: list[str] | list[DecisionLevel] | None +) -> CreateDecisionScorerVersionRequest: + """Build the version request, serializing discrete levels ourselves. + + The generated ``CreateDecisionScorerVersionRequest.to_dict`` has two identical ``isinstance(list)`` + branches for ``list[str] | list[DecisionLevel]``, so it never calls ``DecisionLevel.to_dict`` and would + send raw objects. ``additional_properties`` is merged into the body verbatim, so levels go there. + Fixing the codegen template touches many unrelated generated models, so it is left as a follow-up. + """ + if answer_options and all(isinstance(level, DecisionLevel) for level in answer_options): + request = CreateDecisionScorerVersionRequest(model_name=model_name, question=question) + request.additional_properties["answer_options"] = [ + level.to_dict() for level in answer_options if isinstance(level, DecisionLevel) + ] + return request + return CreateDecisionScorerVersionRequest(model_name=model_name, question=question, answer_options=answer_options) + class Evaluators: config: SplunkAOConfig @@ -114,6 +254,139 @@ def create_custom_llm_evaluator( return version_resp + def create_decision_evaluator( + self, + name: str, + question: str, + node_level: StepType = StepType.llm, + model_name: str = DEFAULT_DECISION_MODEL_NAME, + output_type: OutputTypeEnum | None = None, + answer_options: DecisionAnswerOptions | None = None, + input_type: InputTypeEnum | None = None, + description: str = "", + tags: list[str] | None = None, + ground_truth: bool = False, + ) -> BaseScorerVersionResponse: + """ + Create a decision evaluator. + + Parameters + ---------- + name: str + Name of the evaluator. + question: str + The question the decision model answers (1-2,000 characters). + node_level: StepType + The single node type the evaluator scores. + model_name: str + Decision model registry version, for example ``jev-1.13.0``. + output_type: OutputTypeEnum | None + ``BOOLEAN_PROBABILITY``, ``CATEGORICAL`` or ``DISCRETE``. When ``None``, it is inferred from + ``answer_options``: ``None`` gives ``BOOLEAN_PROBABILITY``, strings give ``CATEGORICAL`` and levels + give ``DISCRETE``. An explicit value that does not match ``answer_options`` raises ``ValueError``. + answer_options: list[str] | list[DecisionLevel | dict] | None + ``None`` for ``BOOLEAN_PROBABILITY``. For ``CATEGORICAL``, a list of 2-255 unique labels. For + ``DISCRETE``, 2-10 ``DecisionLevel`` objects (or ``{level, label, description}`` dicts) numbered + 0, 1, 2 and so on in order. + input_type: InputTypeEnum | None + What the model sees. When ``None``, the server picks a default for the node type. + description: str + Description of the evaluator. + tags: List[str] + Tags associated with the evaluator. + ground_truth: bool + Whether the scorer requires ground truth (``reference_output``) from the dataset. + + Returns + ------- + BaseScorerVersionResponse + Response containing the created evaluator version. + + Raises + ------ + ValueError + If an argument is invalid or the API answers with a validation error. If the scorer was created + but its version was rejected, the scorer is deleted before the error is raised. + SplunkAOAPIError + For a definitive rejection (400, 403, 404, 422), for example 403 when the feature flag is off. + The scorer is deleted before the error is raised. + DecisionVersionOutcomeUnknownError + For a timeout, network error, 5xx or any other failure. The version may have been saved, so the + scorer is kept and the error names its id. + """ + normalized_options = normalize_decision_answer_options(answer_options) + output_type = resolve_decision_output_type(output_type, normalized_options) + _validate_decision_request(output_type, normalized_options) + version_req = _build_decision_version_request( + model_name=model_name, question=question, answer_options=normalized_options + ) + create_scorer_request = CreateScorerRequest( + name=name, + scorer_type=ScorerTypes.DECISION, + description=description, + tags=tags if tags is not None else [], + scoreable_node_types=[node_level], + output_type=output_type, + input_type=input_type if input_type is not None else UNSET, + ground_truth=ground_truth, + ) + + scorer = create_scorers_post.sync(body=create_scorer_request, client=self.config.api_client) + if isinstance(scorer, HTTPValidationError): + raise ValueError(scorer.detail) + if scorer is None: + raise ValueError("Failed to create decision evaluator.") + + try: + version_resp = create_decision_scorer_version_scorers_scorer_id_version_decision_post.sync( + scorer_id=scorer.id, body=version_req, client=self.config.api_client + ) + if isinstance(version_resp, HTTPValidationError): + raise _VersionRejected(version_resp.detail) + if version_resp is None: + raise _VersionRejected("Failed to create decision evaluator version.") + except Exception as error: + if self._was_rejected(error): + self._delete_scorer_quietly(scorer.id) + raise + _logger.warning( + "Decision evaluator version request failed with an unknown outcome; scorer_id=%s was kept: %s", + scorer.id, + error, + ) + raise DecisionVersionOutcomeUnknownError(str(scorer.id), error) from error + + _logger.info("Created decision evaluator: %s", name) + + return version_resp + + @staticmethod + def _was_rejected(error: Exception) -> bool: + """True when the server clearly refused the request, so no version exists. + + Only 400, 403, 404 and 422 are definitive. Timeouts, network errors, 5xx and rate limits are unknown + outcomes: the version may have been saved, so the scorer stays. + """ + if isinstance(error, SplunkAOAPIError): + return error.status_code in REJECTION_STATUS_CODES + return isinstance(error, _VersionRejected) + + def _delete_scorer_quietly(self, scorer_id: str) -> None: + """Best-effort cleanup so a failed create does not leave an empty scorer behind.""" + try: + response = delete_scorer_scorers_scorer_id_delete.sync(scorer_id=scorer_id, client=self.config.api_client) + except Exception as error: + _logger.warning( + "Evaluators.create_decision_evaluator: scorer_id=%s - cleanup delete failed: %s", scorer_id, error + ) + return + if isinstance(response, HTTPValidationError) or response is None: + _logger.warning( + "Evaluators.create_decision_evaluator: scorer_id=%s - cleanup delete was not confirmed: %s", + scorer_id, + getattr(response, "detail", response), + ) + def query( self, project_id: str, @@ -199,6 +472,94 @@ def create_custom_llm_evaluator( ) +def create_decision_evaluator( + name: str, + question: str, + node_level: StepType = StepType.llm, + model_name: str = DEFAULT_DECISION_MODEL_NAME, + output_type: OutputTypeEnum | None = None, + answer_options: DecisionAnswerOptions | None = None, + input_type: InputTypeEnum | None = None, + description: str = "", + tags: list[str] | None = None, + ground_truth: bool = False, +) -> BaseScorerVersionResponse: + """ + Create a decision evaluator. + + A decision evaluator asks a decision model (for example TypeSafe's Jev) one typed question and + scores the node with the probabilities of the answers. + + Parameters + ---------- + name: str + Name of the evaluator. + question: str + The question the decision model answers (1-2,000 characters). + node_level: StepType + The single node type the evaluator scores. + model_name: str + Decision model registry version, for example ``jev-1.13.0``. + output_type: OutputTypeEnum | None + ``BOOLEAN_PROBABILITY``, ``CATEGORICAL`` or ``DISCRETE``. When ``None``, it is inferred from + ``answer_options``: ``None`` gives ``BOOLEAN_PROBABILITY``, strings give ``CATEGORICAL`` and levels + give ``DISCRETE``. An explicit value that does not match ``answer_options`` raises ``ValueError``. + answer_options: list[str] | list[DecisionLevel | dict] | None + ``None`` for ``BOOLEAN_PROBABILITY``. For ``CATEGORICAL``, a list of 2-255 unique labels. For + ``DISCRETE``, 2-10 ``DecisionLevel`` objects (or ``{level, label, description}`` dicts) numbered + 0, 1, 2 and so on in order. + input_type: InputTypeEnum | None + What the model sees. When ``None``, the server picks a default for the node type. + description: str + Description of the evaluator. + tags: List[str] + Tags associated with the evaluator. + ground_truth: bool + Whether the scorer requires ground truth (``reference_output``) from the dataset. + + Returns + ------- + BaseScorerVersionResponse + Response containing the created evaluator version. + + Raises + ------ + ValueError + If an argument is invalid or the API answers with a validation error. If the scorer was created + but its version was rejected, the scorer is deleted before the error is raised. + SplunkAOAPIError + For a definitive rejection (400, 403, 404, 422), for example 403 when the feature flag is off. + The scorer is deleted before the error is raised. + DecisionVersionOutcomeUnknownError + For a timeout, network error, 5xx or any other failure. The version may have been saved, so the + scorer is kept and the error names its id. + + Examples + -------- + from splunk_ao.metrics import create_decision_evaluator + from splunk_ao.resources.models import OutputTypeEnum + + create_decision_evaluator( + name="tone", + question="Is the tone of this response professional?", + output_type=OutputTypeEnum.CATEGORICAL, + answer_options=["professional", "casual", "rude"], + ) + """ + return Evaluators().create_decision_evaluator( + name=name, + question=question, + node_level=node_level, + model_name=model_name, + output_type=output_type, + answer_options=answer_options, + input_type=input_type, + description=description, + tags=tags, + ground_truth=ground_truth, + ) + + def get_evaluators( project_id: str, start_time: datetime.datetime, diff --git a/tests/test_decision_scorers.py b/tests/test_decision_scorers.py new file mode 100644 index 00000000..e2d4d0cd --- /dev/null +++ b/tests/test_decision_scorers.py @@ -0,0 +1,147 @@ +"""Decision scorers decode, and the SDK asks the API for them on every scorer endpoint.""" + +from collections.abc import Callable +from typing import Any +from unittest.mock import Mock, patch + +import pytest + +from splunk_ao.evaluator import DecisionEvaluator, Evaluator +from splunk_ao.resources.api.data.create_decision_scorer_version_scorers_scorer_id_version_decision_post import ( + _get_kwargs as create_decision_version_kwargs, +) +from splunk_ao.resources.api.data.get_scorer_scorers_scorer_id_get import _get_kwargs as get_scorer_kwargs +from splunk_ao.resources.api.data.list_scorers_with_filters_scorers_list_post import _get_kwargs as list_scorers_kwargs +from splunk_ao.resources.api.datasets.get_dataset_datasets_dataset_id_get import _get_kwargs as dataset_get_kwargs +from splunk_ao.resources.api.integrations.list_integrations_integrations_get import ( + _get_kwargs as integrations_get_kwargs, +) +from splunk_ao.resources.api.log_stream.get_metric_settings_projects_project_id_log_streams_log_stream_id_metric_settings_get import ( + _get_kwargs as log_stream_metric_settings_kwargs, +) +from splunk_ao.resources.api.run_scorer_settings.get_settings_projects_project_id_runs_run_id_scorer_settings_get import ( + _get_kwargs as run_scorer_settings_kwargs, +) +from splunk_ao.resources.models import ( + CreateDecisionScorerVersionRequest, + ListScorersRequest, + ListScorersResponse, + ModelType, + ScorerTypes, +) +from splunk_ao.resources.models.run_scorer_settings_response import RunScorerSettingsResponse +from splunk_ao.scorers import Scorers +from splunk_ao.utils.headers_data import get_scorer_capabilities_header + +DECISION_SCORER_ID = "b7933a6d-7a65-4ce3-bfe4-b863109a0413" + + +def _decision_scorer_row() -> dict[str, Any]: + return { + "id": DECISION_SCORER_ID, + "name": "answers_the_question", + "label": "Answers the question", + "scorer_type": "decision", + "model_type": "decision", + "tags": ["custom"], + "created_at": "2026-10-14T18:54:02.848267+00:00", + "created_by": "d351012a-dd92-4d0c-a356-57161b1377cd", + "updated_at": "2026-10-14T18:54:02.848269+00:00", + } + + +def _list_response(*rows: dict[str, Any]) -> ListScorersResponse: + return ListScorersResponse.from_dict( + {"limit": 100, "next_starting_token": None, "paginated": False, "scorers": list(rows), "starting_token": 0} + ) + + +def test_scorer_capabilities_header_advertises_decision() -> None: + # Given: the SDK decodes decision scorers + # When: building the capability header value + # Then: it is the scorer_type value the API matches + assert get_scorer_capabilities_header() == "decision" + + +@pytest.mark.parametrize( + "get_kwargs", + [ + lambda: list_scorers_kwargs(body=ListScorersRequest()), + lambda: get_scorer_kwargs(scorer_id="scorer-id"), + lambda: create_decision_version_kwargs( + scorer_id="scorer-id", body=CreateDecisionScorerVersionRequest(model_name="jev-1.13.0", question="q") + ), + lambda: run_scorer_settings_kwargs(project_id="project-id", run_id="run-id"), + lambda: log_stream_metric_settings_kwargs(project_id="project-id", log_stream_id="log-stream-id"), + ], + ids=[ + "list_scorers", + "get_scorer", + "create_decision_version", + "run_scorer_settings", + "log_stream_evaluator_settings", + ], +) +def test_scorer_endpoints_advertise_decision_scorers(get_kwargs: Callable[[], dict[str, Any]]) -> None: + # Given / When: the SDK builds a request whose response carries scorer_type + headers = get_kwargs()["content_headers"] + + # Then: the request says the SDK can decode decision scorers + assert headers["X-Galileo-Scorer-Capabilities"] == "decision" + assert headers["Splunk-AO-SDK"].startswith("splunk-ao/") + + +def test_other_endpoints_do_not_advertise_scorer_capabilities() -> None: + # Given / When: requests whose responses carry no scorers + requests = [dataset_get_kwargs(dataset_id="dataset-id"), integrations_get_kwargs()] + + # Then: they send no scorer capability header + assert all("X-Galileo-Scorer-Capabilities" not in kwargs["content_headers"] for kwargs in requests) + + +def test_list_scorers_response_decodes_a_decision_scorer() -> None: + # Given / When: the API lists a decision scorer + response = _list_response(_decision_scorer_row()) + + # Then: its type decodes to the enum instead of raising + assert response.scorers[0].scorer_type == ScorerTypes.DECISION + assert response.scorers[0].model_type == ModelType.DECISION + + +@patch("splunk_ao.scorers.list_scorers_with_filters_scorers_list_post") +def test_scorers_list_returns_decision_scorers(list_scorers_mock: Mock) -> None: + # Given: the API returns a decision scorer + list_scorers_mock.sync.return_value = _list_response(_decision_scorer_row()) + + # When: listing scorers + results = Scorers().list() + + # Then: the decision scorer is returned + assert [(result.name, result.scorer_type) for result in results] == [("answers_the_question", ScorerTypes.DECISION)] + + +@patch("splunk_ao.evaluator.Scorers") +def test_evaluator_get_returns_a_decision_evaluator_for_a_decision_scorer(scorers_class: Mock) -> None: + # Given: the API has a decision scorer with this name + scorers_class.return_value.list.return_value = _list_response(_decision_scorer_row()).scorers + + # When: getting the evaluator by name + evaluator = Evaluator.get(name="answers_the_question") + + # Then: it is a DecisionEvaluator of type decision, and it renders + assert isinstance(evaluator, DecisionEvaluator) + assert evaluator.scorer_type == ScorerTypes.DECISION + assert "decision" in str(evaluator) + + +def test_run_scorer_settings_decode_a_decision_scorer() -> None: + # Given / When: run scorer settings that enable a decision scorer + settings = RunScorerSettingsResponse.from_dict( + { + "run_id": "c7933a6d-7a65-4ce3-bfe4-b863109a0414", + "scorers": [{"id": DECISION_SCORER_ID, "name": "answers_the_question", "scorer_type": "decision"}], + } + ) + + # Then: the scorer decodes instead of raising + assert settings.scorers[0].scorer_type == ScorerTypes.DECISION diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py index 82eb42f8..c8db19bd 100644 --- a/tests/test_evaluator.py +++ b/tests/test_evaluator.py @@ -6,8 +6,8 @@ import pytest from galileo_core.schemas.logging.step import StepType -from splunk_ao.evaluator import CodeEvaluator, LlmEvaluator, LocalEvaluator, Evaluator -from splunk_ao.resources.models import HTTPValidationError, OutputTypeEnum, ScorerTypes +from splunk_ao.evaluator import CodeEvaluator, DecisionEvaluator, LlmEvaluator, LocalEvaluator, Evaluator +from splunk_ao.resources.models import HTTPValidationError, OutputTypeEnum, ScorerResponse, ScorerTypes from splunk_ao.resources.models.invalid_result import InvalidResult from splunk_ao.resources.models.task_result_status import TaskResultStatus from splunk_ao.shared.base import SyncState @@ -2025,3 +2025,188 @@ def test_configuration_from_env_vars(self, reset_configuration: None, monkeypatc assert Configuration.code_validation_initial_delay == 3.0 assert Configuration.code_validation_max_delay == 45.0 assert Configuration.code_validation_backoff_multiplier == 1.8 + + +def _decision_scorer(output_type: str, answer_options, *, with_version: bool = True) -> ScorerResponse: + """Decode a scorer the way the API client does, so answer_options arrive in their wire shape.""" + row: dict = { + "id": "s1", + "name": "tone", + "scorer_type": "decision", + "tags": ["t"], + "output_type": output_type, + "scoreable_node_types": ["trace"], + "ground_truth": True, + } + if with_version: + decision: dict = { + "id": str(uuid4()), + "organization_id": str(uuid4()), + "question": "Which tone?", + "created_at": "2025-01-01T00:00:00+00:00", + } + if answer_options is not None: + decision["answer_options"] = answer_options + row["latest_version"] = { + "id": str(uuid4()), + "version": 1, + "scorer_id": "s1", + "model_name": "jev-1.13.0", + "decision_scorer": decision, + } + return ScorerResponse.from_dict(row) + + +class TestDecisionEvaluator: + """Test suite for DecisionEvaluator create and Evaluator.get/list.""" + + @patch("splunk_ao.evaluator.Scorers") + def test_get_returns_decision_evaluator_with_categorical_fields( + self, mock_scorers_class: MagicMock, reset_configuration: None + ) -> None: + # Given: the API returns a categorical decision scorer + mock_scorers_class.return_value.list.return_value = [_decision_scorer("categorical", ["a", "b"])] + + # When: getting it by name + evaluator = Evaluator.get(name="tone") + + # Then: it is a DecisionEvaluator with the version fields filled in + assert isinstance(evaluator, DecisionEvaluator) + assert evaluator.is_synced() + assert evaluator.question == "Which tone?" + assert evaluator.answer_options == ["a", "b"] + assert evaluator.model == "jev-1.13.0" + assert evaluator.output_type == OutputTypeEnum.CATEGORICAL + assert evaluator.node_level == StepType.trace + assert evaluator.ground_truth is True + + @patch("splunk_ao.evaluator.Scorers") + def test_get_populates_discrete_levels_from_raw_dicts( + self, mock_scorers_class: MagicMock, reset_configuration: None + ) -> None: + # Given: a discrete scorer whose levels the decoder leaves as raw dicts + scorer = _decision_scorer("discrete", [{"level": 0, "label": "bad"}, {"level": 1, "label": "good"}]) + mock_scorers_class.return_value.list.return_value = [scorer] + + # When: getting it by id + evaluator = Evaluator.get(id="s1") + + # Then: the levels are DecisionLevel objects + assert isinstance(evaluator, DecisionEvaluator) + assert [(lvl.level, lvl.label) for lvl in evaluator.answer_options] == [(0, "bad"), (1, "good")] + + @patch("splunk_ao.evaluator.Scorers") + def test_get_boolean_probability_without_answer_options( + self, mock_scorers_class: MagicMock, reset_configuration: None + ) -> None: + # Given: a boolean_probability scorer whose decision row omits answer_options + mock_scorers_class.return_value.list.return_value = [_decision_scorer("boolean_probability", None)] + + # When: getting it + evaluator = Evaluator.get(name="tone") + + # Then: answer_options is None and the evaluator is synced + assert isinstance(evaluator, DecisionEvaluator) + assert evaluator.answer_options is None + assert evaluator.is_synced() + + @patch("splunk_ao.evaluator.Scorers") + def test_get_without_version_keeps_attributes_readable( + self, mock_scorers_class: MagicMock, reset_configuration: None + ) -> None: + # Given: a decision scorer with no version in the response + mock_scorers_class.return_value.list.return_value = [ + _decision_scorer("boolean_probability", None, with_version=False) + ] + + # When: getting it + evaluator = Evaluator.get(name="tone") + + # Then: question and answer_options are None and repr works + assert isinstance(evaluator, DecisionEvaluator) + assert evaluator.question is None + assert evaluator.answer_options is None + assert "DecisionEvaluator(name='tone'" in repr(evaluator) + + @patch("splunk_ao.evaluator.Scorers") + def test_list_returns_decision_evaluator(self, mock_scorers_class: MagicMock, reset_configuration: None) -> None: + # Given: the API lists a decision scorer + mock_scorers_class.return_value.list.return_value = [_decision_scorer("categorical", ["a", "b"])] + + # When: listing evaluators + evaluators = Evaluator.list() + + # Then: it is a DecisionEvaluator + assert len(evaluators) == 1 + assert isinstance(evaluators[0], DecisionEvaluator) + + @patch("splunk_ao.evaluator.Evaluators") + @patch("splunk_ao.evaluator.Scorers") + def test_create_sends_fields_and_refreshes( + self, mock_scorers_class: MagicMock, mock_evaluators_class: MagicMock, reset_configuration: None + ) -> None: + # Given: the service creates the evaluator and the list endpoint returns it + version = MagicMock() + version.scorer_id = "s1" + mock_evaluators_class.return_value.create_decision_evaluator.return_value = version + mock_scorers_class.return_value.list.return_value = [_decision_scorer("categorical", ["a", "b"])] + + # When: creating a DecisionEvaluator + evaluator = DecisionEvaluator( + name="tone", + question="Which tone?", + node_level=StepType.trace, + output_type=OutputTypeEnum.CATEGORICAL, + answer_options=["a", "b"], + tags=["t"], + ground_truth=True, + ).create() + + # Then: every field reaches the service and the evaluator is synced + kwargs = mock_evaluators_class.return_value.create_decision_evaluator.call_args.kwargs + assert kwargs["name"] == "tone" + assert kwargs["question"] == "Which tone?" + assert kwargs["node_level"] == StepType.trace + assert kwargs["model_name"] == "jev-1.13.0" + assert kwargs["output_type"] == OutputTypeEnum.CATEGORICAL + assert kwargs["answer_options"] == ["a", "b"] + assert kwargs["ground_truth"] is True + assert evaluator.id == "s1" + assert evaluator.is_synced() + + +class TestDecisionEvaluatorReadPath: + @patch("splunk_ao.evaluator.Scorers") + def test_list_survives_malformed_answer_options( + self, mock_scorers_class: MagicMock, reset_configuration: None + ) -> None: + # Given: one decision scorer with an empty options list and one with a level missing its label + mock_scorers_class.return_value.list.return_value = [ + _decision_scorer("categorical", []), + _decision_scorer("discrete", [{"level": 0}]), + ] + + # When: listing evaluators + evaluators = Evaluator.list() + + # Then: nothing raises and the raw values are kept + assert [type(m) for m in evaluators] == [DecisionEvaluator, DecisionEvaluator] + assert evaluators[0].answer_options == [] + assert evaluators[1].answer_options == [{"level": 0}] + + @patch("splunk_ao.evaluator.Scorers") + def test_latest_version_is_used_when_default_version_has_no_decision_row( + self, mock_scorers_class: MagicMock, reset_configuration: None + ) -> None: + # Given: a default version without decision data and a latest version with it + scorer = _decision_scorer("categorical", ["a", "b"]) + row = scorer.to_dict() + row["default_version"] = {"id": str(uuid4()), "version": 1, "scorer_id": "s1"} + mock_scorers_class.return_value.list.return_value = [ScorerResponse.from_dict(row)] + + # When: getting the evaluator + evaluator = Evaluator.get(name="tone") + + # Then: the question comes from the latest version + assert isinstance(evaluator, DecisionEvaluator) + assert evaluator.question == "Which tone?" diff --git a/tests/test_evaluator_types.py b/tests/test_evaluator_types.py index ea3bebec..56c6396a 100644 --- a/tests/test_evaluator_types.py +++ b/tests/test_evaluator_types.py @@ -10,10 +10,19 @@ from __future__ import annotations +from unittest.mock import patch + import pytest from galileo_core.schemas.logging.step import StepType -from splunk_ao.evaluator import CodeEvaluator, LlmEvaluator, LocalEvaluator, Evaluator, SplunkAOEvaluator +from splunk_ao.evaluator import ( + CodeEvaluator, + DecisionEvaluator, + LlmEvaluator, + LocalEvaluator, + Evaluator, + SplunkAOEvaluator, +) from splunk_ao.resources.models import OutputTypeEnum, ScorerTypes from splunk_ao.shared.exceptions import ValidationError @@ -400,3 +409,85 @@ def my_scorer(trace_or_span): assert StepType.llm in metric.scorable_types assert StepType.workflow in metric.scorable_types assert StepType.agent in metric.scorable_types + + +class TestDecisionEvaluator: + def test_defaults(self): + # When: creating an evaluator with only name and question + evaluator = DecisionEvaluator(name="tone", question="Is it polite?") + + # Then: the decision defaults apply + assert evaluator.scorer_type == ScorerTypes.DECISION + assert evaluator.model == "jev-1.13.0" + assert evaluator.node_level == StepType.llm + assert evaluator.output_type == OutputTypeEnum.BOOLEAN_PROBABILITY + assert evaluator.answer_options is None + assert evaluator.input_type is None + assert evaluator.ground_truth is False + + @pytest.mark.parametrize( + ("output_type", "expected"), + [ + ("boolean_probability", OutputTypeEnum.BOOLEAN_PROBABILITY), + ("Categorical", OutputTypeEnum.CATEGORICAL), + ("discrete", OutputTypeEnum.DISCRETE), + (OutputTypeEnum.DISCRETE, OutputTypeEnum.DISCRETE), + ], + ) + def test_output_type_accepts_decision_types(self, output_type, expected): + # When/Then: decision output types parse from strings and enums + assert DecisionEvaluator(name="q", question="q", output_type=output_type).output_type == expected + + @pytest.mark.parametrize("output_type", ["categorial", "boolean", OutputTypeEnum.PERCENTAGE]) + def test_output_type_rejects_non_decision_types(self, output_type): + # When/Then: typos and non-decision types raise instead of defaulting + with pytest.raises(ValidationError, match="output_type"): + DecisionEvaluator(name="q", question="q", output_type=output_type) + + def test_dict_levels_become_decision_levels(self): + # When: discrete options are given as dicts + evaluator = DecisionEvaluator( + name="q", + question="How good?", + output_type="discrete", + answer_options=[{"level": 0, "label": "bad"}, {"level": 1, "label": "good"}], + ) + + # Then: they are normalized to DecisionLevel objects + assert [(o.level, o.label) for o in evaluator.answer_options] == [(0, "bad"), (1, "good")] + + @pytest.mark.parametrize("question", [None, ""]) + def test_requires_question(self, question): + # When/Then: a missing or empty question is rejected + with pytest.raises(ValidationError, match="question"): + DecisionEvaluator(name="tone", question=question) + + def test_repr(self): + # When/Then: repr shows the name + assert "DecisionEvaluator(name='tone'" in repr(DecisionEvaluator(name="tone", question="Is it polite?")) + + def test_create_failure_marks_failed_sync(self): + # Given: the service rejects the request + evaluator = DecisionEvaluator(name="tone", question="q") + + # When/Then: the error propagates and the evaluator is marked failed + with patch("splunk_ao.evaluator.Evaluators") as evaluators_cls: + evaluators_cls.return_value.create_decision_evaluator.side_effect = ValueError("bad model") + with pytest.raises(ValueError, match="bad model"): + evaluator.create() + assert evaluator.has_failed() + + @pytest.mark.parametrize( + ("answer_options", "expected"), + [ + (None, OutputTypeEnum.BOOLEAN_PROBABILITY), + (["a", "b"], OutputTypeEnum.CATEGORICAL), + ([{"level": 0, "label": "x"}, {"level": 1, "label": "y"}], OutputTypeEnum.DISCRETE), + ], + ) + def test_output_type_is_inferred_from_answer_options(self, answer_options, expected): + # When: no output_type is given + evaluator = DecisionEvaluator(name="q", question="q", answer_options=answer_options) + + # Then: it follows the answer options + assert evaluator.output_type == expected diff --git a/tests/test_evaluators.py b/tests/test_evaluators.py index b8c1c3b0..8afdc9cb 100644 --- a/tests/test_evaluators.py +++ b/tests/test_evaluators.py @@ -6,7 +6,23 @@ import pytest from galileo_core.schemas.logging.step import StepType -from splunk_ao.evaluators import Evaluators, create_custom_llm_evaluator, delete_evaluator, get_evaluators +from splunk_ao.evaluators import ( + DecisionVersionOutcomeUnknownError, + Evaluators, + create_custom_llm_evaluator, + create_decision_evaluator, + delete_evaluator, + get_evaluators, +) +from splunk_ao.exceptions import ( + BadRequestError, + ConflictError, + ForbiddenError, + NotFoundError, + RateLimitError, + ServerError, + SplunkAOAPIError, +) from splunk_ao.resources.models import ( BucketedMetrics, HTTPValidationError, @@ -16,8 +32,11 @@ ValidationError, ) from splunk_ao.resources.models.base_scorer_version_response import BaseScorerVersionResponse +from splunk_ao.resources.models.create_decision_scorer_version_request import CreateDecisionScorerVersionRequest from splunk_ao.resources.models.create_llm_scorer_version_request import CreateLLMScorerVersionRequest from splunk_ao.resources.models.create_scorer_request import CreateScorerRequest +from splunk_ao.resources.models.decision_level import DecisionLevel +from splunk_ao.resources.models.input_type_enum import InputTypeEnum from splunk_ao.resources.models.output_type_enum import OutputTypeEnum from splunk_ao.resources.models.scorer_defaults import ScorerDefaults from splunk_ao.resources.models.scorer_response import ScorerResponse @@ -501,3 +520,322 @@ def test_passes_all_parameters_correctly(self, mock_api_call): assert body.filters == filters assert body.group_by == group_by assert body.interval == interval + + +@pytest.fixture +def decision_scorer_response(): + return ScorerResponse( + id=UUID("12345678-1234-5678-9012-123456789abc"), + name="tone", + scorer_type=ScorerTypes.DECISION, + tags=[], + created_at="2025-01-01T00:00:00Z", + updated_at="2025-01-01T00:00:00Z", + ) + + +@patch("splunk_ao.evaluators.delete_scorer_scorers_scorer_id_delete") +@patch("splunk_ao.evaluators.create_decision_scorer_version_scorers_scorer_id_version_decision_post") +@patch("splunk_ao.evaluators.create_scorers_post") +class TestCreateDecisionEvaluator: + def test_boolean_probability_defaults( + self, + mock_create_scorer, + mock_create_version, + mock_delete, + decision_scorer_response, + mock_scorer_version_response, + ) -> None: + # Given: both endpoints succeed + mock_create_scorer.sync.return_value = decision_scorer_response + mock_create_version.sync.return_value = mock_scorer_version_response + + # When: creating a decision evaluator with only the required arguments + result = create_decision_evaluator(name="tone", question="Is the tone professional?") + + # Then: the scorer and version requests carry the defaults and the version response is returned + assert result is mock_scorer_version_response + scorer_request = mock_create_scorer.sync.call_args.kwargs["body"] + assert isinstance(scorer_request, CreateScorerRequest) + assert scorer_request.name == "tone" + assert scorer_request.scorer_type == ScorerTypes.DECISION + assert scorer_request.output_type == OutputTypeEnum.BOOLEAN_PROBABILITY + assert scorer_request.scoreable_node_types == [StepType.llm] + assert scorer_request.ground_truth is False + assert "input_type" not in scorer_request.to_dict() + assert "defaults" not in scorer_request.to_dict() + + version_call = mock_create_version.sync.call_args + version_request = version_call.kwargs["body"] + assert isinstance(version_request, CreateDecisionScorerVersionRequest) + assert version_request.model_name == "jev-1.13.0" + assert version_request.question == "Is the tone professional?" + assert version_request.answer_options is None + assert version_call.kwargs["scorer_id"] == decision_scorer_response.id + mock_delete.sync.assert_not_called() + + def test_categorical_with_options_and_node_level( + self, + mock_create_scorer, + mock_create_version, + mock_delete, + decision_scorer_response, + mock_scorer_version_response, + ) -> None: + # Given: both endpoints succeed + mock_create_scorer.sync.return_value = decision_scorer_response + mock_create_version.sync.return_value = mock_scorer_version_response + + # When: creating a categorical evaluator on trace nodes with every optional argument set + create_decision_evaluator( + name="tone", + question="Which tone?", + node_level=StepType.trace, + output_type=OutputTypeEnum.CATEGORICAL, + answer_options=["a", "b"], + input_type=InputTypeEnum.TRACE_IO_ONLY, + description="d", + tags=["t"], + ground_truth=True, + ) + + # Then: the node level is a single-element list and the labels are sent as strings + scorer_request = mock_create_scorer.sync.call_args.kwargs["body"] + assert scorer_request.scoreable_node_types == [StepType.trace] + assert scorer_request.output_type == OutputTypeEnum.CATEGORICAL + assert scorer_request.input_type == InputTypeEnum.TRACE_IO_ONLY + assert scorer_request.description == "d" + assert scorer_request.tags == ["t"] + assert scorer_request.ground_truth is True + version_request = mock_create_version.sync.call_args.kwargs["body"] + assert version_request.to_dict()["answer_options"] == ["a", "b"] + + def test_discrete_accepts_levels_and_dicts( + self, + mock_create_scorer, + mock_create_version, + mock_delete, + decision_scorer_response, + mock_scorer_version_response, + ) -> None: + # Given: both endpoints succeed + mock_create_scorer.sync.return_value = decision_scorer_response + mock_create_version.sync.return_value = mock_scorer_version_response + + # When: creating a discrete evaluator from a DecisionLevel and a plain dict + create_decision_evaluator( + name="quality", + question="How good?", + output_type=OutputTypeEnum.DISCRETE, + answer_options=[DecisionLevel(level=0, label="bad"), {"level": 1, "label": "good", "description": "ok"}], + ) + + # Then: the request body holds JSON-ready level dicts + version_request = mock_create_version.sync.call_args.kwargs["body"] + assert version_request.to_dict()["answer_options"] == [ + {"level": 0, "label": "bad"}, + {"level": 1, "label": "good", "description": "ok"}, + ] + + @pytest.mark.parametrize( + ("answer_options", "message"), + [ + ("yes", "single string"), + ({"level": 0, "label": "x"}, "single string or dict"), + ([], "must not be empty"), + ([{"label": "bad"}, {"label": "good"}], r"answer_options\[0\] is missing required key\(s\): level"), + ([1, 2], "got int"), + ], + ) + def test_invalid_answer_options_raise_before_any_request( + self, mock_create_scorer, mock_create_version, mock_delete, answer_options, message + ) -> None: + # When/Then: malformed answer options raise ValueError and no request is sent + with pytest.raises(ValueError, match=message): + create_decision_evaluator( + name="q", question="q", output_type=OutputTypeEnum.DISCRETE, answer_options=answer_options + ) + mock_create_scorer.sync.assert_not_called() + + @pytest.mark.parametrize( + "version_result", + [ + HTTPValidationError(detail="unknown model jev-9"), + None, + BadRequestError(400, b"bad"), + ForbiddenError(403, b"no"), + NotFoundError(404, b"gone"), + SplunkAOAPIError(422, b"unprocessable", "Unprocessable"), + ], + ) + def test_rejected_version_deletes_scorer( + self, mock_create_scorer, mock_create_version, mock_delete, decision_scorer_response, version_result + ) -> None: + # Given: the scorer is created and the version request is clearly rejected + mock_create_scorer.sync.return_value = decision_scorer_response + if isinstance(version_result, Exception): + mock_create_version.sync.side_effect = version_result + else: + mock_create_version.sync.return_value = version_result + + # When/Then: the error surfaces and the scorer is deleted + with pytest.raises((ValueError, SplunkAOAPIError)): + create_decision_evaluator(name="tone", question="q", model_name="jev-9") + mock_delete.sync.assert_called_once() + assert mock_delete.sync.call_args.kwargs["scorer_id"] == decision_scorer_response.id + + def test_http_validation_error_detail_is_in_the_message( + self, mock_create_scorer, mock_create_version, mock_delete, decision_scorer_response + ) -> None: + # Given: the version request fails validation + mock_create_scorer.sync.return_value = decision_scorer_response + mock_create_version.sync.return_value = HTTPValidationError(detail="unknown model jev-9") + + # When/Then: the API detail reaches the caller + with pytest.raises(ValueError, match="unknown model jev-9"): + create_decision_evaluator(name="tone", question="q", model_name="jev-9") + + @pytest.mark.parametrize( + "error", + [ + ServerError(502, b"bad gateway"), + RateLimitError(429, b"slow"), + ConflictError(409, b"conflict"), + TimeoutError("timed out"), + ], + ) + def test_unknown_outcome_keeps_scorer( + self, mock_create_scorer, mock_create_version, mock_delete, decision_scorer_response, error + ) -> None: + # Given: the version request fails in a way that may have saved the version + mock_create_scorer.sync.return_value = decision_scorer_response + mock_create_version.sync.side_effect = error + + # When/Then: the error names the kept scorer, chains the cause, and the scorer is not deleted + with pytest.raises(DecisionVersionOutcomeUnknownError, match=str(decision_scorer_response.id)) as raised: + create_decision_evaluator(name="tone", question="q") + assert raised.value.__cause__ is error + assert raised.value.scorer_id == str(decision_scorer_response.id) + mock_delete.sync.assert_not_called() + + @pytest.mark.parametrize( + "delete_behavior", + [ + {"side_effect": RuntimeError("delete failed")}, + {"return_value": None}, + {"return_value": HTTPValidationError(detail="no")}, + ], + ) + def test_failed_cleanup_does_not_mask_original_error( + self, mock_create_scorer, mock_create_version, mock_delete, decision_scorer_response, delete_behavior + ) -> None: + # Given: the version is rejected and the cleanup delete fails or is not confirmed + mock_create_scorer.sync.return_value = decision_scorer_response + mock_create_version.sync.return_value = HTTPValidationError(detail="bad options") + mock_delete.sync.configure_mock(**delete_behavior) + + # When/Then: the original error still surfaces + with pytest.raises(ValueError, match="bad options"): + create_decision_evaluator(name="tone", question="q") + + @pytest.mark.parametrize("scorer_result", [HTTPValidationError(detail="name taken"), None]) + def test_scorer_failure_raises_without_version_call( + self, mock_create_scorer, mock_create_version, mock_delete, scorer_result + ) -> None: + # Given: the scorer request fails + mock_create_scorer.sync.return_value = scorer_result + + # When/Then: ValueError is raised and neither the version nor the delete endpoint is called + with pytest.raises(ValueError): + create_decision_evaluator(name="tone", question="q") + mock_create_version.sync.assert_not_called() + mock_delete.sync.assert_not_called() + + +@patch("splunk_ao.evaluators.delete_scorer_scorers_scorer_id_delete") +@patch("splunk_ao.evaluators.create_decision_scorer_version_scorers_scorer_id_version_decision_post") +@patch("splunk_ao.evaluators.create_scorers_post") +class TestCreateDecisionEvaluatorGuards: + @pytest.mark.parametrize( + ("output_type", "answer_options"), + [ + (OutputTypeEnum.PERCENTAGE, None), + (OutputTypeEnum.BOOLEAN_PROBABILITY, ["a", "b"]), + (OutputTypeEnum.CATEGORICAL, None), + (OutputTypeEnum.CATEGORICAL, [{"level": 0, "label": "x"}, {"level": 1, "label": "y"}]), + (OutputTypeEnum.DISCRETE, ["a", "b"]), + (OutputTypeEnum.DISCRETE, None), + ], + ) + def test_mismatched_output_type_and_options_create_nothing( + self, mock_create_scorer, mock_create_version, mock_delete, output_type, answer_options + ) -> None: + # When/Then: a request the server would refuse raises before the scorer is created + with pytest.raises(ValueError): + create_decision_evaluator(name="q", question="q", output_type=output_type, answer_options=answer_options) + mock_create_scorer.sync.assert_not_called() + + def test_unparseable_success_response_keeps_scorer( + self, mock_create_scorer, mock_create_version, mock_delete, decision_scorer_response + ) -> None: + # Given: the server saved the version but the 200 body cannot be parsed + mock_create_scorer.sync.return_value = decision_scorer_response + mock_create_version.sync.side_effect = ValueError("Expecting value: line 1 column 1 (char 0)") + + # When/Then: the error surfaces and the scorer is kept, since the version may exist + with pytest.raises(DecisionVersionOutcomeUnknownError, match="Expecting value"): + create_decision_evaluator(name="tone", question="q") + mock_delete.sync.assert_not_called() + + @pytest.mark.parametrize( + ("answer_options", "expected"), + [ + (None, OutputTypeEnum.BOOLEAN_PROBABILITY), + (["a", "b"], OutputTypeEnum.CATEGORICAL), + ([{"level": 0, "label": "x"}, {"level": 1, "label": "y"}], OutputTypeEnum.DISCRETE), + ([DecisionLevel(level=0, label="x"), DecisionLevel(level=1, label="y")], OutputTypeEnum.DISCRETE), + ], + ) + def test_output_type_is_inferred_from_answer_options( + self, + mock_create_scorer, + mock_create_version, + mock_delete, + decision_scorer_response, + mock_scorer_version_response, + answer_options, + expected, + ) -> None: + # Given: both endpoints succeed + mock_create_scorer.sync.return_value = decision_scorer_response + mock_create_version.sync.return_value = mock_scorer_version_response + + # When: creating without an output_type + create_decision_evaluator(name="q", question="q", answer_options=answer_options) + + # Then: the scorer is created with the output type that matches the options + assert mock_create_scorer.sync.call_args.kwargs["body"].output_type == expected + + def test_explicit_output_type_that_matches_options_is_kept( + self, + mock_create_scorer, + mock_create_version, + mock_delete, + decision_scorer_response, + mock_scorer_version_response, + ) -> None: + # Given: both endpoints succeed + mock_create_scorer.sync.return_value = decision_scorer_response + mock_create_version.sync.return_value = mock_scorer_version_response + + # When: output_type is given as a string that matches the options + create_decision_evaluator(name="q", question="q", output_type="categorical", answer_options=["a", "b"]) + + # Then: it is sent as the enum value + assert mock_create_scorer.sync.call_args.kwargs["body"].output_type == OutputTypeEnum.CATEGORICAL + + def test_unknown_output_type_string_raises(self, mock_create_scorer, mock_create_version, mock_delete) -> None: + # When/Then: an unknown output type raises before any request + with pytest.raises(ValueError, match="output_type"): + create_decision_evaluator(name="q", question="q", output_type="categorial", answer_options=["a", "b"]) + mock_create_scorer.sync.assert_not_called() From 6e451ed5845e91678d291084073221e98eb54077 Mon Sep 17 00:00:00 2001 From: John Weiler Date: Fri, 16 Oct 2026 09:52:17 -0400 Subject: [PATCH 4/7] fix(sdk): harden decision evaluator validation and read path (SAO-17973) Co-Authored-By: Claude Sonnet 5.5 --- src/splunk_ao/__future__/__init__.py | 2 ++ src/splunk_ao/__init__.py | 2 ++ src/splunk_ao/evaluator.py | 36 ++++++++++++++++--------- src/splunk_ao/evaluators.py | 30 +++++++++------------ tests/test_evaluator.py | 39 +++++++++++++++++++++++++--- tests/test_evaluator_types.py | 30 ++++++++++++++++----- 6 files changed, 99 insertions(+), 40 deletions(-) diff --git a/src/splunk_ao/__future__/__init__.py b/src/splunk_ao/__future__/__init__.py index 225649df..1c569e28 100644 --- a/src/splunk_ao/__future__/__init__.py +++ b/src/splunk_ao/__future__/__init__.py @@ -6,6 +6,7 @@ from splunk_ao.configuration import Configuration from splunk_ao.dataset import Dataset from splunk_ao.experiment import Experiment +from splunk_ao.evaluators import DecisionVersionOutcomeUnknownError from splunk_ao.integration import Integration from splunk_ao.agent_stream import AgentStream from splunk_ao.evaluator import ( @@ -43,6 +44,7 @@ "ConfigurationError", "Dataset", "DecisionEvaluator", + "DecisionVersionOutcomeUnknownError", "Evaluator", "Experiment", "Integration", diff --git a/src/splunk_ao/__init__.py b/src/splunk_ao/__init__.py index 4b2b0b8f..3acf0294 100644 --- a/src/splunk_ao/__init__.py +++ b/src/splunk_ao/__init__.py @@ -52,6 +52,7 @@ LocalEvaluator, SplunkAOEvaluator, ) +from splunk_ao.evaluators import DecisionVersionOutcomeUnknownError from splunk_ao.exceptions import ( AuthenticationError, BadRequestError, @@ -125,6 +126,7 @@ "ControlSpan", "Dataset", "DecisionEvaluator", + "DecisionVersionOutcomeUnknownError", "Document", "Evaluator", "Experiment", diff --git a/src/splunk_ao/evaluator.py b/src/splunk_ao/evaluator.py index 9a6348bc..cad3c8d4 100644 --- a/src/splunk_ao/evaluator.py +++ b/src/splunk_ao/evaluator.py @@ -22,9 +22,11 @@ from splunk_ao.evaluators import ( DEFAULT_DECISION_MODEL_NAME, DecisionAnswerOptions, + DecisionVersionOutcomeUnknownError, Evaluators, normalize_decision_answer_options, resolve_decision_output_type, + validate_decision_request, ) from splunk_ao.resources.api.data import ( create_code_scorer_version_scorers_scorer_id_version_code_post, @@ -38,9 +40,11 @@ BodyValidateCodeScorerScorersCodeValidatePost, CreateScorerRequest, DecisionLevel, + DecisionScorerDB, HTTPValidationError, InputTypeEnum, OutputTypeEnum, + ScorerResponse, ScorerTypes, TaskResultStatus, UpdateScorerRequest, @@ -407,6 +411,9 @@ def _populate_from_scorer_response(self, scorer_response: Any) -> None: model = None judges = None cot_enabled = None + if model is None and isinstance(self, DecisionEvaluator): + # Decision scorers carry no LLM defaults; keep the model the evaluator already has. + model = getattr(self, "model", None) self._sync_attrs( id=scorer_response.id, @@ -938,13 +945,13 @@ def __init__( self.question = question self.model = model or DEFAULT_DECISION_MODEL_NAME self.node_level = node_level or StepType.llm - normalized_options = normalize_decision_answer_options(answer_options) - if isinstance(output_type, str) and not isinstance(output_type, OutputTypeEnum): - output_type = output_type.lower() try: - self.output_type = resolve_decision_output_type(output_type, normalized_options) + normalized_options = normalize_decision_answer_options(answer_options) + resolved_output_type = resolve_decision_output_type(output_type, normalized_options) + validate_decision_request(resolved_output_type, normalized_options) except ValueError as error: raise ValidationError(str(error)) from error + self.output_type = resolved_output_type self.answer_options = normalized_options self.input_type = input_type self.ground_truth = ground_truth @@ -962,6 +969,8 @@ def create(self) -> DecisionEvaluator: ------ ValueError: If an argument is invalid or the API answers with a validation error. SplunkAOAPIError: For HTTP errors. A scorer created before a clear rejection is deleted. + DecisionVersionOutcomeUnknownError: When the outcome of the version request is unknown (timeout, + network error, 5xx). The scorer is kept and ``id`` is set so it can be refreshed or deleted. Examples -------- @@ -976,9 +985,7 @@ def create(self) -> DecisionEvaluator: question=self.question or "", node_level=self.node_level if self.node_level is not None else StepType.llm, model_name=self.model if self.model is not None else DEFAULT_DECISION_MODEL_NAME, - output_type=self.output_type - if isinstance(self.output_type, OutputTypeEnum) - else OutputTypeEnum.BOOLEAN_PROBABILITY, + output_type=self.output_type, answer_options=self.answer_options, input_type=self.input_type, description=self.description, @@ -998,11 +1005,13 @@ def create(self) -> DecisionEvaluator: logger.info(f"DecisionEvaluator.create: id='{self.id}' - completed") return self except Exception as e: + if isinstance(e, DecisionVersionOutcomeUnknownError): + self._sync_attrs(id=e.scorer_id) self._set_state(SyncState.FAILED_SYNC, error=e) logger.error(f"DecisionEvaluator.create: name='{self.name}' - failed: {e}") raise - def _populate_decision_attrs(self, scorer_response: Any) -> None: + def _populate_decision_attrs(self, scorer_response: ScorerResponse) -> None: """Fill decision fields from a scorer response. The question and answer options live on the scorer's version. When the response carries no version, @@ -1032,7 +1041,8 @@ def _populate_decision_attrs(self, scorer_response: Any) -> None: if version is None or isinstance(version, Unset): continue decision = version.decision_scorer - if decision is None or isinstance(decision, Unset): + # The generated decoder leaves a payload it cannot parse as a raw dict. + if not isinstance(decision, DecisionScorerDB): continue self._sync_attrs(question=decision.question, answer_options=self._read_answer_options(decision)) if not isinstance(version.model_name, Unset) and version.model_name: @@ -1040,13 +1050,15 @@ def _populate_decision_attrs(self, scorer_response: Any) -> None: break @staticmethod - def _read_answer_options(decision: Any) -> Any: + def _read_answer_options(decision: DecisionScorerDB) -> list[str] | list[DecisionLevel] | None: """Convert server answer options leniently: a read must never raise on unexpected data.""" try: return normalize_decision_answer_options(decision.answer_options) except ValueError as error: - logger.warning(f"DecisionEvaluator: keeping raw answer_options for decision_scorer {decision.id}: {error}") - return decision.answer_options + logger.warning( + f"DecisionEvaluator: ignoring unreadable answer_options for decision_scorer {decision.id}: {error}" + ) + return None def __repr__(self) -> str: """Detailed string representation of the evaluator.""" diff --git a/src/splunk_ao/evaluators.py b/src/splunk_ao/evaluators.py index 7990ee26..43ae5b10 100644 --- a/src/splunk_ao/evaluators.py +++ b/src/splunk_ao/evaluators.py @@ -63,11 +63,10 @@ def normalize_decision_answer_options( missing = [key for key in ("level", "label") if key not in option] if missing: raise ValueError(f"answer_options[{index}] is missing required key(s): {', '.join(missing)}.") - levels.append( - DecisionLevel( - level=option["level"], label=option["label"], description=option.get("description") or UNSET - ) - ) + level, label = option["level"], option["label"] + if isinstance(level, bool) or not isinstance(level, int) or not isinstance(label, str): + raise ValueError(f"answer_options[{index}] needs an integer 'level' and a string 'label'.") + levels.append(DecisionLevel(level=level, label=label, description=option.get("description") or UNSET)) else: raise ValueError( "answer_options must be a list of strings (categorical) or a list of DecisionLevel / " @@ -112,7 +111,7 @@ def resolve_decision_output_type( """ if output_type is not None: try: - resolved = OutputTypeEnum(output_type) + resolved = OutputTypeEnum(output_type.lower() if isinstance(output_type, str) else output_type) except ValueError: resolved = None if resolved not in DECISION_OUTPUT_TYPES: @@ -128,15 +127,10 @@ def resolve_decision_output_type( return OutputTypeEnum.DISCRETE -def _validate_decision_request( +def validate_decision_request( output_type: OutputTypeEnum, answer_options: list[str] | list[DecisionLevel] | None ) -> None: - """Reject requests the server would refuse, before anything is created.""" - if output_type not in DECISION_OUTPUT_TYPES: - raise ValueError( - f"output_type must be one of {[t.value for t in DECISION_OUTPUT_TYPES]} for decision evaluators; " - f"got {output_type!r}." - ) + """Reject answer options that do not match the output type, before anything is created.""" if output_type == OutputTypeEnum.BOOLEAN_PROBABILITY: if answer_options is not None: raise ValueError("answer_options must be None for boolean_probability.") @@ -308,7 +302,7 @@ def create_decision_evaluator( If an argument is invalid or the API answers with a validation error. If the scorer was created but its version was rejected, the scorer is deleted before the error is raised. SplunkAOAPIError - For a definitive rejection (400, 403, 404, 422), for example 403 when the feature flag is off. + For a definitive rejection (400, 403, 404, 422), for example 403 when the feature is not enabled. The scorer is deleted before the error is raised. DecisionVersionOutcomeUnknownError For a timeout, network error, 5xx or any other failure. The version may have been saved, so the @@ -316,7 +310,7 @@ def create_decision_evaluator( """ normalized_options = normalize_decision_answer_options(answer_options) output_type = resolve_decision_output_type(output_type, normalized_options) - _validate_decision_request(output_type, normalized_options) + validate_decision_request(output_type, normalized_options) version_req = _build_decision_version_request( model_name=model_name, question=question, answer_options=normalized_options ) @@ -487,7 +481,7 @@ def create_decision_evaluator( """ Create a decision evaluator. - A decision evaluator asks a decision model (for example TypeSafe's Jev) one typed question and + A decision evaluator asks a decision model (for example a decision model) one typed question and scores the node with the probabilities of the answers. Parameters @@ -528,7 +522,7 @@ def create_decision_evaluator( If an argument is invalid or the API answers with a validation error. If the scorer was created but its version was rejected, the scorer is deleted before the error is raised. SplunkAOAPIError - For a definitive rejection (400, 403, 404, 422), for example 403 when the feature flag is off. + For a definitive rejection (400, 403, 404, 422), for example 403 when the feature is not enabled. The scorer is deleted before the error is raised. DecisionVersionOutcomeUnknownError For a timeout, network error, 5xx or any other failure. The version may have been saved, so the @@ -536,7 +530,7 @@ def create_decision_evaluator( Examples -------- - from splunk_ao.metrics import create_decision_evaluator + from splunk_ao.evaluators import create_decision_evaluator from splunk_ao.resources.models import OutputTypeEnum create_decision_evaluator( diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py index c8db19bd..4e676877 100644 --- a/tests/test_evaluator.py +++ b/tests/test_evaluator.py @@ -6,6 +6,7 @@ import pytest from galileo_core.schemas.logging.step import StepType +from splunk_ao.evaluators import DecisionVersionOutcomeUnknownError from splunk_ao.evaluator import CodeEvaluator, DecisionEvaluator, LlmEvaluator, LocalEvaluator, Evaluator from splunk_ao.resources.models import HTTPValidationError, OutputTypeEnum, ScorerResponse, ScorerTypes from splunk_ao.resources.models.invalid_result import InvalidResult @@ -2189,10 +2190,10 @@ def test_list_survives_malformed_answer_options( # When: listing evaluators evaluators = Evaluator.list() - # Then: nothing raises and the raw values are kept + # Then: nothing raises and the unreadable options are dropped assert [type(m) for m in evaluators] == [DecisionEvaluator, DecisionEvaluator] - assert evaluators[0].answer_options == [] - assert evaluators[1].answer_options == [{"level": 0}] + assert evaluators[0].answer_options is None + assert evaluators[1].answer_options is None @patch("splunk_ao.evaluator.Scorers") def test_latest_version_is_used_when_default_version_has_no_decision_row( @@ -2210,3 +2211,35 @@ def test_latest_version_is_used_when_default_version_has_no_decision_row( # Then: the question comes from the latest version assert isinstance(evaluator, DecisionEvaluator) assert evaluator.question == "Which tone?" + + +class TestDecisionEvaluatorOutcomeAndModel: + def test_unknown_outcome_keeps_scorer_id_on_the_evaluator(self, reset_configuration: None) -> None: + # Given: the service keeps the scorer after an unknown outcome + evaluator = DecisionEvaluator(name="tone", question="q") + error = DecisionVersionOutcomeUnknownError("scorer-1", TimeoutError("slow")) + + # When/Then: the id is set so the scorer can be refreshed or deleted, and the state is failed + with patch("splunk_ao.evaluator.Evaluators") as evaluators_cls: + evaluators_cls.return_value.create_decision_evaluator.side_effect = error + with pytest.raises(DecisionVersionOutcomeUnknownError): + evaluator.create() + assert evaluator.id == "scorer-1" + assert evaluator.has_failed() + + @patch("splunk_ao.evaluator.Scorers") + def test_model_is_kept_when_the_response_has_no_decision_version( + self, mock_scorers_class: MagicMock, reset_configuration: None + ) -> None: + # Given: an evaluator with a custom model and a refresh response without a version + evaluator = DecisionEvaluator(name="tone", question="q", model="jev-1.12.0") + mock_scorers_class.return_value.list.return_value = [ + _decision_scorer("boolean_probability", None, with_version=False) + ] + evaluator._sync_attrs(id="s1") + + # When: refreshing + evaluator.refresh() + + # Then: the model survives + assert evaluator.model == "jev-1.12.0" diff --git a/tests/test_evaluator_types.py b/tests/test_evaluator_types.py index 56c6396a..4d68ce01 100644 --- a/tests/test_evaluator_types.py +++ b/tests/test_evaluator_types.py @@ -426,17 +426,33 @@ def test_defaults(self): assert evaluator.ground_truth is False @pytest.mark.parametrize( - ("output_type", "expected"), + ("output_type", "answer_options", "expected"), [ - ("boolean_probability", OutputTypeEnum.BOOLEAN_PROBABILITY), - ("Categorical", OutputTypeEnum.CATEGORICAL), - ("discrete", OutputTypeEnum.DISCRETE), - (OutputTypeEnum.DISCRETE, OutputTypeEnum.DISCRETE), + ("boolean_probability", None, OutputTypeEnum.BOOLEAN_PROBABILITY), + ("Categorical", ["a", "b"], OutputTypeEnum.CATEGORICAL), + ("discrete", [{"level": 0, "label": "x"}, {"level": 1, "label": "y"}], OutputTypeEnum.DISCRETE), + (OutputTypeEnum.CATEGORICAL, ["a", "b"], OutputTypeEnum.CATEGORICAL), ], ) - def test_output_type_accepts_decision_types(self, output_type, expected): + def test_output_type_accepts_decision_types(self, output_type, answer_options, expected): # When/Then: decision output types parse from strings and enums - assert DecisionEvaluator(name="q", question="q", output_type=output_type).output_type == expected + evaluator = DecisionEvaluator(name="q", question="q", output_type=output_type, answer_options=answer_options) + assert evaluator.output_type == expected + + @pytest.mark.parametrize( + ("output_type", "answer_options"), + [ + ("categorical", None), + ("discrete", ["a", "b"]), + ("boolean_probability", ["a", "b"]), + (None, []), + (None, [{"level": "0", "label": "x"}, {"level": 1, "label": "y"}]), + ], + ) + def test_mismatched_or_malformed_options_raise_validation_error(self, output_type, answer_options): + # When/Then: bad options fail at construction as ValidationError, before any request + with pytest.raises(ValidationError): + DecisionEvaluator(name="q", question="q", output_type=output_type, answer_options=answer_options) @pytest.mark.parametrize("output_type", ["categorial", "boolean", OutputTypeEnum.PERCENTAGE]) def test_output_type_rejects_non_decision_types(self, output_type): From 8388e52348a5e224a4073abb9f1539f0f355de79 Mon Sep 17 00:00:00 2001 From: John Weiler Date: Fri, 16 Oct 2026 09:53:24 -0400 Subject: [PATCH 5/7] chore: tidy comments Co-Authored-By: Claude Sonnet 5.5 --- src/splunk_ao/evaluator.py | 2 +- src/splunk_ao/evaluators.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/splunk_ao/evaluator.py b/src/splunk_ao/evaluator.py index cad3c8d4..e552c2ee 100644 --- a/src/splunk_ao/evaluator.py +++ b/src/splunk_ao/evaluator.py @@ -870,7 +870,7 @@ class DecisionEvaluator(Evaluator): """ Decision model evaluator. - A decision evaluator asks a decision model (for example TypeSafe's Jev) one typed question and scores + A decision evaluator asks a decision model one typed question and scores the node with the probabilities of the answers. Attributes diff --git a/src/splunk_ao/evaluators.py b/src/splunk_ao/evaluators.py index 43ae5b10..f2af0a90 100644 --- a/src/splunk_ao/evaluators.py +++ b/src/splunk_ao/evaluators.py @@ -149,7 +149,7 @@ def _build_decision_version_request( The generated ``CreateDecisionScorerVersionRequest.to_dict`` has two identical ``isinstance(list)`` branches for ``list[str] | list[DecisionLevel]``, so it never calls ``DecisionLevel.to_dict`` and would send raw objects. ``additional_properties`` is merged into the body verbatim, so levels go there. - Fixing the codegen template touches many unrelated generated models, so it is left as a follow-up. + Fixing the codegen template would touch many unrelated generated models. """ if answer_options and all(isinstance(level, DecisionLevel) for level in answer_options): request = CreateDecisionScorerVersionRequest(model_name=model_name, question=question) @@ -481,7 +481,7 @@ def create_decision_evaluator( """ Create a decision evaluator. - A decision evaluator asks a decision model (for example a decision model) one typed question and + A decision evaluator asks a decision model one typed question and scores the node with the probabilities of the answers. Parameters From 5945ba89c09eceaeb670e75cde9c0d3ec76b9d80 Mon Sep 17 00:00:00 2001 From: John Weiler Date: Fri, 16 Oct 2026 10:03:35 -0400 Subject: [PATCH 6/7] fix(sdk): validate decision question and answer option limits before creating the scorer (SAO-17973) Co-Authored-By: Claude Sonnet 5.5 --- src/splunk_ao/evaluator.py | 2 +- src/splunk_ao/evaluators.py | 40 ++++++++++++++++++++++++++++--------- tests/test_evaluators.py | 21 +++++++++++++++++++ 3 files changed, 53 insertions(+), 10 deletions(-) diff --git a/src/splunk_ao/evaluator.py b/src/splunk_ao/evaluator.py index e552c2ee..6abcd285 100644 --- a/src/splunk_ao/evaluator.py +++ b/src/splunk_ao/evaluator.py @@ -948,7 +948,7 @@ def __init__( try: normalized_options = normalize_decision_answer_options(answer_options) resolved_output_type = resolve_decision_output_type(output_type, normalized_options) - validate_decision_request(resolved_output_type, normalized_options) + validate_decision_request(resolved_output_type, normalized_options, question) except ValueError as error: raise ValidationError(str(error)) from error self.output_type = resolved_output_type diff --git a/src/splunk_ao/evaluators.py b/src/splunk_ao/evaluators.py index f2af0a90..3ed9f48a 100644 --- a/src/splunk_ao/evaluators.py +++ b/src/splunk_ao/evaluators.py @@ -78,6 +78,9 @@ def normalize_decision_answer_options( class DecisionVersionOutcomeUnknownError(RuntimeError): """The version request failed without a definitive rejection, so the scorer was kept. + A rate limit (429) is also treated as unknown, so it leaves a scorer without a version; delete it by id + before retrying with the same name. + Attributes ---------- scorer_id (str): The scorer that may or may not have a version. @@ -127,18 +130,37 @@ def resolve_decision_output_type( return OutputTypeEnum.DISCRETE +MAX_QUESTION_LENGTH = 2000 + + def validate_decision_request( - output_type: OutputTypeEnum, answer_options: list[str] | list[DecisionLevel] | None + output_type: OutputTypeEnum, answer_options: list[str] | list[DecisionLevel] | None, question: str | None = None ) -> None: - """Reject answer options that do not match the output type, before anything is created.""" + """Reject requests the server would refuse, before anything is created. + + Checks the question length (1-2,000 characters when given) and that the answer options match the output + type: none for boolean_probability, 2-255 unique labels for categorical, 2-10 levels numbered 0, 1, 2 and + so on in order for discrete. + """ + if question is not None and not 1 <= len(question) <= MAX_QUESTION_LENGTH: + raise ValueError(f"question must be 1-{MAX_QUESTION_LENGTH} characters; got {len(question)}.") if output_type == OutputTypeEnum.BOOLEAN_PROBABILITY: if answer_options is not None: raise ValueError("answer_options must be None for boolean_probability.") elif output_type == OutputTypeEnum.CATEGORICAL: if not answer_options or not all(isinstance(option, str) for option in answer_options): raise ValueError("answer_options must be a list of label strings for categorical.") - elif not answer_options or not all(isinstance(option, DecisionLevel) for option in answer_options): - raise ValueError("answer_options must be a list of levels (DecisionLevel or dict) for discrete.") + if not 2 <= len(answer_options) <= 255 or len(set(answer_options)) != len(answer_options): + raise ValueError("answer_options must hold 2-255 unique labels for categorical.") + else: + if not answer_options or not all(isinstance(option, DecisionLevel) for option in answer_options): + raise ValueError("answer_options must be a list of levels (DecisionLevel or dict) for discrete.") + if not 2 <= len(answer_options) <= 10: + raise ValueError("answer_options must hold 2-10 levels for discrete.") + if [level.level for level in answer_options if isinstance(level, DecisionLevel)] != list( + range(len(answer_options)) + ): + raise ValueError("discrete levels must be numbered 0, 1, 2 and so on in order.") def _build_decision_version_request( @@ -254,7 +276,7 @@ def create_decision_evaluator( question: str, node_level: StepType = StepType.llm, model_name: str = DEFAULT_DECISION_MODEL_NAME, - output_type: OutputTypeEnum | None = None, + output_type: OutputTypeEnum | str | None = None, answer_options: DecisionAnswerOptions | None = None, input_type: InputTypeEnum | None = None, description: str = "", @@ -274,7 +296,7 @@ def create_decision_evaluator( The single node type the evaluator scores. model_name: str Decision model registry version, for example ``jev-1.13.0``. - output_type: OutputTypeEnum | None + output_type: OutputTypeEnum | str | None ``BOOLEAN_PROBABILITY``, ``CATEGORICAL`` or ``DISCRETE``. When ``None``, it is inferred from ``answer_options``: ``None`` gives ``BOOLEAN_PROBABILITY``, strings give ``CATEGORICAL`` and levels give ``DISCRETE``. An explicit value that does not match ``answer_options`` raises ``ValueError``. @@ -310,7 +332,7 @@ def create_decision_evaluator( """ normalized_options = normalize_decision_answer_options(answer_options) output_type = resolve_decision_output_type(output_type, normalized_options) - validate_decision_request(output_type, normalized_options) + validate_decision_request(output_type, normalized_options, question) version_req = _build_decision_version_request( model_name=model_name, question=question, answer_options=normalized_options ) @@ -471,7 +493,7 @@ def create_decision_evaluator( question: str, node_level: StepType = StepType.llm, model_name: str = DEFAULT_DECISION_MODEL_NAME, - output_type: OutputTypeEnum | None = None, + output_type: OutputTypeEnum | str | None = None, answer_options: DecisionAnswerOptions | None = None, input_type: InputTypeEnum | None = None, description: str = "", @@ -494,7 +516,7 @@ def create_decision_evaluator( The single node type the evaluator scores. model_name: str Decision model registry version, for example ``jev-1.13.0``. - output_type: OutputTypeEnum | None + output_type: OutputTypeEnum | str | None ``BOOLEAN_PROBABILITY``, ``CATEGORICAL`` or ``DISCRETE``. When ``None``, it is inferred from ``answer_options``: ``None`` gives ``BOOLEAN_PROBABILITY``, strings give ``CATEGORICAL`` and levels give ``DISCRETE``. An explicit value that does not match ``answer_options`` raises ``ValueError``. diff --git a/tests/test_evaluators.py b/tests/test_evaluators.py index 8afdc9cb..663eeee6 100644 --- a/tests/test_evaluators.py +++ b/tests/test_evaluators.py @@ -839,3 +839,24 @@ def test_unknown_output_type_string_raises(self, mock_create_scorer, mock_create with pytest.raises(ValueError, match="output_type"): create_decision_evaluator(name="q", question="q", output_type="categorial", answer_options=["a", "b"]) mock_create_scorer.sync.assert_not_called() + + @pytest.mark.parametrize( + ("question", "answer_options"), + [ + ("", None), + ("x" * 2001, None), + ("q", ["only"]), + ("q", ["a", "a"]), + ("q", [f"l{i}" for i in range(256)]), + ("q", [{"level": 1, "label": "x"}, {"level": 2, "label": "y"}]), + ("q", [{"level": 0, "label": "x"}, {"level": 0, "label": "y"}]), + ("q", [{"level": i, "label": str(i)} for i in range(11)]), + ], + ) + def test_server_limits_are_checked_before_any_request( + self, mock_create_scorer, mock_create_version, mock_delete, question, answer_options + ) -> None: + # When/Then: inputs the server would reject with 422 raise before the scorer is created + with pytest.raises(ValueError): + create_decision_evaluator(name="q", question=question, answer_options=answer_options) + mock_create_scorer.sync.assert_not_called() From f450fb07e836b8b2853a84b6d0efdfc9c1b9bf95 Mon Sep 17 00:00:00 2001 From: John Weiler Date: Fri, 16 Oct 2026 10:40:44 -0400 Subject: [PATCH 7/7] feat(sdk): read the default decision model from Configuration (SAO-17973) DecisionEvaluator falls back to Configuration.default_decision_model (env SPLUNK_AO_DEFAULT_DECISION_MODEL, default jev-1.13.0), the same way LlmEvaluator uses default_scorer_model. create_decision_evaluator keeps a literal default, like create_custom_llm_evaluator. Co-Authored-By: Claude Opus 5.5 --- src/splunk_ao/configuration.py | 6 ++++++ src/splunk_ao/evaluator.py | 8 ++++---- src/splunk_ao/evaluators.py | 6 ++---- tests/test_evaluator.py | 24 ++++++++++++++++++++++++ 4 files changed, 36 insertions(+), 8 deletions(-) diff --git a/src/splunk_ao/configuration.py b/src/splunk_ao/configuration.py index 12d6a65e..e7cfd9d6 100644 --- a/src/splunk_ao/configuration.py +++ b/src/splunk_ao/configuration.py @@ -107,6 +107,12 @@ def parse_log_level(value: str) -> str: value_type=int, parser=int, ), + ConfigKey( + name="default_decision_model", + env_var="SPLUNK_AO_DEFAULT_DECISION_MODEL", + description="Default model for decision evaluators", + default="jev-1.13.0", + ), ConfigKey( name="logging_disabled", env_var="SPLUNK_AO_LOGGING_DISABLED", diff --git a/src/splunk_ao/evaluator.py b/src/splunk_ao/evaluator.py index 6abcd285..06133eea 100644 --- a/src/splunk_ao/evaluator.py +++ b/src/splunk_ao/evaluator.py @@ -20,7 +20,6 @@ from splunk_ao.config import SplunkAOConfig from splunk_ao.configuration import Configuration from splunk_ao.evaluators import ( - DEFAULT_DECISION_MODEL_NAME, DecisionAnswerOptions, DecisionVersionOutcomeUnknownError, Evaluators, @@ -922,7 +921,8 @@ def __init__( Args: name: The name of the evaluator. question: The question the decision model answers (1-2,000 characters). - model: Decision model registry version. Defaults to ``jev-1.13.0``. + model: Decision model registry version. Defaults to Configuration.default_decision_model + (env: SPLUNK_AO_DEFAULT_DECISION_MODEL), which is ``jev-1.13.0`` unless set. node_level: The single node type the evaluator scores. Defaults to StepType.llm. output_type: ``boolean_probability``, ``categorical`` or ``discrete``. Inferred from ``answer_options`` when None. answer_options: None for boolean_probability, a list of labels for categorical, or a list of @@ -943,7 +943,7 @@ def __init__( raise ValidationError("'question' must be provided for decision evaluators.") self.question = question - self.model = model or DEFAULT_DECISION_MODEL_NAME + self.model = model or Configuration.default_decision_model self.node_level = node_level or StepType.llm try: normalized_options = normalize_decision_answer_options(answer_options) @@ -984,7 +984,7 @@ def create(self) -> DecisionEvaluator: name=self.name, question=self.question or "", node_level=self.node_level if self.node_level is not None else StepType.llm, - model_name=self.model if self.model is not None else DEFAULT_DECISION_MODEL_NAME, + model_name=self.model if self.model is not None else Configuration.default_decision_model, output_type=self.output_type, answer_options=self.answer_options, input_type=self.input_type, diff --git a/src/splunk_ao/evaluators.py b/src/splunk_ao/evaluators.py index 3ed9f48a..dbaac659 100644 --- a/src/splunk_ao/evaluators.py +++ b/src/splunk_ao/evaluators.py @@ -33,8 +33,6 @@ _logger = logging.getLogger(__name__) -DEFAULT_DECISION_MODEL_NAME = "jev-1.13.0" - DecisionAnswerOptions = Sequence[str] | Sequence[DecisionLevel | Mapping[str, Any]] @@ -275,7 +273,7 @@ def create_decision_evaluator( name: str, question: str, node_level: StepType = StepType.llm, - model_name: str = DEFAULT_DECISION_MODEL_NAME, + model_name: str = "jev-1.13.0", output_type: OutputTypeEnum | str | None = None, answer_options: DecisionAnswerOptions | None = None, input_type: InputTypeEnum | None = None, @@ -492,7 +490,7 @@ def create_decision_evaluator( name: str, question: str, node_level: StepType = StepType.llm, - model_name: str = DEFAULT_DECISION_MODEL_NAME, + model_name: str = "jev-1.13.0", output_type: OutputTypeEnum | str | None = None, answer_options: DecisionAnswerOptions | None = None, input_type: InputTypeEnum | None = None, diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py index 4e676877..b6bf6855 100644 --- a/tests/test_evaluator.py +++ b/tests/test_evaluator.py @@ -2175,6 +2175,30 @@ def test_create_sends_fields_and_refreshes( assert evaluator.id == "s1" assert evaluator.is_synced() + @patch("splunk_ao.evaluator.Evaluators") + @patch("splunk_ao.evaluator.Scorers") + def test_default_model_comes_from_configuration( + self, + mock_scorers_class: MagicMock, + mock_evaluators_class: MagicMock, + reset_configuration: None, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + # Given: the default decision model is set through the environment + monkeypatch.setenv("SPLUNK_AO_DEFAULT_DECISION_MODEL", "jev-9.9.9") + version = MagicMock() + version.scorer_id = "s1" + mock_evaluators_class.return_value.create_decision_evaluator.return_value = version + mock_scorers_class.return_value.list.return_value = [_decision_scorer("categorical", ["a", "b"])] + + # When: creating a DecisionEvaluator without a model + evaluator = DecisionEvaluator(name="tone", question="Which tone?", answer_options=["a", "b"]) + evaluator.create() + + # Then: the configured model is used + kwargs = mock_evaluators_class.return_value.create_decision_evaluator.call_args.kwargs + assert kwargs["model_name"] == "jev-9.9.9" + class TestDecisionEvaluatorReadPath: @patch("splunk_ao.evaluator.Scorers")