From 1225a56d03384f5d9ad923349eaae3127d743158 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 16 Oct 2026 00:07:30 +0000 Subject: [PATCH 1/3] fix(opencode-agents): skip judge prompts in the handler provides_for stays ('*', 'agent'), so judge selection still lands here. A judge evaluation raises before events() runs, and the judge does not score. --- packages/opencode-agents/README.md | 8 ++--- packages/opencode-agents/agents.md | 2 +- .../handler.py | 30 +++++++++++++++---- .../opencode-agents/tests/test_handler.py | 12 +++++--- 4 files changed, 37 insertions(+), 15 deletions(-) diff --git a/packages/opencode-agents/README.md b/packages/opencode-agents/README.md index f186df8a..43caaa3c 100644 --- a/packages/opencode-agents/README.md +++ b/packages/opencode-agents/README.md @@ -36,9 +36,9 @@ async def session_events(config, user_input, variables, history, session_id): async def main() -> None: await init_client() handler = create_opencode_handler(events=session_events, session_id="ses") - async for event in config( - key="my-agent", handler=handler, skip_judges=True - ).stream("Hello", {"kind": "user", "key": "user-1"}): + async for event in config(key="my-agent", handler=handler).stream( + "Hello", {"kind": "user", "key": "user-1"} + ): print(event) await shutdown() @@ -48,4 +48,4 @@ asyncio.run(main()) `events` is required. Each invocation calls `events(config, user_input, variables, history, session_id)` and the return value must be a fresh async iterable of that session only. The iterable must end: return it after `session.idle`, or close it yourself. The handler yields AG-UI text and tool events. `config().stream()` yields `chunk` then `done`. -Pass `skip_judges=True`. This handler matches every agent-mode provider, including a judge, and it does not run the judge prompt. +This handler matches every agent-mode provider, so judge selection can land here. It does not run the judge prompt: that call raises without calling `events`, and the judge does not score. diff --git a/packages/opencode-agents/agents.md b/packages/opencode-agents/agents.md index 2410195e..6ec51ebe 100644 --- a/packages/opencode-agents/agents.md +++ b/packages/opencode-agents/agents.md @@ -38,6 +38,6 @@ Assistant text is emitted only after `message.updated` marks that message `role The AI Config, user input, variables, and history are arguments to `events`. A callable that ignores them never sees the prompt. -`provides_for` is `('*', 'agent')`, so a judge is routed here too. Pass `skip_judges=True` until this handler runs the prompt it is given. Otherwise `events` runs again on the judge prompt and the score is wrong. +`provides_for` is `('*', 'agent')`, so judge selection still lands here. The handler does not run that prompt. A judge evaluation raises without calling `events`, and the judge does not score. `session_id` is required. The reducer does not adopt the first session on a shared stream. Other sessions are dropped, including their `session.error`. `session.idle` for this session ends the run. If the iterable is a server-wide stream and you neither send `session.idle` nor close it, `done` never arrives. diff --git a/packages/opencode-agents/src/launchdarkly_ai_opencode_agents/handler.py b/packages/opencode-agents/src/launchdarkly_ai_opencode_agents/handler.py index a0aca9ce..05d9e817 100644 --- a/packages/opencode-agents/src/launchdarkly_ai_opencode_agents/handler.py +++ b/packages/opencode-agents/src/launchdarkly_ai_opencode_agents/handler.py @@ -460,6 +460,20 @@ def _session_error_message(properties: object) -> str: return f"The OpenCode session failed ({name})." +def _is_judge_evaluation() -> bool: + """Whether this call is a judge evaluation, not the agent's own session. + + ``provides_for`` stays ``('*', 'agent')``, so judge selection still lands + here. The judge prompt is not this OpenCode session. Calling *events* for + it would answer the judge with that session. + """ + from opentelemetry import context as otel_context + + from launchdarkly_ai_server.conversation import _EVAL_KEY + + return otel_context.get_value(_EVAL_KEY) is not None + + def _input_messages( user_input: str | None, history: list[dict[str, Any]] | None, @@ -508,10 +522,12 @@ def create_opencode_handler( construct an OpenCode SDK client. The stream yields the AG-UI text and tool subset and one ``done``. - ``config().stream()`` still yields ``chunk`` then ``done``. Pass - ``skip_judges=True`` to ``config()``: a judge would otherwise be routed - here and *events* would run again on the judge prompt. *capture_content* - follows the other handlers. + ``config().stream()`` still yields ``chunk`` then ``done``. Judge + selection still lands here, because ``provides_for`` is ``('*', 'agent')``. + The handler does not run that prompt: the call raises without calling + *events*, and the judge does not score. Pass ``skip_judges=True`` to + ``config()`` to skip that attempt before it is selected. + *capture_content* follows the other handlers. """ report_usage("opencode-agents.createOpenCodeHandler", PACKAGE_NAME, __version__) return _create_opencode_handler( @@ -531,8 +547,8 @@ def opencode_agents( *events* is required and is the same callable :func:`create_opencode_handler` takes. *variables*, *history*, *session_id*, and *capture_content* configure - that call. Pass ``skip_judges=True`` so a judged config does not run - *events* again. + that call. A judged config does not call *events* again: a judge + evaluation raises, and the judge does not score. """ report_usage("opencode-agents.opencodeAgents", PACKAGE_NAME, __version__) variables = kwargs.pop("variables", None) @@ -573,6 +589,8 @@ async def _stream_impl( variables: dict[str, Any] | None, history: list[dict[str, Any]] | None = None, ) -> AsyncGenerator[dict[str, Any], None]: + if _is_judge_evaluation(): + raise RuntimeError("The OpenCode handler does not run judge prompts.") capture = effective_capture_content(content_override) span = start_root_span(config, variables or {}) ended: set[int] = set() diff --git a/packages/opencode-agents/tests/test_handler.py b/packages/opencode-agents/tests/test_handler.py index 4e806c23..e9e97cf8 100644 --- a/packages/opencode-agents/tests/test_handler.py +++ b/packages/opencode-agents/tests/test_handler.py @@ -919,10 +919,14 @@ async def events( assert calls == ["q"] calls.clear() - await _drain(config(key="flag", handler=handler).stream("q", CONTEXT)) - assert calls[0] == "q" - assert len(calls) == 2 - assert calls[1] != "q" + judged_run = [ + event + async for event in config(key="flag", handler=handler).stream("q", CONTEXT) + ] + assert calls == ["q"] + assert judged_run[-1]["type"] == "done" + assert judged_run[-1]["response"] == "Hi" + assert not judged_run[-1].get("judge_results") async def test_the_wrapper_forwards_variables_history_and_session( self, mock_ld_client: MagicMock From e54111d894b89684888b65336cb36d8c884f07c7 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 16 Oct 2026 00:07:47 +0000 Subject: [PATCH 2/3] fix(opencode-agents): record block-array history on the span _input_messages kept only string history and always appended user_input, so structured history never reached invoke_agent and an empty turn did. --- .../handler.py | 27 ++++- .../opencode-agents/tests/test_handler.py | 104 ++++++++++++++++++ 2 files changed, 125 insertions(+), 6 deletions(-) diff --git a/packages/opencode-agents/src/launchdarkly_ai_opencode_agents/handler.py b/packages/opencode-agents/src/launchdarkly_ai_opencode_agents/handler.py index 05d9e817..8e36a1f5 100644 --- a/packages/opencode-agents/src/launchdarkly_ai_opencode_agents/handler.py +++ b/packages/opencode-agents/src/launchdarkly_ai_opencode_agents/handler.py @@ -19,6 +19,8 @@ _config, _create_handler, add_cached_tokens_to_input, + compose_history, + content_to_text, end_span_once, set_input_content_attributes, set_output_content_attributes, @@ -478,16 +480,29 @@ def _input_messages( user_input: str | None, history: list[dict[str, Any]] | None, ) -> list[SpanMessage]: + """History and *user_input* as span messages. + + Block-array history is flattened the same way as the other handlers. + System turns stay off this list: they belong on the system instructions. + An empty *user_input* adds nothing, so a history turn that already holds + the user message is not followed by a blank one. + """ + items = [item for item in (history or []) if isinstance(item, dict)] messages: list[SpanMessage] = [] - for item in history or []: - if not isinstance(item, dict): - continue + for item in compose_history(history=items, user_input=user_input): role = item.get("role") content = item.get("content") - if isinstance(role, str) and role and isinstance(content, str): + if not isinstance(role, str) or not role: + continue + if isinstance(content, str): messages.append(text_message(role, content)) - if isinstance(user_input, str): - messages.append(text_message("user", user_input)) + continue + if isinstance(content, list): + text = content_to_text( + [block for block in content if isinstance(block, dict)] + ) + if text: + messages.append(text_message(role, text)) return messages diff --git a/packages/opencode-agents/tests/test_handler.py b/packages/opencode-agents/tests/test_handler.py index e9e97cf8..f9ec586b 100644 --- a/packages/opencode-agents/tests/test_handler.py +++ b/packages/opencode-agents/tests/test_handler.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json from collections.abc import AsyncIterator, Iterator from typing import Any from unittest.mock import AsyncMock, MagicMock @@ -1032,6 +1033,109 @@ def set_span_in_context(self, span: _Span) -> Any: assert tracer.spans[0].attributes["gen_ai.operation.name"] == "invoke_agent" assert tracer.spans[0].attributes["launchdarkly.operation.type"] == "gen_ai" + async def test_block_array_history_reaches_the_span( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from launchdarkly_ai_opencode_agents import spans as span_module + + class _Span: + def __init__(self, name: str, context: Any = None) -> None: + self.name = name + self.context = context + self.attributes: dict[str, Any] = {} + self.events: list[str] = [] + + def set_attribute(self, key: str, value: Any) -> None: + self.attributes[key] = value + + def add_event( + self, name: str, attributes: dict[str, Any] | None = None + ) -> None: + self.events.append(name) + + def set_status(self, code: Any, description: str | None = None) -> None: + return None + + def record_exception(self, exc: BaseException) -> None: + return None + + def end(self) -> None: + return None + + class _Tracer: + def __init__(self) -> None: + self.spans: list[_Span] = [] + + def get_tracer(self, name: str) -> _Tracer: + return self + + def start_span(self, name: str, context: Any = None) -> _Span: + span = _Span(name, context) + self.spans.append(span) + return span + + def set_span_in_context(self, span: _Span) -> Any: + return span + + tracer = _Tracer() + monkeypatch.setattr(span_module, "trace", tracer) + handler = create_opencode_handler( + capture_content=True, + session_id="ses", + events=source( + [ + assistant_open("m1"), + text_delta("m1", "p1", "Hi"), + assistant_completed("m1"), + ] + ), + ) + history = [ + {"role": "system", "content": "do not record"}, + {"role": "user", "content": "earlier"}, + { + "role": "user", + "content": [ + {"type": "text", "text": "see "}, + { + "type": "image", + "source": { + "type": "url", + "url": "https://example.test/a.png", + }, + }, + {"type": "text", "text": "this"}, + ], + }, + { + "role": "assistant", + "content": [{"type": "text", "text": "noted"}], + }, + ] + seen = [ + event async for event in await handler.stream({}, "", None, None, history) + ] + assert seen[-1]["output"] == "Hi" + recorded = json.loads(tracer.spans[0].attributes["gen_ai.input.messages"]) + assert recorded == [ + { + "role": "user", + "parts": [{"type": "text", "content": "earlier"}], + }, + { + "role": "user", + "parts": [{"type": "text", "content": "see this"}], + }, + { + "role": "assistant", + "parts": [{"type": "text", "content": "noted"}], + }, + ] + assert ( + "do not record" not in tracer.spans[0].attributes["gen_ai.input.messages"] + ) + assert "gen_ai.prompt.3.role" not in tracer.spans[0].attributes + async def _drain(stream: Any) -> None: async for _event in stream: From e487401dd25dd72763ff8a9e694be5015de9cc48 Mon Sep 17 00:00:00 2001 From: Alexis Georges Date: Mon, 19 Oct 2026 13:08:06 -0400 Subject: [PATCH 3/3] fix(opencode-agents): keep the skip_judges advice and pin judge tracking Without skip_judges=True the handler's raise is caught by run_judges only after execute_and_track has sent a generation error for the judge config, so every judged request showed the judge as failed. The docs go back to telling callers to pass skip_judges=True, and the raise stays as the safety net. The README also says not to pass this handler to run_judge, which would raise instead of returning None. The judged test now asserts the judge error event, so the chosen behavior is pinned. The private _EVAL_KEY import moves to module top, so a rename in core fails at import time instead of on every stream call. --- packages/opencode-agents/README.md | 10 +++++---- packages/opencode-agents/agents.md | 2 +- .../handler.py | 22 +++++++++---------- .../opencode-agents/tests/test_handler.py | 13 +++++++++++ 4 files changed, 31 insertions(+), 16 deletions(-) diff --git a/packages/opencode-agents/README.md b/packages/opencode-agents/README.md index 43caaa3c..bcbdc2c1 100644 --- a/packages/opencode-agents/README.md +++ b/packages/opencode-agents/README.md @@ -36,9 +36,9 @@ async def session_events(config, user_input, variables, history, session_id): async def main() -> None: await init_client() handler = create_opencode_handler(events=session_events, session_id="ses") - async for event in config(key="my-agent", handler=handler).stream( - "Hello", {"kind": "user", "key": "user-1"} - ): + async for event in config( + key="my-agent", handler=handler, skip_judges=True + ).stream("Hello", {"kind": "user", "key": "user-1"}): print(event) await shutdown() @@ -48,4 +48,6 @@ asyncio.run(main()) `events` is required. Each invocation calls `events(config, user_input, variables, history, session_id)` and the return value must be a fresh async iterable of that session only. The iterable must end: return it after `session.idle`, or close it yourself. The handler yields AG-UI text and tool events. `config().stream()` yields `chunk` then `done`. -This handler matches every agent-mode provider, so judge selection can land here. It does not run the judge prompt: that call raises without calling `events`, and the judge does not score. +Pass `skip_judges=True`. This handler matches every agent-mode provider, so judge selection can land here. It does not run the judge prompt: that call raises without calling `events`, so the judge is tracked as a failed generation and logged as an error. + +Do not pass this handler to `run_judge`. It would be picked as the agent fallback and `run_judge` would raise. diff --git a/packages/opencode-agents/agents.md b/packages/opencode-agents/agents.md index 6ec51ebe..e47fecdd 100644 --- a/packages/opencode-agents/agents.md +++ b/packages/opencode-agents/agents.md @@ -38,6 +38,6 @@ Assistant text is emitted only after `message.updated` marks that message `role The AI Config, user input, variables, and history are arguments to `events`. A callable that ignores them never sees the prompt. -`provides_for` is `('*', 'agent')`, so judge selection still lands here. The handler does not run that prompt. A judge evaluation raises without calling `events`, and the judge does not score. +`provides_for` is `('*', 'agent')`, so judge selection still lands here. Pass `skip_judges=True`. The handler does not run that prompt. A judge evaluation raises without calling `events`, so the judge is tracked as a failed generation. Do not pass this handler to `run_judge`, which would raise. `session_id` is required. The reducer does not adopt the first session on a shared stream. Other sessions are dropped, including their `session.error`. `session.idle` for this session ends the run. If the iterable is a server-wide stream and you neither send `session.idle` nor close it, `done` never arrives. diff --git a/packages/opencode-agents/src/launchdarkly_ai_opencode_agents/handler.py b/packages/opencode-agents/src/launchdarkly_ai_opencode_agents/handler.py index 8e36a1f5..f535f766 100644 --- a/packages/opencode-agents/src/launchdarkly_ai_opencode_agents/handler.py +++ b/packages/opencode-agents/src/launchdarkly_ai_opencode_agents/handler.py @@ -11,6 +11,8 @@ from collections.abc import AsyncGenerator, AsyncIterable, Callable, Mapping from typing import Any +from opentelemetry import context as otel_context + from launchdarkly_ai_server import ( AiConfigRep, ProviderHandler, @@ -27,6 +29,7 @@ text_message, ) from launchdarkly_ai_server.agui import TurnEmitter +from launchdarkly_ai_server.conversation import _EVAL_KEY from launchdarkly_ai_server.lifecycle import effective_capture_content from launchdarkly_ai_server.sdk_usage import report_usage from launchdarkly_ai_server.utils import number_or_zero @@ -469,10 +472,6 @@ def _is_judge_evaluation() -> bool: here. The judge prompt is not this OpenCode session. Calling *events* for it would answer the judge with that session. """ - from opentelemetry import context as otel_context - - from launchdarkly_ai_server.conversation import _EVAL_KEY - return otel_context.get_value(_EVAL_KEY) is not None @@ -537,11 +536,11 @@ def create_opencode_handler( construct an OpenCode SDK client. The stream yields the AG-UI text and tool subset and one ``done``. - ``config().stream()`` still yields ``chunk`` then ``done``. Judge - selection still lands here, because ``provides_for`` is ``('*', 'agent')``. - The handler does not run that prompt: the call raises without calling - *events*, and the judge does not score. Pass ``skip_judges=True`` to - ``config()`` to skip that attempt before it is selected. + ``config().stream()`` still yields ``chunk`` then ``done``. Pass + ``skip_judges=True`` to ``config()``. Judge selection lands here, because + ``provides_for`` is ``('*', 'agent')``, and the handler does not run the + judge prompt: the call raises without calling *events*, so the judge is + tracked as a failed generation. Do not pass this handler to ``run_judge``. *capture_content* follows the other handlers. """ report_usage("opencode-agents.createOpenCodeHandler", PACKAGE_NAME, __version__) @@ -562,8 +561,9 @@ def opencode_agents( *events* is required and is the same callable :func:`create_opencode_handler` takes. *variables*, *history*, *session_id*, and *capture_content* configure - that call. A judged config does not call *events* again: a judge - evaluation raises, and the judge does not score. + that call. Pass ``skip_judges=True``. A judge evaluation raises without + calling *events* again, so a judged config tracks the judge as a failed + generation. """ report_usage("opencode-agents.opencodeAgents", PACKAGE_NAME, __version__) variables = kwargs.pop("variables", None) diff --git a/packages/opencode-agents/tests/test_handler.py b/packages/opencode-agents/tests/test_handler.py index f9ec586b..e7084af6 100644 --- a/packages/opencode-agents/tests/test_handler.py +++ b/packages/opencode-agents/tests/test_handler.py @@ -150,6 +150,15 @@ def mock_ld_client() -> Iterator[MagicMock]: lifecycle_module._reset_for_testing() +def judge_errors(client: MagicMock) -> int: + return sum( + 1 + for call in client.track.call_args_list + if call.args[0] == "$ld:ai:generation:error" + and call.args[2].get("configKey") == "judge-flag" + ) + + class TestSurface: def test_provides_for_is_any_provider_in_agent_mode(self) -> None: handler = create_opencode_handler(events=source([]), session_id="ses") @@ -918,6 +927,7 @@ async def events( ] assert [event["type"] for event in skipped] == ["chunk", "done"] assert calls == ["q"] + assert not judge_errors(mock_ld_client) calls.clear() judged_run = [ @@ -928,6 +938,9 @@ async def events( assert judged_run[-1]["type"] == "done" assert judged_run[-1]["response"] == "Hi" assert not judged_run[-1].get("judge_results") + # Without skip_judges the raise is the safety net: the judge is + # tracked as a failed generation instead of scoring this session. + assert judge_errors(mock_ld_client) == 1 async def test_the_wrapper_forwards_variables_history_and_session( self, mock_ld_client: MagicMock