From a7990adc4b2ab3e589643f57d434fc6d20e2123e Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 21 Jan 2026 17:56:25 +0100 Subject: [PATCH 01/10] ref(openai): Separate input handling to improve typing --- sentry_sdk/integrations/openai.py | 146 ++++++++++++++++++++++++++++-- 1 file changed, 136 insertions(+), 10 deletions(-) diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index 66dc4a1c48..55ec5e4085 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -23,9 +23,20 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from typing import Any, Iterable, List, Optional, Callable, AsyncIterator, Iterator + from typing import ( + Any, + Iterable, + List, + Optional, + Callable, + AsyncIterator, + Iterator, + Union, + ) from sentry_sdk.tracing import Span + from openai.types.responses import ResponseInputParam + try: try: from openai import NotGiven @@ -182,6 +193,124 @@ def _calculate_token_usage( ) +def _get_input_messages(kwargs: "dict[str, Any]") -> "Optional[list[Any] | list[str]]": + # Input messages (the prompt or data sent to the model) + messages = kwargs.get("messages") + if messages is None: + messages = kwargs.get("input") + + if isinstance(messages, str): + messages = [messages] + + return messages + + +def _commmon_set_input_data( + span: "Span", + kwargs: "dict[str, Any]", +): + # Input attributes: Common + set_data_normalized(span, SPANDATA.GEN_AI_SYSTEM, "openai") + + # Input attributes: Optional + kwargs_keys_to_attributes = { + "model": SPANDATA.GEN_AI_REQUEST_MODEL, + "stream": SPANDATA.GEN_AI_RESPONSE_STREAMING, + "max_tokens": SPANDATA.GEN_AI_REQUEST_MAX_TOKENS, + "presence_penalty": SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY, + "frequency_penalty": SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY, + "temperature": SPANDATA.GEN_AI_REQUEST_TEMPERATURE, + "top_p": SPANDATA.GEN_AI_REQUEST_TOP_P, + } + for key, attribute in kwargs_keys_to_attributes.items(): + value = kwargs.get(key) + + if value is not None and _is_given(value): + set_data_normalized(span, attribute, value) + + # Input attributes: Tools + tools = kwargs.get("tools") + if tools is not None and _is_given(tools) and len(tools) > 0: + set_data_normalized( + span, SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, safe_serialize(tools) + ) + + +def _set_responses_api_input_data( + span: "Span", + kwargs: "dict[str, Any]", + integration: "OpenAIIntegration", +): + messages: "Optional[ResponseInputParam | list[str]]" = _get_input_messages(kwargs) + + if ( + messages is not None + and len(messages) > 0 + and should_send_default_pii() + and integration.include_prompts + ): + normalized_messages = normalize_message_roles(messages) + scope = sentry_sdk.get_current_scope() + messages_data = truncate_and_annotate_messages(normalized_messages, span, scope) + if messages_data is not None: + set_data_normalized( + span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False + ) + + set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "responses") + _commmon_set_input_data(span, kwargs) + + +def _set_completions_api_input_data( + span: "Span", + kwargs: "dict[str, Any]", + integration: "OpenAIIntegration", +): + messages: "Optional[ChatCompletionMessageParam]" = _get_input_messages(kwargs) + + if ( + messages is not None + and len(messages) > 0 + and should_send_default_pii() + and integration.include_prompts + ): + normalized_messages = normalize_message_roles(messages) + scope = sentry_sdk.get_current_scope() + messages_data = truncate_and_annotate_messages(normalized_messages, span, scope) + if messages_data is not None: + set_data_normalized( + span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False + ) + + set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "chat") + _commmon_set_input_data(span, kwargs) + + +def _set_embeddings_input_data( + span: "Span", + kwargs: "dict[str, Any]", + integration: "OpenAIIntegration", +): + messages = _get_input_messages(kwargs) + + if ( + messages is not None + and len(messages) > 0 + and should_send_default_pii() + and integration.include_prompts + ): + normalized_messages = normalize_message_roles(messages) + scope = sentry_sdk.get_current_scope() + messages_data = truncate_and_annotate_messages(normalized_messages, span, scope) + if messages_data is not None: + set_data_normalized( + span, SPANDATA.GEN_AI_EMBEDDINGS_INPUT, messages_data, unpack=False + ) + + set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "embeddings") + _commmon_set_input_data(span, kwargs) + + def _set_input_data( span: "Span", kwargs: "dict[str, Any]", @@ -454,16 +583,15 @@ def _new_chat_completion_common(f: "Any", *args: "Any", **kwargs: "Any") -> "Any return f(*args, **kwargs) model = kwargs.get("model") - operation = "chat" span = sentry_sdk.start_span( op=consts.OP.GEN_AI_CHAT, - name=f"{operation} {model}", + name=f"chat {model}", origin=OpenAIIntegration.origin, ) span.__enter__() - _set_input_data(span, kwargs, operation, integration) + _set_completions_api_input_data(span, kwargs, integration) response = yield f, args, kwargs @@ -546,14 +674,13 @@ def _new_embeddings_create_common(f: "Any", *args: "Any", **kwargs: "Any") -> "A return f(*args, **kwargs) model = kwargs.get("model") - operation = "embeddings" with sentry_sdk.start_span( op=consts.OP.GEN_AI_EMBEDDINGS, - name=f"{operation} {model}", + name=f"embeddings {model}", origin=OpenAIIntegration.origin, ) as span: - _set_input_data(span, kwargs, operation, integration) + _set_embeddings_input_data(span, kwargs, integration) response = yield f, args, kwargs @@ -634,16 +761,15 @@ def _new_responses_create_common(f: "Any", *args: "Any", **kwargs: "Any") -> "An return f(*args, **kwargs) model = kwargs.get("model") - operation = "responses" span = sentry_sdk.start_span( op=consts.OP.GEN_AI_RESPONSES, - name=f"{operation} {model}", + name=f"responses {model}", origin=OpenAIIntegration.origin, ) span.__enter__() - _set_input_data(span, kwargs, operation, integration) + _set_responses_api_input_data(span, kwargs, integration) response = yield f, args, kwargs From c6ebc0f353992afb6e37d77082c8e4804238b7e6 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 21 Jan 2026 17:57:12 +0100 Subject: [PATCH 02/10] remove old func --- sentry_sdk/integrations/openai.py | 62 ------------------------------- 1 file changed, 62 deletions(-) diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index 55ec5e4085..a9bccec5ea 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -311,68 +311,6 @@ def _set_embeddings_input_data( _commmon_set_input_data(span, kwargs) -def _set_input_data( - span: "Span", - kwargs: "dict[str, Any]", - operation: str, - integration: "OpenAIIntegration", -) -> None: - # Input messages (the prompt or data sent to the model) - messages = kwargs.get("messages") - if messages is None: - messages = kwargs.get("input") - - if isinstance(messages, str): - messages = [messages] - - if ( - messages is not None - and len(messages) > 0 - and should_send_default_pii() - and integration.include_prompts - ): - normalized_messages = normalize_message_roles(messages) - scope = sentry_sdk.get_current_scope() - messages_data = truncate_and_annotate_messages(normalized_messages, span, scope) - if messages_data is not None: - # Use appropriate field based on operation type - if operation == "embeddings": - set_data_normalized( - span, SPANDATA.GEN_AI_EMBEDDINGS_INPUT, messages_data, unpack=False - ) - else: - set_data_normalized( - span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False - ) - - # Input attributes: Common - set_data_normalized(span, SPANDATA.GEN_AI_SYSTEM, "openai") - set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, operation) - - # Input attributes: Optional - kwargs_keys_to_attributes = { - "model": SPANDATA.GEN_AI_REQUEST_MODEL, - "stream": SPANDATA.GEN_AI_RESPONSE_STREAMING, - "max_tokens": SPANDATA.GEN_AI_REQUEST_MAX_TOKENS, - "presence_penalty": SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY, - "frequency_penalty": SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY, - "temperature": SPANDATA.GEN_AI_REQUEST_TEMPERATURE, - "top_p": SPANDATA.GEN_AI_REQUEST_TOP_P, - } - for key, attribute in kwargs_keys_to_attributes.items(): - value = kwargs.get(key) - - if value is not None and _is_given(value): - set_data_normalized(span, attribute, value) - - # Input attributes: Tools - tools = kwargs.get("tools") - if tools is not None and _is_given(tools) and len(tools) > 0: - set_data_normalized( - span, SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, safe_serialize(tools) - ) - - def _set_output_data( span: "Span", response: "Any", From 8d9fa37482e14f41b55aeaca7ef3f0c5b638ccef Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 21 Jan 2026 18:04:43 +0100 Subject: [PATCH 03/10] . --- sentry_sdk/integrations/openai.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index a9bccec5ea..9c0321d3cd 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -193,7 +193,9 @@ def _calculate_token_usage( ) -def _get_input_messages(kwargs: "dict[str, Any]") -> "Optional[list[Any] | list[str]]": +def _get_input_messages( + kwargs: "dict[str, Any]", +) -> "Optional[Iterable[Any] | list[str]]": # Input messages (the prompt or data sent to the model) messages = kwargs.get("messages") if messages is None: @@ -241,7 +243,7 @@ def _set_responses_api_input_data( kwargs: "dict[str, Any]", integration: "OpenAIIntegration", ): - messages: "Optional[ResponseInputParam | list[str]]" = _get_input_messages(kwargs) + messages: "Optional[ResponseInputParam | list[str]]" = _get_input_messages(kwargs) # type: ignore if ( messages is not None @@ -249,7 +251,7 @@ def _set_responses_api_input_data( and should_send_default_pii() and integration.include_prompts ): - normalized_messages = normalize_message_roles(messages) + normalized_messages = normalize_message_roles(messages) # type: ignore scope = sentry_sdk.get_current_scope() messages_data = truncate_and_annotate_messages(normalized_messages, span, scope) if messages_data is not None: @@ -266,15 +268,17 @@ def _set_completions_api_input_data( kwargs: "dict[str, Any]", integration: "OpenAIIntegration", ): - messages: "Optional[ChatCompletionMessageParam]" = _get_input_messages(kwargs) + messages: "Optional[Iterable[ChatCompletionMessageParam]]" = _get_input_messages( + kwargs + ) if ( messages is not None - and len(messages) > 0 + and len(messages) > 0 # type: ignore and should_send_default_pii() and integration.include_prompts ): - normalized_messages = normalize_message_roles(messages) + normalized_messages = normalize_message_roles(messages) # type: ignore scope = sentry_sdk.get_current_scope() messages_data = truncate_and_annotate_messages(normalized_messages, span, scope) if messages_data is not None: From ab41f1eee58d20e38110237e00aa6088caae0440 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 21 Jan 2026 18:19:45 +0100 Subject: [PATCH 04/10] expand type --- sentry_sdk/integrations/openai.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index 9c0321d3cd..c21b1f8cad 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -9,6 +9,7 @@ normalize_message_roles, truncate_and_annotate_messages, ) +from sentry_sdk.ai._openai_completions_api import _get_system_instructions from sentry_sdk.consts import SPANDATA from sentry_sdk.integrations import DidNotEnable, Integration from sentry_sdk.scope import should_send_default_pii @@ -268,8 +269,8 @@ def _set_completions_api_input_data( kwargs: "dict[str, Any]", integration: "OpenAIIntegration", ): - messages: "Optional[Iterable[ChatCompletionMessageParam]]" = _get_input_messages( - kwargs + messages: "Optional[Iterable[ChatCompletionMessageParam] | list[str]]" = ( + _get_input_messages(kwargs) ) if ( From b3117ac9a0c95cb05a09f737f1b60a064cc836db Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 21 Jan 2026 18:22:31 +0100 Subject: [PATCH 05/10] add ignores --- sentry_sdk/integrations/openai.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index c21b1f8cad..459d782763 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -300,11 +300,11 @@ def _set_embeddings_input_data( if ( messages is not None - and len(messages) > 0 + and len(messages) > 0 # type: ignore and should_send_default_pii() and integration.include_prompts ): - normalized_messages = normalize_message_roles(messages) + normalized_messages = normalize_message_roles(messages) # type: ignore scope = sentry_sdk.get_current_scope() messages_data = truncate_and_annotate_messages(normalized_messages, span, scope) if messages_data is not None: From 847d4b253d37fc75a7029fd84d569c1b8c1f0332 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 21 Jan 2026 18:23:56 +0100 Subject: [PATCH 06/10] revert unrelated change --- sentry_sdk/integrations/openai.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index 459d782763..553d93d195 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -9,7 +9,6 @@ normalize_message_roles, truncate_and_annotate_messages, ) -from sentry_sdk.ai._openai_completions_api import _get_system_instructions from sentry_sdk.consts import SPANDATA from sentry_sdk.integrations import DidNotEnable, Integration from sentry_sdk.scope import should_send_default_pii From 903bbbdb2c1a5dd466cd72a866ba988cbfc45235 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 21 Jan 2026 18:55:00 +0100 Subject: [PATCH 07/10] use union --- sentry_sdk/integrations/openai.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index 553d93d195..75218ff803 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -195,7 +195,7 @@ def _calculate_token_usage( def _get_input_messages( kwargs: "dict[str, Any]", -) -> "Optional[Iterable[Any] | list[str]]": +) -> "Optional[Union[Iterable[Any], list[str]]]": # Input messages (the prompt or data sent to the model) messages = kwargs.get("messages") if messages is None: @@ -243,7 +243,9 @@ def _set_responses_api_input_data( kwargs: "dict[str, Any]", integration: "OpenAIIntegration", ): - messages: "Optional[ResponseInputParam | list[str]]" = _get_input_messages(kwargs) # type: ignore + messages: "Optional[Union[ResponseInputParam, list[str]]]" = _get_input_messages( + kwargs + ) # type: ignore if ( messages is not None @@ -268,7 +270,7 @@ def _set_completions_api_input_data( kwargs: "dict[str, Any]", integration: "OpenAIIntegration", ): - messages: "Optional[Iterable[ChatCompletionMessageParam] | list[str]]" = ( + messages: "Optional[Union[Iterable[ChatCompletionMessageParam], list[str]]]" = ( _get_input_messages(kwargs) ) From f304bece93d306af1966c4e052b908cac0e67f90 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 21 Jan 2026 19:00:54 +0100 Subject: [PATCH 08/10] mypy --- sentry_sdk/integrations/openai.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index 75218ff803..a7af385d12 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -210,7 +210,7 @@ def _get_input_messages( def _commmon_set_input_data( span: "Span", kwargs: "dict[str, Any]", -): +) -> None: # Input attributes: Common set_data_normalized(span, SPANDATA.GEN_AI_SYSTEM, "openai") @@ -242,10 +242,10 @@ def _set_responses_api_input_data( span: "Span", kwargs: "dict[str, Any]", integration: "OpenAIIntegration", -): +) -> None: messages: "Optional[Union[ResponseInputParam, list[str]]]" = _get_input_messages( kwargs - ) # type: ignore + ) if ( messages is not None @@ -269,7 +269,7 @@ def _set_completions_api_input_data( span: "Span", kwargs: "dict[str, Any]", integration: "OpenAIIntegration", -): +) -> None: messages: "Optional[Union[Iterable[ChatCompletionMessageParam], list[str]]]" = ( _get_input_messages(kwargs) ) @@ -296,7 +296,7 @@ def _set_embeddings_input_data( span: "Span", kwargs: "dict[str, Any]", integration: "OpenAIIntegration", -): +) -> None: messages = _get_input_messages(kwargs) if ( From 1e2da155cb67dd8cdd2661dd661dca18b94d2dfc Mon Sep 17 00:00:00 2001 From: Alex Alderman Webb Date: Tue, 27 Jan 2026 15:02:47 +0100 Subject: [PATCH 09/10] feat(openai): Set system instruction attribute for Completions API (#5359) --- sentry_sdk/integrations/openai.py | 100 +++- tests/integrations/openai/test_openai.py | 557 +++++++++++++++++++++-- 2 files changed, 598 insertions(+), 59 deletions(-) diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index a7af385d12..b659059b5d 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -1,5 +1,6 @@ import sys from functools import wraps +from collections.abc import Iterable import sentry_sdk from sentry_sdk import consts @@ -25,7 +26,6 @@ if TYPE_CHECKING: from typing import ( Any, - Iterable, List, Optional, Callable, @@ -34,8 +34,10 @@ Union, ) from sentry_sdk.tracing import Span + from sentry_sdk._types import TextPart - from openai.types.responses import ResponseInputParam + from openai.types.responses import ResponseInputParam, ResponseInputItemParam + from openai import Omit try: try: @@ -52,7 +54,11 @@ from openai.resources import Embeddings, AsyncEmbeddings if TYPE_CHECKING: - from openai.types.chat import ChatCompletionMessageParam, ChatCompletionChunk + from openai.types.chat import ( + ChatCompletionMessageParam, + ChatCompletionChunk, + ChatCompletionSystemMessageParam, + ) except ImportError: raise DidNotEnable("OpenAI not installed") @@ -193,6 +199,45 @@ def _calculate_token_usage( ) +def _is_system_instruction_completions(message: "ChatCompletionMessageParam") -> bool: + return isinstance(message, dict) and message.get("role") == "system" + + +def _get_system_instructions_completions( + messages: "Iterable[ChatCompletionMessageParam]", +) -> "list[ChatCompletionMessageParam]": + if not isinstance(messages, Iterable): + return [] + + return [ + message for message in messages if _is_system_instruction_completions(message) + ] + + +def _transform_system_instructions( + system_instructions: "list[ChatCompletionSystemMessageParam]", +) -> "list[TextPart]": + instruction_text_parts: "list[TextPart]" = [] + + for instruction in system_instructions: + if not isinstance(instruction, dict): + continue + + content = instruction.get("content") + + if isinstance(content, str): + instruction_text_parts.append({"type": "text", "content": content}) + + elif isinstance(content, list): + for part in content: + if isinstance(part, dict) and part.get("type") == "text": + text = part.get("text", "") + if text: + instruction_text_parts.append({"type": "text", "content": text}) + + return instruction_text_parts + + def _get_input_messages( kwargs: "dict[str, Any]", ) -> "Optional[Union[Iterable[Any], list[str]]]": @@ -270,17 +315,48 @@ def _set_completions_api_input_data( kwargs: "dict[str, Any]", integration: "OpenAIIntegration", ) -> None: - messages: "Optional[Union[Iterable[ChatCompletionMessageParam], list[str]]]" = ( - _get_input_messages(kwargs) + messages: "Optional[Union[str, Iterable[ChatCompletionMessageParam]]]" = kwargs.get( + "messages" ) - if ( - messages is not None - and len(messages) > 0 # type: ignore - and should_send_default_pii() - and integration.include_prompts - ): - normalized_messages = normalize_message_roles(messages) # type: ignore + if not should_send_default_pii() or not integration.include_prompts: + set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "responses") + _commmon_set_input_data(span, kwargs) + return + + if messages is None: + set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "chat") + _commmon_set_input_data(span, kwargs) + return + + system_instructions = _get_system_instructions_completions(messages) + if len(system_instructions) > 0: + set_data_normalized( + span, + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, + _transform_system_instructions(system_instructions), + unpack=False, + ) + + if isinstance(messages, str): + normalized_messages = normalize_message_roles([messages]) # type: ignore + scope = sentry_sdk.get_current_scope() + messages_data = truncate_and_annotate_messages(normalized_messages, span, scope) + if messages_data is not None: + set_data_normalized( + span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False + ) + set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "chat") + _commmon_set_input_data(span, kwargs) + return + + non_system_messages = [ + message + for message in messages + if not _is_system_instruction_completions(message) + ] + if len(non_system_messages) > 0: + normalized_messages = normalize_message_roles(non_system_messages) # type: ignore scope = sentry_sdk.get_current_scope() messages_data = truncate_and_annotate_messages(normalized_messages, span, scope) if messages_data is not None: diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index 814289c887..247d0aadfd 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -9,8 +9,10 @@ NOT_GIVEN = None try: from openai import omit + from openai import Omit except ImportError: omit = None + Omit = None from openai import AsyncOpenAI, OpenAI, AsyncStream, Stream, OpenAIError from openai.types import CompletionUsage, CreateEmbeddingResponse, Embedding @@ -44,9 +46,9 @@ OpenAIIntegration, _calculate_token_usage, ) -from sentry_sdk.ai.utils import MAX_GEN_AI_MESSAGE_BYTES from sentry_sdk._types import AnnotatedValue from sentry_sdk.serializer import serialize +from sentry_sdk.utils import safe_serialize from unittest import mock # python 3.3 and above @@ -129,9 +131,13 @@ async def async_iterator(values): @pytest.mark.parametrize( "send_default_pii, include_prompts", - [(True, True), (True, False), (False, True), (False, False)], + [ + (True, False), + (False, True), + (False, False), + ], ) -def test_nonstreaming_chat_completion( +def test_nonstreaming_chat_completion_no_prompts( sentry_init, capture_events, send_default_pii, include_prompts ): sentry_init( @@ -147,7 +153,11 @@ def test_nonstreaming_chat_completion( with start_transaction(name="openai tx"): response = ( client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] + model="some-model", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "hello"}, + ], ) .choices[0] .message.content @@ -159,12 +169,92 @@ def test_nonstreaming_chat_completion( span = tx["spans"][0] assert span["op"] == "gen_ai.chat" - if send_default_pii and include_prompts: - assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert "the model response" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + + assert span["data"]["gen_ai.usage.output_tokens"] == 10 + assert span["data"]["gen_ai.usage.input_tokens"] == 20 + assert span["data"]["gen_ai.usage.total_tokens"] == 30 + + +@pytest.mark.parametrize( + "messages", + [ + pytest.param( + [ + { + "role": "system", + "content": "You are a helpful assistant.", + }, + {"role": "user", "content": "hello"}, + ], + id="blocks", + ), + pytest.param( + [ + { + "role": "system", + "content": [ + {"type": "text", "text": "You are a helpful assistant."}, + {"type": "text", "text": "Be concise and clear."}, + ], + }, + {"role": "user", "content": "hello"}, + ], + id="parts", + ), + ], +) +def test_nonstreaming_chat_completion(sentry_init, capture_events, messages, request): + sentry_init( + integrations=[OpenAIIntegration(include_prompts=True)], + traces_sample_rate=1.0, + send_default_pii=True, + ) + events = capture_events() + + client = OpenAI(api_key="z") + client.chat.completions._post = mock.Mock(return_value=EXAMPLE_CHAT_COMPLETION) + + with start_transaction(name="openai tx"): + response = ( + client.chat.completions.create( + model="some-model", + messages=messages, + ) + .choices[0] + .message.content + ) + + assert response == "the model response" + tx = events[0] + assert tx["type"] == "transaction" + span = tx["spans"][0] + assert span["op"] == "gen_ai.chat" + + param_id = request.node.callspec.id + if "blocks" in param_id: + assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ + { + "type": "text", + "content": "You are a helpful assistant.", + } + ] else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ + { + "type": "text", + "content": "You are a helpful assistant.", + }, + { + "type": "text", + "content": "Be concise and clear.", + }, + ] + + assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + assert "the model response" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] assert span["data"]["gen_ai.usage.output_tokens"] == 10 assert span["data"]["gen_ai.usage.input_tokens"] == 20 @@ -174,9 +264,13 @@ def test_nonstreaming_chat_completion( @pytest.mark.asyncio @pytest.mark.parametrize( "send_default_pii, include_prompts", - [(True, True), (True, False), (False, True), (False, False)], + [ + (True, False), + (False, True), + (False, False), + ], ) -async def test_nonstreaming_chat_completion_async( +async def test_nonstreaming_chat_completion_async_no_prompts( sentry_init, capture_events, send_default_pii, include_prompts ): sentry_init( @@ -186,12 +280,80 @@ async def test_nonstreaming_chat_completion_async( ) events = capture_events() + client = AsyncOpenAI(api_key="z") + client.chat.completions._post = mock.AsyncMock(return_value=EXAMPLE_CHAT_COMPLETION) + + with start_transaction(name="openai tx"): + response = await client.chat.completions.create( + model="some-model", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "hello"}, + ], + ) + response = response.choices[0].message.content + + assert response == "the model response" + tx = events[0] + assert tx["type"] == "transaction" + span = tx["spans"][0] + assert span["op"] == "gen_ai.chat" + + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + + assert span["data"]["gen_ai.usage.output_tokens"] == 10 + assert span["data"]["gen_ai.usage.input_tokens"] == 20 + assert span["data"]["gen_ai.usage.total_tokens"] == 30 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "messages", + [ + pytest.param( + [ + { + "role": "system", + "content": "You are a helpful assistant.", + }, + {"role": "user", "content": "hello"}, + ], + id="blocks", + ), + pytest.param( + [ + { + "role": "system", + "content": [ + {"type": "text", "text": "You are a helpful assistant."}, + {"type": "text", "text": "Be concise and clear."}, + ], + }, + {"role": "user", "content": "hello"}, + ], + id="parts", + ), + ], +) +async def test_nonstreaming_chat_completion_async( + sentry_init, capture_events, messages, request +): + sentry_init( + integrations=[OpenAIIntegration(include_prompts=True)], + traces_sample_rate=1.0, + send_default_pii=True, + ) + events = capture_events() + client = AsyncOpenAI(api_key="z") client.chat.completions._post = AsyncMock(return_value=EXAMPLE_CHAT_COMPLETION) with start_transaction(name="openai tx"): response = await client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] + model="some-model", + messages=messages, ) response = response.choices[0].message.content @@ -201,12 +363,28 @@ async def test_nonstreaming_chat_completion_async( span = tx["spans"][0] assert span["op"] == "gen_ai.chat" - if send_default_pii and include_prompts: - assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert "the model response" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] + param_id = request.node.callspec.id + if "blocks" in param_id: + assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ + { + "type": "text", + "content": "You are a helpful assistant.", + } + ] else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ + { + "type": "text", + "content": "You are a helpful assistant.", + }, + { + "type": "text", + "content": "Be concise and clear.", + }, + ] + + assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + assert "the model response" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] assert span["data"]["gen_ai.usage.output_tokens"] == 10 assert span["data"]["gen_ai.usage.input_tokens"] == 20 @@ -225,9 +403,13 @@ def tiktoken_encoding_if_installed(): # noinspection PyTypeChecker @pytest.mark.parametrize( "send_default_pii, include_prompts", - [(True, True), (True, False), (False, True), (False, False)], + [ + (True, False), + (False, True), + (False, False), + ], ) -def test_streaming_chat_completion( +def test_streaming_chat_completion_no_prompts( sentry_init, capture_events, send_default_pii, include_prompts ): sentry_init( @@ -283,7 +465,11 @@ def test_streaming_chat_completion( client.chat.completions._post = mock.Mock(return_value=returned_stream) with start_transaction(name="openai tx"): response_stream = client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] + model="some-model", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "hello"}, + ], ) response_string = "".join( map(lambda x: x.choices[0].delta.content, response_stream) @@ -294,19 +480,149 @@ def test_streaming_chat_completion( span = tx["spans"][0] assert span["op"] == "gen_ai.chat" - if send_default_pii and include_prompts: - assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert "hello world" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] try: import tiktoken # type: ignore # noqa # pylint: disable=unused-import assert span["data"]["gen_ai.usage.output_tokens"] == 2 - assert span["data"]["gen_ai.usage.input_tokens"] == 1 - assert span["data"]["gen_ai.usage.total_tokens"] == 3 + assert span["data"]["gen_ai.usage.input_tokens"] == 7 + assert span["data"]["gen_ai.usage.total_tokens"] == 9 + except ImportError: + pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly + + +# noinspection PyTypeChecker +@pytest.mark.parametrize( + "messages", + [ + pytest.param( + [ + { + "role": "system", + "content": "You are a helpful assistant.", + }, + {"role": "user", "content": "hello"}, + ], + id="blocks", + ), + pytest.param( + [ + { + "role": "system", + "content": [ + {"type": "text", "text": "You are a helpful assistant."}, + {"type": "text", "text": "Be concise and clear."}, + ], + }, + {"role": "user", "content": "hello"}, + ], + id="parts", + ), + ], +) +def test_streaming_chat_completion(sentry_init, capture_events, messages, request): + sentry_init( + integrations=[ + OpenAIIntegration( + include_prompts=True, + tiktoken_encoding_name=tiktoken_encoding_if_installed(), + ) + ], + traces_sample_rate=1.0, + send_default_pii=True, + ) + events = capture_events() + + client = OpenAI(api_key="z") + returned_stream = Stream(cast_to=None, response=None, client=client) + returned_stream._iterator = [ + ChatCompletionChunk( + id="1", + choices=[ + DeltaChoice( + index=0, delta=ChoiceDelta(content="hel"), finish_reason=None + ) + ], + created=100000, + model="model-id", + object="chat.completion.chunk", + ), + ChatCompletionChunk( + id="1", + choices=[ + DeltaChoice( + index=1, delta=ChoiceDelta(content="lo "), finish_reason=None + ) + ], + created=100000, + model="model-id", + object="chat.completion.chunk", + ), + ChatCompletionChunk( + id="1", + choices=[ + DeltaChoice( + index=2, delta=ChoiceDelta(content="world"), finish_reason="stop" + ) + ], + created=100000, + model="model-id", + object="chat.completion.chunk", + ), + ] + + client.chat.completions._post = mock.Mock(return_value=returned_stream) + with start_transaction(name="openai tx"): + response_stream = client.chat.completions.create( + model="some-model", + messages=messages, + ) + response_string = "".join( + map(lambda x: x.choices[0].delta.content, response_stream) + ) + assert response_string == "hello world" + tx = events[0] + assert tx["type"] == "transaction" + span = tx["spans"][0] + assert span["op"] == "gen_ai.chat" + + param_id = request.node.callspec.id + if "blocks" in param_id: + assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ + { + "type": "text", + "content": "You are a helpful assistant.", + } + ] + else: + assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ + { + "type": "text", + "content": "You are a helpful assistant.", + }, + { + "type": "text", + "content": "Be concise and clear.", + }, + ] + + assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + assert "hello world" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] + + try: + import tiktoken # type: ignore # noqa # pylint: disable=unused-import + + if "blocks" in param_id: + assert span["data"]["gen_ai.usage.output_tokens"] == 2 + assert span["data"]["gen_ai.usage.input_tokens"] == 7 + assert span["data"]["gen_ai.usage.total_tokens"] == 9 + else: + assert span["data"]["gen_ai.usage.output_tokens"] == 2 + assert span["data"]["gen_ai.usage.input_tokens"] == 1 + assert span["data"]["gen_ai.usage.total_tokens"] == 3 except ImportError: pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly @@ -315,9 +631,13 @@ def test_streaming_chat_completion( @pytest.mark.asyncio @pytest.mark.parametrize( "send_default_pii, include_prompts", - [(True, True), (True, False), (False, True), (False, False)], + [ + (True, False), + (False, True), + (False, False), + ], ) -async def test_streaming_chat_completion_async( +async def test_streaming_chat_completion_async_no_prompts( sentry_init, capture_events, send_default_pii, include_prompts ): sentry_init( @@ -377,7 +697,11 @@ async def test_streaming_chat_completion_async( client.chat.completions._post = AsyncMock(return_value=returned_stream) with start_transaction(name="openai tx"): response_stream = await client.chat.completions.create( - model="some-model", messages=[{"role": "system", "content": "hello"}] + model="some-model", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "hello"}, + ], ) response_string = "" @@ -390,19 +714,160 @@ async def test_streaming_chat_completion_async( span = tx["spans"][0] assert span["op"] == "gen_ai.chat" - if send_default_pii and include_prompts: - assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert "hello world" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] try: import tiktoken # type: ignore # noqa # pylint: disable=unused-import assert span["data"]["gen_ai.usage.output_tokens"] == 2 - assert span["data"]["gen_ai.usage.input_tokens"] == 1 - assert span["data"]["gen_ai.usage.total_tokens"] == 3 + assert span["data"]["gen_ai.usage.input_tokens"] == 7 + assert span["data"]["gen_ai.usage.total_tokens"] == 9 + + except ImportError: + pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly + + +# noinspection PyTypeChecker +@pytest.mark.asyncio +@pytest.mark.parametrize( + "messages", + [ + pytest.param( + [ + { + "role": "system", + "content": "You are a helpful assistant.", + }, + {"role": "user", "content": "hello"}, + ], + id="blocks", + ), + pytest.param( + [ + { + "role": "system", + "content": [ + {"type": "text", "text": "You are a helpful assistant."}, + {"type": "text", "text": "Be concise and clear."}, + ], + }, + {"role": "user", "content": "hello"}, + ], + id="parts", + ), + ], +) +async def test_streaming_chat_completion_async( + sentry_init, capture_events, messages, request +): + sentry_init( + integrations=[ + OpenAIIntegration( + include_prompts=True, + tiktoken_encoding_name=tiktoken_encoding_if_installed(), + ) + ], + traces_sample_rate=1.0, + send_default_pii=True, + ) + events = capture_events() + + client = AsyncOpenAI(api_key="z") + returned_stream = AsyncStream(cast_to=None, response=None, client=client) + returned_stream._iterator = async_iterator( + [ + ChatCompletionChunk( + id="1", + choices=[ + DeltaChoice( + index=0, delta=ChoiceDelta(content="hel"), finish_reason=None + ) + ], + created=100000, + model="model-id", + object="chat.completion.chunk", + ), + ChatCompletionChunk( + id="1", + choices=[ + DeltaChoice( + index=1, delta=ChoiceDelta(content="lo "), finish_reason=None + ) + ], + created=100000, + model="model-id", + object="chat.completion.chunk", + ), + ChatCompletionChunk( + id="1", + choices=[ + DeltaChoice( + index=2, + delta=ChoiceDelta(content="world"), + finish_reason="stop", + ) + ], + created=100000, + model="model-id", + object="chat.completion.chunk", + ), + ] + ) + + client.chat.completions._post = AsyncMock(return_value=returned_stream) + with start_transaction(name="openai tx"): + response_stream = await client.chat.completions.create( + model="some-model", + messages=messages, + ) + + response_string = "" + async for x in response_stream: + response_string += x.choices[0].delta.content + + assert response_string == "hello world" + tx = events[0] + assert tx["type"] == "transaction" + span = tx["spans"][0] + assert span["op"] == "gen_ai.chat" + + param_id = request.node.callspec.id + if "blocks" in param_id: + assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ + { + "type": "text", + "content": "You are a helpful assistant.", + } + ] + else: + assert json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]) == [ + { + "type": "text", + "content": "You are a helpful assistant.", + }, + { + "type": "text", + "content": "Be concise and clear.", + }, + ] + + assert "hello" in span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + assert "hello world" in span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] + + try: + import tiktoken # type: ignore # noqa # pylint: disable=unused-import + + if "blocks" in param_id: + assert span["data"]["gen_ai.usage.output_tokens"] == 2 + assert span["data"]["gen_ai.usage.input_tokens"] == 7 + assert span["data"]["gen_ai.usage.total_tokens"] == 9 + else: + assert span["data"]["gen_ai.usage.output_tokens"] == 2 + assert span["data"]["gen_ai.usage.input_tokens"] == 1 + assert span["data"]["gen_ai.usage.total_tokens"] == 3 + except ImportError: pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly @@ -1472,7 +1937,6 @@ def test_openai_message_role_mapping(sentry_init, capture_events): client.chat.completions._post = mock.Mock(return_value=EXAMPLE_CHAT_COMPLETION) # Test messages with mixed roles including "ai" that should be mapped to "assistant" test_messages = [ - {"role": "system", "content": "You are helpful."}, {"role": "user", "content": "Hello"}, {"role": "ai", "content": "Hi there!"}, # Should be mapped to "assistant" {"role": "assistant", "content": "How can I help?"}, # Should stay "assistant" @@ -1492,17 +1956,16 @@ def test_openai_message_role_mapping(sentry_init, capture_events): stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) # Verify that "ai" role was mapped to "assistant" - assert len(stored_messages) == 4 - assert stored_messages[0]["role"] == "system" - assert stored_messages[1]["role"] == "user" + assert len(stored_messages) == 3 + assert stored_messages[0]["role"] == "user" assert ( - stored_messages[2]["role"] == "assistant" + stored_messages[1]["role"] == "assistant" ) # "ai" should be mapped to "assistant" - assert stored_messages[3]["role"] == "assistant" # should stay "assistant" + assert stored_messages[2]["role"] == "assistant" # should stay "assistant" # Verify content is preserved - assert stored_messages[2]["content"] == "Hi there!" - assert stored_messages[3]["content"] == "How can I help?" + assert stored_messages[1]["content"] == "Hi there!" + assert stored_messages[2]["content"] == "How can I help?" # Verify no "ai" roles remain roles = [msg["role"] for msg in stored_messages] From fee4553bdd68c909a92317fda829b135c3b1b0fa Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Tue, 27 Jan 2026 15:21:34 +0100 Subject: [PATCH 10/10] consolidate early returns --- sentry_sdk/integrations/openai.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index ba79fca9ac..b9b24e2028 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -320,12 +320,11 @@ def _set_completions_api_input_data( "messages" ) - if not should_send_default_pii() or not integration.include_prompts: - set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "responses") - _commmon_set_input_data(span, kwargs) - return - - if messages is None: + if ( + not should_send_default_pii() + or not integration.include_prompts + or messages is None + ): set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "chat") _commmon_set_input_data(span, kwargs) return