diff --git a/sentry_sdk/integrations/anthropic.py b/sentry_sdk/integrations/anthropic.py index efc2f70ffd..ca9e60e59d 100644 --- a/sentry_sdk/integrations/anthropic.py +++ b/sentry_sdk/integrations/anthropic.py @@ -438,9 +438,13 @@ def _set_common_input_data( normalized_messages.append(transformed_message) role_normalized_messages = normalize_message_roles(normalized_messages) + + client = sentry_sdk.get_client() scope = sentry_sdk.get_current_scope() - messages_data = truncate_and_annotate_messages( - role_normalized_messages, span, scope + messages_data = ( + role_normalized_messages + if client.options.get("stream_gen_ai_spans", False) + else truncate_and_annotate_messages(role_normalized_messages, span, scope) ) if messages_data is not None: set_data_normalized( diff --git a/sentry_sdk/integrations/google_genai/utils.py b/sentry_sdk/integrations/google_genai/utils.py index 25763ebe07..55a5b80233 100644 --- a/sentry_sdk/integrations/google_genai/utils.py +++ b/sentry_sdk/integrations/google_genai/utils.py @@ -892,9 +892,12 @@ def set_span_data_for_request( if messages: normalized_messages = normalize_message_roles(messages) + client = sentry_sdk.get_client() scope = sentry_sdk.get_current_scope() - messages_data = truncate_and_annotate_messages( - normalized_messages, span, scope + messages_data = ( + normalized_messages + if client.options.get("stream_gen_ai_spans", False) + else truncate_and_annotate_messages(normalized_messages, span, scope) ) if messages_data is not None: set_data_normalized( diff --git a/sentry_sdk/integrations/langchain.py b/sentry_sdk/integrations/langchain.py index 8acf215bfe..4f5a1b4939 100644 --- a/sentry_sdk/integrations/langchain.py +++ b/sentry_sdk/integrations/langchain.py @@ -374,9 +374,15 @@ def on_llm_start( } for prompt in prompts ] + + client = sentry_sdk.get_client() scope = sentry_sdk.get_current_scope() - messages_data = truncate_and_annotate_messages( - normalized_messages, span, scope + messages_data = ( + normalized_messages + if client.options.get("stream_gen_ai_spans", False) + else truncate_and_annotate_messages( + normalized_messages, span, scope + ) ) if messages_data is not None: set_data_normalized( @@ -463,9 +469,15 @@ def on_chat_model_start( self._normalize_langchain_message(message) ) normalized_messages = normalize_message_roles(normalized_messages) + + client = sentry_sdk.get_client() scope = sentry_sdk.get_current_scope() - messages_data = truncate_and_annotate_messages( - normalized_messages, span, scope + messages_data = ( + normalized_messages + if client.options.get("stream_gen_ai_spans", False) + else truncate_and_annotate_messages( + normalized_messages, span, scope + ) ) if messages_data is not None: set_data_normalized( @@ -992,9 +1004,15 @@ def new_invoke(self: "Any", *args: "Any", **kwargs: "Any") -> "Any": and integration.include_prompts ): normalized_messages = normalize_message_roles([input]) + + client = sentry_sdk.get_client() scope = sentry_sdk.get_current_scope() - messages_data = truncate_and_annotate_messages( - normalized_messages, span, scope + messages_data = ( + normalized_messages + if client.options.get("stream_gen_ai_spans", False) + else truncate_and_annotate_messages( + normalized_messages, span, scope + ) ) if messages_data is not None: set_data_normalized( @@ -1049,9 +1067,13 @@ def new_stream(self: "Any", *args: "Any", **kwargs: "Any") -> "Any": and integration.include_prompts ): normalized_messages = normalize_message_roles([input]) + + client = sentry_sdk.get_client() scope = sentry_sdk.get_current_scope() - messages_data = truncate_and_annotate_messages( - normalized_messages, span, scope + messages_data = ( + normalized_messages + if client.options.get("stream_gen_ai_spans", False) + else truncate_and_annotate_messages(normalized_messages, span, scope) ) if messages_data is not None: set_data_normalized( diff --git a/sentry_sdk/integrations/langgraph.py b/sentry_sdk/integrations/langgraph.py index e5ea12b90a..1454d151f4 100644 --- a/sentry_sdk/integrations/langgraph.py +++ b/sentry_sdk/integrations/langgraph.py @@ -181,9 +181,15 @@ def new_invoke(self: "Any", *args: "Any", **kwargs: "Any") -> "Any": input_messages = _parse_langgraph_messages(args[0]) if input_messages: normalized_input_messages = normalize_message_roles(input_messages) + + client = sentry_sdk.get_client() scope = sentry_sdk.get_current_scope() - messages_data = truncate_and_annotate_messages( - normalized_input_messages, span, scope + messages_data = ( + normalized_input_messages + if client.options.get("stream_gen_ai_spans", False) + else truncate_and_annotate_messages( + normalized_input_messages, span, scope + ) ) if messages_data is not None: set_data_normalized( @@ -234,9 +240,15 @@ async def new_ainvoke(self: "Any", *args: "Any", **kwargs: "Any") -> "Any": input_messages = _parse_langgraph_messages(args[0]) if input_messages: normalized_input_messages = normalize_message_roles(input_messages) + + client = sentry_sdk.get_client() scope = sentry_sdk.get_current_scope() - messages_data = truncate_and_annotate_messages( - normalized_input_messages, span, scope + messages_data = ( + normalized_input_messages + if client.options.get("stream_gen_ai_spans", False) + else truncate_and_annotate_messages( + normalized_input_messages, span, scope + ) ) if messages_data is not None: set_data_normalized( diff --git a/sentry_sdk/integrations/litellm.py b/sentry_sdk/integrations/litellm.py index 3cff0fbc23..9561bd61f3 100644 --- a/sentry_sdk/integrations/litellm.py +++ b/sentry_sdk/integrations/litellm.py @@ -119,8 +119,11 @@ def _input_callback(kwargs: "Dict[str, Any]") -> None: if isinstance(embedding_input, list) else [embedding_input] ) - messages_data = truncate_and_annotate_embedding_inputs( - input_list, span, scope + client = sentry_sdk.get_client() + messages_data = ( + input_list + if client.options.get("stream_gen_ai_spans", False) + else truncate_and_annotate_embedding_inputs(input_list, span, scope) ) if messages_data is not None: set_data_normalized( @@ -133,9 +136,14 @@ def _input_callback(kwargs: "Dict[str, Any]") -> None: # For chat, look for the 'messages' parameter messages = kwargs.get("messages", []) if messages: + client = sentry_sdk.get_client() scope = sentry_sdk.get_current_scope() messages = _convert_message_parts(messages) - messages_data = truncate_and_annotate_messages(messages, span, scope) + messages_data = ( + messages + if client.options.get("stream_gen_ai_spans", False) + else truncate_and_annotate_messages(messages, span, scope) + ) if messages_data is not None: set_data_normalized( span, diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index b3919d1a9d..7bb328741e 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -398,8 +398,13 @@ def _set_responses_api_input_data( if isinstance(messages, str): normalized_messages = normalize_message_roles([messages]) # type: ignore + client = sentry_sdk.get_client() scope = sentry_sdk.get_current_scope() - messages_data = truncate_and_annotate_messages(normalized_messages, span, scope) + messages_data = ( + normalized_messages + if client.options.get("stream_gen_ai_spans", False) + else truncate_and_annotate_messages(normalized_messages, span, scope) + ) if messages_data is not None: set_data_normalized( span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False @@ -413,8 +418,13 @@ def _set_responses_api_input_data( ] if len(non_system_messages) > 0: normalized_messages = normalize_message_roles(non_system_messages) + client = sentry_sdk.get_client() scope = sentry_sdk.get_current_scope() - messages_data = truncate_and_annotate_messages(normalized_messages, span, scope) + messages_data = ( + normalized_messages + if client.options.get("stream_gen_ai_spans", False) + else truncate_and_annotate_messages(normalized_messages, span, scope) + ) if messages_data is not None: set_data_normalized( span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False @@ -472,8 +482,13 @@ def _set_completions_api_input_data( if isinstance(messages, str): normalized_messages = normalize_message_roles([messages]) # type: ignore + client = sentry_sdk.get_client() scope = sentry_sdk.get_current_scope() - messages_data = truncate_and_annotate_messages(normalized_messages, span, scope) + messages_data = ( + normalized_messages + if client.options.get("stream_gen_ai_spans", False) + else truncate_and_annotate_messages(normalized_messages, span, scope) + ) if messages_data is not None: set_data_normalized( span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False @@ -503,8 +518,13 @@ def _set_completions_api_input_data( ] if len(non_system_messages) > 0: normalized_messages = normalize_message_roles(non_system_messages) + client = sentry_sdk.get_client() scope = sentry_sdk.get_current_scope() - messages_data = truncate_and_annotate_messages(normalized_messages, span, scope) + messages_data = ( + normalized_messages + if client.options.get("stream_gen_ai_spans", False) + else truncate_and_annotate_messages(normalized_messages, span, scope) + ) if messages_data is not None: set_data_normalized( span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False @@ -539,9 +559,14 @@ def _set_embeddings_input_data( set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "embeddings") normalized_messages = normalize_message_roles([messages]) # type: ignore + client = sentry_sdk.get_client() scope = sentry_sdk.get_current_scope() - messages_data = truncate_and_annotate_embedding_inputs( - normalized_messages, span, scope + messages_data = ( + normalized_messages + if client.options.get("stream_gen_ai_spans", False) + else truncate_and_annotate_embedding_inputs( + normalized_messages, span, scope + ) ) if messages_data is not None: set_data_normalized( @@ -560,9 +585,14 @@ def _set_embeddings_input_data( if len(messages) > 0: normalized_messages = normalize_message_roles(messages) + client = sentry_sdk.get_client() scope = sentry_sdk.get_current_scope() - messages_data = truncate_and_annotate_embedding_inputs( - normalized_messages, span, scope + messages_data = ( + normalized_messages + if client.options.get("stream_gen_ai_spans", False) + else truncate_and_annotate_embedding_inputs( + normalized_messages, span, scope + ) ) if messages_data is not None: set_data_normalized( diff --git a/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py b/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py index 27f9fdab25..2346189a96 100644 --- a/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py +++ b/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py @@ -63,9 +63,12 @@ def invoke_agent_span( if len(messages) > 0: normalized_messages = normalize_message_roles(messages) + client = sentry_sdk.get_client() scope = sentry_sdk.get_current_scope() - messages_data = truncate_and_annotate_messages( - normalized_messages, span, scope + messages_data = ( + normalized_messages + if client.options.get("stream_gen_ai_spans", False) + else truncate_and_annotate_messages(normalized_messages, span, scope) ) if messages_data is not None: set_data_normalized( diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index ee504b3496..ea1faefde7 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -173,8 +173,13 @@ def _set_input_data( ) normalized_messages = normalize_message_roles(request_messages) + client = sentry_sdk.get_client() scope = sentry_sdk.get_current_scope() - messages_data = truncate_and_annotate_messages(normalized_messages, span, scope) + messages_data = ( + normalized_messages + if client.options.get("stream_gen_ai_spans", False) + else truncate_and_annotate_messages(normalized_messages, span, scope) + ) if messages_data is not None: set_data_normalized( span, diff --git a/sentry_sdk/integrations/pydantic_ai/spans/ai_client.py b/sentry_sdk/integrations/pydantic_ai/spans/ai_client.py index dc95acad45..e549083fed 100644 --- a/sentry_sdk/integrations/pydantic_ai/spans/ai_client.py +++ b/sentry_sdk/integrations/pydantic_ai/spans/ai_client.py @@ -182,9 +182,12 @@ def _set_input_messages(span: "sentry_sdk.tracing.Span", messages: "Any") -> Non if formatted_messages: normalized_messages = normalize_message_roles(formatted_messages) + client = sentry_sdk.get_client() scope = sentry_sdk.get_current_scope() - messages_data = truncate_and_annotate_messages( - normalized_messages, span, scope + messages_data = ( + normalized_messages + if client.options.get("stream_gen_ai_spans", False) + else truncate_and_annotate_messages(normalized_messages, span, scope) ) set_data_normalized( span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False diff --git a/sentry_sdk/integrations/pydantic_ai/spans/invoke_agent.py b/sentry_sdk/integrations/pydantic_ai/spans/invoke_agent.py index ee08ca7036..c507315dcd 100644 --- a/sentry_sdk/integrations/pydantic_ai/spans/invoke_agent.py +++ b/sentry_sdk/integrations/pydantic_ai/spans/invoke_agent.py @@ -122,9 +122,12 @@ def invoke_agent_span( if messages: normalized_messages = normalize_message_roles(messages) + client = sentry_sdk.get_client() scope = sentry_sdk.get_current_scope() - messages_data = truncate_and_annotate_messages( - normalized_messages, span, scope + messages_data = ( + normalized_messages + if client.options.get("stream_gen_ai_spans", False) + else truncate_and_annotate_messages(normalized_messages, span, scope) ) set_data_normalized( span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages_data, unpack=False diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index 31f487aef2..d6b2c269d9 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -110,10 +110,14 @@ def test_nonstreaming_create_message( client.messages._post = mock.Mock(return_value=EXAMPLE_MESSAGE) messages = [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, { "role": "user", "content": "Hello, Claude", - } + }, ] if stream_gen_ai_spans: @@ -144,10 +148,16 @@ def test_nonstreaming_create_message( assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" if send_default_pii and include_prompts: - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) == [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + { + "role": "user", + "content": "Hello, Claude", + }, + ] assert ( span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi, I'm Claude." ) @@ -245,10 +255,14 @@ async def test_nonstreaming_create_message_async( client.messages._post = AsyncMock(return_value=EXAMPLE_MESSAGE) messages = [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, { "role": "user", "content": "Hello, Claude", - } + }, ] if stream_gen_ai_spans: @@ -279,10 +293,16 @@ async def test_nonstreaming_create_message_async( assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" if send_default_pii and include_prompts: - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) == [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + { + "role": "user", + "content": "Hello, Claude", + }, + ] assert ( span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi, I'm Claude." ) @@ -413,10 +433,14 @@ def test_streaming_create_message( ) messages = [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, { "role": "user", "content": "Hello, Claude", - } + }, ] if stream_gen_ai_spans: @@ -449,10 +473,16 @@ def test_streaming_create_message( assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" if send_default_pii and include_prompts: - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) == [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + { + "role": "user", + "content": "Hello, Claude", + }, + ] assert ( span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" ) @@ -895,10 +925,14 @@ def test_stream_messages( ) messages = [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, { "role": "user", "content": "Hello, Claude", - } + }, ] if stream_gen_ai_spans: @@ -931,10 +965,16 @@ def test_stream_messages( assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" if send_default_pii and include_prompts: - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) == [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + { + "role": "user", + "content": "Hello, Claude", + }, + ] assert ( span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" ) @@ -1390,10 +1430,14 @@ async def test_streaming_create_message_async( ) messages = [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, { "role": "user", "content": "Hello, Claude", - } + }, ] if stream_gen_ai_spans: @@ -1425,10 +1469,16 @@ async def test_streaming_create_message_async( assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" if send_default_pii and include_prompts: - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) == [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + { + "role": "user", + "content": "Hello, Claude", + }, + ] assert ( span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" ) @@ -1883,10 +1933,14 @@ async def test_stream_message_async( ) messages = [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, { "role": "user", "content": "Hello, Claude", - } + }, ] if stream_gen_ai_spans: @@ -1919,10 +1973,16 @@ async def test_stream_message_async( assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model" if send_default_pii and include_prompts: - assert ( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - == '[{"role": "user", "content": "Hello, Claude"}]' - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) == [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + { + "role": "user", + "content": "Hello, Claude", + }, + ] assert ( span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!" ) @@ -3625,20 +3685,14 @@ def mock_messages_create(*args, **kwargs): assert stored_messages[0]["role"] == expected_role -@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) -def test_anthropic_message_truncation( - sentry_init, - capture_events, - capture_items, - stream_gen_ai_spans, -): +def test_anthropic_message_truncation(sentry_init, capture_events): """Test that large messages are truncated properly in Anthropic integration.""" sentry_init( integrations=[AnthropicIntegration(include_prompts=True)], traces_sample_rate=1.0, send_default_pii=True, - stream_gen_ai_spans=stream_gen_ai_spans, ) + events = capture_events() client = Anthropic(api_key="z") client.messages._post = mock.Mock(return_value=EXAMPLE_MESSAGE) @@ -3654,82 +3708,43 @@ def test_anthropic_message_truncation( {"role": "user", "content": "small message 5"}, ] - if stream_gen_ai_spans: - items = capture_items("transaction", "span") - - with start_transaction(): - client.messages.create(max_tokens=1024, messages=messages, model="model") - - spans = [item.payload for item in items if item.type == "span"] - chat_spans = [ - span - for span in spans - if span["attributes"].get("sentry.op") == OP.GEN_AI_CHAT - ] - - assert len(chat_spans) > 0 + with start_transaction(): + client.messages.create(max_tokens=1024, messages=messages, model="model") - chat_span = chat_spans[0] - assert chat_span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert chat_span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in chat_span["attributes"] + assert len(events) > 0 + tx = events[0] + assert tx["type"] == "transaction" - messages_data = chat_span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert isinstance(messages_data, str) - - parsed_messages = json.loads(messages_data) - assert isinstance(parsed_messages, list) - assert len(parsed_messages) == 1 - assert "small message 5" in str(parsed_messages[0]) - - tx = next(item.payload for item in items if item.type == "transaction") - else: - events = capture_events() - - with start_transaction(): - client.messages.create(max_tokens=1024, messages=messages, model="model") - - assert len(events) > 0 - tx = events[0] - assert tx["type"] == "transaction" - - chat_spans = [ - span for span in tx.get("spans", []) if span.get("op") == OP.GEN_AI_CHAT - ] - - assert len(chat_spans) > 0 + chat_spans = [ + span for span in tx.get("spans", []) if span.get("op") == OP.GEN_AI_CHAT + ] + assert len(chat_spans) > 0 - chat_span = chat_spans[0] - assert chat_span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert chat_span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in chat_span["data"] + chat_span = chat_spans[0] + assert chat_span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert chat_span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert SPANDATA.GEN_AI_REQUEST_MESSAGES in chat_span["data"] - messages_data = chat_span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert isinstance(messages_data, str) + messages_data = chat_span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + assert isinstance(messages_data, str) - parsed_messages = json.loads(messages_data) - assert isinstance(parsed_messages, list) - assert len(parsed_messages) == 1 - assert "small message 5" in str(parsed_messages[0]) + parsed_messages = json.loads(messages_data) + assert isinstance(parsed_messages, list) + assert len(parsed_messages) == 1 + assert "small message 5" in str(parsed_messages[0]) assert tx["_meta"]["spans"]["0"]["data"]["gen_ai.request.messages"][""]["len"] == 5 -@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) @pytest.mark.asyncio -async def test_anthropic_message_truncation_async( - sentry_init, - capture_events, - capture_items, - stream_gen_ai_spans, -): +async def test_anthropic_message_truncation_async(sentry_init, capture_events): """Test that large messages are truncated properly in Anthropic integration.""" sentry_init( integrations=[AnthropicIntegration(include_prompts=True)], traces_sample_rate=1.0, send_default_pii=True, - stream_gen_ai_spans=stream_gen_ai_spans, ) + events = capture_events() client = AsyncAnthropic(api_key="z") client.messages._post = mock.AsyncMock(return_value=EXAMPLE_MESSAGE) @@ -3745,68 +3760,30 @@ async def test_anthropic_message_truncation_async( {"role": "user", "content": "small message 5"}, ] - if stream_gen_ai_spans: - items = capture_items("transaction", "span") - - with start_transaction(): - await client.messages.create( - max_tokens=1024, messages=messages, model="model" - ) - - spans = [item.payload for item in items if item.type == "span"] - chat_spans = [ - span - for span in spans - if span["attributes"].get("sentry.op") == OP.GEN_AI_CHAT - ] - assert len(chat_spans) > 0 - - chat_span = chat_spans[0] - - assert chat_span["attributes"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert chat_span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in chat_span["attributes"] - - messages_data = chat_span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - - assert isinstance(messages_data, str) - - parsed_messages = json.loads(messages_data) - assert isinstance(parsed_messages, list) - assert len(parsed_messages) == 1 - assert "small message 5" in str(parsed_messages[0]) + with start_transaction(): + await client.messages.create(max_tokens=1024, messages=messages, model="model") - tx = next(item.payload for item in items if item.type == "transaction") - else: - events = capture_events() - - with start_transaction(): - await client.messages.create( - max_tokens=1024, messages=messages, model="model" - ) - - assert len(events) > 0 - tx = events[0] - assert tx["type"] == "transaction" - - chat_spans = [ - span for span in tx.get("spans", []) if span.get("op") == OP.GEN_AI_CHAT - ] - assert len(chat_spans) > 0 + assert len(events) > 0 + tx = events[0] + assert tx["type"] == "transaction" - chat_span = chat_spans[0] + chat_spans = [ + span for span in tx.get("spans", []) if span.get("op") == OP.GEN_AI_CHAT + ] + assert len(chat_spans) > 0 - assert chat_span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" - assert chat_span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in chat_span["data"] + chat_span = chat_spans[0] + assert chat_span["data"][SPANDATA.GEN_AI_SYSTEM] == "anthropic" + assert chat_span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" + assert SPANDATA.GEN_AI_REQUEST_MESSAGES in chat_span["data"] - messages_data = chat_span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert isinstance(messages_data, str) + messages_data = chat_span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + assert isinstance(messages_data, str) - parsed_messages = json.loads(messages_data) - assert isinstance(parsed_messages, list) - assert len(parsed_messages) == 1 - assert "small message 5" in str(parsed_messages[0]) + parsed_messages = json.loads(messages_data) + assert isinstance(parsed_messages, list) + assert len(parsed_messages) == 1 + assert "small message 5" in str(parsed_messages[0]) assert tx["_meta"]["spans"]["0"]["data"]["gen_ai.request.messages"][""]["len"] == 5 @@ -5200,21 +5177,14 @@ def test_transform_message_content_list_anthropic(): # Integration tests for binary data in messages -@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) -def test_message_with_base64_image( - sentry_init, - capture_events, - capture_items, - stream_gen_ai_spans, -): +def test_message_with_base64_image(sentry_init, capture_events): """Test that messages with base64 images are properly captured.""" sentry_init( integrations=[AnthropicIntegration(include_prompts=True)], traces_sample_rate=1.0, send_default_pii=True, - stream_gen_ai_spans=stream_gen_ai_spans, ) - + events = capture_events() client = Anthropic(api_key="z") client.messages._post = mock.Mock(return_value=EXAMPLE_MESSAGE) @@ -5235,31 +5205,15 @@ def test_message_with_base64_image( } ] - if stream_gen_ai_spans: - items = capture_items("transaction", "span") - - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") - - spans = [item.payload for item in items if item.type == "span"] - (span,) = spans - - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] - stored_messages = json.loads( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) - else: - events = capture_events() - - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") + with start_transaction(name="anthropic"): + client.messages.create(max_tokens=1024, messages=messages, model="model") - assert len(events) == 1 - (event,) = events - (span,) = event["spans"] + assert len(events) == 1 + (event,) = events + (span,) = event["spans"] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"] - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) + assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"] + stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" @@ -5409,21 +5363,14 @@ def test_message_with_file_image( } -@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) -def test_message_with_base64_pdf( - sentry_init, - capture_events, - capture_items, - stream_gen_ai_spans, -): +def test_message_with_base64_pdf(sentry_init, capture_events): """Test that messages with base64-encoded PDF documents are properly captured.""" sentry_init( integrations=[AnthropicIntegration(include_prompts=True)], traces_sample_rate=1.0, send_default_pii=True, - stream_gen_ai_spans=stream_gen_ai_spans, ) - + events = capture_events() client = Anthropic(api_key="z") client.messages._post = mock.Mock(return_value=EXAMPLE_MESSAGE) @@ -5444,30 +5391,14 @@ def test_message_with_base64_pdf( } ] - if stream_gen_ai_spans: - items = capture_items("transaction", "span") - - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") - - spans = [item.payload for item in items if item.type == "span"] - (span,) = spans - - stored_messages = json.loads( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) - else: - events = capture_events() + with start_transaction(name="anthropic"): + client.messages.create(max_tokens=1024, messages=messages, model="model") - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") - - assert len(events) == 1 - (event,) = events - (span,) = event["spans"] - - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) + assert len(events) == 1 + (event,) = events + (span,) = event["spans"] + stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) content = stored_messages[0]["content"] assert content[1] == { "type": "blob", @@ -5612,21 +5543,14 @@ def test_message_with_file_document( } -@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) -def test_message_with_mixed_content( - sentry_init, - capture_events, - capture_items, - stream_gen_ai_spans, -): +def test_message_with_mixed_content(sentry_init, capture_events): """Test that messages with mixed content (text, images, documents) are properly captured.""" sentry_init( integrations=[AnthropicIntegration(include_prompts=True)], traces_sample_rate=1.0, send_default_pii=True, - stream_gen_ai_spans=stream_gen_ai_spans, ) - + events = capture_events() client = Anthropic(api_key="z") client.messages._post = mock.Mock(return_value=EXAMPLE_MESSAGE) @@ -5663,30 +5587,14 @@ def test_message_with_mixed_content( } ] - if stream_gen_ai_spans: - items = capture_items("transaction", "span") + with start_transaction(name="anthropic"): + client.messages.create(max_tokens=1024, messages=messages, model="model") - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") - - spans = [item.payload for item in items if item.type == "span"] - (span,) = spans - - stored_messages = json.loads( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) - else: - events = capture_events() - - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") - - assert len(events) == 1 - (event,) = events - (span,) = event["spans"] - - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) + assert len(events) == 1 + (event,) = events + (span,) = event["spans"] + stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) content = stored_messages[0]["content"] assert len(content) == 5 @@ -5718,21 +5626,14 @@ def test_message_with_mixed_content( } -@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) -def test_message_with_multiple_images_different_formats( - sentry_init, - capture_events, - capture_items, - stream_gen_ai_spans, -): +def test_message_with_multiple_images_different_formats(sentry_init, capture_events): """Test that messages with multiple images of different source types are handled.""" sentry_init( integrations=[AnthropicIntegration(include_prompts=True)], traces_sample_rate=1.0, send_default_pii=True, - stream_gen_ai_spans=stream_gen_ai_spans, ) - + events = capture_events() client = Anthropic(api_key="z") client.messages._post = mock.Mock(return_value=EXAMPLE_MESSAGE) @@ -5768,30 +5669,14 @@ def test_message_with_multiple_images_different_formats( } ] - if stream_gen_ai_spans: - items = capture_items("transaction", "span") - - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") - - spans = [item.payload for item in items if item.type == "span"] - (span,) = spans - - stored_messages = json.loads( - span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) - else: - events = capture_events() + with start_transaction(name="anthropic"): + client.messages.create(max_tokens=1024, messages=messages, model="model") - with start_transaction(name="anthropic"): - client.messages.create(max_tokens=1024, messages=messages, model="model") - - assert len(events) == 1 - (event,) = events - (span,) = event["spans"] - - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) + assert len(events) == 1 + (event,) = events + (span,) = event["spans"] + stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) content = stored_messages[0]["content"] assert len(content) == 4 diff --git a/tests/integrations/google_genai/test_google_genai.py b/tests/integrations/google_genai/test_google_genai.py index 79318eaea5..723a71959d 100644 --- a/tests/integrations/google_genai/test_google_genai.py +++ b/tests/integrations/google_genai/test_google_genai.py @@ -153,7 +153,12 @@ def test_nonstreaming_generate_content( ), start_transaction(name="google_genai"): config = create_test_config(temperature=0.7, max_output_tokens=100) mock_genai_client.models.generate_content( - model="gemini-1.5-flash", contents="Tell me a joke", config=config + model="gemini-1.5-flash", + contents=[ + "Message demonstrating the absence of truncation.", + "Tell me a joke", + ], + config=config, ) (event,) = (item.payload for item in items if item.type == "transaction") @@ -173,6 +178,24 @@ def test_nonstreaming_generate_content( ) if send_default_pii and include_prompts: + assert json.loads( + chat_span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) == [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Message demonstrating the absence of truncation.", + }, + { + "type": "text", + "text": "Tell me a joke", + }, + ], + } + ] + # Response text is stored as a JSON array response_text = chat_span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] @@ -675,7 +698,12 @@ def test_streaming_generate_content( ), start_transaction(name="google_genai"): config = create_test_config() stream = mock_genai_client.models.generate_content_stream( - model="gemini-1.5-flash", contents="Stream me a response", config=config + model="gemini-1.5-flash", + contents=[ + "Message demonstrating the absence of truncation.", + "Stream me a response", + ], + config=config, ) # Consume the stream (this is what users do with the integration wrapper) @@ -693,6 +721,24 @@ def test_streaming_generate_content( assert len(spans) == 1 chat_span = next(item.payload for item in items if item.type == "span") + assert json.loads( + chat_span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) == [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Message demonstrating the absence of truncation.", + }, + { + "type": "text", + "text": "Stream me a response", + }, + ], + } + ] + # Check that streaming flag is set on both spans assert chat_span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True @@ -1390,21 +1436,16 @@ def test_tool_calls_extraction( assert json.loads(tool_calls[1]["arguments"]) == {"timezone": "PST"} -@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) def test_google_genai_message_truncation( - sentry_init, - capture_events, - capture_items, - mock_genai_client, - stream_gen_ai_spans, + sentry_init, capture_events, mock_genai_client ): """Test that large messages are truncated properly in Google GenAI integration.""" sentry_init( integrations=[GoogleGenAIIntegration(include_prompts=True)], traces_sample_rate=1.0, send_default_pii=True, - stream_gen_ai_spans=stream_gen_ai_spans, ) + events = capture_events() large_content = ( "This is a very long message that will exceed our size limits. " * 1000 @@ -1413,39 +1454,21 @@ def test_google_genai_message_truncation( mock_http_response = create_mock_http_response(EXAMPLE_API_RESPONSE_JSON) - if stream_gen_ai_spans: - items = capture_items("span") - - with mock.patch.object( - mock_genai_client._api_client, "request", return_value=mock_http_response - ), start_transaction(name="google_genai"): - mock_genai_client.models.generate_content( - model="gemini-1.5-flash", - contents=[large_content, small_content], - config=create_test_config(), - ) - - invoke_span = next(item.payload for item in items if item.type == "span") - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in invoke_span["attributes"] - - messages_data = invoke_span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - else: - events = capture_events() - - with mock.patch.object( - mock_genai_client._api_client, "request", return_value=mock_http_response - ), start_transaction(name="google_genai"): + with mock.patch.object( + mock_genai_client._api_client, "request", return_value=mock_http_response + ): + with start_transaction(name="google_genai"): mock_genai_client.models.generate_content( model="gemini-1.5-flash", contents=[large_content, small_content], config=create_test_config(), ) - (event,) = events - invoke_span = event["spans"][0] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in invoke_span["data"] + (event,) = events + invoke_span = event["spans"][0] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES in invoke_span["data"] - messages_data = invoke_span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + messages_data = invoke_span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] assert isinstance(messages_data, str) parsed_messages = json.loads(messages_data) @@ -2528,21 +2551,16 @@ def test_generate_content_with_inline_data( assert messages[0]["content"][1]["content"] == BLOB_DATA_SUBSTITUTE -@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) def test_generate_content_with_function_response( - sentry_init, - capture_events, - capture_items, - mock_genai_client, - stream_gen_ai_spans, + sentry_init, capture_events, mock_genai_client ): """Test generate_content with function_response (tool result).""" sentry_init( integrations=[GoogleGenAIIntegration(include_prompts=True)], traces_sample_rate=1.0, send_default_pii=True, - stream_gen_ai_spans=stream_gen_ai_spans, ) + events = capture_events() mock_http_response = create_mock_http_response(EXAMPLE_API_RESPONSE_JSON) @@ -2568,36 +2586,18 @@ def test_generate_content_with_function_response( ), ] - if stream_gen_ai_spans: - items = capture_items("span") - - with mock.patch.object( - mock_genai_client._api_client, "request", return_value=mock_http_response - ), start_transaction(name="google_genai"): - mock_genai_client.models.generate_content( - model="gemini-1.5-flash", contents=contents, config=create_test_config() - ) - - invoke_span = next(item.payload for item in items if item.type == "span") - - messages = json.loads( - invoke_span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) - else: - events = capture_events() - - with mock.patch.object( - mock_genai_client._api_client, "request", return_value=mock_http_response - ), start_transaction(name="google_genai"): + with mock.patch.object( + mock_genai_client._api_client, "request", return_value=mock_http_response + ): + with start_transaction(name="google_genai"): mock_genai_client.models.generate_content( model="gemini-1.5-flash", contents=contents, config=create_test_config() ) - (event,) = events - invoke_span = event["spans"][0] - - messages = json.loads(invoke_span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) + (event,) = events + invoke_span = event["spans"][0] + messages = json.loads(invoke_span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) assert len(messages) == 1 # First message is user message assert messages[0]["role"] == "tool" @@ -2606,21 +2606,16 @@ def test_generate_content_with_function_response( assert messages[0]["content"]["output"] == "Sunny, 72F" -@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) def test_generate_content_with_mixed_string_and_content( - sentry_init, - capture_events, - capture_items, - mock_genai_client, - stream_gen_ai_spans, + sentry_init, capture_events, mock_genai_client ): """Test generate_content with mixed string and Content objects in list.""" sentry_init( integrations=[GoogleGenAIIntegration(include_prompts=True)], traces_sample_rate=1.0, send_default_pii=True, - stream_gen_ai_spans=stream_gen_ai_spans, ) + events = capture_events() mock_http_response = create_mock_http_response(EXAMPLE_API_RESPONSE_JSON) @@ -2637,36 +2632,18 @@ def test_generate_content_with_mixed_string_and_content( ), ] - if stream_gen_ai_spans: - items = capture_items("span") - - with mock.patch.object( - mock_genai_client._api_client, "request", return_value=mock_http_response - ), start_transaction(name="google_genai"): - mock_genai_client.models.generate_content( - model="gemini-1.5-flash", contents=contents, config=create_test_config() - ) - - invoke_span = next(item.payload for item in items if item.type == "span") - - messages = json.loads( - invoke_span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) - else: - events = capture_events() - - with mock.patch.object( - mock_genai_client._api_client, "request", return_value=mock_http_response - ), start_transaction(name="google_genai"): + with mock.patch.object( + mock_genai_client._api_client, "request", return_value=mock_http_response + ): + with start_transaction(name="google_genai"): mock_genai_client.models.generate_content( model="gemini-1.5-flash", contents=contents, config=create_test_config() ) - (event,) = events - invoke_span = event["spans"][0] - - messages = json.loads(invoke_span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) + (event,) = events + invoke_span = event["spans"][0] + messages = json.loads(invoke_span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) assert len(messages) == 1 # User message assert messages[0]["role"] == "user" @@ -2729,13 +2706,8 @@ def test_generate_content_with_part_object_directly( assert messages[0]["content"] == [{"text": "Direct Part object", "type": "text"}] -@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) def test_generate_content_with_list_of_dicts( - sentry_init, - capture_events, - capture_items, - mock_genai_client, - stream_gen_ai_spans, + sentry_init, capture_events, mock_genai_client ): """ Test generate_content with list of dict format inputs. @@ -2748,8 +2720,8 @@ def test_generate_content_with_list_of_dicts( integrations=[GoogleGenAIIntegration(include_prompts=True)], traces_sample_rate=1.0, send_default_pii=True, - stream_gen_ai_spans=stream_gen_ai_spans, ) + events = capture_events() mock_http_response = create_mock_http_response(EXAMPLE_API_RESPONSE_JSON) @@ -2760,36 +2732,18 @@ def test_generate_content_with_list_of_dicts( {"role": "user", "parts": [{"text": "Second user message"}]}, ] - if stream_gen_ai_spans: - items = capture_items("span") - - with mock.patch.object( - mock_genai_client._api_client, "request", return_value=mock_http_response - ), start_transaction(name="google_genai"): + with mock.patch.object( + mock_genai_client._api_client, "request", return_value=mock_http_response + ): + with start_transaction(name="google_genai"): mock_genai_client.models.generate_content( model="gemini-1.5-flash", contents=contents, config=create_test_config() ) - invoke_span = next(item.payload for item in items if item.type == "span") - - messages = json.loads( - invoke_span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) - else: - events = capture_events() - - with mock.patch.object( - mock_genai_client._api_client, "request", return_value=mock_http_response - ), start_transaction(name="google_genai"): - mock_genai_client.models.generate_content( - model="gemini-1.5-flash", contents=contents, config=create_test_config() - ) - - (event,) = events - invoke_span = event["spans"][0] - - messages = json.loads(invoke_span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) + (event,) = events + invoke_span = event["spans"][0] + messages = json.loads(invoke_span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) assert len(messages) == 1 assert messages[0]["role"] == "user" assert messages[0]["content"] == [{"text": "Second user message", "type": "text"}] diff --git a/tests/integrations/huggingface_hub/test_huggingface_hub.py b/tests/integrations/huggingface_hub/test_huggingface_hub.py index 4772eb368f..85ad55a47c 100644 --- a/tests/integrations/huggingface_hub/test_huggingface_hub.py +++ b/tests/integrations/huggingface_hub/test_huggingface_hub.py @@ -2,6 +2,7 @@ from typing import TYPE_CHECKING from unittest import mock +from sentry_sdk.utils import safe_serialize import pytest import responses from huggingface_hub import InferenceClient @@ -761,7 +762,13 @@ def test_chat_completion( with sentry_sdk.start_transaction(name="test"): client.chat_completion( - messages=[{"role": "user", "content": "Hello!"}], + messages=[ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + {"role": "user", "content": "Hello!"}, + ], stream=False, ) @@ -804,8 +811,14 @@ def test_chat_completion( } if send_default_pii and include_prompts: - expected_data["gen_ai.request.messages"] = ( - '[{"role": "user", "content": "Hello!"}]' + expected_data["gen_ai.request.messages"] = safe_serialize( + [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + {"role": "user", "content": "Hello!"}, + ] ) expected_data["gen_ai.response.text"] = ( "[mocked] Hello! How can I help you today?" @@ -899,7 +912,13 @@ def test_chat_completion_streaming( with sentry_sdk.start_transaction(name="test"): _ = list( client.chat_completion( - [{"role": "user", "content": "Hello!"}], + [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + {"role": "user", "content": "Hello!"}, + ], stream=True, ) ) @@ -945,8 +964,14 @@ def test_chat_completion_streaming( expected_data["gen_ai.usage.total_tokens"] = 197 if send_default_pii and include_prompts: - expected_data["gen_ai.request.messages"] = ( - '[{"role": "user", "content": "Hello!"}]' + expected_data["gen_ai.request.messages"] = safe_serialize( + [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + {"role": "user", "content": "Hello!"}, + ] ) expected_data["gen_ai.response.text"] = "the mocked model response" diff --git a/tests/integrations/langchain/test_langchain.py b/tests/integrations/langchain/test_langchain.py index 79ecc7e96b..3a2ef76a5a 100644 --- a/tests/integrations/langchain/test_langchain.py +++ b/tests/integrations/langchain/test_langchain.py @@ -577,6 +577,9 @@ def test_langchain_create_agent( agent.invoke( { "messages": [ + HumanMessage( + content="Message demonstrating the absence of truncation." + ), HumanMessage(content="How many letters in the word eudca"), ], }, @@ -606,6 +609,19 @@ def test_langchain_create_agent( == "Hello, how can I help you?" ) + assert json.loads( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) == [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + { + "role": "user", + "content": "How many letters in the word eudca", + }, + ] + param_id = request.node.callspec.id if "string" in param_id: assert [ @@ -1343,7 +1359,16 @@ def test_langchain_openai_tools_agent( "send", side_effect=[tool_response, final_response], ) as _, start_transaction(): - list(agent_executor.stream({"input": "How many letters in the word eudca"})) + list( + agent_executor.stream( + { + "input": [ + "Message demonstrating the absence of truncation.", + "How many letters in the word eudca", + ] + } + ) + ) tx = next(item.payload for item in items if item.type == "transaction") assert tx["type"] == "transaction" @@ -1389,6 +1414,15 @@ def test_langchain_openai_tools_agent( assert "word" in tool_exec_span["attributes"][SPANDATA.GEN_AI_TOOL_INPUT] assert 5 == int(tool_exec_span["attributes"][SPANDATA.GEN_AI_TOOL_OUTPUT]) + assert json.loads( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) == [ + { + "role": "user", + "content": "['Message demonstrating the absence of truncation.', 'How many letters in the word eudca']", + } + ] + param_id = request.node.callspec.id if "string" in param_id: assert [ @@ -2011,7 +2045,12 @@ def test_langchain_openai_tools_agent_stream( ) as _, start_transaction(): list( agent_executor.stream( - {"input": "How many letters in the word eudca"}, + { + "input": [ + "Message demonstrating the absence of truncation.", + "How many letters in the word eudca", + ] + }, {"run_name": "my-snazzy-pipeline"}, ) ) @@ -2065,6 +2104,15 @@ def test_langchain_openai_tools_agent_stream( assert "word" in tool_exec_span["attributes"][SPANDATA.GEN_AI_TOOL_INPUT] assert 5 == int(tool_exec_span["attributes"][SPANDATA.GEN_AI_TOOL_OUTPUT]) + assert json.loads( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) == [ + { + "role": "user", + "content": "['Message demonstrating the absence of truncation.', 'How many letters in the word eudca']", + } + ] + param_id = request.node.callspec.id if "string" in param_id: assert [ @@ -2943,13 +2991,7 @@ def test_langchain_message_role_normalization_units(): assert normalized[5] == "string message" # String message unchanged -@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) -def test_langchain_message_truncation( - sentry_init, - capture_events, - capture_items, - stream_gen_ai_spans, -): +def test_langchain_message_truncation(sentry_init, capture_events): """Test that large messages are truncated properly in Langchain integration.""" from langchain_core.outputs import LLMResult, Generation @@ -2957,8 +2999,8 @@ def test_langchain_message_truncation( integrations=[LangchainIntegration(include_prompts=True)], traces_sample_rate=1.0, send_default_pii=True, - stream_gen_ai_spans=stream_gen_ai_spans, ) + events = capture_events() callback = SentryLangchainCallback(max_span_map_size=100, include_prompts=True) @@ -2976,101 +3018,48 @@ def test_langchain_message_truncation( "small message 5", ] - if stream_gen_ai_spans: - items = capture_items("transaction", "span") - - with start_transaction(): - callback.on_llm_start( - serialized=serialized, - prompts=prompts, - run_id=run_id, - name="my_pipeline", - invocation_params={ - "temperature": 0.7, - "max_tokens": 100, - "model": "gpt-3.5-turbo", - }, - ) - - response = LLMResult( - generations=[[Generation(text="The response")]], - llm_output={ - "token_usage": { - "total_tokens": 25, - "prompt_tokens": 10, - "completion_tokens": 15, - } - }, - ) - callback.on_llm_end(response=response, run_id=run_id) - - tx = next(item.payload for item in items if item.type == "transaction") - assert tx["type"] == "transaction" - - spans = [item.payload for item in items if item.type == "span"] - llm_spans = [ - span - for span in spans - if span["attributes"].get("sentry.op") == "gen_ai.text_completion" - ] - - assert len(llm_spans) > 0 - - llm_span = llm_spans[0] - - assert llm_span["attributes"]["gen_ai.operation.name"] == "text_completion" - assert llm_span["attributes"][SPANDATA.GEN_AI_FUNCTION_ID] == "my_pipeline" - - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in llm_span["attributes"] - messages_data = llm_span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - else: - events = capture_events() - - with start_transaction(): - callback.on_llm_start( - serialized=serialized, - prompts=prompts, - run_id=run_id, - name="my_pipeline", - invocation_params={ - "temperature": 0.7, - "max_tokens": 100, - "model": "gpt-3.5-turbo", - }, - ) - - response = LLMResult( - generations=[[Generation(text="The response")]], - llm_output={ - "token_usage": { - "total_tokens": 25, - "prompt_tokens": 10, - "completion_tokens": 15, - } - }, - ) - callback.on_llm_end(response=response, run_id=run_id) - - assert len(events) > 0 - tx = events[0] - assert tx["type"] == "transaction" - - llm_spans = [ - span - for span in tx.get("spans", []) - if span.get("op") == "gen_ai.text_completion" - ] + with start_transaction(): + callback.on_llm_start( + serialized=serialized, + prompts=prompts, + run_id=run_id, + name="my_pipeline", + invocation_params={ + "temperature": 0.7, + "max_tokens": 100, + "model": "gpt-3.5-turbo", + }, + ) - assert len(llm_spans) > 0 + response = LLMResult( + generations=[[Generation(text="The response")]], + llm_output={ + "token_usage": { + "total_tokens": 25, + "prompt_tokens": 10, + "completion_tokens": 15, + } + }, + ) + callback.on_llm_end(response=response, run_id=run_id) - llm_span = llm_spans[0] + assert len(events) > 0 + tx = events[0] + assert tx["type"] == "transaction" - assert llm_span["data"]["gen_ai.operation.name"] == "text_completion" - assert llm_span["data"][SPANDATA.GEN_AI_FUNCTION_ID] == "my_pipeline" + llm_spans = [ + span + for span in tx.get("spans", []) + if span.get("op") == "gen_ai.text_completion" + ] + assert len(llm_spans) > 0 - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in llm_span["data"] - messages_data = llm_span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + llm_span = llm_spans[0] + assert llm_span["data"]["gen_ai.operation.name"] == "text_completion" + assert llm_span["data"][SPANDATA.GEN_AI_FUNCTION_ID] == "my_pipeline" + assert SPANDATA.GEN_AI_REQUEST_MESSAGES in llm_span["data"] + messages_data = llm_span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] assert isinstance(messages_data, str) parsed_messages = json.loads(messages_data) diff --git a/tests/integrations/langgraph/test_langgraph.py b/tests/integrations/langgraph/test_langgraph.py index f8df60739f..b8554f2f60 100644 --- a/tests/integrations/langgraph/test_langgraph.py +++ b/tests/integrations/langgraph/test_langgraph.py @@ -332,8 +332,9 @@ def original_invoke(self, *args, **kwargs): import json request_messages = json.loads(request_messages) - assert len(request_messages) == 1 - assert request_messages[0]["content"] == "Of course! How can I assist you?" + assert len(request_messages) == 2 + assert request_messages[0]["content"] == "Hello, can you help me?" + assert request_messages[1]["content"] == "Of course! How can I assist you?" response_text = invoke_span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] assert response_text == expected_assistant_response @@ -1987,13 +1988,7 @@ def __init__(self, content, message_type="human"): assert "ai" not in roles -@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) -def test_langgraph_message_truncation( - sentry_init, - capture_events, - capture_items, - stream_gen_ai_spans, -): +def test_langgraph_message_truncation(sentry_init, capture_events): """Test that large messages are truncated properly in Langgraph integration.""" import json @@ -2001,8 +1996,8 @@ def test_langgraph_message_truncation( integrations=[LanggraphIntegration(include_prompts=True)], traces_sample_rate=1.0, send_default_pii=True, - stream_gen_ai_spans=stream_gen_ai_spans, ) + events = capture_events() large_content = ( "This is a very long message that will exceed our size limits. " * 1000 @@ -2022,66 +2017,28 @@ def test_langgraph_message_truncation( def original_invoke(self, *args, **kwargs): return {"messages": args[0].get("messages", [])} - if stream_gen_ai_spans: - items = capture_items("transaction", "span") - - with start_transaction(): - wrapped_invoke = _wrap_pregel_invoke(original_invoke) - result = wrapped_invoke(pregel, test_state) - - assert result is not None - - spans = [item.payload for item in items if item.type == "span"] - invoke_spans = [ - span - for span in spans - if span["attributes"].get("sentry.op") == OP.GEN_AI_INVOKE_AGENT - ] - - assert len(invoke_spans) > 0 + with start_transaction(): + wrapped_invoke = _wrap_pregel_invoke(original_invoke) + result = wrapped_invoke(pregel, test_state) - invoke_span = invoke_spans[0] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in invoke_span["attributes"] - - messages_data = invoke_span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - - assert isinstance(messages_data, str) - - parsed_messages = json.loads(messages_data) - assert isinstance(parsed_messages, list) - assert len(parsed_messages) == 1 - assert "small message 5" in str(parsed_messages[0]) - (tx,) = (item.payload for item in items if item.type == "transaction") - else: - events = capture_events() - - with start_transaction(): - wrapped_invoke = _wrap_pregel_invoke(original_invoke) - result = wrapped_invoke(pregel, test_state) - - assert result is not None - - assert len(events) > 0 - tx = events[0] - assert tx["type"] == "transaction" - - invoke_spans = [ - span - for span in tx.get("spans", []) - if span.get("op") == OP.GEN_AI_INVOKE_AGENT - ] - - assert len(invoke_spans) > 0 + assert result is not None + assert len(events) > 0 + tx = events[0] + assert tx["type"] == "transaction" - invoke_span = invoke_spans[0] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in invoke_span["data"] + invoke_spans = [ + span for span in tx.get("spans", []) if span.get("op") == OP.GEN_AI_INVOKE_AGENT + ] + assert len(invoke_spans) > 0 - messages_data = invoke_span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert isinstance(messages_data, str) + invoke_span = invoke_spans[0] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES in invoke_span["data"] - parsed_messages = json.loads(messages_data) - assert isinstance(parsed_messages, list) - assert len(parsed_messages) == 1 - assert "small message 5" in str(parsed_messages[0]) + messages_data = invoke_span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + assert isinstance(messages_data, str) + parsed_messages = json.loads(messages_data) + assert isinstance(parsed_messages, list) + assert len(parsed_messages) == 1 + assert "small message 5" in str(parsed_messages[0]) assert tx["_meta"]["spans"]["0"]["data"]["gen_ai.request.messages"][""]["len"] == 5 diff --git a/tests/integrations/litellm/test_litellm.py b/tests/integrations/litellm/test_litellm.py index 22663f9472..703ae67b1a 100644 --- a/tests/integrations/litellm/test_litellm.py +++ b/tests/integrations/litellm/test_litellm.py @@ -159,7 +159,10 @@ def test_nonstreaming_chat_completion( stream_gen_ai_spans=stream_gen_ai_spans, ) - messages = [{"role": "user", "content": "Hello!"}] + messages = [ + {"role": "user", "content": "Message demonstrating the absence of truncation."}, + {"role": "user", "content": "Hello!"}, + ] client = OpenAI(api_key="test-key") @@ -216,7 +219,13 @@ def test_nonstreaming_chat_completion( assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] + assert json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) == [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + {"role": "user", "content": "Hello!"}, + ] assert SPANDATA.GEN_AI_RESPONSE_TEXT in span["attributes"] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] @@ -302,7 +311,10 @@ async def test_async_nonstreaming_chat_completion( stream_gen_ai_spans=stream_gen_ai_spans, ) - messages = [{"role": "user", "content": "Hello!"}] + messages = [ + {"role": "user", "content": "Message demonstrating the absence of truncation."}, + {"role": "user", "content": "Hello!"}, + ] client = AsyncOpenAI(api_key="test-key") @@ -360,7 +372,13 @@ async def test_async_nonstreaming_chat_completion( assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat" if send_default_pii and include_prompts: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] + assert json.loads(span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) == [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + {"role": "user", "content": "Hello!"}, + ] assert SPANDATA.GEN_AI_RESPONSE_TEXT in span["attributes"] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] @@ -448,7 +466,10 @@ def test_streaming_chat_completion( stream_gen_ai_spans=stream_gen_ai_spans, ) - messages = [{"role": "user", "content": "Hello!"}] + messages = [ + {"role": "user", "content": "Message demonstrating the absence of truncation."}, + {"role": "user", "content": "Hello!"}, + ] client = OpenAI(api_key="test-key") @@ -556,7 +577,10 @@ async def test_async_streaming_chat_completion( stream_gen_ai_spans=stream_gen_ai_spans, ) - messages = [{"role": "user", "content": "Hello!"}] + messages = [ + {"role": "user", "content": "Message demonstrating the absence of truncation."}, + {"role": "user", "content": "Hello!"}, + ] client = AsyncOpenAI(api_key="test-key") @@ -2325,20 +2349,14 @@ def test_integration_setup(sentry_init): assert _failure_callback in (litellm.failure_callback or []) -@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) -def test_litellm_message_truncation( - sentry_init, - capture_events, - capture_items, - stream_gen_ai_spans, -): +def test_litellm_message_truncation(sentry_init, capture_events): """Test that large messages are truncated properly in LiteLLM integration.""" sentry_init( integrations=[LiteLLMIntegration(include_prompts=True)], traces_sample_rate=1.0, send_default_pii=True, - stream_gen_ai_spans=stream_gen_ai_spans, ) + events = capture_events() large_content = ( "This is a very long message that will exceed our size limits. " * 1000 @@ -2352,78 +2370,39 @@ def test_litellm_message_truncation( ] mock_response = MockCompletionResponse() - if stream_gen_ai_spans: - items = capture_items("transaction", "span") - - with start_transaction(name="litellm test"): - kwargs = { - "model": "gpt-3.5-turbo", - "messages": messages, - } - - _input_callback(kwargs) - _success_callback( - kwargs, - mock_response, - datetime.now(), - datetime.now(), - ) - - spans = [item.payload for item in items if item.type == "span"] - chat_spans = [ - span - for span in spans - if span["attributes"].get("sentry.op") == OP.GEN_AI_CHAT - ] - assert len(chat_spans) > 0 - - chat_span = chat_spans[0] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in chat_span["attributes"] - - messages_data = chat_span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert isinstance(messages_data, str) - - parsed_messages = json.loads(messages_data) - assert isinstance(parsed_messages, list) - assert len(parsed_messages) == 1 - assert "small message 5" in str(parsed_messages[0]) - tx = next(item.payload for item in items if item.type == "transaction") - else: - events = capture_events() - - with start_transaction(name="litellm test"): - kwargs = { - "model": "gpt-3.5-turbo", - "messages": messages, - } + with start_transaction(name="litellm test"): + kwargs = { + "model": "gpt-3.5-turbo", + "messages": messages, + } - _input_callback(kwargs) - _success_callback( - kwargs, - mock_response, - datetime.now(), - datetime.now(), - ) + _input_callback(kwargs) + _success_callback( + kwargs, + mock_response, + datetime.now(), + datetime.now(), + ) - assert len(events) > 0 - tx = events[0] - assert tx["type"] == "transaction" + assert len(events) > 0 + tx = events[0] + assert tx["type"] == "transaction" - chat_spans = [ - span for span in tx.get("spans", []) if span.get("op") == OP.GEN_AI_CHAT - ] - assert len(chat_spans) > 0 + chat_spans = [ + span for span in tx.get("spans", []) if span.get("op") == OP.GEN_AI_CHAT + ] + assert len(chat_spans) > 0 - chat_span = chat_spans[0] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in chat_span["data"] + chat_span = chat_spans[0] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES in chat_span["data"] - messages_data = chat_span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert isinstance(messages_data, str) + messages_data = chat_span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + assert isinstance(messages_data, str) - parsed_messages = json.loads(messages_data) - assert isinstance(parsed_messages, list) - assert len(parsed_messages) == 1 - assert "small message 5" in str(parsed_messages[0]) + parsed_messages = json.loads(messages_data) + assert isinstance(parsed_messages, list) + assert len(parsed_messages) == 1 + assert "small message 5" in str(parsed_messages[0]) assert tx["_meta"]["spans"]["0"]["data"]["gen_ai.request.messages"][""]["len"] == 5 diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index 934a0b8f4e..f8be7cdbe1 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -239,6 +239,10 @@ def test_nonstreaming_chat_completion_no_prompts( "role": "system", "content": "You are a helpful assistant.", }, + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, {"role": "user", "content": "hello"}, ], id="blocks", @@ -252,6 +256,10 @@ def test_nonstreaming_chat_completion_no_prompts( {"type": "text", "text": "Be concise and clear."}, ], }, + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, {"role": "user", "content": "hello"}, ], id="parts", @@ -266,6 +274,10 @@ def test_nonstreaming_chat_completion_no_prompts( {"type": "text", "text": "Be concise and clear."}, ], }, + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, {"role": "user", "content": "hello"}, ] ), @@ -360,6 +372,10 @@ def test_nonstreaming_chat_completion( ] assert "hello" in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + assert ( + "Message demonstrating the absence of truncation." + in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) assert "the model response" in span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] assert span["attributes"]["gen_ai.usage.output_tokens"] == 10 @@ -558,6 +574,10 @@ async def test_nonstreaming_chat_completion_async_no_prompts( "role": "system", "content": "You are a helpful assistant.", }, + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, {"role": "user", "content": "hello"}, ], id="blocks", @@ -571,6 +591,10 @@ async def test_nonstreaming_chat_completion_async_no_prompts( {"type": "text", "text": "Be concise and clear."}, ], }, + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, {"role": "user", "content": "hello"}, ], id="parts", @@ -585,6 +609,10 @@ async def test_nonstreaming_chat_completion_async_no_prompts( {"type": "text", "text": "Be concise and clear."}, ], }, + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, {"role": "user", "content": "hello"}, ] ), @@ -676,6 +704,10 @@ async def test_nonstreaming_chat_completion_async( ] assert "hello" in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + assert ( + "Message demonstrating the absence of truncation." + in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) assert "the model response" in span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] assert span["attributes"]["gen_ai.usage.output_tokens"] == 10 @@ -1353,6 +1385,10 @@ async def test_streaming_chat_completion_async_with_usage_in_stream( "role": "system", "content": "You are a helpful assistant.", }, + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, {"role": "user", "content": "hello"}, ], id="blocks", @@ -1366,6 +1402,10 @@ async def test_streaming_chat_completion_async_with_usage_in_stream( {"type": "text", "text": "Be concise and clear."}, ], }, + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, {"role": "user", "content": "hello"}, ], id="parts", @@ -1380,6 +1420,10 @@ async def test_streaming_chat_completion_async_with_usage_in_stream( {"type": "text", "text": "Be concise and clear."}, ], }, + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, {"role": "user", "content": "hello"}, ] ), @@ -1517,6 +1561,10 @@ def test_streaming_chat_completion( assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "model-id" + assert ( + "Message demonstrating the absence of truncation." + in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) assert "hello" in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] assert "hello world" in span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] @@ -1525,12 +1573,12 @@ def test_streaming_chat_completion( if "blocks" in param_id: assert span["attributes"]["gen_ai.usage.output_tokens"] == 2 - assert span["attributes"]["gen_ai.usage.input_tokens"] == 7 - assert span["attributes"]["gen_ai.usage.total_tokens"] == 9 + assert span["attributes"]["gen_ai.usage.input_tokens"] == 15 + assert span["attributes"]["gen_ai.usage.total_tokens"] == 17 else: assert span["attributes"]["gen_ai.usage.output_tokens"] == 2 - assert span["attributes"]["gen_ai.usage.input_tokens"] == 12 - assert span["attributes"]["gen_ai.usage.total_tokens"] == 14 + assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 + assert span["attributes"]["gen_ai.usage.total_tokens"] == 22 except ImportError: pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly @@ -1600,12 +1648,12 @@ def test_streaming_chat_completion( if "blocks" in param_id: assert span["data"]["gen_ai.usage.output_tokens"] == 2 - assert span["data"]["gen_ai.usage.input_tokens"] == 7 - assert span["data"]["gen_ai.usage.total_tokens"] == 9 + assert span["data"]["gen_ai.usage.input_tokens"] == 15 + assert span["data"]["gen_ai.usage.total_tokens"] == 17 else: assert span["data"]["gen_ai.usage.output_tokens"] == 2 - assert span["data"]["gen_ai.usage.input_tokens"] == 12 - assert span["data"]["gen_ai.usage.total_tokens"] == 14 + assert span["data"]["gen_ai.usage.input_tokens"] == 20 + assert span["data"]["gen_ai.usage.total_tokens"] == 22 except ImportError: pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly @@ -1819,6 +1867,10 @@ async def test_streaming_chat_completion_async_no_prompts( "role": "system", "content": "You are a helpful assistant.", }, + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, {"role": "user", "content": "hello"}, ], id="blocks", @@ -1832,6 +1884,10 @@ async def test_streaming_chat_completion_async_no_prompts( {"type": "text", "text": "Be concise and clear."}, ], }, + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, {"role": "user", "content": "hello"}, ], id="parts", @@ -1846,6 +1902,10 @@ async def test_streaming_chat_completion_async_no_prompts( {"type": "text", "text": "Be concise and clear."}, ], }, + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, {"role": "user", "content": "hello"}, ] ), @@ -1989,6 +2049,10 @@ async def test_streaming_chat_completion_async( }, ] + assert ( + "Message demonstrating the absence of truncation." + in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) assert "hello" in span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] assert "hello world" in span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] @@ -1997,12 +2061,12 @@ async def test_streaming_chat_completion_async( if "blocks" in param_id: assert span["attributes"]["gen_ai.usage.output_tokens"] == 2 - assert span["attributes"]["gen_ai.usage.input_tokens"] == 7 - assert span["attributes"]["gen_ai.usage.total_tokens"] == 9 + assert span["attributes"]["gen_ai.usage.input_tokens"] == 15 + assert span["attributes"]["gen_ai.usage.total_tokens"] == 17 else: assert span["attributes"]["gen_ai.usage.output_tokens"] == 2 - assert span["attributes"]["gen_ai.usage.input_tokens"] == 12 - assert span["attributes"]["gen_ai.usage.total_tokens"] == 14 + assert span["attributes"]["gen_ai.usage.input_tokens"] == 20 + assert span["attributes"]["gen_ai.usage.total_tokens"] == 22 except ImportError: pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly @@ -2074,12 +2138,12 @@ async def test_streaming_chat_completion_async( if "blocks" in param_id: assert span["data"]["gen_ai.usage.output_tokens"] == 2 - assert span["data"]["gen_ai.usage.input_tokens"] == 7 - assert span["data"]["gen_ai.usage.total_tokens"] == 9 + assert span["data"]["gen_ai.usage.input_tokens"] == 15 + assert span["data"]["gen_ai.usage.total_tokens"] == 17 else: assert span["data"]["gen_ai.usage.output_tokens"] == 2 - assert span["data"]["gen_ai.usage.input_tokens"] == 12 - assert span["data"]["gen_ai.usage.total_tokens"] == 14 + assert span["data"]["gen_ai.usage.input_tokens"] == 20 + assert span["data"]["gen_ai.usage.total_tokens"] == 22 except ImportError: pass # if tiktoken is not installed, we can't guarantee token usage will be calculated properly @@ -3718,6 +3782,10 @@ def test_ai_client_span_responses_api_no_pii( "role": "system", "content": "You are a helpful assistant.", }, + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, {"role": "user", "content": "hello"}, ], id="blocks_no_type", @@ -3729,6 +3797,11 @@ def test_ai_client_span_responses_api_no_pii( "role": "system", "content": "You are a helpful assistant.", }, + { + "type": "message", + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, {"type": "message", "role": "user", "content": "hello"}, ], id="blocks", @@ -3742,6 +3815,10 @@ def test_ai_client_span_responses_api_no_pii( {"type": "text", "text": "Be concise and clear."}, ], }, + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, {"role": "user", "content": "hello"}, ], id="parts_no_type", @@ -3756,6 +3833,11 @@ def test_ai_client_span_responses_api_no_pii( {"type": "text", "text": "Be concise and clear."}, ], }, + { + "type": "message", + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, {"type": "message", "role": "user", "content": "hello"}, ], id="parts", @@ -3862,7 +3944,13 @@ def test_ai_client_span_responses_api( [{"type": "text", "content": "You are a helpful assistant."}] ), "gen_ai.request.messages": safe_serialize( - [{"role": "user", "content": "hello"}] + [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + {"role": "user", "content": "hello"}, + ] ), } ) @@ -3879,7 +3967,13 @@ def test_ai_client_span_responses_api( ] ), "gen_ai.request.messages": safe_serialize( - [{"role": "user", "content": "hello"}] + [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + {"role": "user", "content": "hello"}, + ] ), } ) @@ -3892,7 +3986,14 @@ def test_ai_client_span_responses_api( [{"type": "text", "content": "You are a helpful assistant."}] ), "gen_ai.request.messages": safe_serialize( - [{"type": "message", "role": "user", "content": "hello"}] + [ + { + "type": "message", + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + {"type": "message", "role": "user", "content": "hello"}, + ] ), } ) @@ -3909,7 +4010,14 @@ def test_ai_client_span_responses_api( ] ), "gen_ai.request.messages": safe_serialize( - [{"type": "message", "role": "user", "content": "hello"}] + [ + { + "type": "message", + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + {"type": "message", "role": "user", "content": "hello"}, + ] ), } ) @@ -3925,7 +4033,13 @@ def test_ai_client_span_responses_api( ] ), "gen_ai.request.messages": safe_serialize( - [{"role": "user", "content": "hello"}] + [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + {"role": "user", "content": "hello"}, + ] ), } ) @@ -3943,7 +4057,13 @@ def test_ai_client_span_responses_api( ] ), "gen_ai.request.messages": safe_serialize( - [{"role": "user", "content": "hello"}] + [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + {"role": "user", "content": "hello"}, + ] ), } ) @@ -3957,7 +4077,14 @@ def test_ai_client_span_responses_api( ] ), "gen_ai.request.messages": safe_serialize( - [{"type": "message", "role": "user", "content": "hello"}] + [ + { + "type": "message", + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + {"type": "message", "role": "user", "content": "hello"}, + ] ), } ) @@ -3975,7 +4102,14 @@ def test_ai_client_span_responses_api( ] ), "gen_ai.request.messages": safe_serialize( - [{"type": "message", "role": "user", "content": "hello"}] + [ + { + "type": "message", + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + {"type": "message", "role": "user", "content": "hello"}, + ] ), } ) @@ -4330,6 +4464,10 @@ def test_error_in_responses_api( "role": "system", "content": "You are a helpful assistant.", }, + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, {"role": "user", "content": "hello"}, ], id="blocks_no_type", @@ -4341,6 +4479,11 @@ def test_error_in_responses_api( "role": "system", "content": "You are a helpful assistant.", }, + { + "type": "message", + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, {"type": "message", "role": "user", "content": "hello"}, ], id="blocks", @@ -4354,6 +4497,10 @@ def test_error_in_responses_api( {"type": "text", "text": "Be concise and clear."}, ], }, + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, {"role": "user", "content": "hello"}, ], id="parts_no_type", @@ -4368,6 +4515,11 @@ def test_error_in_responses_api( {"type": "text", "text": "Be concise and clear."}, ], }, + { + "type": "message", + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, {"type": "message", "role": "user", "content": "hello"}, ], id="parts", @@ -4474,7 +4626,13 @@ async def test_ai_client_span_responses_async_api( [{"type": "text", "content": "You are a helpful assistant."}] ), "gen_ai.request.messages": safe_serialize( - [{"role": "user", "content": "hello"}] + [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + {"role": "user", "content": "hello"}, + ] ), } ) @@ -4491,7 +4649,13 @@ async def test_ai_client_span_responses_async_api( ] ), "gen_ai.request.messages": safe_serialize( - [{"role": "user", "content": "hello"}] + [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + {"role": "user", "content": "hello"}, + ] ), } ) @@ -4504,7 +4668,14 @@ async def test_ai_client_span_responses_async_api( [{"type": "text", "content": "You are a helpful assistant."}] ), "gen_ai.request.messages": safe_serialize( - [{"type": "message", "role": "user", "content": "hello"}] + [ + { + "type": "message", + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + {"type": "message", "role": "user", "content": "hello"}, + ] ), } ) @@ -4521,7 +4692,14 @@ async def test_ai_client_span_responses_async_api( ] ), "gen_ai.request.messages": safe_serialize( - [{"type": "message", "role": "user", "content": "hello"}] + [ + { + "type": "message", + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + {"type": "message", "role": "user", "content": "hello"}, + ] ), } ) @@ -4537,7 +4715,13 @@ async def test_ai_client_span_responses_async_api( ] ), "gen_ai.request.messages": safe_serialize( - [{"role": "user", "content": "hello"}] + [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + {"role": "user", "content": "hello"}, + ] ), } ) @@ -4555,7 +4739,13 @@ async def test_ai_client_span_responses_async_api( ] ), "gen_ai.request.messages": safe_serialize( - [{"role": "user", "content": "hello"}] + [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + {"role": "user", "content": "hello"}, + ] ), } ) @@ -4569,7 +4759,14 @@ async def test_ai_client_span_responses_async_api( ] ), "gen_ai.request.messages": safe_serialize( - [{"type": "message", "role": "user", "content": "hello"}] + [ + { + "type": "message", + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + {"type": "message", "role": "user", "content": "hello"}, + ] ), } ) @@ -4587,7 +4784,14 @@ async def test_ai_client_span_responses_async_api( ] ), "gen_ai.request.messages": safe_serialize( - [{"type": "message", "role": "user", "content": "hello"}] + [ + { + "type": "message", + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + {"type": "message", "role": "user", "content": "hello"}, + ] ), } ) @@ -4812,6 +5016,10 @@ async def test_ai_client_span_responses_async_api( "role": "system", "content": "You are a helpful assistant.", }, + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, {"role": "user", "content": "hello"}, ], id="blocks_no_type", @@ -4823,6 +5031,11 @@ async def test_ai_client_span_responses_async_api( "role": "system", "content": "You are a helpful assistant.", }, + { + "type": "message", + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, {"type": "message", "role": "user", "content": "hello"}, ], id="blocks", @@ -4836,6 +5049,10 @@ async def test_ai_client_span_responses_async_api( {"type": "text", "text": "Be concise and clear."}, ], }, + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, {"role": "user", "content": "hello"}, ], id="parts_no_type", @@ -4850,6 +5067,11 @@ async def test_ai_client_span_responses_async_api( {"type": "text", "text": "Be concise and clear."}, ], }, + { + "type": "message", + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, {"type": "message", "role": "user", "content": "hello"}, ], id="parts", @@ -4974,7 +5196,13 @@ async def test_ai_client_span_streaming_responses_async_api( [{"type": "text", "content": "You are a helpful assistant."}] ), "gen_ai.request.messages": safe_serialize( - [{"role": "user", "content": "hello"}] + [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + {"role": "user", "content": "hello"}, + ] ), } ) @@ -4991,7 +5219,13 @@ async def test_ai_client_span_streaming_responses_async_api( ] ), "gen_ai.request.messages": safe_serialize( - [{"role": "user", "content": "hello"}] + [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + {"role": "user", "content": "hello"}, + ] ), } ) @@ -5004,7 +5238,14 @@ async def test_ai_client_span_streaming_responses_async_api( [{"type": "text", "content": "You are a helpful assistant."}] ), "gen_ai.request.messages": safe_serialize( - [{"type": "message", "role": "user", "content": "hello"}] + [ + { + "type": "message", + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + {"type": "message", "role": "user", "content": "hello"}, + ] ), } ) @@ -5021,7 +5262,14 @@ async def test_ai_client_span_streaming_responses_async_api( ] ), "gen_ai.request.messages": safe_serialize( - [{"type": "message", "role": "user", "content": "hello"}] + [ + { + "type": "message", + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + {"type": "message", "role": "user", "content": "hello"}, + ] ), } ) @@ -5037,7 +5285,13 @@ async def test_ai_client_span_streaming_responses_async_api( ] ), "gen_ai.request.messages": safe_serialize( - [{"role": "user", "content": "hello"}] + [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + {"role": "user", "content": "hello"}, + ] ), } ) @@ -5055,7 +5309,13 @@ async def test_ai_client_span_streaming_responses_async_api( ] ), "gen_ai.request.messages": safe_serialize( - [{"role": "user", "content": "hello"}] + [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + {"role": "user", "content": "hello"}, + ] ), } ) @@ -5069,7 +5329,14 @@ async def test_ai_client_span_streaming_responses_async_api( ] ), "gen_ai.request.messages": safe_serialize( - [{"type": "message", "role": "user", "content": "hello"}] + [ + { + "type": "message", + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + {"type": "message", "role": "user", "content": "hello"}, + ] ), } ) @@ -5087,7 +5354,14 @@ async def test_ai_client_span_streaming_responses_async_api( ] ), "gen_ai.request.messages": safe_serialize( - [{"type": "message", "role": "user", "content": "hello"}] + [ + { + "type": "message", + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + {"type": "message", "role": "user", "content": "hello"}, + ] ), } ) @@ -5835,20 +6109,16 @@ def test_openai_message_role_mapping( assert stored_messages[0]["role"] == expected_role -@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) def test_openai_message_truncation( sentry_init, capture_events, - capture_items, nonstreaming_chat_completions_model_response, - stream_gen_ai_spans, ): """Test that large messages are truncated properly in OpenAI integration.""" sentry_init( integrations=[OpenAIIntegration(include_prompts=True)], traces_sample_rate=1.0, send_default_pii=True, - stream_gen_ai_spans=stream_gen_ai_spans, ) client = OpenAI(api_key="z") @@ -5876,45 +6146,24 @@ def test_openai_message_truncation( {"role": "user", "content": large_content}, ] - if stream_gen_ai_spans: - items = capture_items("transaction", "span") - - with start_transaction(name="openai tx"): - client.chat.completions.create( - model="some-model", - messages=large_messages, - ) - - span = next(item.payload for item in items if item.type == "span") - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["attributes"] - - messages_data = span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert isinstance(messages_data, str) + events = capture_events() - parsed_messages = json.loads(messages_data) - assert isinstance(parsed_messages, list) - assert len(parsed_messages) <= len(large_messages) - - (event,) = (item.payload for item in items if item.type == "transaction") - else: - events = capture_events() - - with start_transaction(name="openai tx"): - client.chat.completions.create( - model="some-model", - messages=large_messages, - ) + with start_transaction(name="openai tx"): + client.chat.completions.create( + model="some-model", + messages=large_messages, + ) - (event,) = events - span = event["spans"][0] - assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"] + (event,) = events + span = event["spans"][0] + assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"] - messages_data = span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert isinstance(messages_data, str) + messages_data = span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + assert isinstance(messages_data, str) - parsed_messages = json.loads(messages_data) - assert isinstance(parsed_messages, list) - assert len(parsed_messages) <= len(large_messages) + parsed_messages = json.loads(messages_data) + assert isinstance(parsed_messages, list) + assert len(parsed_messages) <= len(large_messages) meta_path = event["_meta"] span_meta = meta_path["spans"]["0"]["data"] diff --git a/tests/integrations/openai_agents/test_openai_agents.py b/tests/integrations/openai_agents/test_openai_agents.py index bf44562b14..430d60ac01 100644 --- a/tests/integrations/openai_agents/test_openai_agents.py +++ b/tests/integrations/openai_agents/test_openai_agents.py @@ -313,6 +313,10 @@ async def test_agent_invocation_span_no_pii( "role": "system", "content": "You are a helpful assistant.", }, + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, { "role": "user", "content": "Test input", @@ -327,6 +331,11 @@ async def test_agent_invocation_span_no_pii( "role": "system", "content": "You are a helpful assistant.", }, + { + "type": "message", + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, { "type": "message", "role": "user", @@ -344,6 +353,10 @@ async def test_agent_invocation_span_no_pii( {"type": "text", "text": "Be concise and clear."}, ], }, + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, { "role": "user", "content": "Test input", @@ -361,6 +374,11 @@ async def test_agent_invocation_span_no_pii( {"type": "text", "text": "Be concise and clear."}, ], }, + { + "type": "message", + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, { "type": "message", "role": "user", @@ -462,6 +480,21 @@ async def test_agent_invocation_span( {"type": "text", "content": "You are a helpful assistant."}, ] ) + + assert json.loads( + ai_client_span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) == [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Message demonstrating the absence of truncation.", + } + ], + }, + {"role": "user", "content": [{"type": "text", "text": "Test input"}]}, + ] elif "blocks_no_type" in param_id: assert ai_client_span["attributes"][ "gen_ai.system_instructions" @@ -474,6 +507,21 @@ async def test_agent_invocation_span( {"type": "text", "content": "You are a helpful assistant."}, ] ) + + assert json.loads( + ai_client_span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) == [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Message demonstrating the absence of truncation.", + } + ], + }, + {"role": "user", "content": [{"type": "text", "text": "Test input"}]}, + ] elif "blocks" in param_id and instructions is None: # type: ignore assert ai_client_span["attributes"][ "gen_ai.system_instructions" @@ -482,6 +530,21 @@ async def test_agent_invocation_span( {"type": "text", "content": "You are a helpful assistant."}, ] ) + + assert json.loads( + ai_client_span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) == [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Message demonstrating the absence of truncation.", + } + ], + }, + {"role": "user", "content": [{"type": "text", "text": "Test input"}]}, + ] elif "blocks" in param_id: assert ai_client_span["attributes"][ "gen_ai.system_instructions" @@ -494,6 +557,21 @@ async def test_agent_invocation_span( {"type": "text", "content": "You are a helpful assistant."}, ] ) + + assert json.loads( + ai_client_span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) == [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Message demonstrating the absence of truncation.", + } + ], + }, + {"role": "user", "content": [{"type": "text", "text": "Test input"}]}, + ] elif "parts_no_type" in param_id and instructions is None: assert ai_client_span["attributes"][ "gen_ai.system_instructions" @@ -503,6 +581,21 @@ async def test_agent_invocation_span( {"type": "text", "content": "Be concise and clear."}, ] ) + + assert json.loads( + ai_client_span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) == [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Message demonstrating the absence of truncation.", + } + ], + }, + {"role": "user", "content": [{"type": "text", "text": "Test input"}]}, + ] elif "parts_no_type" in param_id: assert ai_client_span["attributes"][ "gen_ai.system_instructions" @@ -516,6 +609,21 @@ async def test_agent_invocation_span( {"type": "text", "content": "Be concise and clear."}, ] ) + + assert json.loads( + ai_client_span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) == [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Message demonstrating the absence of truncation.", + } + ], + }, + {"role": "user", "content": [{"type": "text", "text": "Test input"}]}, + ] elif instructions is None: # type: ignore assert ai_client_span["attributes"][ "gen_ai.system_instructions" @@ -525,6 +633,21 @@ async def test_agent_invocation_span( {"type": "text", "content": "Be concise and clear."}, ] ) + + assert json.loads( + ai_client_span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) == [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Message demonstrating the absence of truncation.", + } + ], + }, + {"role": "user", "content": [{"type": "text", "text": "Test input"}]}, + ] else: assert ai_client_span["attributes"][ "gen_ai.system_instructions" @@ -539,6 +662,21 @@ async def test_agent_invocation_span( ] ) + assert json.loads( + ai_client_span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) == [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Message demonstrating the absence of truncation.", + } + ], + }, + {"role": "user", "content": [{"type": "text", "text": "Test input"}]}, + ] + assert ( invoke_agent_span["attributes"]["gen_ai.response.text"] == "Hello, how can I help you?" @@ -960,6 +1098,10 @@ def test_agent_invocation_span_sync_no_pii( "role": "system", "content": "You are a helpful assistant.", }, + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, { "role": "user", "content": "Test input", @@ -974,6 +1116,11 @@ def test_agent_invocation_span_sync_no_pii( "role": "system", "content": "You are a helpful assistant.", }, + { + "type": "message", + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, { "type": "message", "role": "user", @@ -991,6 +1138,10 @@ def test_agent_invocation_span_sync_no_pii( {"type": "text", "text": "Be concise and clear."}, ], }, + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, { "role": "user", "content": "Test input", @@ -1008,6 +1159,11 @@ def test_agent_invocation_span_sync_no_pii( {"type": "text", "text": "Be concise and clear."}, ], }, + { + "type": "message", + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, { "type": "message", "role": "user", @@ -1114,6 +1270,21 @@ def test_agent_invocation_span_sync( {"type": "text", "content": "You are a helpful assistant."}, ] ) + + assert json.loads( + ai_client_span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) == [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Message demonstrating the absence of truncation.", + } + ], + }, + {"role": "user", "content": [{"type": "text", "text": "Test input"}]}, + ] elif "blocks_no_type" in param_id: assert ai_client_span["attributes"][ "gen_ai.system_instructions" @@ -1126,6 +1297,21 @@ def test_agent_invocation_span_sync( {"type": "text", "content": "You are a helpful assistant."}, ] ) + + assert json.loads( + ai_client_span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) == [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Message demonstrating the absence of truncation.", + } + ], + }, + {"role": "user", "content": [{"type": "text", "text": "Test input"}]}, + ] elif "blocks" in param_id and instructions is None: # type: ignore assert ai_client_span["attributes"][ "gen_ai.system_instructions" @@ -1134,6 +1320,21 @@ def test_agent_invocation_span_sync( {"type": "text", "content": "You are a helpful assistant."}, ] ) + + assert json.loads( + ai_client_span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) == [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Message demonstrating the absence of truncation.", + } + ], + }, + {"role": "user", "content": [{"type": "text", "text": "Test input"}]}, + ] elif "blocks" in param_id: assert ai_client_span["attributes"][ "gen_ai.system_instructions" @@ -1146,6 +1347,21 @@ def test_agent_invocation_span_sync( {"type": "text", "content": "You are a helpful assistant."}, ] ) + + assert json.loads( + ai_client_span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) == [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Message demonstrating the absence of truncation.", + } + ], + }, + {"role": "user", "content": [{"type": "text", "text": "Test input"}]}, + ] elif "parts_no_type" in param_id and instructions is None: assert ai_client_span["attributes"][ "gen_ai.system_instructions" @@ -1155,6 +1371,21 @@ def test_agent_invocation_span_sync( {"type": "text", "content": "Be concise and clear."}, ] ) + + assert json.loads( + ai_client_span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) == [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Message demonstrating the absence of truncation.", + } + ], + }, + {"role": "user", "content": [{"type": "text", "text": "Test input"}]}, + ] elif "parts_no_type" in param_id: assert ai_client_span["attributes"][ "gen_ai.system_instructions" @@ -1168,6 +1399,21 @@ def test_agent_invocation_span_sync( {"type": "text", "content": "Be concise and clear."}, ] ) + + assert json.loads( + ai_client_span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) == [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Message demonstrating the absence of truncation.", + } + ], + }, + {"role": "user", "content": [{"type": "text", "text": "Test input"}]}, + ] elif instructions is None: # type: ignore assert ai_client_span["attributes"][ "gen_ai.system_instructions" @@ -1177,6 +1423,21 @@ def test_agent_invocation_span_sync( {"type": "text", "content": "Be concise and clear."}, ] ) + + assert json.loads( + ai_client_span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) == [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Message demonstrating the absence of truncation.", + } + ], + }, + {"role": "user", "content": [{"type": "text", "text": "Test input"}]}, + ] else: assert ai_client_span["attributes"][ "gen_ai.system_instructions" @@ -1190,6 +1451,21 @@ def test_agent_invocation_span_sync( {"type": "text", "content": "Be concise and clear."}, ] ) + + assert json.loads( + ai_client_span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) == [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Message demonstrating the absence of truncation.", + } + ], + }, + {"role": "user", "content": [{"type": "text", "text": "Test input"}]}, + ] else: with patch.object( agent.model._client._client, @@ -1932,6 +2208,24 @@ def simple_test_tool(message: str) -> str: "gen_ai.request.messages" ] == safe_serialize( [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Please use the simple test tool"} + ], + }, + { + "role": "assistant", + "content": [ + { + "arguments": '{"message": "hello"}', + "call_id": "call_123", + "name": "simple_test_tool", + "type": "function_call", + "id": "call_123", + } + ], + }, { "role": "tool", "content": [ diff --git a/tests/integrations/pydantic_ai/test_pydantic_ai.py b/tests/integrations/pydantic_ai/test_pydantic_ai.py index 23850c3233..c6bbbfc24b 100644 --- a/tests/integrations/pydantic_ai/test_pydantic_ai.py +++ b/tests/integrations/pydantic_ai/test_pydantic_ai.py @@ -76,7 +76,9 @@ async def test_agent_run_async( if stream_gen_ai_spans: items = capture_items("transaction", "span") - result = await test_agent.run("Test input") + result = await test_agent.run( + ["Message demonstrating the absence of truncation.", "Test input"] + ) assert result is not None assert result.output is not None @@ -102,7 +104,23 @@ async def test_agent_run_async( assert "chat" in chat_span["name"] assert chat_span["attributes"]["gen_ai.operation.name"] == "chat" assert chat_span["attributes"]["gen_ai.response.streaming"] is False - assert "gen_ai.request.messages" in chat_span["attributes"] + assert json.loads( + chat_span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) == [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Message demonstrating the absence of truncation.", + }, + { + "type": "text", + "text": "Test input", + }, + ], + } + ] assert "gen_ai.usage.input_tokens" in chat_span["attributes"] assert "gen_ai.usage.output_tokens" in chat_span["attributes"] else: @@ -275,7 +293,9 @@ def test_agent_run_sync( if stream_gen_ai_spans: items = capture_items("transaction", "span") - result = test_agent.run_sync("Test input") + result = test_agent.run_sync( + ["Message demonstrating the absence of truncation.", "Test input"] + ) assert result is not None assert result.output is not None @@ -394,7 +414,9 @@ async def test_agent_run_stream( if stream_gen_ai_spans: items = capture_items("transaction", "span") - async with test_agent.run_stream("Test input") as result: + async with test_agent.run_stream( + ["Message demonstrating the absence of truncation.", "Test input"] + ) as result: # Consume the stream async for _ in result.stream_output(): pass @@ -416,7 +438,23 @@ async def test_agent_run_stream( # Verify streaming flag is True for streaming for chat_span in chat_spans: assert chat_span["attributes"]["gen_ai.response.streaming"] is True - assert "gen_ai.request.messages" in chat_span["attributes"] + assert json.loads( + chat_span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) == [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Message demonstrating the absence of truncation.", + }, + { + "type": "text", + "text": "Test input", + }, + ], + } + ] assert "gen_ai.usage.input_tokens" in chat_span["attributes"] # Streaming responses should still have output data assert ( @@ -479,7 +517,9 @@ async def test_agent_run_stream_events( if stream_gen_ai_spans: items = capture_items("transaction", "span") - async for _ in test_agent.run_stream_events("Test input"): + async for _ in test_agent.run_stream_events( + ["Message demonstrating the absence of truncation.", "Test input"] + ): pass # Verify transaction