diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index dff264f9f8..47679f6012 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -695,6 +695,12 @@ class SPANDATA: Example: 0.1 """ + GEN_AI_REQUEST_REASONING_LEVEL = "gen_ai.request.reasoning.level" + """ + The reasoning or thinking effort level requested for a GenAI model. + Example: "high" + """ + GEN_AI_REQUEST_SEED = "gen_ai.request.seed" """ The seed, ideally models given the same seed and same other parameters will produce the exact same output. diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index 186c665ed1..8a77668329 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -362,6 +362,13 @@ def _set_responses_api_input_data( if conversation_id is not None: set_on_span(SPANDATA.GEN_AI_CONVERSATION_ID, conversation_id) + reasoning = kwargs.get("reasoning") + if isinstance(reasoning, dict) and "effort" in reasoning: + set_on_span( + SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL, + reasoning["effort"], + ) + if not should_send_default_pii() or not integration.include_prompts: set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "responses") return @@ -489,6 +496,10 @@ def _set_completions_api_input_data( if top_p is not None and _is_given(top_p): set_on_span(SPANDATA.GEN_AI_REQUEST_TOP_P, top_p) + reasoning_level = kwargs.get("reasoning_effort") + if reasoning_level is not None and _is_given(reasoning_level): + set_on_span(SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL, reasoning_level) + if ( not should_send_default_pii() or not integration.include_prompts diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index b9572815f8..522a24b8fe 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -3890,6 +3890,7 @@ def test_ai_client_span_responses_api_no_pii( max_output_tokens=100, temperature=0.7, top_p=0.9, + reasoning={"effort": "high"}, ) sentry_sdk.flush() @@ -3901,6 +3902,7 @@ def test_ai_client_span_responses_api_no_pii( "gen_ai.request.max_tokens": 100, "gen_ai.request.temperature": 0.7, "gen_ai.request.top_p": 0.9, + "gen_ai.request.reasoning.level": "high", "gen_ai.request.model": "gpt-4o", "gen_ai.response.model": "response-model-id", "gen_ai.response.streaming": False, @@ -3933,6 +3935,7 @@ def test_ai_client_span_responses_api_no_pii( max_output_tokens=100, temperature=0.7, top_p=0.9, + reasoning={"effort": "high"}, ) spans = [item.payload for item in items] @@ -3943,6 +3946,7 @@ def test_ai_client_span_responses_api_no_pii( "gen_ai.request.max_tokens": 100, "gen_ai.request.temperature": 0.7, "gen_ai.request.top_p": 0.9, + "gen_ai.request.reasoning.level": "high", "gen_ai.request.model": "gpt-4o", "gen_ai.response.model": "response-model-id", "gen_ai.response.streaming": False, @@ -3974,6 +3978,7 @@ def test_ai_client_span_responses_api_no_pii( max_output_tokens=100, temperature=0.7, top_p=0.9, + reasoning={"effort": "high"}, ) (transaction,) = events @@ -3987,6 +3992,7 @@ def test_ai_client_span_responses_api_no_pii( "gen_ai.request.max_tokens": 100, "gen_ai.request.temperature": 0.7, "gen_ai.request.top_p": 0.9, + "gen_ai.request.reasoning.level": "high", "gen_ai.request.model": "gpt-4o", "gen_ai.response.model": "response-model-id", "gen_ai.response.streaming": False, @@ -4186,6 +4192,7 @@ def test_ai_client_span_responses_api( max_output_tokens=100, temperature=0.7, top_p=0.9, + reasoning={"effort": "high"}, ) sentry_sdk.flush() @@ -4198,6 +4205,7 @@ def test_ai_client_span_responses_api( "gen_ai.request.max_tokens": 100, "gen_ai.request.temperature": 0.7, "gen_ai.request.top_p": 0.9, + "gen_ai.request.reasoning.level": "high", "gen_ai.system": "openai", "gen_ai.response.model": "response-model-id", "gen_ai.response.streaming": False, @@ -4233,6 +4241,7 @@ def test_ai_client_span_responses_api( max_output_tokens=100, temperature=0.7, top_p=0.9, + reasoning={"effort": "high"}, ) spans = [item.payload for item in items] @@ -4244,6 +4253,7 @@ def test_ai_client_span_responses_api( "gen_ai.request.max_tokens": 100, "gen_ai.request.temperature": 0.7, "gen_ai.request.top_p": 0.9, + "gen_ai.request.reasoning.level": "high", "gen_ai.system": "openai", "gen_ai.response.model": "response-model-id", "gen_ai.response.streaming": False, @@ -4279,6 +4289,7 @@ def test_ai_client_span_responses_api( max_output_tokens=100, temperature=0.7, top_p=0.9, + reasoning={"effort": "high"}, ) (transaction,) = events @@ -4293,6 +4304,7 @@ def test_ai_client_span_responses_api( "gen_ai.request.max_tokens": 100, "gen_ai.request.temperature": 0.7, "gen_ai.request.top_p": 0.9, + "gen_ai.request.reasoning.level": "high", "gen_ai.system": "openai", "gen_ai.response.model": "response-model-id", "gen_ai.response.streaming": False, @@ -4383,6 +4395,80 @@ def test_responses_api_conversation_id( assert span["data"]["gen_ai.conversation.id"] == expected_id +@pytest.mark.parametrize("span_streaming", [True, False]) +@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) +@pytest.mark.parametrize( + "reasoning, expected_level", + [ + pytest.param(omit, None, id="omit"), + pytest.param(None, None, id="none"), + pytest.param({"summary": "auto"}, None, id="dict_without_effort"), + pytest.param({"effort": "high"}, "high", id="dict"), + ], +) +@pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") +def test_responses_api_reasoning_level( + sentry_init, + capture_events, + capture_items, + reasoning, + expected_level, + stream_gen_ai_spans, + span_streaming, +): + sentry_init( + integrations=[OpenAIIntegration()], + disabled_integrations=[StdlibIntegration], + traces_sample_rate=1.0, + stream_gen_ai_spans=stream_gen_ai_spans, + trace_lifecycle="stream" if span_streaming else "static", + ) + + client = OpenAI(api_key="z") + client.responses._post = mock.Mock(return_value=EXAMPLE_RESPONSE) + + if span_streaming or stream_gen_ai_spans: + items = capture_items("span") + + with start_transaction(name="openai tx"): + client.responses.create( + model="gpt-4o", + input="hello", + reasoning=reasoning, + ) + + sentry_sdk.flush() + span = next(item.payload for item in items if item.type == "span") + + if expected_level is None: + assert SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL not in span["attributes"] + else: + assert ( + span["attributes"][SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL] + == expected_level + ) + + else: + events = capture_events() + + with start_transaction(name="openai tx"): + client.responses.create( + model="gpt-4o", + input="hello", + reasoning=reasoning, + ) + + (transaction,) = events + span = transaction["spans"][0] + + if expected_level is None: + assert SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL not in span["data"] + else: + assert ( + span["data"][SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL] == expected_level + ) + + @pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available") @@ -4660,6 +4746,7 @@ async def test_ai_client_span_responses_async_api( max_output_tokens=100, temperature=0.7, top_p=0.9, + reasoning={"effort": "high"}, ) sentry_sdk.flush() @@ -4672,6 +4759,7 @@ async def test_ai_client_span_responses_async_api( "gen_ai.request.max_tokens": 100, "gen_ai.request.temperature": 0.7, "gen_ai.request.top_p": 0.9, + "gen_ai.request.reasoning.level": "high", "gen_ai.request.messages": safe_serialize(expected_request_messages), "gen_ai.request.model": "gpt-4o", "gen_ai.response.model": "response-model-id", @@ -4707,6 +4795,7 @@ async def test_ai_client_span_responses_async_api( max_output_tokens=100, temperature=0.7, top_p=0.9, + reasoning={"effort": "high"}, ) spans = [item.payload for item in items] @@ -4718,6 +4807,7 @@ async def test_ai_client_span_responses_async_api( "gen_ai.request.max_tokens": 100, "gen_ai.request.temperature": 0.7, "gen_ai.request.top_p": 0.9, + "gen_ai.request.reasoning.level": "high", "gen_ai.request.messages": safe_serialize(expected_request_messages), "gen_ai.request.model": "gpt-4o", "gen_ai.response.model": "response-model-id", @@ -4753,6 +4843,7 @@ async def test_ai_client_span_responses_async_api( max_output_tokens=100, temperature=0.7, top_p=0.9, + reasoning={"effort": "high"}, ) (transaction,) = events @@ -4767,6 +4858,7 @@ async def test_ai_client_span_responses_async_api( "gen_ai.request.max_tokens": 100, "gen_ai.request.temperature": 0.7, "gen_ai.request.top_p": 0.9, + "gen_ai.request.reasoning.level": "high", "gen_ai.request.messages": safe_serialize(expected_request_messages[-1:]), "gen_ai.request.model": "gpt-4o", "gen_ai.response.model": "response-model-id", @@ -4985,6 +5077,7 @@ async def test_ai_client_span_streaming_responses_async_api( max_output_tokens=100, temperature=0.7, top_p=0.9, + reasoning={"effort": "high"}, ) async for _ in result: pass @@ -5005,6 +5098,7 @@ async def test_ai_client_span_streaming_responses_async_api( "gen_ai.request.messages": safe_serialize(expected_request_messages), "gen_ai.request.temperature": 0.7, "gen_ai.request.top_p": 0.9, + "gen_ai.request.reasoning.level": "high", "gen_ai.response.model": "response-model-id", "gen_ai.response.streaming": True, "gen_ai.system": "openai", @@ -5046,6 +5140,7 @@ async def test_ai_client_span_streaming_responses_async_api( max_output_tokens=100, temperature=0.7, top_p=0.9, + reasoning={"effort": "high"}, ) async for _ in result: pass @@ -5064,6 +5159,7 @@ async def test_ai_client_span_streaming_responses_async_api( "gen_ai.request.messages": safe_serialize(expected_request_messages[-1:]), "gen_ai.request.temperature": 0.7, "gen_ai.request.top_p": 0.9, + "gen_ai.request.reasoning.level": "high", "gen_ai.response.model": "response-model-id", "gen_ai.response.streaming": True, "gen_ai.system": "openai", @@ -5310,6 +5406,7 @@ def test_streaming_responses_api( max_output_tokens=100, temperature=0.7, top_p=0.9, + reasoning={"effort": "high"}, ) response_string = "" @@ -5326,6 +5423,7 @@ def test_streaming_responses_api( assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL] == "high" assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "response-model-id" @@ -5354,6 +5452,7 @@ def test_streaming_responses_api( max_output_tokens=100, temperature=0.7, top_p=0.9, + reasoning={"effort": "high"}, ) response_string = "" @@ -5370,6 +5469,7 @@ def test_streaming_responses_api( assert span["data"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 assert span["data"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 assert span["data"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 + assert span["data"][SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL] == "high" assert span["data"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "response-model-id" @@ -5438,6 +5538,7 @@ async def test_streaming_responses_api_async( max_output_tokens=100, temperature=0.7, top_p=0.9, + reasoning={"effort": "high"}, ) response_string = "" @@ -5454,6 +5555,7 @@ async def test_streaming_responses_api_async( assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 + assert span["attributes"][SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL] == "high" assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "response-model-id" @@ -5482,6 +5584,7 @@ async def test_streaming_responses_api_async( max_output_tokens=100, temperature=0.7, top_p=0.9, + reasoning={"effort": "high"}, ) response_string = "" @@ -5498,6 +5601,7 @@ async def test_streaming_responses_api_async( assert span["data"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 100 assert span["data"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 assert span["data"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9 + assert span["data"][SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL] == "high" assert span["data"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "response-model-id" @@ -5585,6 +5689,96 @@ def test_empty_tools_in_chat_completion( assert "gen_ai.request.available_tools" not in span["data"] +@pytest.mark.parametrize("span_streaming", [True, False]) +@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) +# Feature added in https://github.com/openai/openai-python/pull/1952 +@pytest.mark.skipif( + OPENAI_VERSION is None or OPENAI_VERSION < (1, 58, 0), + reason="OpenAI versions <1.58.0 do not support the reasoning_effort parameter.", +) +@pytest.mark.parametrize( + "reasoning_effort,expected_level", + [ + pytest.param(omit, None, id="omit"), + pytest.param(None, None, id="none"), + pytest.param("high", "high", id="high"), + pytest.param("minimal", "minimal", id="minimal"), + ], +) +def test_chat_completion_reasoning_level( + sentry_init, + capture_events, + capture_items, + reasoning_effort, + expected_level, + nonstreaming_chat_completions_model_response, + stream_gen_ai_spans, + span_streaming, +): + sentry_init( + integrations=[OpenAIIntegration()], + disabled_integrations=[StdlibIntegration], + traces_sample_rate=1.0, + stream_gen_ai_spans=stream_gen_ai_spans, + trace_lifecycle="stream" if span_streaming else "static", + ) + + client = OpenAI(api_key="z") + client.chat.completions._post = mock.Mock( + return_value=nonstreaming_chat_completions_model_response( + response_id="chat-id", + response_model="gpt-3.5-turbo", + message_content="the model response", + created=10000000, + usage=CompletionUsage( + prompt_tokens=20, + completion_tokens=10, + total_tokens=30, + ), + ) + ) + + if span_streaming or stream_gen_ai_spans: + items = capture_items("span") + + with start_transaction(name="openai tx"): + client.chat.completions.create( + model="some-model", + messages=[{"role": "system", "content": "hello"}], + reasoning_effort=reasoning_effort, + ) + + sentry_sdk.flush() + span = next(item.payload for item in items if item.type == "span") + + if expected_level is None: + assert SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL not in span["attributes"] + else: + assert ( + span["attributes"][SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL] + == expected_level + ) + else: + events = capture_events() + + with start_transaction(name="openai tx"): + client.chat.completions.create( + model="some-model", + messages=[{"role": "system", "content": "hello"}], + reasoning_effort=reasoning_effort, + ) + + (event,) = events + span = event["spans"][0] + + if expected_level is None: + assert SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL not in span["data"] + else: + assert ( + span["data"][SPANDATA.GEN_AI_REQUEST_REASONING_LEVEL] == expected_level + ) + + # Test messages with mixed roles including "ai" that should be mapped to "assistant" @pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize("stream_gen_ai_spans", [True, False])