diff --git a/.changeset/lemon-regions-shine.md b/.changeset/lemon-regions-shine.md new file mode 100644 index 000000000..ec49649aa --- /dev/null +++ b/.changeset/lemon-regions-shine.md @@ -0,0 +1,5 @@ +--- +"braintrust": patch +--- + +fix: Capture reasoning in mistral diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.log-payloads.json index e02cea9c9..fe7fc96b4 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.log-payloads.json @@ -98,12 +98,101 @@ "has_output": false, "input": null, "metadata": { - "operation": "chat-tool-call" + "operation": "chat-stream-reasoning" }, "metric_keys": [], "output": null, "span_id": "" }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral", + "reasoning_effort": "high" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": null, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-stream-thinking" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "magistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "content_part_types": [ + "thinking", + "text" + ], + "finish_reason": null, + "has_content": true, + "role": "assistant", + "text_block_count": 1, + "thinking_block_count": 1, + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-tool-call" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, { "has_input": true, "has_output": true, @@ -132,7 +221,7 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -162,7 +251,7 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -193,7 +282,7 @@ "tool_call_count": 2, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -204,7 +293,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -228,7 +317,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -239,7 +328,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -263,7 +352,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -274,7 +363,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -304,7 +393,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -315,7 +404,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -345,7 +434,7 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -356,7 +445,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -386,7 +475,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -397,7 +486,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -416,6 +505,6 @@ "embedding_length": "", "type": "embedding" }, - "span_id": "" + "span_id": "" } ] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-events.json index ab1e0df51..837183598 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-events.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-10-0.span-events.json @@ -84,6 +84,79 @@ ], "type": "llm" }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-stream-reasoning" + }, + "metric_keys": [], + "name": "mistral-chat-reasoning-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral", + "reasoning_effort": "high" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-stream-thinking" + }, + "metric_keys": [], + "name": "mistral-chat-thinking-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "magistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, { "has_input": false, "has_output": false, @@ -93,7 +166,7 @@ "metric_keys": [], "name": "mistral-chat-tool-call-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -114,9 +187,9 @@ ], "name": "mistral.chat.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -135,9 +208,9 @@ ], "name": "mistral.chat.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -156,9 +229,9 @@ ], "name": "mistral.chat.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -171,7 +244,7 @@ "metric_keys": [], "name": "mistral-fim-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -192,9 +265,9 @@ ], "name": "mistral.fim.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -207,7 +280,7 @@ "metric_keys": [], "name": "mistral-fim-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -228,9 +301,9 @@ ], "name": "mistral.fim.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -243,7 +316,7 @@ "metric_keys": [], "name": "mistral-agents-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -264,9 +337,9 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -279,7 +352,7 @@ "metric_keys": [], "name": "mistral-agents-tool-call-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -300,9 +373,9 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -315,7 +388,7 @@ "metric_keys": [], "name": "mistral-agents-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -336,9 +409,9 @@ ], "name": "mistral.agents.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -351,7 +424,7 @@ "metric_keys": [], "name": "mistral-embeddings-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -371,9 +444,9 @@ ], "name": "mistral.embeddings.create", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" } diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.log-payloads.json index e02cea9c9..fe7fc96b4 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.log-payloads.json @@ -98,12 +98,101 @@ "has_output": false, "input": null, "metadata": { - "operation": "chat-tool-call" + "operation": "chat-stream-reasoning" }, "metric_keys": [], "output": null, "span_id": "" }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral", + "reasoning_effort": "high" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": null, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-stream-thinking" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "magistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "content_part_types": [ + "thinking", + "text" + ], + "finish_reason": null, + "has_content": true, + "role": "assistant", + "text_block_count": 1, + "thinking_block_count": 1, + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-tool-call" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, { "has_input": true, "has_output": true, @@ -132,7 +221,7 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -162,7 +251,7 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -193,7 +282,7 @@ "tool_call_count": 2, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -204,7 +293,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -228,7 +317,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -239,7 +328,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -263,7 +352,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -274,7 +363,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -304,7 +393,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -315,7 +404,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -345,7 +434,7 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -356,7 +445,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -386,7 +475,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -397,7 +486,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -416,6 +505,6 @@ "embedding_length": "", "type": "embedding" }, - "span_id": "" + "span_id": "" } ] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-events.json index ab1e0df51..837183598 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-events.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-14-1.span-events.json @@ -84,6 +84,79 @@ ], "type": "llm" }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-stream-reasoning" + }, + "metric_keys": [], + "name": "mistral-chat-reasoning-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral", + "reasoning_effort": "high" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-stream-thinking" + }, + "metric_keys": [], + "name": "mistral-chat-thinking-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "magistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, { "has_input": false, "has_output": false, @@ -93,7 +166,7 @@ "metric_keys": [], "name": "mistral-chat-tool-call-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -114,9 +187,9 @@ ], "name": "mistral.chat.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -135,9 +208,9 @@ ], "name": "mistral.chat.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -156,9 +229,9 @@ ], "name": "mistral.chat.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -171,7 +244,7 @@ "metric_keys": [], "name": "mistral-fim-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -192,9 +265,9 @@ ], "name": "mistral.fim.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -207,7 +280,7 @@ "metric_keys": [], "name": "mistral-fim-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -228,9 +301,9 @@ ], "name": "mistral.fim.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -243,7 +316,7 @@ "metric_keys": [], "name": "mistral-agents-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -264,9 +337,9 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -279,7 +352,7 @@ "metric_keys": [], "name": "mistral-agents-tool-call-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -300,9 +373,9 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -315,7 +388,7 @@ "metric_keys": [], "name": "mistral-agents-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -336,9 +409,9 @@ ], "name": "mistral.agents.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -351,7 +424,7 @@ "metric_keys": [], "name": "mistral-embeddings-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -371,9 +444,9 @@ ], "name": "mistral.embeddings.create", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" } diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.log-payloads.json index e02cea9c9..fe7fc96b4 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.log-payloads.json @@ -98,12 +98,101 @@ "has_output": false, "input": null, "metadata": { - "operation": "chat-tool-call" + "operation": "chat-stream-reasoning" }, "metric_keys": [], "output": null, "span_id": "" }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral", + "reasoning_effort": "high" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": null, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-stream-thinking" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "magistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "content_part_types": [ + "thinking", + "text" + ], + "finish_reason": null, + "has_content": true, + "role": "assistant", + "text_block_count": 1, + "thinking_block_count": 1, + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-tool-call" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, { "has_input": true, "has_output": true, @@ -132,7 +221,7 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -162,7 +251,7 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -193,7 +282,7 @@ "tool_call_count": 2, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -204,7 +293,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -228,7 +317,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -239,7 +328,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -263,7 +352,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -274,7 +363,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -304,7 +393,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -315,7 +404,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -345,7 +434,7 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -356,7 +445,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -386,7 +475,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -397,7 +486,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -416,6 +505,6 @@ "embedding_length": "", "type": "embedding" }, - "span_id": "" + "span_id": "" } ] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-events.json index ab1e0df51..837183598 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-events.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-15-1.span-events.json @@ -84,6 +84,79 @@ ], "type": "llm" }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-stream-reasoning" + }, + "metric_keys": [], + "name": "mistral-chat-reasoning-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral", + "reasoning_effort": "high" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-stream-thinking" + }, + "metric_keys": [], + "name": "mistral-chat-thinking-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "magistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, { "has_input": false, "has_output": false, @@ -93,7 +166,7 @@ "metric_keys": [], "name": "mistral-chat-tool-call-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -114,9 +187,9 @@ ], "name": "mistral.chat.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -135,9 +208,9 @@ ], "name": "mistral.chat.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -156,9 +229,9 @@ ], "name": "mistral.chat.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -171,7 +244,7 @@ "metric_keys": [], "name": "mistral-fim-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -192,9 +265,9 @@ ], "name": "mistral.fim.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -207,7 +280,7 @@ "metric_keys": [], "name": "mistral-fim-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -228,9 +301,9 @@ ], "name": "mistral.fim.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -243,7 +316,7 @@ "metric_keys": [], "name": "mistral-agents-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -264,9 +337,9 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -279,7 +352,7 @@ "metric_keys": [], "name": "mistral-agents-tool-call-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -300,9 +373,9 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -315,7 +388,7 @@ "metric_keys": [], "name": "mistral-agents-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -336,9 +409,9 @@ ], "name": "mistral.agents.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -351,7 +424,7 @@ "metric_keys": [], "name": "mistral-embeddings-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -371,9 +444,9 @@ ], "name": "mistral.embeddings.create", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" } diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json index 6fde9fa53..089c640c1 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.log-payloads.json @@ -95,12 +95,51 @@ "has_output": false, "input": null, "metadata": { - "operation": "chat-tool-call" + "operation": "chat-stream-reasoning" }, "metric_keys": [], "output": null, "span_id": "" }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral", + "reasoning_effort": "high" + }, + "metric_keys": [ + "time_to_first_token" + ], + "output": { + "choice_count": 1, + "finish_reason": null, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-tool-call" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, { "has_input": true, "has_output": true, @@ -129,7 +168,7 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -159,7 +198,7 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -190,7 +229,7 @@ "tool_call_count": 2, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -201,7 +240,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -225,7 +264,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -236,7 +275,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -257,7 +296,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -268,7 +307,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -298,7 +337,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -309,7 +348,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -339,7 +378,7 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -350,7 +389,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -380,7 +419,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -391,7 +430,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -410,6 +449,6 @@ "embedding_length": "", "type": "embedding" }, - "span_id": "" + "span_id": "" } ] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-events.json index 09caba8c2..e9b4e1042 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-events.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1-3-4.span-events.json @@ -84,6 +84,43 @@ ], "type": "llm" }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-stream-reasoning" + }, + "metric_keys": [], + "name": "mistral-chat-reasoning-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral", + "reasoning_effort": "high" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, { "has_input": false, "has_output": false, @@ -93,7 +130,7 @@ "metric_keys": [], "name": "mistral-chat-tool-call-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -114,9 +151,9 @@ ], "name": "mistral.chat.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -135,9 +172,9 @@ ], "name": "mistral.chat.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -156,9 +193,9 @@ ], "name": "mistral.chat.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -171,7 +208,7 @@ "metric_keys": [], "name": "mistral-fim-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -192,9 +229,9 @@ ], "name": "mistral.fim.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -207,7 +244,7 @@ "metric_keys": [], "name": "mistral-fim-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -225,9 +262,9 @@ ], "name": "mistral.fim.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -240,7 +277,7 @@ "metric_keys": [], "name": "mistral-agents-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -261,9 +298,9 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -276,7 +313,7 @@ "metric_keys": [], "name": "mistral-agents-tool-call-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -297,9 +334,9 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -312,7 +349,7 @@ "metric_keys": [], "name": "mistral-agents-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -333,9 +370,9 @@ ], "name": "mistral.agents.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -348,7 +385,7 @@ "metric_keys": [], "name": "mistral-embeddings-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -368,9 +405,9 @@ ], "name": "mistral.embeddings.create", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" } diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.log-payloads.json index e02cea9c9..fe7fc96b4 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.log-payloads.json @@ -98,12 +98,101 @@ "has_output": false, "input": null, "metadata": { - "operation": "chat-tool-call" + "operation": "chat-stream-reasoning" }, "metric_keys": [], "output": null, "span_id": "" }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral", + "reasoning_effort": "high" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": null, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-stream-thinking" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "magistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "content_part_types": [ + "thinking", + "text" + ], + "finish_reason": null, + "has_content": true, + "role": "assistant", + "text_block_count": 1, + "thinking_block_count": 1, + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-tool-call" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, { "has_input": true, "has_output": true, @@ -132,7 +221,7 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -162,7 +251,7 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -193,7 +282,7 @@ "tool_call_count": 2, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -204,7 +293,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -228,7 +317,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -239,7 +328,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -263,7 +352,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -274,7 +363,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -304,7 +393,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -315,7 +404,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -345,7 +434,7 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -356,7 +445,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -386,7 +475,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -397,7 +486,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -416,6 +505,6 @@ "embedding_length": "", "type": "embedding" }, - "span_id": "" + "span_id": "" } ] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-events.json index ab1e0df51..837183598 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-events.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v1.span-events.json @@ -84,6 +84,79 @@ ], "type": "llm" }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-stream-reasoning" + }, + "metric_keys": [], + "name": "mistral-chat-reasoning-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral", + "reasoning_effort": "high" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-stream-thinking" + }, + "metric_keys": [], + "name": "mistral-chat-thinking-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "magistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, { "has_input": false, "has_output": false, @@ -93,7 +166,7 @@ "metric_keys": [], "name": "mistral-chat-tool-call-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -114,9 +187,9 @@ ], "name": "mistral.chat.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -135,9 +208,9 @@ ], "name": "mistral.chat.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -156,9 +229,9 @@ ], "name": "mistral.chat.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -171,7 +244,7 @@ "metric_keys": [], "name": "mistral-fim-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -192,9 +265,9 @@ ], "name": "mistral.fim.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -207,7 +280,7 @@ "metric_keys": [], "name": "mistral-fim-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -228,9 +301,9 @@ ], "name": "mistral.fim.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -243,7 +316,7 @@ "metric_keys": [], "name": "mistral-agents-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -264,9 +337,9 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -279,7 +352,7 @@ "metric_keys": [], "name": "mistral-agents-tool-call-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -300,9 +373,9 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -315,7 +388,7 @@ "metric_keys": [], "name": "mistral-agents-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -336,9 +409,9 @@ ], "name": "mistral.agents.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -351,7 +424,7 @@ "metric_keys": [], "name": "mistral-embeddings-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -371,9 +444,9 @@ ], "name": "mistral.embeddings.create", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" } diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.log-payloads.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.log-payloads.json index dec74bdb6..f549ead00 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.log-payloads.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.log-payloads.json @@ -100,12 +100,103 @@ "has_output": false, "input": null, "metadata": { - "operation": "chat-tool-call" + "operation": "chat-stream-reasoning" }, "metric_keys": [], "output": null, "span_id": "" }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral", + "reasoning_effort": "high" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "finish_reason": null, + "has_content": true, + "role": "assistant", + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-stream-thinking" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, + { + "has_input": true, + "has_output": true, + "input": { + "item_count": 1, + "roles": [ + "user" + ], + "type": "messages" + }, + "metadata": { + "model": "magistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "output": { + "choice_count": 1, + "content_part_types": [ + "thinking", + "text" + ], + "finish_reason": null, + "has_content": true, + "role": "assistant", + "text_block_count": 1, + "thinking_block_count": 1, + "tool_call_count": 0, + "type": "choices" + }, + "span_id": "" + }, + { + "has_input": false, + "has_output": false, + "input": null, + "metadata": { + "operation": "chat-tool-call" + }, + "metric_keys": [], + "output": null, + "span_id": "" + }, { "has_input": true, "has_output": true, @@ -135,7 +226,7 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -166,7 +257,7 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -198,7 +289,7 @@ "tool_call_count": 2, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -209,7 +300,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -234,7 +325,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -245,7 +336,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -270,7 +361,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -281,7 +372,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -312,7 +403,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -323,7 +414,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -354,7 +445,7 @@ "tool_call_count": 1, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -365,7 +456,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -396,7 +487,7 @@ "tool_call_count": 0, "type": "choices" }, - "span_id": "" + "span_id": "" }, { "has_input": false, @@ -407,7 +498,7 @@ }, "metric_keys": [], "output": null, - "span_id": "" + "span_id": "" }, { "has_input": true, @@ -426,6 +517,6 @@ "embedding_length": "", "type": "embedding" }, - "span_id": "" + "span_id": "" } ] diff --git a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-events.json b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-events.json index 1f09eecd5..df04617bf 100644 --- a/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-events.json +++ b/e2e/scenarios/mistral-instrumentation/__snapshots__/mistral-v2.span-events.json @@ -86,6 +86,81 @@ ], "type": "llm" }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-stream-reasoning" + }, + "metric_keys": [], + "name": "mistral-chat-reasoning-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "mistral-small-latest", + "provider": "mistral", + "reasoning_effort": "high" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "chat-stream-thinking" + }, + "metric_keys": [], + "name": "mistral-chat-thinking-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "magistral-small-latest", + "provider": "mistral" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "mistral.chat.stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, { "has_input": false, "has_output": false, @@ -95,7 +170,7 @@ "metric_keys": [], "name": "mistral-chat-tool-call-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -117,9 +192,9 @@ ], "name": "mistral.chat.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -139,9 +214,9 @@ ], "name": "mistral.chat.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -161,9 +236,9 @@ ], "name": "mistral.chat.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -176,7 +251,7 @@ "metric_keys": [], "name": "mistral-fim-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -198,9 +273,9 @@ ], "name": "mistral.fim.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -213,7 +288,7 @@ "metric_keys": [], "name": "mistral-fim-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -235,9 +310,9 @@ ], "name": "mistral.fim.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -250,7 +325,7 @@ "metric_keys": [], "name": "mistral-agents-complete-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -272,9 +347,9 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -287,7 +362,7 @@ "metric_keys": [], "name": "mistral-agents-tool-call-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -309,9 +384,9 @@ ], "name": "mistral.agents.complete", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -324,7 +399,7 @@ "metric_keys": [], "name": "mistral-agents-stream-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -346,9 +421,9 @@ ], "name": "mistral.agents.stream", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" }, @@ -361,7 +436,7 @@ "metric_keys": [], "name": "mistral-embeddings-operation", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ "" ], @@ -381,9 +456,9 @@ ], "name": "mistral.embeddings.create", "root_span_id": "", - "span_id": "", + "span_id": "", "span_parents": [ - "" + "" ], "type": "llm" } diff --git a/e2e/scenarios/mistral-instrumentation/assertions.ts b/e2e/scenarios/mistral-instrumentation/assertions.ts index 13f31b1b1..0f8077ec1 100644 --- a/e2e/scenarios/mistral-instrumentation/assertions.ts +++ b/e2e/scenarios/mistral-instrumentation/assertions.ts @@ -17,10 +17,12 @@ import { summarizeWrapperContract, } from "../../helpers/wrapper-contract"; import { + ADJUSTABLE_REASONING_MODEL, FIM_MODEL, CHAT_MODEL, AGENT_MODEL, EMBEDDING_MODEL, + NATIVE_REASONING_MODEL, ROOT_NAME, SCENARIO_NAME, } from "./constants.mjs"; @@ -145,18 +147,43 @@ function summarizeOutput(output: unknown): Json { (Array.isArray(message.tool_calls) && message.tool_calls) || (Array.isArray(message.toolCalls) && message.toolCalls) || []; - - return { + const contentParts = Array.isArray(message.content) ? message.content : []; + const contentPartTypes = contentParts + .map((part) => + isRecord(part) && typeof part.type === "string" ? part.type : null, + ) + .filter((part): part is string => part !== null); + const summary = { choice_count: output.length, finish_reason: finishReason, has_content: typeof message.content === "string" ? message.content.length > 0 - : false, + : contentPartTypes.includes("text") + ? true + : false, role: typeof message.role === "string" ? message.role : null, tool_call_count: toolCalls.length, type: "choices", }; + + if (Array.isArray(message.content)) { + const hasThinkingContent = contentPartTypes.includes("thinking"); + + return { + ...summary, + content_part_types: contentPartTypes, + finish_reason: hasThinkingContent ? null : finishReason, + thinking_block_count: contentPartTypes.filter( + (partType) => partType === "thinking", + ).length, + text_block_count: contentPartTypes.filter( + (partType) => partType === "text", + ).length, + }; + } + + return summary; } if (!isRecord(output as Json)) { @@ -189,7 +216,7 @@ function summarizePayloadRow(row: CapturedLogRow): Json { input: summarizeInput(row.input), metadata: pickMetadata( row.metadata as Record | undefined, - ["model", "operation", "provider", "scenario"], + ["model", "operation", "provider", "reasoning_effort", "scenario"], ), metric_keys: Object.keys(metrics) .filter((key) => key !== "start" && key !== "end") @@ -236,29 +263,52 @@ function normalizeLegacyV134PayloadSummaryRow( "mistral.chat.stream", "mistral.fim.stream", ]); + let normalizedSummaryRow = summaryRow; + if ( - snapshotName !== "mistral-v1-3-4" || - !spanName || - !unstableLegacyV134SpanNames.has(spanName) || - !isRecord(summaryRow) + snapshotName === "mistral-v1-3-4" && + spanName && + unstableLegacyV134SpanNames.has(spanName) && + isRecord(summaryRow) ) { - return summaryRow; + const output = isRecord(summaryRow.output) ? summaryRow.output : null; + + normalizedSummaryRow = { + ...summaryRow, + metric_keys: normalizeLegacyV134MetricKeys(summaryRow.metric_keys), + ...(output + ? { + output: { + ...output, + finish_reason: null, + }, + } + : {}), + }; + } + + if (!isRecord(normalizedSummaryRow)) { + return normalizedSummaryRow; } - const output = isRecord(summaryRow.output) ? summaryRow.output : null; + const output = isRecord(normalizedSummaryRow.output) + ? normalizedSummaryRow.output + : null; + const metadata = isRecord(normalizedSummaryRow.metadata) + ? normalizedSummaryRow.metadata + : null; - return { - ...summaryRow, - metric_keys: normalizeLegacyV134MetricKeys(summaryRow.metric_keys), - ...(output - ? { - output: { - ...output, - finish_reason: null, - }, - } - : {}), - }; + if (output && metadata?.reasoning_effort === "high") { + return { + ...normalizedSummaryRow, + output: { + ...output, + finish_reason: null, + }, + }; + } + + return normalizedSummaryRow; } function mergeRecordValues( @@ -341,6 +391,14 @@ function buildSpanSummary( events, "mistral-chat-stream-operation", ); + const chatReasoningStreamOperation = findLatestSpan( + events, + "mistral-chat-reasoning-stream-operation", + ); + const chatThinkingStreamOperation = findLatestSpan( + events, + "mistral-chat-thinking-stream-operation", + ); const chatToolCallOperation = findLatestSpan( events, "mistral-chat-tool-call-operation", @@ -393,6 +451,14 @@ function buildSpanSummary( findMistralSpan(events, chatStreamOperation?.span.id, [ "mistral.chat.stream", ]), + chatReasoningStreamOperation, + findMistralSpan(events, chatReasoningStreamOperation?.span.id, [ + "mistral.chat.stream", + ]), + chatThinkingStreamOperation, + findMistralSpan(events, chatThinkingStreamOperation?.span.id, [ + "mistral.chat.stream", + ]), chatToolCallOperation, ...selectedChatToolCallSpans, fimCompleteOperation, @@ -417,17 +483,20 @@ function buildSpanSummary( findMistralSpan(events, embeddingsOperation?.span.id, [ "mistral.embeddings.create", ]), - ].map((event) => - normalizeLegacyV134SpanSummaryRow( - summarizeWrapperContract(event!, [ - "model", - "operation", - "provider", - "scenario", - ]), - snapshotName, - ), - ) as Json, + ] + .filter((event): event is CapturedLogEvent => event !== undefined) + .map((event) => + normalizeLegacyV134SpanSummaryRow( + summarizeWrapperContract(event, [ + "model", + "operation", + "provider", + "reasoning_effort", + "scenario", + ]), + snapshotName, + ), + ) as Json, ); } @@ -521,6 +590,7 @@ export function defineMistralInstrumentationAssertions(options: { name: string; runScenario: RunMistralScenario; snapshotName: string; + supportsThinkingStream?: boolean; testFileUrl: string; timeoutMs: number; }): void { @@ -532,6 +602,7 @@ export function defineMistralInstrumentationAssertions(options: { options.testFileUrl, `${options.snapshotName}.log-payloads.json`, ); + const supportsThinkingStream = options.supportsThinkingStream ?? true; const testConfig = { timeout: options.timeoutMs, }; @@ -597,6 +668,93 @@ export function defineMistralInstrumentationAssertions(options: { expect(span?.output).toBeDefined(); }); + test( + "captures trace for chat.stream() reasoning metadata", + testConfig, + () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan( + events, + "mistral-chat-reasoning-stream-operation", + ); + const span = findMistralSpan(events, operation?.span.id, [ + "mistral.chat.stream", + ]); + const metadata = span?.row.metadata as + | Record + | undefined; + + expect(operation).toBeDefined(); + expect(span).toBeDefined(); + expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(span?.span.type).toBe("llm"); + expect(metadata).toMatchObject({ + model: ADJUSTABLE_REASONING_MODEL, + provider: "mistral", + reasoning_effort: "high", + }); + expect(span?.metrics).toMatchObject({ + time_to_first_token: expect.any(Number), + prompt_tokens: expect.any(Number), + completion_tokens: expect.any(Number), + }); + expect(span?.output).toBeDefined(); + }, + ); + + if (supportsThinkingStream) { + test( + "captures trace for chat.stream() thinking content", + testConfig, + () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan( + events, + "mistral-chat-thinking-stream-operation", + ); + const span = findMistralSpan(events, operation?.span.id, [ + "mistral.chat.stream", + ]); + const metadata = span?.row.metadata as + | Record + | undefined; + const output = span?.output as + | Array<{ + message?: { + content?: + | Array<{ + thinking?: Array<{ text?: string; type?: string }>; + text?: string; + type?: string; + }> + | string + | null; + }; + }> + | undefined; + const content = Array.isArray(output?.[0]?.message?.content) + ? output[0].message.content + : []; + + expect(operation).toBeDefined(); + expect(span).toBeDefined(); + expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(span?.span.type).toBe("llm"); + expect(metadata).toMatchObject({ + model: NATIVE_REASONING_MODEL, + provider: "mistral", + }); + expect(span?.metrics).toMatchObject({ + time_to_first_token: expect.any(Number), + prompt_tokens: expect.any(Number), + completion_tokens: expect.any(Number), + }); + expect(content.some((part) => part.type === "thinking")).toBe(true); + expect(content.some((part) => part.type === "text")).toBe(true); + }, + ); + } + test("captures trace for chat.complete() tool calling", testConfig, () => { const root = findLatestSpan(events, ROOT_NAME); const operation = findLatestSpan( diff --git a/e2e/scenarios/mistral-instrumentation/constants.mjs b/e2e/scenarios/mistral-instrumentation/constants.mjs index c3266b467..e57644f74 100644 --- a/e2e/scenarios/mistral-instrumentation/constants.mjs +++ b/e2e/scenarios/mistral-instrumentation/constants.mjs @@ -1,4 +1,6 @@ const CHAT_MODEL = "mistral-small-2506"; +const ADJUSTABLE_REASONING_MODEL = "mistral-small-latest"; +const NATIVE_REASONING_MODEL = "magistral-small-latest"; const EMBEDDING_MODEL = "mistral-embed"; const FIM_MODEL = "codestral-2508"; const AGENT_MODEL = CHAT_MODEL; @@ -6,10 +8,12 @@ const ROOT_NAME = "mistral-root"; const SCENARIO_NAME = "mistral-instrumentation"; export { + ADJUSTABLE_REASONING_MODEL, AGENT_MODEL, CHAT_MODEL, EMBEDDING_MODEL, FIM_MODEL, + NATIVE_REASONING_MODEL, ROOT_NAME, SCENARIO_NAME, }; diff --git a/e2e/scenarios/mistral-instrumentation/scenario.impl.mjs b/e2e/scenarios/mistral-instrumentation/scenario.impl.mjs index 47d0fe2e4..d8b73adf8 100644 --- a/e2e/scenarios/mistral-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/mistral-instrumentation/scenario.impl.mjs @@ -5,10 +5,12 @@ import { runTracedScenario, } from "../../helpers/provider-runtime.mjs"; import { + ADJUSTABLE_REASONING_MODEL, AGENT_MODEL, CHAT_MODEL, EMBEDDING_MODEL, FIM_MODEL, + NATIVE_REASONING_MODEL, ROOT_NAME, SCENARIO_NAME, } from "./constants.mjs"; @@ -20,43 +22,55 @@ const MISTRAL_REQUEST_RETRY_OPTIONS = { delayMs: 2_000, maxDelayMs: 10_000, }; + +const MISTRAL_THINKING_STREAM_OPTOUTS = new Set(["mistral-sdk-v1-3-4"]); + +function createMistralScenarioSpec(spec) { + return { + ...spec, + ...(MISTRAL_THINKING_STREAM_OPTOUTS.has(spec.dependencyName) + ? { supportsThinkingStream: false } + : {}), + }; +} + export const MISTRAL_SCENARIO_SPECS = [ - { + createMistralScenarioSpec({ autoEntry: "scenario.mistral-v1-3-4.mjs", dependencyName: "mistral-sdk-v1-3-4", snapshotName: "mistral-v1-3-4", wrapperEntry: "scenario.mistral-v1-3-4.ts", - }, - { + }), + createMistralScenarioSpec({ autoEntry: "scenario.mistral-v1-10-0.mjs", dependencyName: "mistral-sdk-v1-10-0", snapshotName: "mistral-v1-10-0", wrapperEntry: "scenario.mistral-v1-10-0.ts", - }, - { + }), + createMistralScenarioSpec({ autoEntry: "scenario.mistral-v1-14-1.mjs", dependencyName: "mistral-sdk-v1-14-1", snapshotName: "mistral-v1-14-1", wrapperEntry: "scenario.mistral-v1-14-1.ts", - }, - { + }), + createMistralScenarioSpec({ autoEntry: "scenario.mistral-v1-15-1.mjs", dependencyName: "mistral-sdk-v1-15-1", snapshotName: "mistral-v1-15-1", wrapperEntry: "scenario.mistral-v1-15-1.ts", - }, - { + }), + createMistralScenarioSpec({ autoEntry: "scenario.mistral-v1.mjs", dependencyName: "mistral-sdk-v1", snapshotName: "mistral-v1", wrapperEntry: "scenario.mistral-v1.ts", - }, - { + }), + createMistralScenarioSpec({ autoEntry: "scenario.mjs", dependencyName: "mistral-sdk-v2", snapshotName: "mistral-v2", wrapperEntry: "scenario.ts", - }, + }), ]; function nonEmptyString(value) { @@ -340,7 +354,7 @@ async function resolveAgentRuntime(client) { async function runMistralInstrumentationScenario( Mistral, - { decorateClient } = {}, + { decorateClient, supportsThinkingStream = true } = {}, ) { const baseClient = new Mistral({ apiKey: process.env.MISTRAL_API_KEY, @@ -400,6 +414,54 @@ async function runMistralInstrumentationScenario( }, ); + await runOperation( + "mistral-chat-reasoning-stream-operation", + "chat-stream-reasoning", + async () => { + await withRetry(async () => { + const stream = await client.chat.stream({ + model: ADJUSTABLE_REASONING_MODEL, + messages: [ + { + role: "user", + content: + "John is one of 4 children. The first sister is 4 years old. Next year, the second sister will be twice as old as the first sister. The third sister is two years older than the second sister. The third sister is half the age of her older brother. How old is John? Reply with just the number.", + }, + ], + maxTokens: 256, + reasoning_effort: "high", + stream: true, + temperature: 0, + }); + await collectAsync(stream); + }, MISTRAL_REQUEST_RETRY_OPTIONS); + }, + ); + + if (supportsThinkingStream) { + await runOperation( + "mistral-chat-thinking-stream-operation", + "chat-stream-thinking", + async () => { + await withRetry(async () => { + const stream = await client.chat.stream({ + model: NATIVE_REASONING_MODEL, + messages: [ + { + role: "user", + content: "What is 2+2? Reply with just the number.", + }, + ], + maxTokens: 1024, + stream: true, + temperature: 0, + }); + await collectAsync(stream); + }, MISTRAL_REQUEST_RETRY_OPTIONS); + }, + ); + } + await runOperation( "mistral-chat-tool-call-operation", "chat-tool-call", @@ -670,12 +732,13 @@ async function runMistralInstrumentationScenario( } } -export async function runWrappedMistralInstrumentation(Mistral) { +export async function runWrappedMistralInstrumentation(Mistral, options) { await runMistralInstrumentationScenario(Mistral, { + ...options, decorateClient: wrapMistral, }); } -export async function runAutoMistralInstrumentation(Mistral) { - await runMistralInstrumentationScenario(Mistral); +export async function runAutoMistralInstrumentation(Mistral, options) { + await runMistralInstrumentationScenario(Mistral, options); } diff --git a/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-3-4.mjs b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-3-4.mjs index 4f772e22d..3ecf3d47f 100644 --- a/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-3-4.mjs +++ b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-3-4.mjs @@ -2,4 +2,8 @@ import { Mistral } from "mistral-sdk-v1-3-4"; import { runMain } from "../../helpers/provider-runtime.mjs"; import { runAutoMistralInstrumentation } from "./scenario.impl.mjs"; -runMain(async () => runAutoMistralInstrumentation(Mistral)); +runMain(async () => + runAutoMistralInstrumentation(Mistral, { + supportsThinkingStream: false, + }), +); diff --git a/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-3-4.ts b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-3-4.ts index 1db41c17f..349d45244 100644 --- a/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-3-4.ts +++ b/e2e/scenarios/mistral-instrumentation/scenario.mistral-v1-3-4.ts @@ -2,4 +2,8 @@ import { Mistral } from "mistral-sdk-v1-3-4"; import { runMain } from "../../helpers/scenario-runtime"; import { runWrappedMistralInstrumentation } from "./scenario.impl.mjs"; -runMain(async () => runWrappedMistralInstrumentation(Mistral)); +runMain(async () => + runWrappedMistralInstrumentation(Mistral, { + supportsThinkingStream: false, + }), +); diff --git a/e2e/scenarios/mistral-instrumentation/scenario.test.ts b/e2e/scenarios/mistral-instrumentation/scenario.test.ts index 77b1da4f6..f12eed393 100644 --- a/e2e/scenarios/mistral-instrumentation/scenario.test.ts +++ b/e2e/scenarios/mistral-instrumentation/scenario.test.ts @@ -37,6 +37,9 @@ for (const scenario of mistralScenarios) { }); }, snapshotName: scenario.snapshotName, + ...(scenario.supportsThinkingStream === false + ? { supportsThinkingStream: false } + : {}), testFileUrl: import.meta.url, timeoutMs: MISTRAL_SCENARIO_TIMEOUT_MS, }); @@ -53,6 +56,9 @@ for (const scenario of mistralScenarios) { }); }, snapshotName: scenario.snapshotName, + ...(scenario.supportsThinkingStream === false + ? { supportsThinkingStream: false } + : {}), testFileUrl: import.meta.url, timeoutMs: MISTRAL_SCENARIO_TIMEOUT_MS, }); diff --git a/js/src/instrumentation/plugins/mistral-plugin.test.ts b/js/src/instrumentation/plugins/mistral-plugin.test.ts index e73d63076..96d410440 100644 --- a/js/src/instrumentation/plugins/mistral-plugin.test.ts +++ b/js/src/instrumentation/plugins/mistral-plugin.test.ts @@ -12,6 +12,7 @@ describe("extractMistralRequestMetadata", () => { extractMistralRequestMetadata({ model: "mistral-large-latest", maxTokens: 128, + reasoning_effort: "high", temperature: 0.4, n: 2, safe_prompt: true, @@ -24,6 +25,7 @@ describe("extractMistralRequestMetadata", () => { ).toEqual({ model: "mistral-large-latest", maxTokens: 128, + reasoning_effort: "high", temperature: 0.4, n: 2, safe_prompt: true, @@ -367,4 +369,72 @@ describe("aggregateMistralStreamChunks", () => { }, ]); }); + + it("preserves thinking content blocks from reasoning streams", () => { + const aggregated = aggregateMistralStreamChunks([ + { + data: { + choices: [ + { + delta: { + role: "assistant", + content: [ + { + type: "thinking", + thinking: [{ type: "text", text: "Let me" }], + }, + ], + }, + }, + ], + }, + }, + { + data: { + choices: [ + { + delta: { + content: [ + { + type: "thinking", + thinking: [{ type: "text", text: " think this through." }], + }, + { + type: "text", + text: "4", + }, + ], + }, + finishReason: "stop", + }, + ], + }, + }, + ]); + + expect(aggregated.output).toEqual([ + { + index: 0, + message: { + role: "assistant", + content: [ + { + type: "thinking", + thinking: [ + { + type: "text", + text: "Let me think this through.", + }, + ], + }, + { + type: "text", + text: "4", + }, + ], + }, + finishReason: "stop", + }, + ]); + }); }); diff --git a/js/src/instrumentation/plugins/mistral-plugin.ts b/js/src/instrumentation/plugins/mistral-plugin.ts index 88ae25f46..da9f122b3 100644 --- a/js/src/instrumentation/plugins/mistral-plugin.ts +++ b/js/src/instrumentation/plugins/mistral-plugin.ts @@ -13,6 +13,9 @@ import type { MistralChatCompletionChunkChoice, MistralChatCompletionEvent, MistralChatCompletionResponse, + MistralContentPart, + MistralTextContentPart, + MistralThinkingContentPart, MistralToolCallDelta, } from "../../vendor-sdk-types/mistral"; @@ -166,6 +169,8 @@ const MISTRAL_REQUEST_METADATA_ALLOWLIST = new Set([ "presence_penalty", "randomSeed", "random_seed", + "reasoningEffort", + "reasoning_effort", "responseFormat", "response_format", "safePrompt", @@ -377,6 +382,165 @@ function extractDeltaText(content: unknown): string | undefined { return textParts.length > 0 ? textParts.join("") : undefined; } +function normalizeMistralTextContentPart( + part: unknown, +): MistralTextContentPart | undefined { + if ( + !isObject(part) || + part.type !== "text" || + typeof part.text !== "string" + ) { + return undefined; + } + + return { + type: "text", + text: part.text, + }; +} + +function normalizeMistralThinkingContentPart( + part: unknown, +): MistralThinkingContentPart | undefined { + if (!isObject(part) || part.type !== "thinking") { + return undefined; + } + + const thinking = Array.isArray(part.thinking) + ? part.thinking + .map((thinkingPart) => normalizeMistralTextContentPart(thinkingPart)) + .filter( + (thinkingPart): thinkingPart is MistralTextContentPart => + thinkingPart !== undefined && typeof thinkingPart.text === "string", + ) + : []; + + return { + type: "thinking", + thinking, + }; +} + +function normalizeMistralContentParts(content: unknown): MistralContentPart[] { + if (!Array.isArray(content)) { + return []; + } + + return content + .map((part) => { + return ( + normalizeMistralTextContentPart(part) || + normalizeMistralThinkingContentPart(part) + ); + }) + .filter((part): part is MistralContentPart => part !== undefined); +} + +function mergeMistralTextSegments( + left: MistralTextContentPart[], + right: MistralTextContentPart[], +): MistralTextContentPart[] { + const merged = left.map((part) => ({ ...part })); + + for (const part of right) { + const lastPart = merged[merged.length - 1]; + if ( + lastPart && + lastPart.type === "text" && + typeof lastPart.text === "string" && + typeof part.text === "string" + ) { + lastPart.text += part.text; + continue; + } + + merged.push({ ...part }); + } + + return merged; +} + +function mergeMistralContentParts( + left: MistralContentPart[] | undefined, + right: MistralContentPart[], +): MistralContentPart[] { + const merged = [...(left || []).map((part) => structuredClone(part))]; + + for (const part of right) { + const lastPart = merged[merged.length - 1]; + if ( + part.type === "text" && + lastPart?.type === "text" && + typeof lastPart.text === "string" && + typeof part.text === "string" + ) { + lastPart.text += part.text; + continue; + } + + if ( + part.type === "thinking" && + lastPart?.type === "thinking" && + Array.isArray(lastPart.thinking) && + Array.isArray(part.thinking) + ) { + lastPart.thinking = mergeMistralTextSegments( + lastPart.thinking, + part.thinking, + ); + continue; + } + + merged.push(structuredClone(part)); + } + + return merged; +} + +function appendMistralContent( + accumulator: MistralChoiceAccumulator, + content: unknown, +): void { + if (typeof content === "string") { + if (accumulator.contentParts) { + accumulator.contentParts = mergeMistralContentParts( + accumulator.contentParts, + [{ type: "text", text: content }], + ); + return; + } + + accumulator.content = `${accumulator.content || ""}${content}`; + return; + } + + const normalizedContentParts = normalizeMistralContentParts(content); + if (normalizedContentParts.length === 0) { + return; + } + + const hasStructuredContent = normalizedContentParts.some( + (part) => part.type !== "text", + ); + + if (!accumulator.contentParts && !hasStructuredContent) { + const text = extractDeltaText(content); + if (text) { + accumulator.content = `${accumulator.content || ""}${text}`; + } + return; + } + + accumulator.contentParts = mergeMistralContentParts( + accumulator.contentParts || + (accumulator.content + ? [{ type: "text", text: accumulator.content }] + : []), + normalizedContentParts, + ); + delete accumulator.content; +} + function getDeltaToolCalls( delta: Record, ): MistralToolCallDelta[] { @@ -522,6 +686,7 @@ function getChoiceFinishReason( type MistralChoiceAccumulator = { content?: string; + contentParts?: MistralContentPart[]; finishReason?: string | null; index: number; order: number; @@ -642,10 +807,7 @@ export function aggregateMistralStreamChunks( accumulator.role = delta.role; } - const deltaText = extractDeltaText(delta.content); - if (deltaText) { - accumulator.content = `${accumulator.content || ""}${deltaText}`; - } + appendMistralContent(accumulator, delta.content); accumulator.toolCalls = mergeToolCallDeltas( accumulator.toolCalls, @@ -670,7 +832,7 @@ export function aggregateMistralStreamChunks( index: choice.index, message: { ...(choice.role ? { role: choice.role } : {}), - content: choice.content ?? null, + content: choice.contentParts ?? choice.content ?? null, ...(choice.toolCalls ? { toolCalls: choice.toolCalls } : {}), }, ...(choice.finishReason !== undefined diff --git a/js/src/vendor-sdk-types/mistral.ts b/js/src/vendor-sdk-types/mistral.ts index 90fd390fd..1a7edc8d8 100644 --- a/js/src/vendor-sdk-types/mistral.ts +++ b/js/src/vendor-sdk-types/mistral.ts @@ -8,9 +8,25 @@ export type MistralToolCallDelta = { [key: string]: unknown; }; +export type MistralTextContentPart = { + type: "text"; + text?: string; + [key: string]: unknown; +}; + +export type MistralThinkingContentPart = { + type: "thinking"; + thinking?: MistralTextContentPart[] | null; + [key: string]: unknown; +}; + +export type MistralContentPart = + | MistralTextContentPart + | MistralThinkingContentPart; + export type MistralChatMessageDelta = { role?: string; - content?: string | null; + content?: string | MistralContentPart[] | null; toolCalls?: MistralToolCallDelta[] | null; tool_calls?: MistralToolCallDelta[] | null; [key: string]: unknown; @@ -20,7 +36,7 @@ export type MistralChatCompletionChoice = { index?: number; message?: { role?: string; - content?: string | null; + content?: string | MistralContentPart[] | null; toolCalls?: unknown; tool_calls?: unknown; };