Skip to content

Commit ef23ed5

Browse files
isaacsclaude
andauthored
feat(server-utils): Record AI SDK v7 runtime and tool context on Vercel AI spans (#25774)
On `ai` 7 the `telemetry.metadata` option is gone. Per-call metadata goes in `runtimeContext`, and the app opts keys into telemetry with `telemetry.includeRuntimeContext` (and `includeToolsContext` for tool context). The channel subscriber did not read either field, so there was no way to put request-level data on gen_ai spans. The subscriber now records each included `runtimeContext` key as `vercel.ai.settings.context.<key>` on the operation span and on its `generate_content` and `execute_tool` child spans. It records each included `toolContext` key as `vercel.ai.toolCall.context.<key>` on the `execute_tool` span. The `runtimeContext` names match `@ai-sdk/otel` (`ai.settings.context.*`) under the usual `ai.*` to `vercel.ai.*` renaming. `ai` before 7.0.114 published the full context to the channel and ignored the include lists. A filtered context looks the same as an unfiltered one, so the subscriber only reads context from events that carry `aiVersion`, which `ai` added in 7.0.154. Older 7.x versions record no context, so keys the app did not opt in never leave it. Fixes #24706 Fixes JS-3788 Refs getsentry/sentry-conventions#756 Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
1 parent c16633e commit ef23ed5

4 files changed

Lines changed: 366 additions & 2 deletions

File tree

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
import * as Sentry from '@sentry/node';
2+
import { generateText, stepCountIs, streamText, tool } from 'ai';
3+
import { MockLanguageModelV3, simulateReadableStream } from 'ai/test';
4+
import { z } from 'zod';
5+
6+
const usage = {
7+
inputTokens: { total: 10, noCache: 10, cached: 0 },
8+
outputTokens: { total: 20, noCache: 20, cached: 0 },
9+
totalTokens: { total: 30, noCache: 30, cached: 0 },
10+
};
11+
12+
const tools = {
13+
getWeather: tool({
14+
description: 'Get the current weather for a location',
15+
inputSchema: z.object({ location: z.string() }),
16+
contextSchema: z.object({ units: z.string(), apiKey: z.string() }),
17+
execute: async ({ location }) => `Weather in ${location}: Sunny`,
18+
}),
19+
};
20+
21+
async function run() {
22+
await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
23+
let callCount = 0;
24+
await generateText({
25+
model: new MockLanguageModelV3({
26+
doGenerate: async () =>
27+
callCount++ === 0
28+
? {
29+
finishReason: { unified: 'tool-calls', raw: 'tool_calls' },
30+
usage,
31+
content: [
32+
{
33+
type: 'tool-call',
34+
toolCallId: 'call-1',
35+
toolName: 'getWeather',
36+
input: JSON.stringify({ location: 'Vienna' }),
37+
},
38+
],
39+
warnings: [],
40+
}
41+
: {
42+
finishReason: { unified: 'stop', raw: 'stop' },
43+
usage,
44+
content: [{ type: 'text', text: 'Sunny' }],
45+
warnings: [],
46+
},
47+
}),
48+
tools,
49+
stopWhen: stepCountIs(3),
50+
prompt: 'Weather in Vienna?',
51+
runtimeContext: { requestId: 'req_generate', tenantId: 'acme', apiToken: 'secret-runtime' },
52+
toolsContext: { getWeather: { units: 'metric', apiKey: 'secret-tool' } },
53+
telemetry: {
54+
isEnabled: true,
55+
functionId: 'support-chat',
56+
includeRuntimeContext: { requestId: true, tenantId: true },
57+
includeToolsContext: { getWeather: { units: true } },
58+
},
59+
});
60+
61+
const result = streamText({
62+
model: new MockLanguageModelV3({
63+
doStream: async () => ({
64+
stream: simulateReadableStream({
65+
chunks: [
66+
{ type: 'stream-start', warnings: [] },
67+
{ type: 'text-start', id: '0' },
68+
{ type: 'text-delta', id: '0', delta: 'Hello!' },
69+
{ type: 'text-end', id: '0' },
70+
{ type: 'finish', finishReason: { unified: 'stop', raw: 'stop' }, usage },
71+
],
72+
}),
73+
}),
74+
}),
75+
prompt: 'Hi',
76+
runtimeContext: { requestId: 'req_stream', apiToken: 'secret-runtime' },
77+
telemetry: {
78+
isEnabled: true,
79+
functionId: 'support-stream',
80+
includeRuntimeContext: { requestId: true },
81+
},
82+
});
83+
for await (const _part of result.fullStream) {
84+
void _part;
85+
}
86+
});
87+
88+
await Sentry.flush(2000);
89+
}
90+
91+
run();

‎dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts‎

Lines changed: 65 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,12 @@ import {
2626
SENTRY_OP,
2727
SENTRY_ORIGIN,
2828
} from '@sentry/conventions/attributes';
29-
import { GEN_AI_DECIDE } from '@sentry/conventions/op';
29+
import {
30+
GEN_AI_DECIDE,
31+
GEN_AI_EXECUTE_TOOL,
32+
GEN_AI_GENERATE_CONTENT,
33+
GEN_AI_INVOKE_AGENT,
34+
} from '@sentry/conventions/op';
3035
import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../../../packages/server-utils/src/ai/core/gen-ai-attributes';
3136
import { cleanupChildProcesses, createEsmAndCjsTests, createEsmTests } from '../../../../utils/runner';
3237

@@ -1429,3 +1434,62 @@ describe('Vercel AI telemetry metadata (v6)', () => {
14291434
{ additionalDependencies: { ai: '^6.0.0' } },
14301435
);
14311436
});
1437+
1438+
describe('Vercel AI runtime and tool context (v7)', () => {
1439+
afterAll(() => {
1440+
cleanupChildProcesses();
1441+
});
1442+
1443+
createEsmTests(
1444+
__dirname,
1445+
'scenario-runtime-context.mjs',
1446+
'instrument.mjs',
1447+
(createRunner, test) => {
1448+
test('records the context keys included in telemetry as span attributes', async () => {
1449+
await createRunner()
1450+
.expect({
1451+
span: container => {
1452+
const spans = container.items.filter(
1453+
span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.vercelai',
1454+
);
1455+
// Keys left out of `includeRuntimeContext`/`includeToolsContext` must not reach any span.
1456+
expect(JSON.stringify(spans)).not.toMatch(/secret-runtime|secret-tool|apiToken|apiKey/);
1457+
1458+
const generateSpans = spans.filter(
1459+
span => span.attributes['vercel.ai.settings.context.requestId']?.value === 'req_generate',
1460+
);
1461+
expect(generateSpans.map(span => span.attributes[SENTRY_OP]?.value).sort()).toEqual([
1462+
GEN_AI_EXECUTE_TOOL,
1463+
GEN_AI_GENERATE_CONTENT,
1464+
GEN_AI_GENERATE_CONTENT,
1465+
GEN_AI_INVOKE_AGENT,
1466+
]);
1467+
for (const span of generateSpans) {
1468+
expect(span.attributes['vercel.ai.settings.context.tenantId']).toEqual({
1469+
type: 'string',
1470+
value: 'acme',
1471+
});
1472+
}
1473+
1474+
const toolSpan = generateSpans.find(span => span.attributes[SENTRY_OP]?.value === GEN_AI_EXECUTE_TOOL);
1475+
expect(toolSpan?.attributes['vercel.ai.tool_call.context.units']).toEqual({
1476+
type: 'string',
1477+
value: 'metric',
1478+
});
1479+
1480+
const streamSpans = spans.filter(
1481+
span => span.attributes['vercel.ai.settings.context.requestId']?.value === 'req_stream',
1482+
);
1483+
expect(streamSpans.map(span => span.attributes[SENTRY_OP]?.value).sort()).toEqual([
1484+
GEN_AI_GENERATE_CONTENT,
1485+
GEN_AI_INVOKE_AGENT,
1486+
]);
1487+
},
1488+
})
1489+
.start()
1490+
.completed();
1491+
});
1492+
},
1493+
{ additionalDependencies: { ai: '^7.0.154' } },
1494+
);
1495+
});

‎packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,8 @@ const VERCEL_AI_OPERATION_ID_ATTRIBUTE = 'vercel.ai.operationId';
8686
const VERCEL_AI_MODEL_PROVIDER_ATTRIBUTE = 'vercel.ai.model.provider';
8787
const VERCEL_AI_SETTINGS_MAX_RETRIES_ATTRIBUTE = 'vercel.ai.settings.maxRetries';
8888
const VERCEL_AI_TELEMETRY_METADATA_ATTRIBUTE_PREFIX = 'vercel.ai.telemetry.metadata.';
89+
const VERCEL_AI_SETTINGS_CONTEXT_ATTRIBUTE_PREFIX = 'vercel.ai.settings.context.';
90+
const VERCEL_AI_TOOL_CALL_CONTEXT_ATTRIBUTE_PREFIX = 'vercel.ai.tool_call.context.';
8991

9092
// Tracks the top-level operationId (and whether it streams) per `callId` so a model-call span can
9193
// name its `doGenerate`/`doStream` operation the same way the OTel integration does. `isStream` is
@@ -108,6 +110,10 @@ const toolDescriptionsByCallId = new Map<string, Map<string, string>>();
108110
// before the operation's own span ends. The running token sum lives on the span's own attributes.
109111
const invokeAgentSpanByCallId = new Map<string, Span>();
110112

113+
// Only an operation's start event carries `runtimeContext`; its model-call and tool events copy the
114+
// resulting attributes from here. Cleared with the operation.
115+
const runtimeContextAttributesByCallId = new Map<string, SpanAttributes>();
116+
111117
// Only top-level operations own the `callId` → operationId mapping; `step`/`languageModelCall`/
112118
// `executeTool` share the parent's `callId`, so they must not clear it.
113119
const ROOT_OPERATION_TYPES = new Set<ChannelEventType>([
@@ -140,6 +146,7 @@ export function clearOperationCallId(callId: string): void {
140146
operationIdByCallId.delete(callId);
141147
toolDescriptionsByCallId.delete(callId);
142148
invokeAgentSpanByCallId.delete(callId);
149+
runtimeContextAttributesByCallId.delete(callId);
143150
}
144151

145152
/**
@@ -450,9 +457,17 @@ export function createSpanFromMessage(
450457
getOpenAiConversationId(event.providerOptions) ??
451458
(ROOT_OPERATION_TYPES.has(type) ? undefined : getActiveSpanConversationId());
452459

460+
const runtimeContextAttributes =
461+
(callId ? runtimeContextAttributesByCallId.get(callId) : undefined) ??
462+
contextAttributes(event, event.runtimeContext, VERCEL_AI_SETTINGS_CONTEXT_ATTRIBUTE_PREFIX);
463+
if (callId && ROOT_OPERATION_TYPES.has(type)) {
464+
runtimeContextAttributesByCallId.set(callId, runtimeContextAttributes);
465+
}
466+
453467
const baseAttributes: SpanAttributes = {
454468
[SENTRY_ORIGIN]: ORIGIN,
455469
...telemetryMetadataAttributes(event.telemetryMetadata),
470+
...runtimeContextAttributes,
456471
[GEN_AI_PROVIDER_NAME]: provider,
457472
[VERCEL_AI_MODEL_PROVIDER_ATTRIBUTE]: provider,
458473
[GEN_AI_REQUEST_MODEL]: modelId,
@@ -473,7 +488,7 @@ export function createSpanFromMessage(
473488

474489
return buildModelCallSpan(event, baseAttributes, recordInputs, callId, modelId);
475490
case 'executeTool':
476-
return buildToolSpan(event, recordInputs, conversationId);
491+
return buildToolSpan(event, recordInputs, conversationId, runtimeContextAttributes);
477492
case 'embed':
478493
case 'embedMany': {
479494
// `embed` carries a single `value`; `embedMany` a `values` array — both map to the embeddings input.
@@ -520,6 +535,26 @@ function telemetryMetadataAttributes(metadata: unknown): Record<string, string>
520535
return attributes;
521536
}
522537

538+
/**
539+
* One string attribute per context key, as `<prefix><key>`. `ai` < 7.0.114 published the full context to the
540+
* channel, ignoring `telemetry.includeRuntimeContext`/`includeToolsContext`, and a filtered context
541+
* looks the same as an unfiltered one. `aiVersion` (added in 7.0.154) is the only reliable sign that
542+
* the AI SDK filtered it, so without it we record nothing.
543+
*/
544+
function contextAttributes(event: Record<string, unknown>, context: unknown, prefix: string): SpanAttributes {
545+
const attributes: SpanAttributes = {};
546+
if (typeof event.aiVersion !== 'string' || !isObjectLike(context)) {
547+
return attributes;
548+
}
549+
for (const [key, value] of Object.entries(context)) {
550+
const attributeValue = stringify(value);
551+
if (value !== null && attributeValue !== undefined) {
552+
attributes[`${prefix}${key}`] = attributeValue;
553+
}
554+
}
555+
return attributes;
556+
}
557+
523558
/** Start a `gen_ai.<operation>` span named `<operation> <suffix>` (or just `<operation>` when no suffix). */
524559
function startGenAiSpan(operation: GenAiOperation, suffix: string | undefined, attributes: SpanAttributes): Span {
525560
return startInactiveSpan({
@@ -581,6 +616,7 @@ function buildToolSpan(
581616
event: Record<string, unknown>,
582617
recordInputs: boolean,
583618
conversationId: string | undefined,
619+
runtimeContextAttributes: SpanAttributes,
584620
): Span {
585621
const toolCall = isObjectLike(event.toolCall) ? event.toolCall : {};
586622
const toolName = asString(toolCall.toolName);
@@ -592,6 +628,8 @@ function buildToolSpan(
592628
recordInputs && toolName ? resolveToolDescription(asString(event.callId), toolName, event.tools) : undefined;
593629
return startGenAiSpan('execute_tool', toolName, {
594630
[SENTRY_ORIGIN]: ORIGIN,
631+
...runtimeContextAttributes,
632+
...contextAttributes(event, event.toolContext, VERCEL_AI_TOOL_CALL_CONTEXT_ATTRIBUTE_PREFIX),
595633
[GEN_AI_TOOL_NAME]: toolName,
596634
[GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: toolCallId,
597635
[GEN_AI_TOOL_DESCRIPTION]: description,

0 commit comments

Comments
 (0)