Skip to content

Commit fd6dc8e

Browse files
mydeaclaude
mydea
and
claude
committed
test(e2e): Move OpenAI and Anthropic send-to-sentry cases into gen-ai-libraries
gen-ai-libraries already runs chat and tool calls for both SDKs against real OpenRouter through the event proxy. Moves the cases only the send-to-sentry apps had (streaming usage, Responses API, embeddings, the Anthropic stream helper) into it, so they also run on Cloudflare, adds a (latest) variant for both SDKs, and removes node-openai-send-to-sentry and node-anthropic-send-to-sentry. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent fddeafc commit fd6dc8e

22 files changed

Lines changed: 241 additions & 966 deletions

File tree

‎dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "gen-ai-libraries",
3-
"description": "Real gen_ai spans for every instrumented AI library (OpenAI, Anthropic, Mistral, Together, Vercel AI), each driven through OpenRouter with a chat query and a tool call, on Node and on Cloudflare",
3+
"description": "Real gen_ai spans for every instrumented AI library (OpenAI, Anthropic, Mistral, Together, Vercel AI), each driven through OpenRouter with a chat query and a tool call, plus provider-specific calls (streaming, the OpenAI Responses and embeddings APIs, the Anthropic stream helper), on Node and on Cloudflare",
44
"version": "1.0.0",
55
"private": true,
66
"type": "module",
@@ -13,7 +13,8 @@
1313
"test:build": "pnpm install",
1414
"test:build:cloudflare": "pnpm install && vite build",
1515
"test:assert": "pnpm test",
16-
"test:assert:cloudflare": "RUNTIME=cloudflare pnpm test"
16+
"test:assert:cloudflare": "RUNTIME=cloudflare pnpm test",
17+
"test:build-latest": "pnpm install && pnpm add openai@latest && pnpm update '@anthropic-ai/sdk@<1'"
1718
},
1819
"dependencies": {
1920
"@anthropic-ai/sdk": "0.63.0",
@@ -46,6 +47,10 @@
4647
"build-command": "pnpm test:build:cloudflare",
4748
"assert-command": "pnpm test:assert:cloudflare",
4849
"label": "gen-ai-libraries (cloudflare)"
50+
},
51+
{
52+
"build-command": "pnpm test:build-latest",
53+
"label": "gen-ai-libraries (latest)"
4954
}
5055
]
5156
}

‎dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.cloudflare.ts‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,9 @@ export default Sentry.withSentry(
1717
const url = new URL(request.url);
1818
const [, id, action] = url.pathname.split('/');
1919
const library = id ? byId.get(id) : undefined;
20+
const extra = library && action ? library.extras?.[action] : undefined;
2021

21-
if (!library || (action !== 'chat' && action !== 'tools')) {
22+
if (!library || !action || (action !== 'chat' && action !== 'tools' && !extra)) {
2223
return new Response('Not found', { status: 404 });
2324
}
2425

@@ -28,8 +29,9 @@ export default Sentry.withSentry(
2829
}
2930

3031
try {
31-
const spanName = action === 'tools' ? 'ai-tool-workflow' : 'ai-workflow';
32-
const result = await Sentry.startSpan({ name: spanName, op: 'function' }, () => library[action](apiKey));
32+
const spanName = extra ? `${library.id}/${action}` : action === 'tools' ? 'ai-tool-workflow' : 'ai-workflow';
33+
const call = extra ?? library[action as 'chat' | 'tools'];
34+
const result = await Sentry.startSpan({ name: spanName, op: 'function' }, () => call(apiKey));
3335
return Response.json({ result });
3436
} catch (error) {
3537
return Response.json({ message: (error as Error).message }, { status: 500 });

‎dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.node.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,17 @@ for (const library of libraries) {
3333
next(error);
3434
}
3535
});
36+
37+
for (const [name, call] of Object.entries(library.extras ?? {})) {
38+
app.get(`/${library.id}/${name}`, async (_req, res, next) => {
39+
try {
40+
const result = await Sentry.startSpan({ name: `${library.id}/${name}`, op: 'function' }, () => call(apiKey));
41+
res.send({ result });
42+
} catch (error) {
43+
next(error);
44+
}
45+
});
46+
}
3647
}
3748

3849
Sentry.setupExpressErrorHandler(app);

‎dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/libraries.ts‎

Lines changed: 122 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,53 @@ export interface Library {
6060
origin?: string;
6161
chat: (apiKey: string) => Promise<unknown>;
6262
tools: (apiKey: string) => Promise<unknown>;
63+
/** Calls only this library's SDK has (streaming, other endpoints), served as `/:lib/:name`. */
64+
extras?: Record<string, (apiKey: string) => Promise<unknown>>;
65+
}
66+
67+
const EMBEDDINGS_MODEL = 'openai/text-embedding-3-small';
68+
69+
const openAiClient = (apiKey: string): OpenAI => new OpenAI({ apiKey, baseURL: OPENROUTER_V1 });
70+
71+
// OpenRouter's Anthropic skin authenticates with a bearer token, so the key goes in `authToken`
72+
// (Authorization: Bearer) rather than `apiKey` (x-api-key).
73+
const anthropicClient = (apiKey: string): Anthropic =>
74+
new Anthropic({ authToken: apiKey, baseURL: OPENROUTER_BASE, fetch: fetchWithoutDone });
75+
76+
/**
77+
* OpenRouter ends each stream with an OpenAI-style `data: [DONE]` frame, which is not part of
78+
* Anthropic's protocol. Since 0.136.0 the SDK JSON-parses every named event, so this frame
79+
* makes it throw. Drop the frame before the SDK sees it.
80+
*/
81+
async function fetchWithoutDone(input: string | URL | Request, init?: RequestInit): Promise<Response> {
82+
const response = await fetch(input, init);
83+
if (!response.body || !response.headers.get('content-type')?.includes('text/event-stream')) {
84+
return response;
85+
}
86+
87+
let buffer = '';
88+
const keepFrame = (frame: string): boolean => !/^data: \[DONE\]$/m.test(frame);
89+
const body = response.body.pipeThrough(new TextDecoderStream()).pipeThrough(
90+
new TransformStream<string, string>({
91+
transform(chunk, controller) {
92+
buffer += chunk;
93+
const frames = buffer.split(/(?<=\r?\n\r?\n)/);
94+
buffer = /\r?\n\r?\n$/.test(frames[frames.length - 1] ?? '') ? '' : (frames.pop() ?? '');
95+
controller.enqueue(frames.filter(keepFrame).join(''));
96+
},
97+
flush(controller) {
98+
if (keepFrame(buffer)) {
99+
controller.enqueue(buffer);
100+
}
101+
},
102+
}),
103+
);
104+
105+
return new Response(body.pipeThrough(new TextEncoderStream()), {
106+
status: response.status,
107+
statusText: response.statusText,
108+
headers: response.headers,
109+
});
63110
}
64111

65112
/** Chat + forced tool call for an OpenAI-compatible chat-completions client (OpenAI, Together). */
@@ -100,7 +147,47 @@ function openAiCompatible(
100147
}
101148

102149
export const libraries: Library[] = [
103-
openAiCompatible('openai', 'openai', 'auto.ai.openai', apiKey => new OpenAI({ apiKey, baseURL: OPENROUTER_V1 })),
150+
{
151+
...openAiCompatible('openai', 'openai', 'auto.ai.openai', openAiClient),
152+
extras: {
153+
'chat-stream': async apiKey => {
154+
const stream = await openAiClient(apiKey).chat.completions.create({
155+
model: MODEL,
156+
messages: [
157+
{ role: 'system', content: SYSTEM },
158+
{ role: 'user', content: CHAT_PROMPT },
159+
],
160+
temperature: 0,
161+
max_tokens: 32,
162+
stream: true,
163+
// Ask for the usage in the final chunk, so the streamed span carries token counts.
164+
stream_options: { include_usage: true },
165+
});
166+
let answer = '';
167+
for await (const chunk of stream) {
168+
answer += chunk.choices[0]?.delta.content ?? '';
169+
}
170+
return answer;
171+
},
172+
responses: async apiKey => {
173+
const response = await openAiClient(apiKey).responses.create({
174+
model: MODEL,
175+
instructions: SYSTEM,
176+
input: CHAT_PROMPT,
177+
temperature: 0,
178+
max_output_tokens: 32,
179+
});
180+
return response.output_text;
181+
},
182+
embeddings: async apiKey => {
183+
const embedding = await openAiClient(apiKey).embeddings.create({
184+
model: EMBEDDINGS_MODEL,
185+
input: 'The capital of France is Paris.',
186+
});
187+
return embedding.data[0]?.embedding.length ?? 0;
188+
},
189+
},
190+
},
104191
openAiCompatible(
105192
'together',
106193
'together_ai',
@@ -146,10 +233,7 @@ export const libraries: Library[] = [
146233
provider: 'anthropic',
147234
origin: 'auto.ai.anthropic',
148235
chat: async apiKey => {
149-
// OpenRouter's Anthropic skin authenticates with a bearer token, so the key goes in `authToken`
150-
// (Authorization: Bearer) rather than `apiKey` (x-api-key).
151-
const client = new Anthropic({ authToken: apiKey, baseURL: OPENROUTER_BASE });
152-
const message = await client.messages.create({
236+
const message = await anthropicClient(apiKey).messages.create({
153237
model: MODEL,
154238
max_tokens: 32,
155239
temperature: 0,
@@ -160,8 +244,7 @@ export const libraries: Library[] = [
160244
return first && first.type === 'text' ? first.text : '';
161245
},
162246
tools: async apiKey => {
163-
const client = new Anthropic({ authToken: apiKey, baseURL: OPENROUTER_BASE });
164-
const message = await client.messages.create({
247+
const message = await anthropicClient(apiKey).messages.create({
165248
model: MODEL,
166249
max_tokens: 64,
167250
messages: [{ role: 'user', content: WEATHER_PROMPT }],
@@ -180,6 +263,38 @@ export const libraries: Library[] = [
180263
});
181264
return (message.content ?? []).filter(block => block.type === 'tool_use');
182265
},
266+
extras: {
267+
// `messages.create({ stream: true })` returns an async iterable of events the caller drains.
268+
'chat-stream': async apiKey => {
269+
const stream = await anthropicClient(apiKey).messages.create({
270+
model: MODEL,
271+
max_tokens: 32,
272+
system: SYSTEM,
273+
messages: [{ role: 'user', content: CHAT_PROMPT }],
274+
stream: true,
275+
});
276+
let answer = '';
277+
for await (const event of stream) {
278+
if (event.type === 'content_block_delta' && event.delta.type === 'text_delta') {
279+
answer += event.delta.text;
280+
}
281+
}
282+
return answer;
283+
},
284+
// The SDK's streaming helper, which the integration instruments separately from `create`.
285+
'stream-helper': async apiKey => {
286+
const message = await anthropicClient(apiKey)
287+
.messages.stream({
288+
model: MODEL,
289+
max_tokens: 32,
290+
system: SYSTEM,
291+
messages: [{ role: 'user', content: CHAT_PROMPT }],
292+
})
293+
.finalMessage();
294+
const first = message.content?.[0];
295+
return first && first.type === 'text' ? first.text : '';
296+
},
297+
},
183298
},
184299

185300
{

‎dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/extras.test.ts‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
import { expect, test } from '@playwright/test';
2+
import type { SerializedStreamedSpan } from '@sentry-internal/test-utils';
3+
import { collectStreamedSpans, getSpanOp } from '@sentry-internal/test-utils';
4+
import { APP } from './constants';
5+
import { attr, describeTree, expectCommonChatAttributes, LIBRARIES } from './utils';
6+
7+
const OPENAI = LIBRARIES.find(library => library.id === 'openai')!;
8+
const ANTHROPIC = LIBRARIES.find(library => library.id === 'anthropic')!;
9+
10+
/**
11+
* Calls `/:lib/:extra` and returns its trace once the request span (the segment) has arrived. The
12+
* segment ends last, so every model-call span of the request is in by then.
13+
*/
14+
async function requestExtra(baseURL: string | undefined, path: string): Promise<SerializedStreamedSpan[]> {
15+
const spansPromise = collectStreamedSpans(
16+
APP,
17+
spansOfTrace => spansOfTrace.some(span => span.name === path) && spansOfTrace.some(span => span.is_segment),
18+
);
19+
20+
const response = await fetch(`${baseURL}/${path}`);
21+
expect(response.status, await response.text()).toBe(200);
22+
23+
return spansPromise;
24+
}
25+
26+
/** The only span of the trace with `op`; finding exactly one also catches a call instrumented twice. */
27+
function onlySpanWithOp(spans: SerializedStreamedSpan[], op: string): SerializedStreamedSpan {
28+
const matching = spans.filter(span => getSpanOp(span) === op);
29+
expect(matching, `expected exactly one ${op} span in:\n${describeTree(spans)}`).toHaveLength(1);
30+
return matching[0]!;
31+
}
32+
33+
/** `gen_ai.response.finish_reasons` is a JSON array; a streamed answer can repeat the reason per chunk. */
34+
function finishReasons(span: SerializedStreamedSpan): string[] {
35+
const raw = attr(span, 'gen_ai.response.finish_reasons');
36+
expect(typeof raw, 'gen_ai.response.finish_reasons').toBe('string');
37+
return JSON.parse(raw as string);
38+
}
39+
40+
test('openai: a streamed chat completion records its token usage', async ({ baseURL }) => {
41+
const spans = await requestExtra(baseURL, 'openai/chat-stream');
42+
const span = onlySpanWithOp(spans, 'gen_ai.chat');
43+
44+
// The usage only arrives in the final chunk, which the streaming instrumentation has to pick up.
45+
expectCommonChatAttributes(span, OPENAI);
46+
expect(attr(span, 'gen_ai.response.streaming')).toBe(true);
47+
expect(finishReasons(span)).toContain('stop');
48+
});
49+
50+
test('openai: a Responses API call emits a gen_ai.chat span', async ({ baseURL }) => {
51+
const spans = await requestExtra(baseURL, 'openai/responses');
52+
const span = onlySpanWithOp(spans, 'gen_ai.chat');
53+
54+
expectCommonChatAttributes(span, OPENAI);
55+
expect(finishReasons(span)).toEqual(['completed']);
56+
expect(attr(span, 'gen_ai.input.messages')).toContain('capital of France');
57+
});
58+
59+
test('openai: an embeddings call emits a gen_ai.embeddings span', async ({ baseURL }) => {
60+
const spans = await requestExtra(baseURL, 'openai/embeddings');
61+
const span = onlySpanWithOp(spans, 'gen_ai.embeddings');
62+
63+
expect(span.name).toBe('embeddings openai/text-embedding-3-small');
64+
expect(span.status).toBe('ok');
65+
expect(attr(span, 'sentry.origin')).toBe('auto.ai.openai');
66+
expect(attr(span, 'gen_ai.operation.name')).toBe('embeddings');
67+
expect(attr(span, 'gen_ai.provider.name')).toBe('openai');
68+
expect(attr(span, 'gen_ai.embeddings.input')).toContain('Paris');
69+
expect(attr(span, 'gen_ai.usage.input_tokens')).toBeGreaterThan(0);
70+
});
71+
72+
test('anthropic: a streamed message records its output usage', async ({ baseURL }) => {
73+
const spans = await requestExtra(baseURL, 'anthropic/chat-stream');
74+
const span = onlySpanWithOp(spans, 'gen_ai.chat');
75+
76+
expect(attr(span, 'gen_ai.response.streaming')).toBe(true);
77+
expect(finishReasons(span)).toContain('end_turn');
78+
expect(attr(span, 'gen_ai.usage.output_tokens')).toBeGreaterThan(0);
79+
// The input usage is not asserted: the instrumentation takes it from `message_start`, where
80+
// OpenRouter reports 0, and ignores the real count OpenRouter (and the Anthropic API) put in
81+
// `message_delta`.
82+
expect(attr(span, 'gen_ai.provider.name')).toBe(ANTHROPIC.provider);
83+
expect(attr(span, 'sentry.origin')).toBe(ANTHROPIC.origin);
84+
});
85+
86+
test('anthropic: the stream helper emits a single gen_ai.chat span', async ({ baseURL }) => {
87+
const spans = await requestExtra(baseURL, 'anthropic/stream-helper');
88+
// `messages.stream()` calls `create` underneath, which must not produce a second span.
89+
const span = onlySpanWithOp(spans, 'gen_ai.chat');
90+
91+
expect(attr(span, 'gen_ai.response.streaming')).toBe(true);
92+
expect(finishReasons(span)).toContain('end_turn');
93+
expect(attr(span, 'gen_ai.usage.output_tokens')).toBeGreaterThan(0);
94+
expect(attr(span, 'gen_ai.provider.name')).toBe(ANTHROPIC.provider);
95+
expect(attr(span, 'sentry.origin')).toBe(ANTHROPIC.origin);
96+
});

‎dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/.gitignore‎

Lines changed: 0 additions & 5 deletions
This file was deleted.

‎dev-packages/e2e-tests/test-applications/node-anthropic-send-to-sentry/package.json‎

Lines changed: 0 additions & 36 deletions
This file was deleted.

0 commit comments

Comments
 (0)