From 1bb8cfcd15072e883daabddba7186c91434916bd Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Thu, 5 Nov 2026 16:03:12 +0100 Subject: [PATCH] test(node): collect TypeSafe spans across streamed envelopes Co-Authored-By: OpenAI Codex --- .../suites/tracing/typesafe/test.ts | 192 +++++++++--------- 1 file changed, 97 insertions(+), 95 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts b/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts index ab0b4b68bae2..56e7840fe759 100644 --- a/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts @@ -26,110 +26,112 @@ describe('TypeSafe integration', () => { ])('%s instrumentation', (_, scenarioFile, instrumentFile, noRecordingInstrumentFile) => { createEsmAndCjsTests(__dirname, scenarioFile, instrumentFile, (createRunner, test) => { test('creates decide spans for systemOne', async () => { - await createRunner() - .unordered() - .expect({ - span: container => { - const decideSpans = container.items.filter(span => span.attributes[SENTRY_OP]?.value === GEN_AI_DECIDE); - expect(decideSpans).toHaveLength(4); - for (const span of decideSpans) { - expect(span.attributes[SENTRY_ORIGIN]?.value).toBe('auto.ai.typesafe'); - expect(span.attributes[GEN_AI_OPERATION_NAME]?.value).toBe('decide'); - expect(span.attributes[GEN_AI_PROVIDER_NAME]?.value).toBe('typesafe'); - } + const runner = createRunner(); + const spansPromise = runner.collectStreamedSpans( + spans => + spans.some(span => span.is_segment && span.name === 'main') && + spans.filter(span => span.attributes[SENTRY_OP]?.value === GEN_AI_DECIDE).length >= 4, + ); - const defaultModelSpan = decideSpans.find(span => span.name === 'decide jev-latest')!; - expect(defaultModelSpan).toBeDefined(); - expect(defaultModelSpan.status).toBe('ok'); - expect(defaultModelSpan.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('jev-latest'); - expect(defaultModelSpan.attributes[GEN_AI_RESPONSE_MODEL]?.value).toBe('jev-1.13.0'); - expect(defaultModelSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(275); - expect(defaultModelSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(20); - expect(defaultModelSpan.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(295); - expect(JSON.parse(defaultModelSpan.attributes[GEN_AI_INPUT_MESSAGES]?.value as string)).toEqual([ - { - type: 'evaluation', - state: 'I cannot log in, and I also want a refund for last month.', - questions: { - authIssue: { type: 'noul', instructions: 'Is there a login problem?' }, - department: { - type: 'choice', - instructions: 'Which team should handle this?', - criteria: { billing: 'Charges and refunds', technical: 'Bugs and outages' }, - }, - urgency: { - type: 'score', - instructions: 'How urgent is this ticket?', - criteria: ['low', 'medium', 'high'], - }, - }, - }, - ]); - expect(JSON.parse(defaultModelSpan.attributes[GEN_AI_OUTPUT_MESSAGES]?.value as string)).toEqual([ - { - type: 'evaluation', - answers: { - authIssue: { type: 'noul', noul: 0.98 }, - department: { - type: 'choice', - choice: 'billing', - confidence: 0.28, - probabilities: { billing: 0.64, technical: 0.36 }, - }, - urgency: { - type: 'score', - score: 1.58, - confidence: 0.9, - legend: { 0: 'low', 1: 'medium', 2: 'high' }, - probabilities: { 0: 0, 1: 0.42, 2: 0.58 }, - }, - }, - }, - ]); + await runner.start().completed(); - const asResponseSpan = decideSpans.find(span => span.name === 'decide jev-1.13')!; - expect(asResponseSpan).toBeDefined(); - expect(asResponseSpan.status).toBe('ok'); - expect(asResponseSpan.attributes[GEN_AI_RESPONSE_MODEL]?.value).toBe('jev-1.13.0'); + const decideSpans = (await spansPromise).filter(span => span.attributes[SENTRY_OP]?.value === GEN_AI_DECIDE); + expect(decideSpans).toHaveLength(4); + for (const span of decideSpans) { + expect(span.attributes[SENTRY_ORIGIN]?.value).toBe('auto.ai.typesafe'); + expect(span.attributes[GEN_AI_OPERATION_NAME]?.value).toBe('decide'); + expect(span.attributes[GEN_AI_PROVIDER_NAME]?.value).toBe('typesafe'); + } - const errorSpan = decideSpans.find(span => span.name === 'decide error-model')!; - expect(errorSpan).toBeDefined(); - expect(errorSpan.status).toBe('error'); - expect(errorSpan.attributes[GEN_AI_RESPONSE_MODEL]).toBeUndefined(); - expect(errorSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); - - const validationErrorSpan = decideSpans.find(span => span.name === 'decide validation-error')!; - expect(validationErrorSpan).toBeDefined(); - expect(validationErrorSpan.status).toBe('error'); - expect(validationErrorSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); + const defaultModelSpan = decideSpans.find(span => span.name === 'decide jev-latest')!; + expect(defaultModelSpan).toBeDefined(); + expect(defaultModelSpan.status).toBe('ok'); + expect(defaultModelSpan.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('jev-latest'); + expect(defaultModelSpan.attributes[GEN_AI_RESPONSE_MODEL]?.value).toBe('jev-1.13.0'); + expect(defaultModelSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(275); + expect(defaultModelSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(20); + expect(defaultModelSpan.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(295); + expect(JSON.parse(defaultModelSpan.attributes[GEN_AI_INPUT_MESSAGES]?.value as string)).toEqual([ + { + type: 'evaluation', + state: 'I cannot log in, and I also want a refund for last month.', + questions: { + authIssue: { type: 'noul', instructions: 'Is there a login problem?' }, + department: { + type: 'choice', + instructions: 'Which team should handle this?', + criteria: { billing: 'Charges and refunds', technical: 'Bugs and outages' }, + }, + urgency: { + type: 'score', + instructions: 'How urgent is this ticket?', + criteria: ['low', 'medium', 'high'], + }, + }, + }, + ]); + expect(JSON.parse(defaultModelSpan.attributes[GEN_AI_OUTPUT_MESSAGES]?.value as string)).toEqual([ + { + type: 'evaluation', + answers: { + authIssue: { type: 'noul', noul: 0.98 }, + department: { + type: 'choice', + choice: 'billing', + confidence: 0.28, + probabilities: { billing: 0.64, technical: 0.36 }, + }, + urgency: { + type: 'score', + score: 1.58, + confidence: 0.9, + legend: { 0: 'low', 1: 'medium', 2: 'high' }, + probabilities: { 0: 0, 1: 0.42, 2: 0.58 }, + }, }, - }) - .start() - .completed(); + }, + ]); + + const asResponseSpan = decideSpans.find(span => span.name === 'decide jev-1.13')!; + expect(asResponseSpan).toBeDefined(); + expect(asResponseSpan.status).toBe('ok'); + expect(asResponseSpan.attributes[GEN_AI_RESPONSE_MODEL]?.value).toBe('jev-1.13.0'); + + const errorSpan = decideSpans.find(span => span.name === 'decide error-model')!; + expect(errorSpan).toBeDefined(); + expect(errorSpan.status).toBe('error'); + expect(errorSpan.attributes[GEN_AI_RESPONSE_MODEL]).toBeUndefined(); + expect(errorSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); + + const validationErrorSpan = decideSpans.find(span => span.name === 'decide validation-error')!; + expect(validationErrorSpan).toBeDefined(); + expect(validationErrorSpan.status).toBe('error'); + expect(validationErrorSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); }); }); createEsmAndCjsTests(__dirname, scenarioFile, noRecordingInstrumentFile, (createRunner, test) => { test('does not record inputs or outputs when recording is off', async () => { - await createRunner() - .unordered() - .expect({ - span: container => { - const decideSpans = container.items.filter(span => span.attributes[SENTRY_OP]?.value === GEN_AI_DECIDE); - expect(decideSpans).toHaveLength(4); - expect(decideSpans.filter(span => span.status === 'error')).toHaveLength(2); - for (const span of decideSpans) { - expect(span.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); - expect(span.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); - // State, questions and answers must not come back through another attribute (e.g. an error message). - // Only the attributes are checked (timestamps could match a number), and `probabilities` only - // occurs in answers. - expect(JSON.stringify(span.attributes)).not.toMatch(/cannot log in|Charges and refunds|probabilities/); - } - }, - }) - .start() - .completed(); + const runner = createRunner(); + const spansPromise = runner.collectStreamedSpans( + spans => + spans.some(span => span.is_segment && span.name === 'main') && + spans.filter(span => span.attributes[SENTRY_OP]?.value === GEN_AI_DECIDE).length >= 4, + ); + + await runner.start().completed(); + + const decideSpans = (await spansPromise).filter(span => span.attributes[SENTRY_OP]?.value === GEN_AI_DECIDE); + expect(decideSpans).toHaveLength(4); + expect(decideSpans.filter(span => span.status === 'error')).toHaveLength(2); + for (const span of decideSpans) { + expect(span.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); + expect(span.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); + // State, questions and answers must not come back through another attribute (e.g. an error message). + // Only the attributes are checked (timestamps could match a number), and `probabilities` only + // occurs in answers. + expect(JSON.stringify(span.attributes)).not.toMatch(/cannot log in|Charges and refunds|probabilities/); + } }); }); });