Skip to content

Commit 3e47add

Browse files
AbhiPrasadAbhiPrasad
authored
feat(node): Update vercel ai spans as per new conventions (#16497)
resolves #16454 As per https://www.notion.so/sentry/WIP-Span-layout-of-AI-agents-1fa8b10e4b5d80f0a83dd251a5ea4ab1, we update the span names and ops to better match OpenTelemetry. This should make them more easily accessible to the new agents module view we are building. Now instead of a span name like `generateText.doGenerate`, we will show a name like `generate_text gpt-4o-mini` (which is the OTEL convention for the gen ai span name). Another example, tool calls will show the tool name so you'll get `execute_tool getWeather` instead of `ai.toolCall`.
1 parent e040002 commit 3e47add

File tree

4 files changed

+300
-93
lines changed

4 files changed

+300
-93
lines changed

‎dev-packages/node-integration-tests/suites/tracing/vercelai/scenario.mjs‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import * as Sentry from '@sentry/node';
22
import { generateText } from 'ai';
33
import { MockLanguageModelV1 } from 'ai/test';
4+
import { z } from 'zod';
45

56
async function run() {
67
await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
@@ -30,6 +31,35 @@ async function run() {
3031
prompt: 'Where is the second span?',
3132
});
3233

34+
// This span should include tool calls and tool results
35+
await generateText({
36+
model: new MockLanguageModelV1({
37+
doGenerate: async () => ({
38+
rawCall: { rawPrompt: null, rawSettings: {} },
39+
finishReason: 'tool-calls',
40+
usage: { promptTokens: 15, completionTokens: 25 },
41+
text: 'Tool call completed!',
42+
toolCalls: [
43+
{
44+
toolCallType: 'function',
45+
toolCallId: 'call-1',
46+
toolName: 'getWeather',
47+
args: '{ "location": "San Francisco" }',
48+
},
49+
],
50+
}),
51+
}),
52+
tools: {
53+
getWeather: {
54+
parameters: z.object({ location: z.string() }),
55+
execute: async args => {
56+
return `Weather in ${args.location}: Sunny, 72°F`;
57+
},
58+
},
59+
},
60+
prompt: 'What is the weather in San Francisco?',
61+
});
62+
3363
// This span should not be captured because we've disabled telemetry
3464
await generateText({
3565
experimental_telemetry: { isEnabled: false },

‎dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts‎

Lines changed: 174 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -26,19 +26,19 @@ describe('Vercel AI integration', () => {
2626
'gen_ai.usage.output_tokens': 20,
2727
'gen_ai.usage.total_tokens': 30,
2828
'operation.name': 'ai.generateText',
29-
'sentry.op': 'ai.pipeline.generateText',
29+
'sentry.op': 'ai.pipeline.generate_text',
3030
'sentry.origin': 'auto.vercelai.otel',
3131
},
3232
description: 'generateText',
33-
op: 'ai.pipeline.generateText',
33+
op: 'ai.pipeline.generate_text',
3434
origin: 'auto.vercelai.otel',
3535
status: 'ok',
3636
}),
3737
// Second span - explicitly enabled telemetry but recordInputs/recordOutputs not set, should not record when sendDefaultPii: false
3838
expect.objectContaining({
3939
data: {
4040
'sentry.origin': 'auto.vercelai.otel',
41-
'sentry.op': 'ai.run.doGenerate',
41+
'sentry.op': 'gen_ai.generate_text',
4242
'operation.name': 'ai.generateText.doGenerate',
4343
'ai.operationId': 'ai.generateText.doGenerate',
4444
'ai.model.provider': 'mock-provider',
@@ -59,8 +59,8 @@ describe('Vercel AI integration', () => {
5959
'gen_ai.response.model': 'mock-model-id',
6060
'gen_ai.usage.total_tokens': 30,
6161
},
62-
description: 'generateText.doGenerate',
63-
op: 'ai.run.doGenerate',
62+
description: 'generate_text mock-model-id',
63+
op: 'gen_ai.generate_text',
6464
origin: 'auto.vercelai.otel',
6565
status: 'ok',
6666
}),
@@ -83,19 +83,19 @@ describe('Vercel AI integration', () => {
8383
'gen_ai.usage.output_tokens': 20,
8484
'gen_ai.usage.total_tokens': 30,
8585
'operation.name': 'ai.generateText',
86-
'sentry.op': 'ai.pipeline.generateText',
86+
'sentry.op': 'ai.pipeline.generate_text',
8787
'sentry.origin': 'auto.vercelai.otel',
8888
},
8989
description: 'generateText',
90-
op: 'ai.pipeline.generateText',
90+
op: 'ai.pipeline.generate_text',
9191
origin: 'auto.vercelai.otel',
9292
status: 'ok',
9393
}),
9494
// Fourth span - doGenerate for explicit telemetry enabled call
9595
expect.objectContaining({
9696
data: {
9797
'sentry.origin': 'auto.vercelai.otel',
98-
'sentry.op': 'ai.run.doGenerate',
98+
'sentry.op': 'gen_ai.generate_text',
9999
'operation.name': 'ai.generateText.doGenerate',
100100
'ai.operationId': 'ai.generateText.doGenerate',
101101
'ai.model.provider': 'mock-provider',
@@ -119,8 +119,79 @@ describe('Vercel AI integration', () => {
119119
'gen_ai.response.model': 'mock-model-id',
120120
'gen_ai.usage.total_tokens': 30,
121121
},
122-
description: 'generateText.doGenerate',
123-
op: 'ai.run.doGenerate',
122+
description: 'generate_text mock-model-id',
123+
op: 'gen_ai.generate_text',
124+
origin: 'auto.vercelai.otel',
125+
status: 'ok',
126+
}),
127+
// Fifth span - tool call generateText span
128+
expect.objectContaining({
129+
data: {
130+
'ai.model.id': 'mock-model-id',
131+
'ai.model.provider': 'mock-provider',
132+
'ai.operationId': 'ai.generateText',
133+
'ai.pipeline.name': 'generateText',
134+
'ai.response.finishReason': 'tool-calls',
135+
'ai.settings.maxRetries': 2,
136+
'ai.settings.maxSteps': 1,
137+
'ai.streaming': false,
138+
'gen_ai.response.model': 'mock-model-id',
139+
'gen_ai.usage.input_tokens': 15,
140+
'gen_ai.usage.output_tokens': 25,
141+
'gen_ai.usage.total_tokens': 40,
142+
'operation.name': 'ai.generateText',
143+
'sentry.op': 'ai.pipeline.generate_text',
144+
'sentry.origin': 'auto.vercelai.otel',
145+
},
146+
description: 'generateText',
147+
op: 'ai.pipeline.generate_text',
148+
origin: 'auto.vercelai.otel',
149+
status: 'ok',
150+
}),
151+
// Sixth span - tool call doGenerate span
152+
expect.objectContaining({
153+
data: {
154+
'ai.model.id': 'mock-model-id',
155+
'ai.model.provider': 'mock-provider',
156+
'ai.operationId': 'ai.generateText.doGenerate',
157+
'ai.pipeline.name': 'generateText.doGenerate',
158+
'ai.response.finishReason': 'tool-calls',
159+
'ai.response.id': expect.any(String),
160+
'ai.response.model': 'mock-model-id',
161+
'ai.response.timestamp': expect.any(String),
162+
'ai.settings.maxRetries': 2,
163+
'ai.streaming': false,
164+
'gen_ai.request.model': 'mock-model-id',
165+
'gen_ai.response.finish_reasons': ['tool-calls'],
166+
'gen_ai.response.id': expect.any(String),
167+
'gen_ai.response.model': 'mock-model-id',
168+
'gen_ai.system': 'mock-provider',
169+
'gen_ai.usage.input_tokens': 15,
170+
'gen_ai.usage.output_tokens': 25,
171+
'gen_ai.usage.total_tokens': 40,
172+
'operation.name': 'ai.generateText.doGenerate',
173+
'sentry.op': 'gen_ai.generate_text',
174+
'sentry.origin': 'auto.vercelai.otel',
175+
},
176+
description: 'generate_text mock-model-id',
177+
op: 'gen_ai.generate_text',
178+
origin: 'auto.vercelai.otel',
179+
status: 'ok',
180+
}),
181+
// Seventh span - tool call execution span
182+
expect.objectContaining({
183+
data: {
184+
'ai.operationId': 'ai.toolCall',
185+
'ai.toolCall.id': 'call-1',
186+
'ai.toolCall.name': 'getWeather',
187+
'gen_ai.tool.call.id': 'call-1',
188+
'gen_ai.tool.name': 'getWeather',
189+
'operation.name': 'ai.toolCall',
190+
'sentry.op': 'gen_ai.execute_tool',
191+
'sentry.origin': 'auto.vercelai.otel',
192+
},
193+
description: 'execute_tool getWeather',
194+
op: 'gen_ai.execute_tool',
124195
origin: 'auto.vercelai.otel',
125196
status: 'ok',
126197
}),
@@ -149,11 +220,11 @@ describe('Vercel AI integration', () => {
149220
'gen_ai.usage.output_tokens': 20,
150221
'gen_ai.usage.total_tokens': 30,
151222
'operation.name': 'ai.generateText',
152-
'sentry.op': 'ai.pipeline.generateText',
223+
'sentry.op': 'ai.pipeline.generate_text',
153224
'sentry.origin': 'auto.vercelai.otel',
154225
},
155226
description: 'generateText',
156-
op: 'ai.pipeline.generateText',
227+
op: 'ai.pipeline.generate_text',
157228
origin: 'auto.vercelai.otel',
158229
status: 'ok',
159230
}),
@@ -182,11 +253,11 @@ describe('Vercel AI integration', () => {
182253
'gen_ai.usage.output_tokens': 20,
183254
'gen_ai.usage.total_tokens': 30,
184255
'operation.name': 'ai.generateText.doGenerate',
185-
'sentry.op': 'ai.run.doGenerate',
256+
'sentry.op': 'gen_ai.generate_text',
186257
'sentry.origin': 'auto.vercelai.otel',
187258
},
188-
description: 'generateText.doGenerate',
189-
op: 'ai.run.doGenerate',
259+
description: 'generate_text mock-model-id',
260+
op: 'gen_ai.generate_text',
190261
origin: 'auto.vercelai.otel',
191262
status: 'ok',
192263
}),
@@ -209,19 +280,19 @@ describe('Vercel AI integration', () => {
209280
'gen_ai.usage.output_tokens': 20,
210281
'gen_ai.usage.total_tokens': 30,
211282
'operation.name': 'ai.generateText',
212-
'sentry.op': 'ai.pipeline.generateText',
283+
'sentry.op': 'ai.pipeline.generate_text',
213284
'sentry.origin': 'auto.vercelai.otel',
214285
},
215286
description: 'generateText',
216-
op: 'ai.pipeline.generateText',
287+
op: 'ai.pipeline.generate_text',
217288
origin: 'auto.vercelai.otel',
218289
status: 'ok',
219290
}),
220291
// Fourth span - doGenerate for explicitly enabled telemetry call
221292
expect.objectContaining({
222293
data: {
223294
'sentry.origin': 'auto.vercelai.otel',
224-
'sentry.op': 'ai.run.doGenerate',
295+
'sentry.op': 'gen_ai.generate_text',
225296
'operation.name': 'ai.generateText.doGenerate',
226297
'ai.operationId': 'ai.generateText.doGenerate',
227298
'ai.model.provider': 'mock-provider',
@@ -245,8 +316,91 @@ describe('Vercel AI integration', () => {
245316
'gen_ai.response.model': 'mock-model-id',
246317
'gen_ai.usage.total_tokens': 30,
247318
},
248-
description: 'generateText.doGenerate',
249-
op: 'ai.run.doGenerate',
319+
description: 'generate_text mock-model-id',
320+
op: 'gen_ai.generate_text',
321+
origin: 'auto.vercelai.otel',
322+
status: 'ok',
323+
}),
324+
// Fifth span - tool call generateText span (should include prompts when sendDefaultPii: true)
325+
expect.objectContaining({
326+
data: {
327+
'ai.model.id': 'mock-model-id',
328+
'ai.model.provider': 'mock-provider',
329+
'ai.operationId': 'ai.generateText',
330+
'ai.pipeline.name': 'generateText',
331+
'ai.prompt': '{"prompt":"What is the weather in San Francisco?"}',
332+
'ai.response.finishReason': 'tool-calls',
333+
'ai.response.text': 'Tool call completed!',
334+
'ai.response.toolCalls': expect.any(String),
335+
'ai.settings.maxRetries': 2,
336+
'ai.settings.maxSteps': 1,
337+
'ai.streaming': false,
338+
'gen_ai.prompt': '{"prompt":"What is the weather in San Francisco?"}',
339+
'gen_ai.response.model': 'mock-model-id',
340+
'gen_ai.usage.input_tokens': 15,
341+
'gen_ai.usage.output_tokens': 25,
342+
'gen_ai.usage.total_tokens': 40,
343+
'operation.name': 'ai.generateText',
344+
'sentry.op': 'ai.pipeline.generate_text',
345+
'sentry.origin': 'auto.vercelai.otel',
346+
},
347+
description: 'generateText',
348+
op: 'ai.pipeline.generate_text',
349+
origin: 'auto.vercelai.otel',
350+
status: 'ok',
351+
}),
352+
// Sixth span - tool call doGenerate span (should include prompts when sendDefaultPii: true)
353+
expect.objectContaining({
354+
data: {
355+
'ai.model.id': 'mock-model-id',
356+
'ai.model.provider': 'mock-provider',
357+
'ai.operationId': 'ai.generateText.doGenerate',
358+
'ai.pipeline.name': 'generateText.doGenerate',
359+
'ai.prompt.format': expect.any(String),
360+
'ai.prompt.messages': expect.any(String),
361+
'ai.prompt.toolChoice': expect.any(String),
362+
'ai.prompt.tools': expect.any(Array),
363+
'ai.response.finishReason': 'tool-calls',
364+
'ai.response.id': expect.any(String),
365+
'ai.response.model': 'mock-model-id',
366+
'ai.response.text': 'Tool call completed!',
367+
'ai.response.timestamp': expect.any(String),
368+
'ai.response.toolCalls': expect.any(String),
369+
'ai.settings.maxRetries': 2,
370+
'ai.streaming': false,
371+
'gen_ai.request.model': 'mock-model-id',
372+
'gen_ai.response.finish_reasons': ['tool-calls'],
373+
'gen_ai.response.id': expect.any(String),
374+
'gen_ai.response.model': 'mock-model-id',
375+
'gen_ai.system': 'mock-provider',
376+
'gen_ai.usage.input_tokens': 15,
377+
'gen_ai.usage.output_tokens': 25,
378+
'gen_ai.usage.total_tokens': 40,
379+
'operation.name': 'ai.generateText.doGenerate',
380+
'sentry.op': 'gen_ai.generate_text',
381+
'sentry.origin': 'auto.vercelai.otel',
382+
},
383+
description: 'generate_text mock-model-id',
384+
op: 'gen_ai.generate_text',
385+
origin: 'auto.vercelai.otel',
386+
status: 'ok',
387+
}),
388+
// Seventh span - tool call execution span
389+
expect.objectContaining({
390+
data: {
391+
'ai.operationId': 'ai.toolCall',
392+
'ai.toolCall.args': expect.any(String),
393+
'ai.toolCall.id': 'call-1',
394+
'ai.toolCall.name': 'getWeather',
395+
'ai.toolCall.result': expect.any(String),
396+
'gen_ai.tool.call.id': 'call-1',
397+
'gen_ai.tool.name': 'getWeather',
398+
'operation.name': 'ai.toolCall',
399+
'sentry.op': 'gen_ai.execute_tool',
400+
'sentry.origin': 'auto.vercelai.otel',
401+
},
402+
description: 'execute_tool getWeather',
403+
op: 'gen_ai.execute_tool',
250404
origin: 'auto.vercelai.otel',
251405
status: 'ok',
252406
}),

‎packages/node/src/integrations/tracing/vercelai/attributes.ts‎ renamed to ‎packages/node/src/integrations/tracing/vercelai/ai_sdk_attributes.ts‎

File renamed without changes.

0 commit comments

Comments
 (0)