Skip to content

Commit fd4f990

Browse files
nicohrubecnicohrubec
committed
Split embeddings test into separate file
1 parent dbebf0c commit fd4f990

File tree

3 files changed

+166
-115
lines changed

3 files changed

+166
-115
lines changed

‎dev-packages/node-integration-tests/suites/tracing/openai/scenario.mjs‎ renamed to ‎dev-packages/node-integration-tests/suites/tracing/openai/scenario-chat.mjs‎

Lines changed: 0 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -74,35 +74,6 @@ class MockOpenAI {
7474
};
7575
},
7676
};
77-
78-
this.embeddings = {
79-
create: async params => {
80-
await new Promise(resolve => setTimeout(resolve, 10));
81-
82-
if (params.model === 'error-model') {
83-
const error = new Error('Model not found');
84-
error.status = 404;
85-
error.headers = { 'x-request-id': 'mock-request-123' };
86-
throw error;
87-
}
88-
89-
return {
90-
object: 'list',
91-
data: [
92-
{
93-
object: 'embedding',
94-
embedding: [0.1, 0.2, 0.3],
95-
index: 0,
96-
},
97-
],
98-
model: params.model,
99-
usage: {
100-
prompt_tokens: 10,
101-
total_tokens: 10,
102-
},
103-
};
104-
},
105-
};
10677
}
10778

10879
// Create a mock streaming response for chat completions
@@ -341,24 +312,6 @@ async function run() {
341312
} catch {
342313
// Error is expected and handled
343314
}
344-
345-
// Seventh test: embeddings API
346-
await client.embeddings.create({
347-
input: 'Embedding test!',
348-
model: 'text-embedding-3-small',
349-
dimensions: 1536,
350-
encoding_format: 'float',
351-
});
352-
353-
// Eighth test: embeddings API error model
354-
try {
355-
await client.embeddings.create({
356-
input: 'Error embedding test!',
357-
model: 'error-model',
358-
});
359-
} catch {
360-
// Error is expected and handled
361-
}
362315
});
363316
}
364317

‎dev-packages/node-integration-tests/suites/tracing/openai/scenario-embeddings.mjs‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
import { instrumentOpenAiClient } from '@sentry/core';
2+
import * as Sentry from '@sentry/node';
3+
4+
class MockOpenAI {
5+
constructor(config) {
6+
this.apiKey = config.apiKey;
7+
8+
this.embeddings = {
9+
create: async params => {
10+
await new Promise(resolve => setTimeout(resolve, 10));
11+
12+
if (params.model === 'error-model') {
13+
const error = new Error('Model not found');
14+
error.status = 404;
15+
error.headers = { 'x-request-id': 'mock-request-123' };
16+
throw error;
17+
}
18+
19+
return {
20+
object: 'list',
21+
data: [
22+
{
23+
object: 'embedding',
24+
embedding: [0.1, 0.2, 0.3],
25+
index: 0,
26+
},
27+
],
28+
model: params.model,
29+
usage: {
30+
prompt_tokens: 10,
31+
total_tokens: 10,
32+
},
33+
};
34+
},
35+
};
36+
}
37+
}
38+
39+
async function run() {
40+
await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
41+
const mockClient = new MockOpenAI({
42+
apiKey: 'mock-api-key',
43+
});
44+
45+
const client = instrumentOpenAiClient(mockClient);
46+
47+
// First test: embeddings API
48+
await client.embeddings.create({
49+
input: 'Embedding test!',
50+
model: 'text-embedding-3-small',
51+
dimensions: 1536,
52+
encoding_format: 'float',
53+
});
54+
55+
// Second test: embeddings API error model
56+
try {
57+
await client.embeddings.create({
58+
input: 'Error embedding test!',
59+
model: 'error-model',
60+
});
61+
} catch {
62+
// Error is expected and handled
63+
}
64+
});
65+
}
66+
67+
run();

‎dev-packages/node-integration-tests/suites/tracing/openai/test.ts‎

Lines changed: 99 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ describe('OpenAI integration', () => {
66
cleanupChildProcesses();
77
});
88

9-
const EXPECTED_TRANSACTION_DEFAULT_PII_FALSE = {
9+
const EXPECTED_TRANSACTION_DEFAULT_PII_FALSE_CHAT = {
1010
transaction: 'main',
1111
spans: expect.arrayContaining([
1212
// First span - basic chat completion without PII
@@ -144,45 +144,10 @@ describe('OpenAI integration', () => {
144144
origin: 'auto.ai.openai',
145145
status: 'internal_error',
146146
}),
147-
// Seventh span - embeddings API
148-
expect.objectContaining({
149-
data: {
150-
'gen_ai.operation.name': 'embeddings',
151-
'sentry.op': 'gen_ai.embeddings',
152-
'sentry.origin': 'auto.ai.openai',
153-
'gen_ai.system': 'openai',
154-
'gen_ai.request.model': 'text-embedding-3-small',
155-
'gen_ai.request.encoding_format': 'float',
156-
'gen_ai.request.dimensions': 1536,
157-
'gen_ai.response.model': 'text-embedding-3-small',
158-
'gen_ai.usage.input_tokens': 10,
159-
'gen_ai.usage.total_tokens': 10,
160-
'openai.response.model': 'text-embedding-3-small',
161-
'openai.usage.prompt_tokens': 10,
162-
},
163-
description: 'embeddings text-embedding-3-small',
164-
op: 'gen_ai.embeddings',
165-
origin: 'auto.ai.openai',
166-
status: 'ok',
167-
}),
168-
// Eighth span - embeddings API error model
169-
expect.objectContaining({
170-
data: {
171-
'gen_ai.operation.name': 'embeddings',
172-
'sentry.op': 'gen_ai.embeddings',
173-
'sentry.origin': 'auto.ai.openai',
174-
'gen_ai.system': 'openai',
175-
'gen_ai.request.model': 'error-model',
176-
},
177-
description: 'embeddings error-model',
178-
op: 'gen_ai.embeddings',
179-
origin: 'auto.ai.openai',
180-
status: 'internal_error',
181-
}),
182147
]),
183148
};
184149

185-
const EXPECTED_TRANSACTION_DEFAULT_PII_TRUE = {
150+
const EXPECTED_TRANSACTION_DEFAULT_PII_TRUE_CHAT = {
186151
transaction: 'main',
187152
spans: expect.arrayContaining([
188153
// First span - basic chat completion with PII
@@ -332,7 +297,64 @@ describe('OpenAI integration', () => {
332297
origin: 'auto.ai.openai',
333298
status: 'internal_error',
334299
}),
335-
// Seventh span - embeddings API with PII
300+
]),
301+
};
302+
303+
const EXPECTED_TRANSACTION_WITH_OPTIONS = {
304+
transaction: 'main',
305+
spans: expect.arrayContaining([
306+
// Check that custom options are respected
307+
expect.objectContaining({
308+
data: expect.objectContaining({
309+
'gen_ai.request.messages': expect.any(String), // Should include messages when recordInputs: true
310+
'gen_ai.response.text': expect.any(String), // Should include response text when recordOutputs: true
311+
}),
312+
}),
313+
// Check that custom options are respected for streaming
314+
expect.objectContaining({
315+
data: expect.objectContaining({
316+
'gen_ai.request.messages': expect.any(String), // Should include messages when recordInputs: true
317+
'gen_ai.response.text': expect.any(String), // Should include response text when recordOutputs: true
318+
'gen_ai.request.stream': true, // Should be marked as stream
319+
}),
320+
}),
321+
]),
322+
};
323+
324+
createEsmAndCjsTests(__dirname, 'scenario-chat.mjs', 'instrument.mjs', (createRunner, test) => {
325+
test('creates openai related spans with sendDefaultPii: false', async () => {
326+
await createRunner()
327+
.ignore('event')
328+
.expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE_CHAT })
329+
.start()
330+
.completed();
331+
});
332+
});
333+
334+
createEsmAndCjsTests(__dirname, 'scenario-chat.mjs', 'instrument-with-pii.mjs', (createRunner, test) => {
335+
test('creates openai related spans with sendDefaultPii: true', async () => {
336+
await createRunner()
337+
.ignore('event')
338+
.expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_TRUE_CHAT })
339+
.start()
340+
.completed();
341+
});
342+
});
343+
344+
createEsmAndCjsTests(__dirname, 'scenario-chat.mjs', 'instrument-with-options.mjs', (createRunner, test) => {
345+
test('creates openai related spans with custom options', async () => {
346+
await createRunner()
347+
.ignore('event')
348+
.expect({ transaction: EXPECTED_TRANSACTION_WITH_OPTIONS })
349+
.start()
350+
.completed();
351+
});
352+
});
353+
354+
const EXPECTED_TRANSACTION_DEFAULT_PII_FALSE_EMBEDDINGS = {
355+
transaction: 'main',
356+
spans: expect.arrayContaining([
357+
// First span - embeddings API
336358
expect.objectContaining({
337359
data: {
338360
'gen_ai.operation.name': 'embeddings',
@@ -342,7 +364,6 @@ describe('OpenAI integration', () => {
342364
'gen_ai.request.model': 'text-embedding-3-small',
343365
'gen_ai.request.encoding_format': 'float',
344366
'gen_ai.request.dimensions': 1536,
345-
'gen_ai.request.messages': 'Embedding test!',
346367
'gen_ai.response.model': 'text-embedding-3-small',
347368
'gen_ai.usage.input_tokens': 10,
348369
'gen_ai.usage.total_tokens': 10,
@@ -354,15 +375,14 @@ describe('OpenAI integration', () => {
354375
origin: 'auto.ai.openai',
355376
status: 'ok',
356377
}),
357-
// Eighth span - embeddings API error model with PII
378+
// Second span - embeddings API error model
358379
expect.objectContaining({
359380
data: {
360381
'gen_ai.operation.name': 'embeddings',
361382
'sentry.op': 'gen_ai.embeddings',
362383
'sentry.origin': 'auto.ai.openai',
363384
'gen_ai.system': 'openai',
364385
'gen_ai.request.model': 'error-model',
365-
'gen_ai.request.messages': 'Error embedding test!',
366386
},
367387
description: 'embeddings error-model',
368388
op: 'gen_ai.embeddings',
@@ -372,52 +392,63 @@ describe('OpenAI integration', () => {
372392
]),
373393
};
374394

375-
const EXPECTED_TRANSACTION_WITH_OPTIONS = {
395+
const EXPECTED_TRANSACTION_DEFAULT_PII_TRUE_EMBEDDINGS = {
376396
transaction: 'main',
377397
spans: expect.arrayContaining([
378-
// Check that custom options are respected
398+
// First span - embeddings API with PII
379399
expect.objectContaining({
380-
data: expect.objectContaining({
381-
'gen_ai.request.messages': expect.any(String), // Should include messages when recordInputs: true
382-
'gen_ai.response.text': expect.any(String), // Should include response text when recordOutputs: true
383-
}),
400+
data: {
401+
'gen_ai.operation.name': 'embeddings',
402+
'sentry.op': 'gen_ai.embeddings',
403+
'sentry.origin': 'auto.ai.openai',
404+
'gen_ai.system': 'openai',
405+
'gen_ai.request.model': 'text-embedding-3-small',
406+
'gen_ai.request.encoding_format': 'float',
407+
'gen_ai.request.dimensions': 1536,
408+
'gen_ai.request.messages': 'Embedding test!',
409+
'gen_ai.response.model': 'text-embedding-3-small',
410+
'gen_ai.usage.input_tokens': 10,
411+
'gen_ai.usage.total_tokens': 10,
412+
'openai.response.model': 'text-embedding-3-small',
413+
'openai.usage.prompt_tokens': 10,
414+
},
415+
description: 'embeddings text-embedding-3-small',
416+
op: 'gen_ai.embeddings',
417+
origin: 'auto.ai.openai',
418+
status: 'ok',
384419
}),
385-
// Check that custom options are respected for streaming
420+
// Second span - embeddings API error model with PII
386421
expect.objectContaining({
387-
data: expect.objectContaining({
388-
'gen_ai.request.messages': expect.any(String), // Should include messages when recordInputs: true
389-
'gen_ai.response.text': expect.any(String), // Should include response text when recordOutputs: true
390-
'gen_ai.request.stream': true, // Should be marked as stream
391-
}),
422+
data: {
423+
'gen_ai.operation.name': 'embeddings',
424+
'sentry.op': 'gen_ai.embeddings',
425+
'sentry.origin': 'auto.ai.openai',
426+
'gen_ai.system': 'openai',
427+
'gen_ai.request.model': 'error-model',
428+
'gen_ai.request.messages': 'Error embedding test!',
429+
},
430+
description: 'embeddings error-model',
431+
op: 'gen_ai.embeddings',
432+
origin: 'auto.ai.openai',
433+
status: 'internal_error',
392434
}),
393435
]),
394436
};
395-
396-
createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test) => {
437+
createEsmAndCjsTests(__dirname, 'scenario-embeddings.mjs', 'instrument.mjs', (createRunner, test) => {
397438
test('creates openai related spans with sendDefaultPii: false', async () => {
398439
await createRunner()
399440
.ignore('event')
400-
.expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE })
441+
.expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE_EMBEDDINGS })
401442
.start()
402443
.completed();
403444
});
404445
});
405446

406-
createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-with-pii.mjs', (createRunner, test) => {
447+
createEsmAndCjsTests(__dirname, 'scenario-embeddings.mjs', 'instrument-with-pii.mjs', (createRunner, test) => {
407448
test('creates openai related spans with sendDefaultPii: true', async () => {
408449
await createRunner()
409450
.ignore('event')
410-
.expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_TRUE })
411-
.start()
412-
.completed();
413-
});
414-
});
415-
416-
createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-with-options.mjs', (createRunner, test) => {
417-
test('creates openai related spans with custom options', async () => {
418-
await createRunner()
419-
.ignore('event')
420-
.expect({ transaction: EXPECTED_TRANSACTION_WITH_OPTIONS })
451+
.expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_TRUE_EMBEDDINGS })
421452
.start()
422453
.completed();
423454
});

0 commit comments

Comments
 (0)