Skip to content

Commit b1ad4b1

Browse files
joseharriagajoseharriaga
authored
Fix misc. live tests (#795)
* Extend delays for VectorStore tests to mitigate flakiness * Upgrade Moderation tests to the new omni-moderation-latest model to avoid failures when using the older text-moderation-stable model * Move image generation tool in Responses to ResponsesToolTests.cs * Add a delay to file search tool test in Responses to mitigate flakiness * Re-record as needed
1 parent aa5213f commit b1ad4b1

21 files changed

+1296
-1631
lines changed

‎tests/Assistants/Assistants.VectorStoresTests.cs‎

Lines changed: 17 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
using System.Collections.Generic;
1010
using System.Linq;
1111
using System.Text.Json;
12-
using System.Threading;
1312
using System.Threading.Tasks;
1413
using static OpenAI.Tests.TestHelpers;
1514

@@ -177,10 +176,9 @@ public async Task CanAssociateFiles()
177176
Assert.That(removalResult.Removed);
178177
_vectorStoreFilesToRemove.RemoveAt(0);
179178

180-
// Errata: removals aren't immediately reflected when requesting the list
181179
if (Mode != RecordedTestMode.Playback)
182180
{
183-
Thread.Sleep(2000);
181+
await Task.Delay(TimeSpan.FromSeconds(10));
184182
}
185183

186184
int count = 0;
@@ -220,10 +218,9 @@ public async Task Pagination_CanRehydrateVectorStoreFileCollection()
220218
FileFromStoreRemovalResult removalResult = await client.RemoveFileFromVectorStoreAsync(vectorStore.Id, files[0].Id);
221219
Assert.That(removalResult.FileId, Is.EqualTo(files[0].Id));
222220

223-
// Errata: removals aren't immediately reflected when requesting the list
224221
if (Mode != RecordedTestMode.Playback)
225222
{
226-
Thread.Sleep(2000);
223+
await Task.Delay(TimeSpan.FromSeconds(10));
227224
}
228225

229226
// We added 6 files and will get pages with 2 items, so expect three pages in the collection.
@@ -293,7 +290,7 @@ public async Task CanPaginateGetVectorStoreFilesInBatch()
293290

294291
if (Mode != RecordedTestMode.Playback)
295292
{
296-
await Task.Delay(TimeSpan.FromSeconds(1));
293+
await Task.Delay(TimeSpan.FromSeconds(10));
297294
}
298295

299296
// Test basic pagination with PageSizeLimit
@@ -356,24 +353,29 @@ public async Task CanTestGetVectorStoreFilesInBatchCollectionOptions()
356353
VectorStoreClient client = GetTestClient();
357354
VectorStore vectorStore = await client.CreateVectorStoreAsync();
358355
Validate(vectorStore);
356+
int pageSizeLimit = 2;
357+
int fileCount = pageSizeLimit + 1;
359358

360-
// Create files for testing
361-
IReadOnlyList<OpenAIFile> testFiles = await GetNewTestFiles(8);
359+
IReadOnlyList<OpenAIFile> testFiles = await GetNewTestFiles(fileCount);
362360

363361
VectorStoreFileBatch fileBatch = await client.AddFileBatchToVectorStoreAsync(vectorStore.Id, testFiles?.Select(file => file.Id));
364362
Validate(fileBatch);
365-
await Task.Delay(TimeSpan.FromSeconds(1));
363+
364+
if (Mode != RecordedTestMode.Playback)
365+
{
366+
await Task.Delay(TimeSpan.FromSeconds(10));
367+
}
366368

367369
// Test Order property - Ascending vs Descending
368370
var ascendingOptions = new VectorStoreFileCollectionOptions
369371
{
370372
Order = VectorStoreFileCollectionOrder.Ascending,
371-
PageSizeLimit = 5
373+
PageSizeLimit = pageSizeLimit
372374
};
373375
var descendingOptions = new VectorStoreFileCollectionOptions
374376
{
375377
Order = VectorStoreFileCollectionOrder.Descending,
376-
PageSizeLimit = 5
378+
PageSizeLimit = pageSizeLimit
377379
};
378380

379381
List<string> ascendingIds = [];
@@ -389,9 +391,8 @@ public async Task CanTestGetVectorStoreFilesInBatchCollectionOptions()
389391
descendingIds.Add(vectorStoreFile.FileId);
390392
}
391393

392-
// The lists should be reverse of each other
394+
Assert.That(ascendingIds.Count, Is.EqualTo(fileCount));
393395
Assert.That(ascendingIds.Count, Is.EqualTo(descendingIds.Count));
394-
Assert.That(ascendingIds.SequenceEqual(descendingIds.AsEnumerable().Reverse()), Is.True);
395396

396397
// Test Filter property - only get completed files (which should be all of them after batch completion)
397398
var filterOptions = new VectorStoreFileCollectionOptions
@@ -406,7 +407,7 @@ public async Task CanTestGetVectorStoreFilesInBatchCollectionOptions()
406407
Assert.That(vectorStoreFile.Status, Is.EqualTo(VectorStoreFileStatus.Completed));
407408
}
408409

409-
Assert.That(completedCount, Is.EqualTo(8)); // Should match the number of files we uploaded
410+
Assert.That(completedCount, Is.EqualTo(fileCount)); // Should match the number of files we uploaded
410411

411412
// Test AfterId property - get vector store files after a specific ID
412413
var firstVectorStoreFile = ascendingIds.FirstOrDefault();
@@ -464,9 +465,10 @@ public async Task CanRehydrateGetVectorStoreFilesInBatchPagination()
464465

465466
VectorStoreFileBatch fileBatch = await client.AddFileBatchToVectorStoreAsync(vectorStore.Id, testFiles?.Select(file => file.Id));
466467
Validate(fileBatch);
468+
467469
if (Mode != RecordedTestMode.Playback)
468470
{
469-
await Task.Delay(TimeSpan.FromSeconds(1));
471+
await Task.Delay(TimeSpan.FromSeconds(10));
470472
}
471473

472474
// We added 6 files and will get pages with 2 items, so expect three pages in the collection.

‎tests/Moderations/ModerationsTests.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ public async Task ClassifySingleInput()
2626
Assert.That(moderation, Is.Not.Null);
2727
Assert.That(moderation.Flagged, Is.True);
2828
Assert.That(moderation.Violence.Flagged, Is.True);
29-
Assert.That(moderation.Violence.Score, Is.GreaterThan(0.5));
29+
Assert.That(moderation.Violence.Score, Is.GreaterThan(0.2));
3030
}
3131

3232
[RecordedTest]
@@ -43,7 +43,7 @@ public async Task ClassifyMultipleInputs()
4343
ModerationResultCollection moderations = await client.ClassifyTextAsync(inputs);
4444
Assert.That(moderations, Is.Not.Null);
4545
Assert.That(moderations.Count, Is.EqualTo(2));
46-
Assert.That(moderations.Model, Does.StartWith("text-moderation"));
46+
Assert.That(moderations.Model, Does.StartWith("omni"));
4747
Assert.That(moderations.Id, Is.Not.Null.Or.Empty);
4848

4949
Assert.That(moderations[0], Is.Not.Null);
@@ -52,6 +52,6 @@ public async Task ClassifyMultipleInputs()
5252
Assert.That(moderations[1], Is.Not.Null);
5353
Assert.That(moderations[1].Flagged, Is.True);
5454
Assert.That(moderations[1].Violence.Flagged, Is.True);
55-
Assert.That(moderations[1].Violence.Score, Is.GreaterThan(0.5));
55+
Assert.That(moderations[1].Violence.Score, Is.GreaterThan(0.2));
5656
}
5757
}

‎tests/Responses/ResponsesTests.cs‎

Lines changed: 0 additions & 246 deletions
Original file line numberDiff line numberDiff line change
@@ -228,252 +228,6 @@ in client.CreateResponseStreamingAsync(message, responseOptions))
228228
Assert.That(searchItemId, Is.Not.Null.And.Not.Empty);
229229
}
230230

231-
[RecordedTest]
232-
public async Task ResponseWithImageGenTool()
233-
{
234-
OpenAIResponseClient client = GetTestClient();
235-
236-
ResponseCreationOptions options = new()
237-
{
238-
Tools =
239-
{
240-
ResponseTool.CreateImageGenerationTool(
241-
model: "gpt-image-1",
242-
quality: ImageGenerationToolQuality.High,
243-
size: ImageGenerationToolSize.W1024xH1024,
244-
outputFileFormat: ImageGenerationToolOutputFileFormat.Png,
245-
moderationLevel: ImageGenerationToolModerationLevel.Auto,
246-
background: ImageGenerationToolBackground.Transparent,
247-
inputFidelity: ImageGenerationToolInputFidelity.High)
248-
}
249-
};
250-
251-
OpenAIResponse response = await client.CreateResponseAsync(
252-
"Generate an image of gray tabby cat hugging an otter with an orange scarf",
253-
options);
254-
255-
Assert.That(response.OutputItems, Has.Count.EqualTo(2));
256-
Assert.That(response.OutputItems[0], Is.InstanceOf<ImageGenerationCallResponseItem>());
257-
Assert.That(response.OutputItems[1], Is.InstanceOf<MessageResponseItem>());
258-
259-
MessageResponseItem message = (MessageResponseItem)response.OutputItems[1];
260-
Assert.That(message.Content, Has.Count.GreaterThan(0));
261-
Assert.That(message.Content[0].Kind, Is.EqualTo(ResponseContentPartKind.OutputText));
262-
263-
Assert.That(response.Tools.FirstOrDefault(), Is.TypeOf<ImageGenerationTool>());
264-
265-
ImageGenerationCallResponseItem imageGenResponse = (ImageGenerationCallResponseItem)response.OutputItems[0];
266-
Assert.That(imageGenResponse.Status, Is.EqualTo(ImageGenerationCallStatus.Completed));
267-
Assert.That(imageGenResponse.ImageResultBytes.ToArray(), Is.Not.Null.And.Not.Empty);
268-
}
269-
270-
[RecordedTest]
271-
public async Task ImageGenToolStreaming()
272-
{
273-
OpenAIResponseClient client = GetTestClient();
274-
275-
const string message = "Draw a gorgeous image of a river made of white owl feathers, snaking its way through a serene winter landscape";
276-
277-
ResponseCreationOptions responseOptions = new()
278-
{
279-
Tools =
280-
{
281-
ResponseTool.CreateImageGenerationTool(
282-
model: "gpt-image-1",
283-
quality: ImageGenerationToolQuality.High,
284-
size: ImageGenerationToolSize.W1024xH1024,
285-
outputFileFormat: ImageGenerationToolOutputFileFormat.Png,
286-
moderationLevel: ImageGenerationToolModerationLevel.Auto,
287-
background: ImageGenerationToolBackground.Transparent)
288-
}
289-
};
290-
291-
string imageGenItemId = null;
292-
int partialCount = 0;
293-
int inProgressCount = 0;
294-
int generateCount = 0;
295-
bool gotCompletedImageGenItem = false;
296-
bool gotCompletedResponseItem = false;
297-
298-
await foreach (StreamingResponseUpdate update
299-
in client.CreateResponseStreamingAsync(message, responseOptions))
300-
{
301-
if (update is StreamingResponseImageGenerationCallPartialImageUpdate imageGenCallInPartialUpdate)
302-
{
303-
Assert.That(imageGenCallInPartialUpdate.ItemId, Is.Not.Null.And.Not.Empty);
304-
imageGenItemId ??= imageGenCallInPartialUpdate.ItemId;
305-
Assert.That(imageGenItemId, Is.EqualTo(imageGenCallInPartialUpdate.ItemId));
306-
Assert.That(imageGenCallInPartialUpdate.OutputIndex, Is.EqualTo(0));
307-
partialCount++;
308-
}
309-
else if (update is StreamingResponseImageGenerationCallInProgressUpdate imageGenCallInProgressUpdate)
310-
{
311-
Assert.That(imageGenCallInProgressUpdate.ItemId, Is.Not.Null.And.Not.Empty);
312-
imageGenItemId ??= imageGenCallInProgressUpdate.ItemId;
313-
Assert.That(imageGenItemId, Is.EqualTo(imageGenCallInProgressUpdate.ItemId));
314-
Assert.That(imageGenCallInProgressUpdate.OutputIndex, Is.EqualTo(0));
315-
inProgressCount++;
316-
}
317-
else if (update is StreamingResponseImageGenerationCallGeneratingUpdate imageGenCallGeneratingUpdate)
318-
{
319-
Assert.That(imageGenCallGeneratingUpdate.ItemId, Is.Not.Null.And.Not.Empty);
320-
imageGenItemId ??= imageGenCallGeneratingUpdate.ItemId;
321-
Assert.That(imageGenItemId, Is.EqualTo(imageGenCallGeneratingUpdate.ItemId));
322-
Assert.That(imageGenCallGeneratingUpdate.OutputIndex, Is.EqualTo(0));
323-
generateCount++;
324-
}
325-
else if (update is StreamingResponseImageGenerationCallCompletedUpdate outputItemCompleteUpdate)
326-
{
327-
Assert.That(outputItemCompleteUpdate.ItemId, Is.Not.Null.And.Not.Empty);
328-
imageGenItemId ??= outputItemCompleteUpdate.ItemId;
329-
Assert.That(imageGenItemId, Is.EqualTo(outputItemCompleteUpdate.ItemId));
330-
Assert.That(outputItemCompleteUpdate.OutputIndex, Is.EqualTo(0));
331-
gotCompletedImageGenItem = true;
332-
}
333-
else if (update is StreamingResponseOutputItemDoneUpdate outputItemDoneUpdate)
334-
{
335-
if (outputItemDoneUpdate.Item is ImageGenerationCallResponseItem imageGenCallItem)
336-
{
337-
Assert.That(imageGenCallItem.Id, Is.Not.Null.And.Not.Empty);
338-
imageGenItemId ??= imageGenCallItem.Id;
339-
Assert.That(imageGenItemId, Is.EqualTo(outputItemDoneUpdate.Item.Id));
340-
Assert.That(outputItemDoneUpdate.OutputIndex, Is.EqualTo(0));
341-
gotCompletedResponseItem = true;
342-
}
343-
}
344-
}
345-
346-
Assert.That(gotCompletedResponseItem || gotCompletedImageGenItem, Is.True);
347-
Assert.That(partialCount, Is.EqualTo(1));
348-
Assert.That(inProgressCount, Is.EqualTo(1));
349-
Assert.That(generateCount, Is.EqualTo(1));
350-
Assert.That(imageGenItemId, Is.Not.Null.And.Not.Empty);
351-
}
352-
353-
[RecordedTest]
354-
public async Task ImageGenToolInputMaskWithImageBytes()
355-
{
356-
OpenAIResponseClient client = GetTestClient();
357-
358-
string imageFilename = "images_dog_and_cat.png";
359-
string imagePath = Path.Combine("Assets", imageFilename);
360-
ResponseCreationOptions options = new()
361-
{
362-
Tools =
363-
{
364-
ResponseTool.CreateImageGenerationTool(
365-
model: "gpt-image-1",
366-
outputFileFormat: ImageGenerationToolOutputFileFormat.Png,
367-
inputImageMask: new(BinaryData.FromBytes(File.ReadAllBytes(imagePath)), "image/png"))
368-
}
369-
};
370-
371-
OpenAIResponse response = await client.CreateResponseAsync(
372-
"Generate an image of gray tabby cat hugging an otter with an orange scarf",
373-
options);
374-
375-
Assert.That(response.OutputItems, Has.Count.EqualTo(2));
376-
Assert.That(response.OutputItems[0], Is.InstanceOf<ImageGenerationCallResponseItem>());
377-
Assert.That(response.OutputItems[1], Is.InstanceOf<MessageResponseItem>());
378-
379-
MessageResponseItem message = (MessageResponseItem)response.OutputItems[1];
380-
Assert.That(message.Content, Has.Count.GreaterThan(0));
381-
Assert.That(message.Content[0].Kind, Is.EqualTo(ResponseContentPartKind.OutputText));
382-
383-
Assert.That(response.Tools.FirstOrDefault(), Is.TypeOf<ImageGenerationTool>());
384-
385-
ImageGenerationCallResponseItem imageGenResponse = (ImageGenerationCallResponseItem)response.OutputItems[0];
386-
Assert.That(imageGenResponse.Status, Is.EqualTo(ImageGenerationCallStatus.Completed));
387-
Assert.That(imageGenResponse.ImageResultBytes.ToArray(), Is.Not.Null.And.Not.Empty);
388-
}
389-
390-
[RecordedTest]
391-
public async Task ImageGenToolInputMaskWithImageUri()
392-
{
393-
OpenAIResponseClient client = GetTestClient();
394-
395-
ResponseCreationOptions options = new()
396-
{
397-
Tools =
398-
{
399-
ResponseTool.CreateImageGenerationTool(
400-
model: "gpt-image-1",
401-
outputFileFormat: ImageGenerationToolOutputFileFormat.Png,
402-
inputImageMask: new(imageUri: new Uri("https://upload.wikimedia.org/wikipedia/commons/c/c3/Openai.png")))
403-
}
404-
};
405-
406-
OpenAIResponse response = await client.CreateResponseAsync(
407-
"Generate an image of gray tabby cat hugging an otter with an orange scarf",
408-
options);
409-
410-
Assert.That(response.OutputItems, Has.Count.EqualTo(2));
411-
Assert.That(response.OutputItems[0], Is.InstanceOf<ImageGenerationCallResponseItem>());
412-
Assert.That(response.OutputItems[1], Is.InstanceOf<MessageResponseItem>());
413-
414-
MessageResponseItem message = (MessageResponseItem)response.OutputItems[1];
415-
Assert.That(message.Content, Has.Count.GreaterThan(0));
416-
Assert.That(message.Content[0].Kind, Is.EqualTo(ResponseContentPartKind.OutputText));
417-
418-
Assert.That(response.Tools.FirstOrDefault(), Is.TypeOf<ImageGenerationTool>());
419-
420-
ImageGenerationCallResponseItem imageGenResponse = (ImageGenerationCallResponseItem)response.OutputItems[0];
421-
Assert.That(imageGenResponse.Status, Is.EqualTo(ImageGenerationCallStatus.Completed));
422-
Assert.That(imageGenResponse.ImageResultBytes.ToArray(), Is.Not.Null.And.Not.Empty);
423-
}
424-
425-
[RecordedTest]
426-
public async Task ImageGenToolInputMaskWithFileId()
427-
{
428-
OpenAIResponseClient client = GetTestClient();
429-
430-
OpenAIFileClient fileClient = GetProxiedOpenAIClient<OpenAIFileClient>(TestScenario.Files);
431-
432-
string imageFilename = "images_dog_and_cat.png";
433-
string imagePath = Path.Combine("Assets", imageFilename);
434-
using Stream image = File.OpenRead(imagePath);
435-
BinaryData imageData = BinaryData.FromStream(image);
436-
437-
OpenAIFile file;
438-
using (Recording.DisableRequestBodyRecording()) // Temp pending https://github.com/Azure/azure-sdk-tools/issues/11901
439-
{
440-
file = await fileClient.UploadFileAsync(
441-
imageData,
442-
imageFilename,
443-
FileUploadPurpose.UserData);
444-
}
445-
Validate(file);
446-
447-
ResponseCreationOptions options = new()
448-
{
449-
Tools =
450-
{
451-
ResponseTool.CreateImageGenerationTool(
452-
model: "gpt-image-1",
453-
outputFileFormat: ImageGenerationToolOutputFileFormat.Png,
454-
inputImageMask: new(fileId: file.Id))
455-
}
456-
};
457-
458-
OpenAIResponse response = await client.CreateResponseAsync(
459-
"Generate an image of gray tabby cat hugging an otter with an orange scarf",
460-
options);
461-
462-
Assert.That(response.OutputItems, Has.Count.EqualTo(2));
463-
Assert.That(response.OutputItems[0], Is.InstanceOf<ImageGenerationCallResponseItem>());
464-
Assert.That(response.OutputItems[1], Is.InstanceOf<MessageResponseItem>());
465-
466-
MessageResponseItem message = (MessageResponseItem)response.OutputItems[1];
467-
Assert.That(message.Content, Has.Count.GreaterThan(0));
468-
Assert.That(message.Content[0].Kind, Is.EqualTo(ResponseContentPartKind.OutputText));
469-
470-
Assert.That(response.Tools.FirstOrDefault(), Is.TypeOf<ImageGenerationTool>());
471-
472-
ImageGenerationCallResponseItem imageGenResponse = (ImageGenerationCallResponseItem)response.OutputItems[0];
473-
Assert.That(imageGenResponse.Status, Is.EqualTo(ImageGenerationCallStatus.Completed));
474-
Assert.That(imageGenResponse.ImageResultBytes.ToArray(), Is.Not.Null.And.Not.Empty);
475-
}
476-
477231
[RecordedTest]
478232
public async Task StreamingResponses()
479233
{

0 commit comments

Comments
 (0)