Skip to content
This repository was archived by the owner on Nov 27, 2024. It is now read-only.

Commit 2ad5ae0

Browse files
saddam213saddam213
authored
Merge pull request #120 from saddam213/Stream
Video Stream Support
2 parents ba6475f + 38028b8 commit 2ad5ae0

File tree

17 files changed

+561
-186
lines changed

17 files changed

+561
-186
lines changed

‎OnnxStack.Console/Examples/FeatureExtractorVideoExample.cs‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
using OnnxStack.Core.Video;
2+
using OnnxStack.FeatureExtractor.Pipelines;
3+
using OnnxStack.StableDiffusion.Config;
4+
5+
namespace OnnxStack.Console.Runner
6+
{
7+
public sealed class FeatureExtractorVideoExample : IExampleRunner
8+
{
9+
private readonly string _outputDirectory;
10+
private readonly StableDiffusionConfig _configuration;
11+
12+
public FeatureExtractorVideoExample(StableDiffusionConfig configuration)
13+
{
14+
_configuration = configuration;
15+
_outputDirectory = Path.Combine(Directory.GetCurrentDirectory(), "Examples", nameof(FeatureExtractorVideoExample));
16+
Directory.CreateDirectory(_outputDirectory);
17+
}
18+
19+
public int Index => 13;
20+
21+
public string Name => "Feature Extractor Video Example";
22+
23+
public string Description => "Video exmaple using basic feature extractor";
24+
25+
/// <summary>
26+
/// ControlNet Example
27+
/// </summary>
28+
public async Task RunAsync()
29+
{
30+
// Read Video
31+
var videoFile = "C:\\Users\\Deven\\Pictures\\parrot.mp4";
32+
var videoInfo = await VideoHelper.ReadVideoInfoAsync(videoFile);
33+
34+
// Create pipeline
35+
var pipeline = FeatureExtractorPipeline.CreatePipeline("D:\\Repositories\\controlnet_onnx\\annotators\\canny.onnx");
36+
37+
// Create Video Stream
38+
var videoStream = VideoHelper.ReadVideoStreamAsync(videoFile, videoInfo.FrameRate);
39+
40+
// Create Pipeline Stream
41+
var pipelineStream = pipeline.RunAsync(videoStream);
42+
43+
// Write Video Stream
44+
await VideoHelper.WriteVideoStreamAsync(videoInfo, pipelineStream, Path.Combine(_outputDirectory, $"Result.mp4"));
45+
46+
//Unload
47+
await pipeline.UnloadAsync();
48+
}
49+
}
50+
}

‎OnnxStack.Console/Examples/UpscaleExample.cs‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,13 +29,10 @@ public async Task RunAsync()
2929

3030
// Run pipeline
3131
var result = await pipeline.RunAsync(inputImage);
32-
33-
// Create Image from Tensor result
34-
var image = new OnnxImage(result, ImageNormalizeType.ZeroToOne);
35-
32+
3633
// Save Image File
3734
var outputFilename = Path.Combine(_outputDirectory, $"Upscaled.png");
38-
await image.SaveAsync(outputFilename);
35+
await result.SaveAsync(outputFilename);
3936

4037
// Unload
4138
await pipeline.UnloadAsync();

‎OnnxStack.Console/Examples/UpscaleStreamExample.cs‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
using OnnxStack.Core.Video;
2+
using OnnxStack.FeatureExtractor.Pipelines;
3+
4+
namespace OnnxStack.Console.Runner
5+
{
6+
public sealed class UpscaleStreamExample : IExampleRunner
7+
{
8+
private readonly string _outputDirectory;
9+
10+
public UpscaleStreamExample()
11+
{
12+
_outputDirectory = Path.Combine(Directory.GetCurrentDirectory(), "Examples", nameof(UpscaleStreamExample));
13+
Directory.CreateDirectory(_outputDirectory);
14+
}
15+
16+
public int Index => 10;
17+
18+
public string Name => "Streaming Video Upscale Demo";
19+
20+
public string Description => "Upscales a video stream";
21+
22+
public async Task RunAsync()
23+
{
24+
// Read Video
25+
var videoFile = "C:\\Users\\Deven\\Pictures\\parrot.mp4";
26+
var videoInfo = await VideoHelper.ReadVideoInfoAsync(videoFile);
27+
28+
// Create pipeline
29+
var pipeline = ImageUpscalePipeline.CreatePipeline("D:\\Repositories\\upscaler\\SwinIR\\003_realSR_BSRGAN_DFO_s64w8_SwinIR-M_x4_GAN.onnx", 4);
30+
31+
// Load pipeline
32+
await pipeline.LoadAsync();
33+
34+
// Create Video Stream
35+
var videoStream = VideoHelper.ReadVideoStreamAsync(videoFile, videoInfo.FrameRate);
36+
37+
// Create Pipeline Stream
38+
var pipelineStream = pipeline.RunAsync(videoStream);
39+
40+
// Write Video Stream
41+
await VideoHelper.WriteVideoStreamAsync(videoInfo, pipelineStream, Path.Combine(_outputDirectory, $"Result.mp4"));
42+
43+
//Unload
44+
await pipeline.UnloadAsync();
45+
}
46+
47+
}
48+
}

‎OnnxStack.Console/Examples/VideoToVideoStreamExample.cs‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
using OnnxStack.Core.Video;
2+
using OnnxStack.FeatureExtractor.Pipelines;
3+
using OnnxStack.StableDiffusion.Config;
4+
using OnnxStack.StableDiffusion.Enums;
5+
using OnnxStack.StableDiffusion.Pipelines;
6+
7+
namespace OnnxStack.Console.Runner
8+
{
9+
public sealed class VideoToVideoStreamExample : IExampleRunner
10+
{
11+
private readonly string _outputDirectory;
12+
private readonly StableDiffusionConfig _configuration;
13+
14+
public VideoToVideoStreamExample(StableDiffusionConfig configuration)
15+
{
16+
_configuration = configuration;
17+
_outputDirectory = Path.Combine(Directory.GetCurrentDirectory(), "Examples", nameof(VideoToVideoStreamExample));
18+
Directory.CreateDirectory(_outputDirectory);
19+
}
20+
21+
public int Index => 4;
22+
23+
public string Name => "Video To Video Stream Demo";
24+
25+
public string Description => "Video Stream Stable Diffusion Inference";
26+
27+
public async Task RunAsync()
28+
{
29+
30+
// Read Video
31+
var videoFile = "C:\\Users\\Deven\\Pictures\\gidsgphy.gif";
32+
var videoInfo = await VideoHelper.ReadVideoInfoAsync(videoFile);
33+
34+
// Loop though the appsettings.json model sets
35+
foreach (var modelSet in _configuration.ModelSets)
36+
{
37+
OutputHelpers.WriteConsole($"Loading Model `{modelSet.Name}`...", ConsoleColor.Cyan);
38+
39+
// Create Pipeline
40+
var pipeline = PipelineBase.CreatePipeline(modelSet);
41+
42+
// Preload Models (optional)
43+
await pipeline.LoadAsync();
44+
45+
// Add text and video to prompt
46+
var promptOptions = new PromptOptions
47+
{
48+
Prompt = "Iron Man",
49+
DiffuserType = DiffuserType.ImageToImage
50+
};
51+
52+
53+
// Create Video Stream
54+
var videoStream = VideoHelper.ReadVideoStreamAsync(videoFile, videoInfo.FrameRate);
55+
56+
// Create Pipeline Stream
57+
var pipelineStream = pipeline.GenerateVideoStreamAsync(videoStream, promptOptions, progressCallback:OutputHelpers.ProgressCallback);
58+
59+
// Write Video Stream
60+
await VideoHelper.WriteVideoStreamAsync(videoInfo, pipelineStream, Path.Combine(_outputDirectory, $"{modelSet.PipelineType}.mp4"));
61+
62+
//Unload
63+
await pipeline.UnloadAsync();
64+
}
65+
}
66+
}
67+
}

‎OnnxStack.Core/Extensions/TensorExtension.cs‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
using Microsoft.ML.OnnxRuntime.Tensors;
22
using System;
33
using System.Collections.Generic;
4+
using System.Linq;
45

56
namespace OnnxStack.Core
67
{
@@ -75,6 +76,33 @@ public static IEnumerable<DenseTensor<float>> SplitBatch(this DenseTensor<float>
7576
}
7677

7778

79+
/// <summary>
80+
/// Joins the tensors across the 0 axis.
81+
/// </summary>
82+
/// <param name="tensors">The tensors.</param>
83+
/// <param name="axis">The axis.</param>
84+
/// <returns></returns>
85+
/// <exception cref="System.NotImplementedException">Only axis 0 is supported</exception>
86+
public static DenseTensor<float> Join(this IList<DenseTensor<float>> tensors, int axis = 0)
87+
{
88+
if (axis != 0)
89+
throw new NotImplementedException("Only axis 0 is supported");
90+
91+
var tensor = tensors.First();
92+
var dimensions = tensor.Dimensions.ToArray();
93+
dimensions[0] *= tensors.Count;
94+
95+
var newLength = (int)tensor.Length;
96+
var buffer = new float[newLength * tensors.Count].AsMemory();
97+
for (int i = 0; i < tensors.Count(); i++)
98+
{
99+
var start = i * newLength;
100+
tensors[i].Buffer.CopyTo(buffer[start..]);
101+
}
102+
return new DenseTensor<float>(buffer, dimensions);
103+
}
104+
105+
78106
/// <summary>
79107
/// Concatenates the specified tensors along the specified axis.
80108
/// </summary>

‎OnnxStack.Core/Image/OnnxImage.cs‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,20 @@ public byte[] GetImageBytes()
144144
}
145145

146146

147+
/// <summary>
148+
/// Gets the image as bytes.
149+
/// </summary>
150+
/// <returns></returns>
151+
public async Task<byte[]> GetImageBytesAsync()
152+
{
153+
using (var memoryStream = new MemoryStream())
154+
{
155+
await _imageData.SaveAsPngAsync(memoryStream);
156+
return memoryStream.ToArray();
157+
}
158+
}
159+
160+
147161
/// <summary>
148162
/// Gets the image as stream.
149163
/// </summary>
@@ -156,6 +170,40 @@ public Stream GetImageStream()
156170
}
157171

158172

173+
/// <summary>
174+
/// Gets the image as stream.
175+
/// </summary>
176+
/// <returns></returns>
177+
public async Task<Stream> GetImageStreamAsync()
178+
{
179+
var memoryStream = new MemoryStream();
180+
await _imageData.SaveAsPngAsync(memoryStream);
181+
return memoryStream;
182+
}
183+
184+
185+
/// <summary>
186+
/// Copies the image to stream.
187+
/// </summary>
188+
/// <param name="destination">The destination.</param>
189+
/// <returns></returns>
190+
public void CopyToStream(Stream destination)
191+
{
192+
_imageData.SaveAsPng(destination);
193+
}
194+
195+
196+
/// <summary>
197+
/// Copies the image to stream.
198+
/// </summary>
199+
/// <param name="destination">The destination.</param>
200+
/// <returns></returns>
201+
public Task CopyToStreamAsync(Stream destination)
202+
{
203+
return _imageData.SaveAsPngAsync(destination);
204+
}
205+
206+
159207
/// <summary>
160208
/// Gets the image as tensor.
161209
/// </summary>

‎OnnxStack.Core/Video/OnnxVideo.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ public OnnxVideo(VideoInfo info, IEnumerable<DenseTensor<float>> videoTensors)
8888
/// <summary>
8989
/// Gets the aspect ratio.
9090
/// </summary>
91-
public double AspectRatio => (double)_info.Width / _info.Height;
91+
public double AspectRatio => _info.AspectRatio;
9292

9393
/// <summary>
9494
/// Gets a value indicating whether this instance has video.

0 commit comments

Comments
 (0)