Skip to content
This repository was archived by the owner on Nov 27, 2024. It is now read-only.

Commit 4aad8a7

Browse files
saddam213saddam213
authored
Merge pull request #103 from TheyCallMeHex/master
VideoToVideo Example Runners
2 parents 86f7c31 + 798d87e commit 4aad8a7

File tree

3 files changed

+221
-4
lines changed

3 files changed

+221
-4
lines changed

‎OnnxStack.Console/Examples/VideoToVideoExample.cs‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
using OnnxStack.Console;
2+
using OnnxStack.Core.Image;
3+
using OnnxStack.StableDiffusion.Common;
4+
using OnnxStack.StableDiffusion.Config;
5+
using OnnxStack.StableDiffusion.Enums;
6+
using SixLabors.ImageSharp.PixelFormats;
7+
using SixLabors.ImageSharp;
8+
using System;
9+
using System.Collections.Generic;
10+
using System.Diagnostics;
11+
using System.Text;
12+
using OnnxStack.Core.Video;
13+
using OnnxStack.Core.Services;
14+
15+
namespace OnnxStack.Console.Runner
16+
{
17+
public sealed class VideoToVideoExample : IExampleRunner
18+
{
19+
private readonly string _outputDirectory;
20+
private readonly StableDiffusionConfig _configuration;
21+
private readonly IStableDiffusionService _stableDiffusionService;
22+
private readonly IVideoService _videoService;
23+
24+
public VideoToVideoExample(StableDiffusionConfig configuration, IStableDiffusionService stableDiffusionService, IVideoService videoService)
25+
{
26+
_configuration = configuration;
27+
_stableDiffusionService = stableDiffusionService;
28+
_videoService = videoService;
29+
_outputDirectory = Path.Combine(Directory.GetCurrentDirectory(), "Examples", nameof(UpscaleExample));
30+
Directory.CreateDirectory(_outputDirectory);
31+
}
32+
33+
public string Name => "Video To Video Demo";
34+
35+
public string Description => "Vidio Stable Diffusion Inference";
36+
37+
public async Task RunAsync()
38+
{
39+
var model = _configuration.ModelSets.FirstOrDefault(x => x.Name == "LCM-Dreamshaper-V7");
40+
OutputHelpers.WriteConsole("Loading Model...", ConsoleColor.Cyan);
41+
await _stableDiffusionService.LoadModelAsync(model);
42+
OutputHelpers.WriteConsole("Model Loaded.", ConsoleColor.Cyan);
43+
string inputVideoPath = "C:\\Users\\Hex\\Downloads\\doit.mp4";
44+
string outputVideoPath = "C:\\Users\\Hex\\Downloads\\doitdiffused.mp4";
45+
46+
47+
var prompt = "Iron Man";
48+
var negativePrompt = "";
49+
50+
var inputVideo = File.ReadAllBytes(inputVideoPath);
51+
52+
var promptOptions = new PromptOptions
53+
{
54+
Prompt = prompt,
55+
NegativePrompt = negativePrompt,
56+
DiffuserType = DiffuserType.ImageToImage,
57+
InputVideo = new VideoInput(inputVideo)
58+
};
59+
60+
var schedulerOptions = new SchedulerOptions
61+
{
62+
SchedulerType = SchedulerType.LCM,
63+
GuidanceScale = 1f,
64+
InferenceSteps = 10,
65+
Strength = 0.35f,
66+
Height = 512,
67+
Width = 512
68+
};
69+
70+
var result = await _stableDiffusionService.GenerateAsBytesAsync(new ModelOptions(model), promptOptions, schedulerOptions);
71+
File.WriteAllBytes(outputVideoPath, result);
72+
}
73+
}
74+
}

‎OnnxStack.Console/Examples/VideoToVideoExampleFromScratch.cs‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
using OnnxStack.Core.Image;
2+
using OnnxStack.ImageUpscaler.Config;
3+
using OnnxStack.ImageUpscaler.Services;
4+
using OnnxStack.StableDiffusion.Common;
5+
using OnnxStack.StableDiffusion.Config;
6+
using SixLabors.ImageSharp.PixelFormats;
7+
using System;
8+
using System.Collections.Generic;
9+
using System.Diagnostics;
10+
using System.Text;
11+
using SixLabors.ImageSharp;
12+
using SixLabors.ImageSharp.Processing;
13+
using OnnxStack.StableDiffusion.Enums;
14+
15+
namespace OnnxStack.Console.Runner
16+
{
17+
public sealed class VideoToVideoExampleFromScratch : IExampleRunner
18+
{
19+
private readonly string _outputDirectory;
20+
private readonly StableDiffusionConfig _configuration;
21+
private readonly IStableDiffusionService _stableDiffusionService;
22+
23+
public VideoToVideoExampleFromScratch(StableDiffusionConfig configuration, IStableDiffusionService stableDiffusionService)
24+
{
25+
_configuration = configuration;
26+
_stableDiffusionService = stableDiffusionService;
27+
_outputDirectory = Path.Combine(Directory.GetCurrentDirectory(), "Examples", nameof(UpscaleExample));
28+
Directory.CreateDirectory(_outputDirectory);
29+
}
30+
31+
public string Name => "Video To Video Demo From Scratch";
32+
33+
public string Description => "Vidio Stable Diffusion Inference From Scratch";
34+
35+
public async Task RunAsync()
36+
{
37+
var model = _configuration.ModelSets.FirstOrDefault(x => x.Name == "LCM-Dreamshaper-V7");
38+
OutputHelpers.WriteConsole("Loading Model...", ConsoleColor.Cyan);
39+
await _stableDiffusionService.LoadModelAsync(model);
40+
OutputHelpers.WriteConsole("Model Loaded.", ConsoleColor.Cyan);
41+
string inputVideoPath = "C:\\Users\\Hex\\Downloads\\doit.mp4";
42+
string outputFramesPath = "C:\\Users\\Hex\\Desktop\\frames\\frame_%04d.png";
43+
string ffmpegCommand = $"-i \"{inputVideoPath}\" -vf fps=30 -c:v png -y \"{outputFramesPath}\"";
44+
string ffmpeg = @"C:\Users\Hex\Desktop\OnnxStack\ffmpeg.exe";
45+
46+
Process process = new Process
47+
{
48+
StartInfo = new ProcessStartInfo
49+
{
50+
FileName = ffmpeg,
51+
Arguments = ffmpegCommand,
52+
RedirectStandardOutput = true,
53+
RedirectStandardError = true,
54+
UseShellExecute = false,
55+
CreateNoWindow = true
56+
}
57+
};
58+
59+
process.OutputDataReceived += (sender, e) => OutputHelpers.WriteConsole(e.Data, ConsoleColor.Cyan);
60+
process.ErrorDataReceived += (sender, e) => OutputHelpers.WriteConsole(e.Data, ConsoleColor.Cyan);
61+
62+
process.Start();
63+
64+
process.BeginOutputReadLine();
65+
process.BeginErrorReadLine();
66+
67+
process.WaitForExit();
68+
69+
outputFramesPath = outputFramesPath.Replace("\\frame_%04d.png", "");
70+
71+
string[] files = Directory.GetFiles(outputFramesPath);
72+
73+
var prompt = "Iron Man";
74+
var negativePrompt = "";
75+
76+
var promptOptions = new PromptOptions
77+
{
78+
Prompt = prompt,
79+
NegativePrompt = negativePrompt,
80+
DiffuserType = DiffuserType.ImageToImage
81+
};
82+
83+
var schedulerOptions = new SchedulerOptions
84+
{
85+
SchedulerType = SchedulerType.LCM,
86+
GuidanceScale = 1f,
87+
InferenceSteps = 10,
88+
Strength = 0.35f,
89+
Height = 512,
90+
Width = 512
91+
};
92+
93+
foreach (string filePath in files)
94+
{
95+
OutputHelpers.WriteConsole($"Defusing {filePath}", ConsoleColor.Cyan);
96+
Image<Rgba32> frameDestination = new(schedulerOptions.Width, schedulerOptions.Height);
97+
var frameSource = await Image.LoadAsync(filePath);
98+
frameSource.Mutate(x => x.Resize(frameDestination.Size));
99+
promptOptions.InputImage = new InputImage(frameSource.CloneAs<Rgba32>());
100+
frameDestination = await _stableDiffusionService.GenerateAsImageAsync(new ModelOptions(model), promptOptions, schedulerOptions);
101+
await frameDestination.SaveAsPngAsync(filePath);
102+
OutputHelpers.WriteConsole($"{filePath} saved", ConsoleColor.Cyan);
103+
}
104+
105+
string outputVideoPath = "C:\\Users\\Hex\\Downloads\\doitdefused.mp4";
106+
107+
ffmpegCommand = $"ffmpeg -framerate 30 -i \"{outputFramesPath}\\frame_%04d.png\" -c:v libx264 -pix_fmt yuv420p -y \"{outputVideoPath}\"";
108+
109+
// Create a process to run the FFmpeg command
110+
process = new Process
111+
{
112+
StartInfo = new ProcessStartInfo
113+
{
114+
FileName = ffmpeg,
115+
Arguments = ffmpegCommand,
116+
RedirectStandardOutput = true,
117+
RedirectStandardError = true,
118+
UseShellExecute = false,
119+
CreateNoWindow = true
120+
}
121+
};
122+
123+
process.OutputDataReceived += (sender, e) => OutputHelpers.WriteConsole(e.Data, ConsoleColor.Cyan);
124+
process.ErrorDataReceived += (sender, e) => OutputHelpers.WriteConsole(e.Data, ConsoleColor.Cyan);
125+
126+
// Start the process
127+
process.Start();
128+
129+
process.BeginOutputReadLine();
130+
process.BeginErrorReadLine();
131+
132+
// Wait for the process to exit
133+
process.WaitForExit();
134+
}
135+
}
136+
}

‎OnnxStack.StableDiffusion/Services/StableDiffusionService.cs‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -333,12 +333,19 @@ private async Task GenerateInputVideoFrames(PromptOptions promptOptions, Action<
333333
if (!promptOptions.HasInputVideo || promptOptions.InputVideo.VideoFrames is not null)
334334
return;
335335

336-
// Already has VideoFrames
337-
if (promptOptions.InputVideo.VideoFrames is not null)
338-
return;
336+
if (promptOptions.VideoInputFPS == 0 || promptOptions.VideoOutputFPS == 0)
337+
{
338+
var videoInfo = await _videoService.GetVideoInfoAsync(promptOptions.InputVideo);
339+
if (promptOptions.VideoInputFPS == 0)
340+
promptOptions.VideoInputFPS = videoInfo.FPS;
341+
342+
if (promptOptions.VideoOutputFPS == 0)
343+
promptOptions.VideoOutputFPS = videoInfo.FPS;
344+
}
339345

346+
var videoFrame = await _videoService.CreateFramesAsync(promptOptions.InputVideo, promptOptions.VideoInputFPS);
340347
progress?.Invoke(new DiffusionProgress($"Generating video frames @ {promptOptions.VideoInputFPS}fps"));
341-
promptOptions.InputVideo.VideoFrames = await _videoService.CreateFramesAsync(promptOptions.InputVideo, promptOptions.VideoInputFPS);
348+
promptOptions.InputVideo.VideoFrames = videoFrame;
342349
}
343350
}
344351
}

0 commit comments

Comments
 (0)