ControlNet VideoToVideo Support

Author: saddam213

OnnxStack.StableDiffusion/Diffusers/DiffuserBase.cs

@@ -124,7 +124,8 @@ public virtual async Task<DenseTensor<float>> DiffuseAsync(ModelOptions modelOpt
                 foreach (var videoFrame in videoFrames)
                 {
                     frameIndex++;
-                    promptOptions.InputImage = new InputImage(videoFrame);
+                    promptOptions.InputImage = promptOptions.DiffuserType == DiffuserType.ControlNet ? default : new InputImage(videoFrame);
+                    promptOptions.InputContolImage = promptOptions.DiffuserType == DiffuserType.ImageToImage ? default : new InputImage(videoFrame);
                     var frameResultTensor = await SchedulerStepAsync(modelOptions, promptOptions, schedulerOptions, promptEmbeddings, performGuidance, schedulerFrameCallback, cancellationToken);
 
                     // Frame Progress

OnnxStack.UI/Views/VideoToVideoView.xaml

@@ -20,11 +20,18 @@
                 <Button Content="Generate" Command="{Binding GenerateCommand}" IsEnabled="{Binding SelectedModel.IsLoaded, FallbackValue=False, TargetNullValue=False}" IsDefault="True" Margin="1,0,0,0"/>
             </UniformGrid>
             <DockPanel>
-                <userControls:ModelPickerControl DockPanel.Dock="Top"
-                      UISettings="{Binding UISettings}"
-                      SupportedDiffusers="{Binding SupportedDiffusers}"
-                      IsEnabled="{Binding IsGenerating, Converter={StaticResource InverseBoolConverter}}"
-                      SelectedModel="{Binding SelectedModel, Mode=TwoWay}"/>
+                <StackPanel DockPanel.Dock="Top">
+                    <userControls:ModelPickerControl 
+                        UISettings="{Binding UISettings}"
+                        SupportedDiffusers="{Binding SupportedDiffusers}"
+                        IsEnabled="{Binding IsGenerating, Converter={StaticResource InverseBoolConverter}}"
+                        SelectedModel="{Binding SelectedModel, Mode=TwoWay}"/>
+                    <userControls:ControlNetPickerControl 
+                          UISettings="{Binding UISettings}"
+                          IsEnabled="{Binding IsGenerating, Converter={StaticResource InverseBoolConverter}}"
+                          SelectedModel="{Binding SelectedControlNetModel, Mode=TwoWay}"
+                          Visibility="{Binding SelectedModel.IsControlNet, Converter={StaticResource BooleanToVisibilityConverter}, FallbackValue=Collapsed}"/>
+                </StackPanel>
                 <DockPanel IsEnabled="{Binding SelectedModel.IsLoaded, FallbackValue=False, TargetNullValue=False}">
                     <userControls:SchedulerControl DockPanel.Dock="Bottom" Margin="0, 10, 0 ,0"
                         SelectedModel="{Binding SelectedModel}"

OnnxStack.UI/Views/VideoToVideoView.xaml.cs

@@ -44,6 +44,7 @@ public partial class VideoToVideoView : UserControl, INavigatable, INotifyProper
         private VideoInputModel _inputVideo;
         private VideoInputModel _resultVideo;
         private StableDiffusionModelSetViewModel _selectedModel;
+        private ControlNetModelSetViewModel _selectedControlNetModel;
         private PromptOptionsModel _promptOptionsModel;
         private SchedulerOptionsModel _schedulerOptions;
         private CancellationTokenSource _cancelationTokenSource;
@@ -63,7 +64,7 @@ public VideoToVideoView()
                 _stableDiffusionService = App.GetService<IStableDiffusionService>();
             }
 
-            SupportedDiffusers = new() { DiffuserType.ImageToImage };
+            SupportedDiffusers = new() { DiffuserType.ImageToImage, DiffuserType.ControlNet };
             CancelCommand = new AsyncRelayCommand(Cancel, CanExecuteCancel);
             GenerateCommand = new AsyncRelayCommand(Generate, CanExecuteGenerate);
             ClearHistoryCommand = new AsyncRelayCommand(ClearHistory, CanExecuteClearHistory);
@@ -95,6 +96,12 @@ public StableDiffusionModelSetViewModel SelectedModel
             set { _selectedModel = value; NotifyPropertyChanged(); }
         }
 
+        public ControlNetModelSetViewModel SelectedControlNetModel
+        {
+            get { return _selectedControlNetModel; }
+            set { _selectedControlNetModel = value; NotifyPropertyChanged(); }
+        }
+
         public PromptOptionsModel PromptOptions
         {
             get { return _promptOptionsModel; }
@@ -215,7 +222,7 @@ private async Task Generate()
                 var promptOptions = GetPromptOptions(PromptOptions, _videoFrames);
 
                 var timestamp = Stopwatch.GetTimestamp();
-                var result = await _stableDiffusionService.GenerateAsBytesAsync(new ModelOptions(_selectedModel.ModelSet), promptOptions, schedulerOptions, ProgressCallback(), _cancelationTokenSource.Token);
+                var result = await _stableDiffusionService.GenerateAsBytesAsync(new ModelOptions(_selectedModel.ModelSet, _selectedControlNetModel?.ModelSet), promptOptions, schedulerOptions, ProgressCallback(), _cancelationTokenSource.Token);
                 var resultVideo = await GenerateResultAsync(result, promptOptions, schedulerOptions, timestamp);
                 if (resultVideo != null)
                 {
@@ -247,7 +254,9 @@ private async Task Generate()
         /// </returns>
         private bool CanExecuteGenerate()
         {
-            return !IsGenerating && HasInputResult;
+            return !IsGenerating 
+                && HasInputResult
+                && (!SelectedModel.IsControlNet || (SelectedModel.IsControlNet && SelectedControlNetModel?.IsLoaded == true));
         }
 
 
@@ -314,11 +323,19 @@ private void Reset()
 
         private PromptOptions GetPromptOptions(PromptOptionsModel promptOptionsModel, VideoFrames videoFrames)
         {
+            var diffuserType = DiffuserType.ImageToImage;
+            if (_selectedModel.IsControlNet)
+            {
+                diffuserType = _schedulerOptions.Strength >= 1
+                     ? DiffuserType.ControlNet
+                     : DiffuserType.ControlNetImage;
+            }
+  
             return new PromptOptions
             {
                 Prompt = promptOptionsModel.Prompt,
                 NegativePrompt = promptOptionsModel.NegativePrompt,
-                DiffuserType = DiffuserType.ImageToImage,
+                DiffuserType = diffuserType,
                 InputVideo = new VideoInput(videoFrames),
                 VideoInputFPS = promptOptionsModel.VideoInputFPS,
                 VideoOutputFPS = promptOptionsModel.VideoOutputFPS,