ConditionScale added, Support non normalized image input

Author: saddam213

OnnxStack.Core/Config/OnnxModelType.cs

@@ -10,6 +10,7 @@ public enum OnnxModelType
         VaeEncoder = 30,
         VaeDecoder = 40,
         Control = 50,
-        Upscaler = 1000
+        Annotation = 100,
+        Upscaler = 1000,
     }
 }

OnnxStack.Core/Extensions/OrtValueExtensions.cs

@@ -59,6 +59,11 @@ public static OrtValue ToOrtValue(this DenseTensor<long> tensor, OnnxNamedMetada
             return OrtValue.CreateTensorValueFromMemory(OrtMemoryInfo.DefaultInstance, tensor.Buffer, tensor.Dimensions.ToLong());
         }
 
+        public static OrtValue ToOrtValue(this DenseTensor<double> tensor, OnnxNamedMetadata metadata)
+        {
+            return OrtValue.CreateTensorValueFromMemory(OrtMemoryInfo.DefaultInstance, tensor.Buffer, tensor.Dimensions.ToLong());
+        }
+
 
         /// <summary>
         /// Creates and allocates the output tensors buffer.

OnnxStack.Core/Model/OnnxInferenceParameters.cs

@@ -45,6 +45,12 @@ public void AddInputTensor(DenseTensor<float> value)
             _inputs.Add(metaData, value.ToOrtValue(metaData));
         }
 
+        public void AddInputTensor(DenseTensor<double> value)
+        {
+            var metaData = GetNextInputMetadata();
+            _inputs.Add(metaData, value.ToOrtValue(metaData));
+        }
+
 
         /// <summary>
         /// Adds the input tensor.

OnnxStack.StableDiffusion/Config/SchedulerOptions.cs

@@ -83,6 +83,7 @@ public record SchedulerOptions
 
         public float AestheticScore { get; set; } = 6f;
         public float AestheticNegativeScore { get; set; } = 2.5f;
+        public float ConditioningScale { get; set; } = 0.7f;
 
         public bool IsKarrasScheduler
         {

OnnxStack.StableDiffusion/Diffusers/ControlNet/ControlNetDiffuser.cs

@@ -35,7 +35,15 @@ public ControlNetDiffuser(IOnnxModelService onnxModelService, IPromptService pro
         public override DiffuserPipelineType PipelineType => DiffuserPipelineType.ControlNet;
 
 
-
+        /// <summary>
+        /// Creates the Conditioning Scale tensor.
+        /// </summary>
+        /// <param name="conditioningScale">The conditioningScale.</param>
+        /// <returns></returns>
+        protected static DenseTensor<double> CreateConditioningScaleTensor(float conditioningScale)
+        {
+            return TensorHelper.CreateTensor(new double[] { conditioningScale }, new int[] { 1 });
+        }
 
 
         /// <summary>

OnnxStack.StableDiffusion/Diffusers/ControlNet/ImageDiffuser.cs renamed to OnnxStack.StableDiffusion/Diffusers/ControlNet/TextDiffuser.cs

@@ -9,7 +9,6 @@
 using OnnxStack.StableDiffusion.Enums;
 using OnnxStack.StableDiffusion.Helpers;
 using OnnxStack.StableDiffusion.Models;
-using SixLabors.ImageSharp;
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
@@ -19,14 +18,14 @@
 
 namespace OnnxStack.StableDiffusion.Diffusers.ControlNet
 {
-    public sealed class ImageDiffuser : ControlNetDiffuser
+    public sealed class TextDiffuser : ControlNetDiffuser
     {
         /// <summary>
-        /// Initializes a new instance of the <see cref="ImageDiffuser"/> class.
+        /// Initializes a new instance of the <see cref="TextDiffuser"/> class.
         /// </summary>
         /// <param name="configuration">The configuration.</param>
         /// <param name="onnxModelService">The onnx model service.</param>
-        public ImageDiffuser(IOnnxModelService onnxModelService, IPromptService promptService, ILogger<ImageDiffuser> logger)
+        public TextDiffuser(IOnnxModelService onnxModelService, IPromptService promptService, ILogger<TextDiffuser> logger)
             : base(onnxModelService, promptService, logger)
         {
         }
@@ -65,9 +64,9 @@ protected override async Task<DenseTensor<float>> SchedulerStepAsync(StableDiffu
 
                 // Get Model metadata
                 var controlNetMetadata = _onnxModelService.GetModelMetadata(modelOptions, OnnxModelType.Control);
-
-                // TODO: do we need to pre-process?
-                var controlImage = promptOptions.InputImage.ToDenseTensor(new[] { 1, 3, schedulerOptions.Height, schedulerOptions.Width });
+ 
+                // Control Image
+                var controlImage = promptOptions.InputImage.ToDenseTensor(new[] { 1, 3, schedulerOptions.Height, schedulerOptions.Width }, false);
 
                 // Loop though the timesteps
                 var step = 0;
@@ -81,6 +80,8 @@ protected override async Task<DenseTensor<float>> SchedulerStepAsync(StableDiffu
                     var inputLatent = performGuidance ? latents.Repeat(2) : latents;
                     var inputTensor = scheduler.ScaleInput(inputLatent, timestep);
                     var timestepTensor = CreateTimestepTensor(timestep);
+                    var controlImageTensor = performGuidance ? controlImage.Repeat(2) : controlImage;
+                    var conditioningScale = CreateConditioningScaleTensor(schedulerOptions.ConditioningScale);
 
                     var outputChannels = performGuidance ? 2 : 1;
                     var outputDimension = schedulerOptions.GetScaledDimension(outputChannels);
@@ -97,34 +98,35 @@ protected override async Task<DenseTensor<float>> SchedulerStepAsync(StableDiffu
                             controlNetParameters.AddInputTensor(timestepTensor);
                             controlNetParameters.AddInputTensor(promptEmbeddings.PromptEmbeds);
                             controlNetParameters.AddInputTensor(controlImage);
+                            controlNetParameters.AddInputTensor(conditioningScale);
+
+                            // Optimization: Pre-allocate device buffers for inputs
                             foreach (var item in controlNetMetadata.Outputs)
                                 controlNetParameters.AddOutputBuffer();
 
+                            // ControlNet inference
                             var controlNetResults = _onnxModelService.RunInference(modelOptions, OnnxModelType.Control, controlNetParameters);
-                            if (controlNetResults.IsNullOrEmpty())
-                                throw new Exception("Control model produced no result");
 
                             // Add ControlNet outputs to Unet input
                             foreach (var item in controlNetResults)
-                                inferenceParameters.AddInputTensor(item.ToDenseTensor());
-                        }
+                                inferenceParameters.AddInput(item);
 
+                            // Add output buffer
+                            inferenceParameters.AddOutputBuffer(outputDimension);
 
-                        // Add output buffer
-                        inferenceParameters.AddOutputBuffer(outputDimension);
-
-                        // Unet
-                        var results = await _onnxModelService.RunInferenceAsync(modelOptions, OnnxModelType.Unet, inferenceParameters);
-                        using (var result = results.First())
-                        {
-                            var noisePred = result.ToDenseTensor();
+                            // Unet inference
+                            var results = await _onnxModelService.RunInferenceAsync(modelOptions, OnnxModelType.Unet, inferenceParameters);
+                            using (var result = results.First())
+                            {
+                                var noisePred = result.ToDenseTensor();
 
-                            // Perform guidance
-                            if (performGuidance)
-                                noisePred = PerformGuidance(noisePred, schedulerOptions.GuidanceScale);
+                                // Perform guidance
+                                if (performGuidance)
+                                    noisePred = PerformGuidance(noisePred, schedulerOptions.GuidanceScale);
 
-                            // Scheduler Step
-                            latents = scheduler.Step(noisePred, timestep, latents).Result;
+                                // Scheduler Step
+                                latents = scheduler.Step(noisePred, timestep, latents).Result;
+                            }
                         }
                     }
 
@@ -137,50 +139,14 @@ protected override async Task<DenseTensor<float>> SchedulerStepAsync(StableDiffu
             }
         }
 
-
-        /// <summary>
-        /// Gets the timesteps.
-        /// </summary>
-        /// <param name="prompt">The prompt.</param>
-        /// <param name="options">The options.</param>
-        /// <param name="scheduler">The scheduler.</param>
-        /// <returns></returns>
         protected override IReadOnlyList<int> GetTimesteps(SchedulerOptions options, IScheduler scheduler)
         {
-            var inittimestep = Math.Min((int)(options.InferenceSteps * options.Strength), options.InferenceSteps);
-            var start = Math.Max(options.InferenceSteps - inittimestep, 0);
-            return scheduler.Timesteps.Skip(start).ToList();
+            return scheduler.Timesteps;
         }
 
-
-        /// <summary>
-        /// Prepares the latents for inference.
-        /// </summary>
-        /// <param name="prompt">The prompt.</param>
-        /// <param name="options">The options.</param>
-        /// <param name="scheduler">The scheduler.</param>
-        /// <returns></returns>
-        protected override async Task<DenseTensor<float>> PrepareLatentsAsync(StableDiffusionModelSet model, PromptOptions prompt, SchedulerOptions options, IScheduler scheduler, IReadOnlyList<int> timesteps)
+        protected override Task<DenseTensor<float>> PrepareLatentsAsync(StableDiffusionModelSet model, PromptOptions prompt, SchedulerOptions options, IScheduler scheduler, IReadOnlyList<int> timesteps)
         {
-            var imageTensor = prompt.InputImage.ToDenseTensor(new[] { 1, 3, options.Height, options.Width });
-
-            //TODO: Model Config, Channels
-            var outputDimension = options.GetScaledDimension();
-            var metadata = _onnxModelService.GetModelMetadata(model, OnnxModelType.VaeEncoder);
-            using (var inferenceParameters = new OnnxInferenceParameters(metadata))
-            {
-                inferenceParameters.AddInputTensor(imageTensor);
-                inferenceParameters.AddOutputBuffer(outputDimension);
-
-                var results = await _onnxModelService.RunInferenceAsync(model, OnnxModelType.VaeEncoder, inferenceParameters);
-                using (var result = results.First())
-                {
-                    var outputResult = result.ToDenseTensor();
-                    var scaledSample = outputResult.MultiplyBy(model.ScaleFactor);
-                    return scheduler.AddNoise(scaledSample, scheduler.CreateRandomSample(scaledSample.Dimensions), timesteps);
-                }
-            }
+            return Task.FromResult(scheduler.CreateRandomSample(options.GetScaledDimension(), scheduler.InitNoiseSigma));
         }
-
     }
 }

OnnxStack.StableDiffusion/Helpers/ImageHelpers.cs

@@ -1,12 +1,10 @@
 using Microsoft.ML.OnnxRuntime.Tensors;
 using OnnxStack.Core.Image;
-using OnnxStack.StableDiffusion.Models;
 using SixLabors.ImageSharp;
 using SixLabors.ImageSharp.PixelFormats;
 using SixLabors.ImageSharp.Processing;
 using System;
 using System.IO;
-using System.Threading.Tasks;
 
 namespace OnnxStack.StableDiffusion.Helpers
 {
@@ -55,18 +53,18 @@ public static Stream ToImageStream(this DenseTensor<float> imageTensor)
         /// <param name="imageData">The image data.</param>
         /// <param name="dimensions">The dimensions.</param>
         /// <returns></returns>
-        public static DenseTensor<float> ToDenseTensor(this InputImage imageData, ReadOnlySpan<int> dimensions)
+        public static DenseTensor<float> ToDenseTensor(this InputImage imageData, ReadOnlySpan<int> dimensions, bool normalize = true)
         {
             if (!string.IsNullOrEmpty(imageData.ImageBase64))
-                return TensorFromBase64(imageData.ImageBase64, dimensions);
+                return TensorFromBase64(imageData.ImageBase64, dimensions, normalize);
             if (imageData.ImageBytes != null)
-                return TensorFromBytes(imageData.ImageBytes, dimensions);
+                return TensorFromBytes(imageData.ImageBytes, dimensions, normalize);
             if (imageData.ImageStream != null)
-                return TensorFromStream(imageData.ImageStream, dimensions);
+                return TensorFromStream(imageData.ImageStream, dimensions, normalize);
             if (imageData.ImageTensor != null)
                 return imageData.ImageTensor.ToDenseTensor(); // Note: Tensor Copy // TODO: Reshape to dimensions
 
-            return TensorFromImage(imageData.Image, dimensions);
+            return TensorFromImage(imageData.Image, dimensions, normalize);
         }
 
 
@@ -147,12 +145,12 @@ public static void TensorToImageDebug2(DenseTensor<float> imageTensor, string fi
         /// <param name="image">The image.</param>
         /// <param name="dimensions">The dimensions.</param>
         /// <returns></returns>
-        public static DenseTensor<float> TensorFromImage(Image<Rgba32> image, ReadOnlySpan<int> dimensions)
+        public static DenseTensor<float> TensorFromImage(Image<Rgba32> image, ReadOnlySpan<int> dimensions, bool normalize)
         {
             using (image)
             {
                 Resize(image, dimensions);
-                return ProcessPixels(image, dimensions);
+                return ProcessPixels(image, dimensions, normalize);
             }
         }
 
@@ -164,12 +162,12 @@ public static DenseTensor<float> TensorFromImage(Image<Rgba32> image, ReadOnlySp
         /// <param name="width">The width.</param>
         /// <param name="height">The height.</param>
         /// <returns></returns>
-        public static DenseTensor<float> TensorFromFile(string filename, ReadOnlySpan<int> dimensions)
+        public static DenseTensor<float> TensorFromFile(string filename, ReadOnlySpan<int> dimensions, bool normalize)
         {
             using (var image = Image.Load<Rgba32>(filename))
             {
                 Resize(image, dimensions);
-                return ProcessPixels(image, dimensions);
+                return ProcessPixels(image, dimensions, normalize);
             }
         }
 
@@ -180,12 +178,12 @@ public static DenseTensor<float> TensorFromFile(string filename, ReadOnlySpan<in
         /// <param name="base64Image">The base64 image.</param>
         /// <param name="width">The width.</param>
         /// <param name="height">The height.</param>
-        public static DenseTensor<float> TensorFromBase64(string base64Image, ReadOnlySpan<int> dimensions)
+        public static DenseTensor<float> TensorFromBase64(string base64Image, ReadOnlySpan<int> dimensions, bool normalize)
         {
             using (var image = Image.Load<Rgba32>(Convert.FromBase64String(base64Image.Split(',')[1])))
             {
                 Resize(image, dimensions);
-                return ProcessPixels(image, dimensions);
+                return ProcessPixels(image, dimensions, normalize);
             }
         }
 
@@ -198,12 +196,12 @@ public static DenseTensor<float> TensorFromBase64(string base64Image, ReadOnlySp
         /// <param name="height">The height.</param>
         /// <returns>
         /// </returns>
-        public static DenseTensor<float> TensorFromBytes(byte[] imageBytes, ReadOnlySpan<int> dimensions)
+        public static DenseTensor<float> TensorFromBytes(byte[] imageBytes, ReadOnlySpan<int> dimensions, bool normalize)
         {
             using (var image = Image.Load<Rgba32>(imageBytes))
             {
                 Resize(image, dimensions);
-                return ProcessPixels(image, dimensions);
+                return ProcessPixels(image, dimensions, normalize);
             }
         }
 
@@ -215,12 +213,12 @@ public static DenseTensor<float> TensorFromBytes(byte[] imageBytes, ReadOnlySpan
         /// <param name="width">The width.</param>
         /// <param name="height">The height.</param>
         /// <returns></returns>
-        public static DenseTensor<float> TensorFromStream(Stream imageStream, ReadOnlySpan<int> dimensions)
+        public static DenseTensor<float> TensorFromStream(Stream imageStream, ReadOnlySpan<int> dimensions, bool normalize)
         {
             using (var image = Image.Load<Rgba32>(imageStream))
             {
                 Resize(image, dimensions);
-                return ProcessPixels(image, dimensions);
+                return ProcessPixels(image, dimensions, normalize);
             }
         }
 
@@ -231,7 +229,7 @@ public static DenseTensor<float> TensorFromStream(Stream imageStream, ReadOnlySp
         /// <param name="image">The image.</param>
         /// <param name="dimensions">The dimensions.</param>
         /// <returns></returns>
-        private static DenseTensor<float> ProcessPixels(Image<Rgba32> image, ReadOnlySpan<int> dimensions)
+        private static DenseTensor<float> ProcessPixels(Image<Rgba32> image, ReadOnlySpan<int> dimensions, bool normalize)
         {
             var width = dimensions[3];
             var height = dimensions[2];
@@ -244,11 +242,11 @@ private static DenseTensor<float> ProcessPixels(Image<Rgba32> image, ReadOnlySpa
                     for (int y = 0; y < height; y++)
                     {
                         var pixelSpan = img.GetRowSpan(y);
-                        imageArray[0, 0, y, x] = (pixelSpan[x].R / 255.0f) * 2.0f - 1.0f;
-                        imageArray[0, 1, y, x] = (pixelSpan[x].G / 255.0f) * 2.0f - 1.0f;
-                        imageArray[0, 2, y, x] = (pixelSpan[x].B / 255.0f) * 2.0f - 1.0f;
+                        imageArray[0, 0, y, x] = normalize ? (pixelSpan[x].R / 255.0f) * 2.0f - 1.0f : (pixelSpan[x].R / 255.0f);
+                        imageArray[0, 1, y, x] = normalize ? (pixelSpan[x].G / 255.0f) * 2.0f - 1.0f : (pixelSpan[x].G / 255.0f);
+                        imageArray[0, 2, y, x] = normalize ? (pixelSpan[x].B / 255.0f) * 2.0f - 1.0f : (pixelSpan[x].B / 255.0f);
                         if (channels == 4)
-                            imageArray[0, 3, y, x] = (pixelSpan[x].A / 255.0f) * 2.0f - 1.0f;
+                            imageArray[0, 3, y, x] = normalize ? (pixelSpan[x].A / 255.0f) * 2.0f - 1.0f : (pixelSpan[x].A / 255.0f);
                     }
                 }
             });

OnnxStack.StableDiffusion/Registration.cs

@@ -82,7 +82,7 @@ private static void RegisterServices(this IServiceCollection serviceCollection)
             serviceCollection.AddSingleton<IDiffuser, StableDiffusion.Diffusers.InstaFlow.TextDiffuser>();
 
             //ControlNet
-            serviceCollection.AddSingleton<IDiffuser, StableDiffusion.Diffusers.ControlNet.ImageDiffuser>();
+            serviceCollection.AddSingleton<IDiffuser, StableDiffusion.Diffusers.ControlNet.TextDiffuser>();
         }