Background remove pipeline, improve video processing performance

Author: saddam213

TensorStack.Common/Extensions/Extensions.cs

@@ -4,6 +4,8 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Runtime.CompilerServices;
+using System.Threading;
+using System.Threading.Tasks;
 
 namespace TensorStack.Common
 {
@@ -141,5 +143,29 @@ public static float ZeroIfNan(this float value)
         {
             return float.IsNaN(value) ? 0f : value;
         }
+
+
+        public static bool CanBeCanceled(this CancellationTokenSource cancellationTokenSource)
+        {
+            if (cancellationTokenSource is null)
+                return false;
+
+            try
+            {
+                return cancellationTokenSource.Token.CanBeCanceled;
+            }
+            catch { }
+            return false;
+        }
+
+
+        public static async Task SafeCancelAsync(this CancellationTokenSource cancellationTokenSource)
+        {
+            try
+            {
+                await cancellationTokenSource.CancelAsync();
+            }
+            catch { }
+        }
     }
 }

TensorStack.Common/Extensions/TensorExtensions.cs

@@ -581,16 +581,16 @@ public static Tensor<float> Join(this IEnumerable<Tensor<float>> tensors, int ax
             dimensions[0] *= count;
 
             var newLength = (int)tensor.Length;
-            var buffer = new float[newLength * count].AsMemory();
+            var buffer = new Tensor<float>(dimensions);
 
             var index = 0;
             foreach (var item in tensors)
             {
                 var start = index * newLength;
-                item.Memory.CopyTo(buffer[start..]);
+                item.Memory.CopyTo(buffer.Memory[start..]);
                 index++;
             }
-            return new Tensor<float>(buffer, dimensions);
+            return buffer;
         }
 
 
@@ -997,7 +997,7 @@ public static Tensor<float> WithGuidance(this Tensor<float> tensor, bool applyGu
         /// <param name="resizeMode">The resize mode.</param>
         /// <param name="resizeMethod">The resize method.</param>
         /// <returns>ImageTensor.</returns>
-        public static ImageTensor ResizeImage(this ImageTensor sourceImage, int targetWidth, int targetHeight, ResizeMode resizeMode = ResizeMode.Stretch, ResizeMethod resizeMethod = ResizeMethod.Bicubic)
+        public static ImageTensor ResizeImage(this ImageTensor sourceImage, int targetWidth, int targetHeight, ResizeMode resizeMode = ResizeMode.Stretch, ResizeMethod resizeMethod = ResizeMethod.Bilinear)
         {
             return resizeMethod switch
             {

TensorStack.Common/Video/VideoFrame.cs

@@ -15,8 +15,9 @@ public class VideoFrame
         /// </summary>
         /// <param name="frame">The frame.</param>
         /// <param name="sourceFrameRate">The source frame rate.</param>
-        public VideoFrame(ImageTensor frame, float sourceFrameRate, ImageTensor auxFrame = default)
+        public VideoFrame(int index,ImageTensor frame, float sourceFrameRate, ImageTensor auxFrame = default)
         {
+            Index = index;
             Frame = frame;
             AuxFrame = auxFrame;
             SourceFrameRate = sourceFrameRate;
@@ -28,9 +29,13 @@ public VideoFrame(ImageTensor frame, float sourceFrameRate, ImageTensor auxFrame
         /// </summary>
         /// <param name="frame">The frame.</param>
         /// <param name="sourceFrameRate">The source frame rate.</param>
-        public VideoFrame(Tensor<float> frame, float sourceFrameRate)
-            : this(new ImageTensor(frame), sourceFrameRate) { }
+        public VideoFrame(int index, Tensor<float> frame, float sourceFrameRate)
+            : this(index, new ImageTensor(frame), sourceFrameRate) { }
 
+        /// <summary>
+        /// Gets the index.
+        /// </summary>
+        public int Index { get; }
 
         /// <summary>
         /// Gets the frame.

TensorStack.Extractors/Common/BackgroundImageOptions.cs

@@ -0,0 +1,33 @@
+// Copyright (c) TensorStack. All rights reserved.
+// Licensed under the Apache 2.0 License.
+using TensorStack.Common;
+using TensorStack.Common.Pipeline;
+using TensorStack.Common.Tensor;
+
+namespace TensorStack.Extractors.Common
+{
+    /// <summary>
+    /// Default ExtractorOptions.
+    /// </summary>
+    public record BackgroundImageOptions : IRunOptions
+    {
+        /// <summary>
+        /// Gets a value indicating whether the output is inverted.
+        /// </summary>
+        public BackgroundMode Mode { get; init; }
+
+        /// <summary>
+        /// Gets the input.
+        /// </summary>
+        public ImageTensor Input { get; init; }
+    }
+
+    public enum BackgroundMode
+    {
+        MaskBackground = 0,
+        MaskForeground = 1,
+
+        RemoveBackground = 10,
+        RemoveForeground = 11,
+    }
+}

TensorStack.Extractors/Common/ExtractorConfig.cs

@@ -26,11 +26,6 @@ public record ExtractorConfig : ModelConfig
         /// </summary>
         public Normalization OutputNormalization { get; init; }
 
-        /// <summary>
-        /// If the result should ne inverted
-        /// </summary>
-        public bool IsOutputInverted { get; init; }
-
         /// <summary>
         /// The channels the model supports 1 = Greyscale, RGB = 3, RGBA = 4.
         /// </summary>

TensorStack.Extractors/Common/ExtractorImageOptions.cs

@@ -1,18 +1,11 @@
 // Copyright (c) TensorStack. All rights reserved.
 // Licensed under the Apache 2.0 License.
-using TensorStack.Common;
 using TensorStack.Common.Tensor;
 
 namespace TensorStack.Extractors.Common
 {
     public record ExtractorImageOptions : ExtractorOptions
     {
-        public ExtractorImageOptions(ImageTensor input, bool mergeInput = false, TileMode tileMode = TileMode.None, int maxTileSize = 512, int tileOverlap = 16)
-            : base(mergeInput, tileMode, maxTileSize, tileOverlap)
-        {
-            Input = input;
-        }
-
-        public ImageTensor Input { get; }
+        public ImageTensor Input { get; init; }
     }
 }

TensorStack.Extractors/Common/ExtractorOptions.cs

@@ -10,38 +10,29 @@ namespace TensorStack.Extractors.Common
     /// </summary>
     public record ExtractorOptions : IRunOptions
     {
-        /// <summary>
-        /// Initializes a new instance of the <see cref="ExtractorOptions"/> class.
-        /// </summary>
-        /// <param name="tileMode">Enable/Disable TileMode, splitting image into smaller tiles to save memory.</param>
-        /// <param name="maxTileSize">The maximum size of the tile for TileMode</param>
-        /// <param name="tileOverlap">The tile overlap in pixels to avoid visible seams.</param>
-        public ExtractorOptions(bool mergeInput = false, TileMode tileMode = TileMode.None, int maxTileSize = 512, int tileOverlap = 16)
-        {
-            MergeInput = mergeInput;
-            TileMode = tileMode;
-            MaxTileSize = maxTileSize;
-            TileOverlap = tileOverlap;
-        }
-
         /// <summary>
         /// Megre the input and output result into a new tensor.
         /// </summary>
-        public bool MergeInput { get; set; }
+        public bool MergeInput { get; init; }
 
         /// <summary>
         /// Enable/Disable TileMode, splitting image into smaller tiles to save memory.
         /// </summary>
-        public TileMode TileMode { get; }
+        public TileMode TileMode { get; init; }
 
         /// <summary>
         /// The maximum size of the tile.
         /// </summary>
-        public int MaxTileSize { get; }
+        public int MaxTileSize { get; init; }
 
         /// <summary>
         /// The tile overlap in pixels to avoid visible seams.
         /// </summary>
-        public int TileOverlap { get; }
+        public int TileOverlap { get; init; }
+
+        /// <summary>
+        /// Gets a value indicating whether the output is inverted.
+        /// </summary>
+        public bool IsInverted { get; init; }
     }
 }

TensorStack.Extractors/Common/ExtractorStreamOptions.cs

@@ -8,12 +8,6 @@ namespace TensorStack.Extractors.Common
 {
     public record ExtractorStreamOptions : ExtractorOptions
     {
-        public ExtractorStreamOptions(IAsyncEnumerable<VideoFrame> input, bool mergeInput = false, TileMode tileMode = TileMode.None, int maxTileSize = 512, int tileOverlap = 16)
-            : base(mergeInput, tileMode, maxTileSize, tileOverlap)
-        {
-            Input = input;
-        }
-
         public IAsyncEnumerable<VideoFrame> Input { get; }
     }
 }

TensorStack.Extractors/Common/ExtractorVideoOptions.cs

@@ -7,12 +7,6 @@ namespace TensorStack.Extractors.Common
 {
     public record ExtractorVideoOptions : ExtractorOptions
     {
-        public ExtractorVideoOptions(VideoTensor input, bool mergeInput = false, TileMode tileMode = TileMode.None, int maxTileSize = 512, int tileOverlap = 16)
-            : base(mergeInput, tileMode, maxTileSize, tileOverlap)
-        {
-            Input = input;
-        }
-
         public VideoTensor Input { get; }
     }
 }

TensorStack.Extractors/Models/ExtractorModel.cs

@@ -34,11 +34,6 @@ private ExtractorModel(ExtractorConfig configuration)
         /// </summary>
         public Normalization OutputNormalization => Configuration.OutputNormalization;
 
-        /// <summary>
-        /// If the result should ne inverted
-        /// </summary>
-        public bool IsOutputInverted => Configuration.IsOutputInverted;
-
         /// <summary>
         /// The channels the model supports 1 = Greyscale, RGB = 3, RGBA = 4.
         /// </summary>