Support Video + Audio

Author: saddam213

TensorStack.Common/Common/FileHelper.cs

@@ -0,0 +1,46 @@
+using System;
+using System.IO;
+
+namespace TensorStack.Common.Common
+{
+    public static class FileHelper
+    {
+        public static bool DeleteFile(string filename)
+        {
+            try
+            {
+                if (File.Exists(filename))
+                    return false;
+
+                File.Delete(filename);
+                return true;
+            }
+            catch (Exception)
+            {
+                return false;
+            }
+        }
+
+
+        public static void DeleteFiles(params string[] filenames)
+        {
+            foreach (string filename in filenames)
+            {
+                DeleteFile(filename);
+            }
+        }
+
+
+        public static string RandomFileName(string extension)
+        {
+            var ext = Path.HasExtension(extension) ? Path.GetExtension(extension) : extension;
+            return $"{Path.GetFileNameWithoutExtension(Path.GetRandomFileName())}.{ext.Trim('.')}";
+        }
+
+
+        public static string RandomFileName(string directory, string extension)
+        {
+            return Path.Combine(directory, RandomFileName(extension));
+        }
+    }
+}

TensorStack.Video.Windows/Extensions.cs

@@ -0,0 +1,96 @@
+using System;
+using System.Diagnostics;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+using TensorStack.Common.Common;
+using TensorStack.Common.Video;
+
+namespace TensorStack.Video
+{
+    public class VideoService : IVideoService
+    {
+        private readonly IVideoConfiguration _configuration;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="VideoService"/> class.
+        /// </summary>
+        /// <param name="configuration">The configuration.</param>
+        public VideoService(IVideoConfiguration configuration)
+        {
+            _configuration = configuration;
+        }
+
+        /// <summary>
+        /// Get video information
+        /// </summary>
+        /// <param name="filename">The filename.</param>
+        public async Task<VideoInfo> GetInfoAsync(string filename)
+        {
+            return await VideoManager.LoadVideoInfoAsync(filename);
+        }
+
+
+        /// <summary>
+        /// Get the Video stream
+        /// </summary>
+        /// <param name="filename">The filename.</param>
+        public async Task<VideoInputStream> GetStreamAsync(string filename)
+        {
+            return await VideoInputStream.CreateAsync(filename);
+        }
+
+
+        /// <summary>
+        /// Saves the stream aync.
+        /// </summary>
+        /// <param name="videoInput">The video input.</param>
+        /// <param name="videoFile">The video file.</param>
+        /// <param name="frameProcessor">The frame processor.</param>
+        /// <param name="cancellationToken">The cancellation token that can be used by other objects or threads to receive notice of cancellation.</param>
+        public async Task<VideoInputStream> SaveStreamAync(VideoInputStream videoInput, string videoOutputFile, Func<VideoFrame, Task<VideoFrame>> frameProcessor, CancellationToken cancellationToken = default)
+        {
+            var videoFrames = videoInput.GetAsync(cancellationToken: cancellationToken);
+            await VideoManager.WriteVideoStreamAsync(videoOutputFile, videoFrames, frameProcessor, _configuration.ReadBuffer, _configuration.ReadBuffer, _configuration.VideoCodec, cancellationToken: cancellationToken);
+            await AddAudioAsync(videoOutputFile, videoInput.Filename, cancellationToken);
+            return await VideoInputStream.CreateAsync(videoOutputFile);
+        }
+
+
+
+        private async Task AddAudioAsync(string target, string source, CancellationToken cancellationToken = default)
+        {
+            var tempFile = FileHelper.RandomFileName(_configuration.DirectoryTemp, target);
+            var arguments = $"-i \"{target}\" -i \"{source}\" -c:v copy -c:a copy -map 0:v:0 -map 1:a:0 -y \"{tempFile}\"";
+
+            try
+            {
+                using (var videoWriter = CreateProcess(arguments))
+                {
+                    videoWriter.Start();
+                    await videoWriter.WaitForExitAsync(cancellationToken);
+                }
+
+                if (File.Exists(tempFile))
+                    File.Move(tempFile, target, true);
+            }
+            finally
+            {
+                FileHelper.DeleteFile(tempFile);
+            }
+        }
+
+
+        private Process CreateProcess(string arguments)
+        {
+            var process = new Process();
+            process.StartInfo.FileName = _configuration.FFmpegPath;
+            process.StartInfo.Arguments = arguments;
+            process.StartInfo.RedirectStandardInput = true;
+            process.StartInfo.UseShellExecute = false;
+            process.StartInfo.CreateNoWindow = true;
+            return process;
+        }
+
+    }
+}

TensorStack.Video.Windows/VideoService.cs

@@ -0,0 +1,15 @@
+// Copyright (c) TensorStack. All rights reserved.
+// Licensed under the Apache 2.0 License.
+namespace TensorStack.Video
+{
+    public interface IVideoConfiguration
+    {
+        string FFmpegPath { get; set; }
+        string FFprobePath { get; set; }
+        string DirectoryTemp { get; set; }
+
+        int ReadBuffer { get; set; }
+        int WriteBuffer { get; set; }
+        string VideoCodec { get; set; }
+    }
+}

TensorStack.Video/Common/IVideoConfiguration.cs

@@ -0,0 +1,14 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using TensorStack.Common.Video;
+
+namespace TensorStack.Video
+{
+    public interface IVideoService
+    {
+        Task<VideoInfo> GetInfoAsync(string filename);
+        Task<VideoInputStream> GetStreamAsync(string filename);
+        Task<VideoInputStream> SaveStreamAync(VideoInputStream videoInput, string videoOutputFile, Func<VideoFrame, Task<VideoFrame>> frameProcessor, CancellationToken cancellationToken = default);
+    }
+}

TensorStack.Video/Common/IVideoService.cs

@@ -2,82 +2,23 @@
 // Licensed under the Apache 2.0 License.
 using OpenCvSharp;
 using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Runtime.CompilerServices;
-using System.Threading;
-using System.Threading.Tasks;
 using TensorStack.Common.Tensor;
-using TensorStack.Common.Video;
 
 namespace TensorStack.Video
 {
     public static class Extensions
     {
         /// <summary>
-        /// Saves the video.
+        /// Converts Matrix to Tensor.
         /// </summary>
-        /// <param name="imageFrames">The image frames.</param>
-        /// <param name="videoFile">The video file.</param>
-        /// <param name="framerate">The framerate.</param>
-        /// <param name="width">The width.</param>
-        /// <param name="height">The height.</param>
-        /// <param name="videoCodec">The video codec.</param>
-        /// <param name="cancellationToken">The cancellation token that can be used by other objects or threads to receive notice of cancellation.</param>
-        public static async Task SaveAync(this IAsyncEnumerable<ImageTensor> imageFrames, string videoFile, float framerate, string videoCodec = "mp4v", int? widthOverride = null, int? heightOverride = null, CancellationToken cancellationToken = default)
-        {
-            var videoFrames = imageFrames.AsVideoFrames(framerate, cancellationToken);
-            await VideoService.WriteVideoStreamAsync(videoFile, videoFrames, videoCodec, widthOverride, heightOverride, framerate, cancellationToken);
-        }
-
-
-        /// <summary>
-        /// Saves the video.
-        /// </summary>
-        /// <param name="imageFrames">The image frames.</param>
-        /// <param name="videoFile">The video file.</param>
-        /// <param name="framerate">The framerate.</param>
-        /// <param name="videoCodec">The video codec.</param>
-        /// <param name="cancellationToken">The cancellation token that can be used by other objects or threads to receive notice of cancellation.</param>
-        public static async Task SaveAync(this IAsyncEnumerable<VideoFrame> videoFrames, string videoFile, string videoCodec = "mp4v", int? widthOverride = null, int? heightOverride = null, float? frameRateOverride = null, CancellationToken cancellationToken = default)
-        {
-            await VideoService.WriteVideoStreamAsync(videoFile, videoFrames, videoCodec, widthOverride, heightOverride, frameRateOverride, cancellationToken);
-        }
-
-
-        /// <summary>
-        /// Saves the video frames processing each frame [Read -> Process -> Write].
-        /// Reads an writes are buffered allowing higher processing thoughput
-        /// </summary>
-        /// <param name="videoInput">The VideoInputStream.</param>
-        /// <param name="videoFile">The video file.</param>
-        /// <param name="frameProcessor">The frame processor.</param>
-        /// <param name="readBuffer">The read buffer (frames).</param>
-        /// <param name="writeBuffer">The write buffer (frames).</param>
-        /// <param name="widthOverride">The output width override.</param>
-        /// <param name="heightOverride">The output height override.</param>
-        /// <param name="frameRateOverride">The output frame rate override.</param>
-        /// <param name="videoCodec">The video codec.</param>
-        /// <param name="cancellationToken">The cancellation token that can be used by other objects or threads to receive notice of cancellation.</param>
-        public static async Task<VideoInputStream> SaveAync(this VideoInputStream videoInput, string videoFile, Func<VideoFrame, Task<VideoFrame>> frameProcessor, int readBuffer = 16, int writeBuffer = 16, string videoCodec = "mp4v", int? widthOverride = null, int? heightOverride = null, float? frameRateOverride = null, CancellationToken cancellationToken = default)
-        {
-            var videoFrames = videoInput.GetAsync(cancellationToken: cancellationToken);
-            await VideoService.WriteVideoStreamAsync(videoFile, videoFrames, frameProcessor, readBuffer, writeBuffer, videoCodec, widthOverride, heightOverride, frameRateOverride, cancellationToken);
-            return await VideoInputStream.CreateAsync(videoFile);
-        }
-
-
-        /// <summary>
-        /// Converts Mat to Tensor.
-        /// </summary>
-        /// <param name="mat">The mat.</param>
+        /// <param name="matrix">The matrix.</param>
         /// <returns>Tensor&lt;System.Single&gt;.</returns>
-        internal static unsafe ImageTensor ToTensor(this Mat mat, Size cropSize = default)
+        internal static unsafe ImageTensor ToTensor(this Mat matrix, Size cropSize = default)
         {
             int cropX = 0;
             int cropY = 0;
-            int height = mat.Rows;
-            int width = mat.Cols;
+            int height = matrix.Rows;
+            int width = matrix.Cols;
 
             if (cropSize != default)
             {
@@ -98,14 +39,14 @@ internal static unsafe ImageTensor ToTensor(this Mat mat, Size cropSize = defaul
 
             unsafe
             {
-                var source = mat.DataPointer;
-                int srcStride = mat.Cols * 3;
+                var source = matrix.DataPointer;
+                int srcStride = matrix.Cols * 3;
                 int dstStride = height * width;
                 for (int y = 0; y < height; y++)
                 {
                     for (int x = 0; x < width; x++)
                     {
-                        int srcIndex = ((y + cropY) * mat.Cols + (x + cropX)) * 3;
+                        int srcIndex = ((y + cropY) * matrix.Cols + (x + cropX)) * 3;
                         int dstIndex = y * width + x;
 
                         destination[0 * dstStride + dstIndex] = GetFloatValue(source[srcIndex + 2]); // R
@@ -193,14 +134,5 @@ internal static float GetFloatValue(this byte value)
             return value;
         }
 
-
-        internal static async IAsyncEnumerable<VideoFrame> AsVideoFrames(this IAsyncEnumerable<ImageTensor> videoFrames, float frameRate, [EnumeratorCancellation] CancellationToken cancellationToken = default)
-        {
-            var frameIndex = 0;
-            await foreach (var videoFrame in videoFrames.WithCancellation(cancellationToken))
-            {
-                yield return new VideoFrame(frameIndex++, videoFrame, frameRate);
-            }
-        }
     }
 }

TensorStack.Video/Extensions.cs

@@ -23,7 +23,7 @@ public class VideoInput : VideoTensor
         /// <param name="frameRate">The frame rate.</param>
         /// <param name="videoCodec">The video codec.</param>
         public VideoInput(string filename, int? widthOverride = default, int? heightOverride = default, float? frameRateOverride = default, ResizeMode resizeMode = ResizeMode.Crop)
-            : this(VideoService.LoadVideoTensor(filename, widthOverride, heightOverride, frameRateOverride, resizeMode))
+            : this(VideoManager.LoadVideoTensor(filename, widthOverride, heightOverride, frameRateOverride, resizeMode))
         {
             _filename = filename;
         }
@@ -50,7 +50,7 @@ public VideoInput(VideoTensor videoTensor) : base(videoTensor, videoTensor.Frame
         /// <returns>A Task representing the asynchronous operation.</returns>
         public async Task SaveAsync(string filename, string videoCodec = "mp4v", float? frameRateOverride = default, CancellationToken cancellationToken = default)
         {
-            await VideoService.SaveVideoTensorAync(filename, this, videoCodec, frameRateOverride, cancellationToken);
+            await VideoManager.SaveVideoTensorAync(filename, this, videoCodec, frameRateOverride, cancellationToken);
         }
 
 
@@ -95,7 +95,7 @@ public void SetFilename(string filename)
         /// <returns>A Task&lt;VideoInput&gt; representing the asynchronous operation.</returns>
         public static async Task<VideoInput> CreateAsync(string filename, int? widthOverride = default, int? heightOverride = default, float? frameRateOverride = default, ResizeMode resizeMode = ResizeMode.Crop, CancellationToken cancellationToken = default)
         {
-            return new VideoInput(await VideoService.LoadVideoTensorAsync(filename, widthOverride, heightOverride, frameRateOverride, resizeMode, cancellationToken));
+            return new VideoInput(await VideoManager.LoadVideoTensorAsync(filename, widthOverride, heightOverride, frameRateOverride, resizeMode, cancellationToken));
         }
     }
 }

TensorStack.Video/VideoInput.cs

@@ -32,7 +32,8 @@ public VideoInputStream(VideoInfo videoInfo)
         /// <param name="filename">The filename.</param>
         /// <param name="videoCodec">The video codec.</param>
         /// <exception cref="System.Exception">Failed to open video file.</exception>
-        public VideoInputStream(string filename) : this(VideoService.LoadVideoInfo(filename)) { }
+        public VideoInputStream(string filename) 
+            : this(VideoManager.LoadVideoInfo(filename)) { }
 
         /// <summary>
         /// Gets the filename.
@@ -86,7 +87,7 @@ public VideoInputStream(string filename) : this(VideoService.LoadVideoInfo(filen
         /// <returns>IAsyncEnumerable&lt;ImageFrame&gt;.</returns>
         public IAsyncEnumerable<VideoFrame> GetAsync(int? widthOverride = default, int? heightOverride = default, float? frameRateOverride = default, ResizeMode resizeMode = ResizeMode.Stretch, CancellationToken cancellationToken = default)
         {
-            return VideoService.ReadStreamAsync(_videoInfo.FileName, frameRateOverride, widthOverride, heightOverride, resizeMode, cancellationToken);
+            return VideoManager.ReadStreamAsync(_videoInfo.FileName, frameRateOverride, widthOverride, heightOverride, resizeMode, cancellationToken);
         }
 
 
@@ -102,7 +103,7 @@ public IAsyncEnumerable<VideoFrame> GetAsync(int? widthOverride = default, int?
         /// <returns>Task.</returns>
         public Task SaveAsync(IAsyncEnumerable<VideoFrame> stream, string filename, string videoCodec = "mp4v", int? widthOverride = null, int? heightOverride = null, float? frameRateOverride = null, CancellationToken cancellationToken = default)
         {
-            return VideoService.WriteVideoStreamAsync(_videoInfo.FileName, stream, videoCodec, widthOverride, heightOverride, frameRateOverride, cancellationToken);
+            return VideoManager.WriteVideoStreamAsync(_videoInfo.FileName, stream, videoCodec, widthOverride, heightOverride, frameRateOverride, cancellationToken);
         }
 
 
@@ -167,7 +168,7 @@ public async Task<VideoInputStream> CopyAsync(string newFilename, bool overwrite
         /// <returns>A Task&lt;VideoInputStream&gt; representing the asynchronous operation.</returns>
         public static async Task<VideoInputStream> CreateAsync(string filename)
         {
-            var videoInfo = await VideoService.LoadVideoInfoAsync(filename);
+            var videoInfo = await VideoManager.LoadVideoInfoAsync(filename);
             return new VideoInputStream(videoInfo);
         }
     }