VideoToVideo example

OnnxStack.Console/Examples/VideoToVideoExample.cs

@@ -29,10 +29,9 @@ public VideoToVideoExample(StableDiffusionConfig configuration, IVideoService vi
 
         public async Task RunAsync()
         {
-            string inputVideoPath = "C:\\Users\\Deven\\Pictures\\gidsgphy.gif";
-            var inputFile = File.ReadAllBytes(inputVideoPath);
-            var videoInfo = await _videoService.GetVideoInfoAsync(inputFile);
-            var videoInput = await _videoService.CreateFramesAsync(inputFile, videoInfo.FPS);
+            // Load Video
+            var targetFPS = 15;
+            var videoInput = await VideoInput.FromFileAsync("C:\\Users\\Deven\\Pictures\\gidsgphy.gif", targetFPS);
 
             // Loop though the appsettings.json model sets
             foreach (var modelSet in _configuration.ModelSets)
@@ -50,18 +49,15 @@ public async Task RunAsync()
                 {
                     Prompt = "Iron Man",
                     DiffuserType = DiffuserType.ImageToImage,
-                    InputVideo = new VideoInput(videoInput)
+                    InputVideo = videoInput
                 };
 
                 // Run pipeline
                 var result = await pipeline.RunAsync(promptOptions, progressCallback: OutputHelpers.FrameProgressCallback);
 
-                // Create Video from Tensor result
-                var videoResult = await _videoService.CreateVideoAsync(result, videoInfo.FPS);
-
                 // Save Video File
                 var outputFilename = Path.Combine(_outputDirectory, $"{modelSet.Name}.mp4");
-                await File.WriteAllBytesAsync(outputFilename, videoResult.Data);
+                await VideoInput.SaveFileAsync(result, outputFilename, targetFPS);
             }
         }
     }

OnnxStack.Core/Services/IVideoService.cs

@@ -45,7 +45,7 @@ public interface IVideoService
         /// <param name="videoFPS">The video FPS.</param>
         /// <param name="cancellationToken">The cancellation token.</param>
         /// <returns></returns>
-        Task<VideoFrames> CreateFramesAsync(byte[] videoBytes, float videoFPS, CancellationToken cancellationToken = default);
+        Task<VideoFrames> CreateFramesAsync(byte[] videoBytes, float? videoFPS = default, CancellationToken cancellationToken = default);
 
 
         /// <summary>
@@ -55,7 +55,7 @@ public interface IVideoService
         /// <param name="videoFPS">The video FPS.</param>
         /// <param name="cancellationToken">The cancellation token.</param>
         /// <returns></returns>
-        Task<VideoFrames> CreateFramesAsync(Stream videoStream, float videoFPS, CancellationToken cancellationToken = default);
+        Task<VideoFrames> CreateFramesAsync(Stream videoStream, float? videoFPS = default, CancellationToken cancellationToken = default);
 
 
         /// <summary>
@@ -67,7 +67,7 @@ public interface IVideoService
         /// <returns></returns>
         /// <exception cref="NotSupportedException">VideoTensor not supported</exception>
         /// <exception cref="ArgumentException">No video data found</exception>
-        Task<VideoFrames> CreateFramesAsync(VideoInput videoInput, float videoFPS, CancellationToken cancellationToken = default);
+        Task<VideoFrames> CreateFramesAsync(VideoInput videoInput, float? videoFPS = default, CancellationToken cancellationToken = default);
 
 
         /// <summary>

OnnxStack.Core/Services/VideoService.cs

@@ -135,7 +135,7 @@ public async Task<VideoOutput> CreateVideoAsync(IEnumerable<byte[]> videoFrames,
         /// <returns></returns>
         /// <exception cref="NotSupportedException">VideoTensor not supported</exception>
         /// <exception cref="ArgumentException">No video data found</exception>
-        public async Task<VideoFrames> CreateFramesAsync(VideoInput videoInput, float videoFPS, CancellationToken cancellationToken = default)
+        public async Task<VideoFrames> CreateFramesAsync(VideoInput videoInput, float? videoFPS = default, CancellationToken cancellationToken = default)
         {
 
             if (videoInput.VideoBytes is not null)
@@ -158,11 +158,12 @@ public async Task<VideoFrames> CreateFramesAsync(VideoInput videoInput, float vi
         /// <param name="videoFPS">The video FPS.</param>
         /// <param name="cancellationToken">The cancellation token.</param>
         /// <returns></returns>
-        public async Task<VideoFrames> CreateFramesAsync(byte[] videoBytes, float videoFPS, CancellationToken cancellationToken = default)
+        public async Task<VideoFrames> CreateFramesAsync(byte[] videoBytes, float? videoFPS = default, CancellationToken cancellationToken = default)
         {
             var videoInfo = await GetVideoInfoAsync(videoBytes, cancellationToken);
-            var videoFrames = await CreateFramesInternalAsync(videoBytes, videoFPS, cancellationToken).ToListAsync(cancellationToken);
-            videoInfo = videoInfo with { FPS = videoFPS };
+            var targetFPS = videoFPS ?? videoInfo.FPS;
+            var videoFrames = await CreateFramesInternalAsync(videoBytes, targetFPS, cancellationToken).ToListAsync(cancellationToken);
+            videoInfo = videoInfo with { FPS = targetFPS };
             return new VideoFrames(videoInfo, videoFrames);
         }
 
@@ -174,14 +175,16 @@ public async Task<VideoFrames> CreateFramesAsync(byte[] videoBytes, float videoF
         /// <param name="videoFPS">The video FPS.</param>
         /// <param name="cancellationToken">The cancellation token.</param>
         /// <returns></returns>
-        public async Task<VideoFrames> CreateFramesAsync(Stream videoStream, float videoFPS, CancellationToken cancellationToken = default)
+        public async Task<VideoFrames> CreateFramesAsync(Stream videoStream, float? videoFPS = default, CancellationToken cancellationToken = default)
         {
             using (var memoryStream = new MemoryStream())
             {
                 await memoryStream.CopyToAsync(videoStream, cancellationToken).ConfigureAwait(false);
                 var videoBytes = memoryStream.ToArray();
                 var videoInfo = await GetVideoInfoAsync(videoBytes, cancellationToken);
-                var videoFrames = await CreateFramesInternalAsync(videoBytes, videoFPS, cancellationToken).ToListAsync(cancellationToken);
+                var targetFPS = videoFPS ?? videoInfo.FPS;
+                var videoFrames = await CreateFramesInternalAsync(videoBytes, targetFPS, cancellationToken).ToListAsync(cancellationToken);
+                videoInfo = videoInfo with { FPS = targetFPS };
                 return new VideoFrames(videoInfo, videoFrames);
             }
         }

OnnxStack.Core/Video/VideoInput.cs

@@ -1,6 +1,11 @@
-using Microsoft.ML.OnnxRuntime.Tensors;
+using Microsoft.Extensions.Primitives;
+using Microsoft.ML.OnnxRuntime.Tensors;
+using OnnxStack.Core.Config;
+using OnnxStack.Core.Services;
 using System.IO;
 using System.Text.Json.Serialization;
+using System.Threading;
+using System.Threading.Tasks;
 
 namespace OnnxStack.Core.Video
 {
@@ -34,7 +39,7 @@ public VideoInput() { }
         /// </summary>
         /// <param name="videoFrames">The video frames.</param>
         public VideoInput(VideoFrames videoFrames) => VideoFrames = videoFrames;
-        
+
 
         /// <summary>
         /// Gets the video bytes.
@@ -75,5 +80,39 @@ public VideoInput() { }
             || VideoStream != null
             || VideoTensor != null
             || VideoFrames != null;
+
+
+
+        /// <summary>
+        /// Create a VideoInput from file
+        /// </summary>
+        /// <param name="videoFile">The video file.</param>
+        /// <param name="targetFPS">The target FPS.</param>
+        /// <param name="config">The configuration.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <returns></returns>
+        public static async Task<VideoInput> FromFileAsync(string videoFile, float? targetFPS = default, OnnxStackConfig config = default, CancellationToken cancellationToken = default)
+        {
+            var videoBytes = await File.ReadAllBytesAsync(videoFile, cancellationToken);
+            var videoService = new VideoService(config ?? new OnnxStackConfig());
+            var videoFrames = await videoService.CreateFramesAsync(videoBytes, targetFPS, cancellationToken);
+            return new VideoInput(videoFrames);
+        }
+
+
+        /// <summary>
+        /// Saves the video file
+        /// </summary>
+        /// <param name="videoTensor">The video tensor.</param>
+        /// <param name="videoFile">The video file.</param>
+        /// <param name="targetFPS">The target FPS.</param>
+        /// <param name="config">The configuration.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        public static async Task SaveFileAsync(DenseTensor<float> videoTensor, string videoFile, float targetFPS, OnnxStackConfig config = default, CancellationToken cancellationToken = default)
+        {
+            var videoService = new VideoService(config ?? new OnnxStackConfig());
+            var videoOutput = await videoService.CreateVideoAsync(videoTensor, targetFPS, cancellationToken);
+            await File.WriteAllBytesAsync(videoFile, videoOutput.Data, cancellationToken);
+        }
     }
 }

OnnxStack.StableDiffusion/README.md

@@ -175,4 +175,47 @@ await pipeline.UnloadAsync();
 ```
 | Input  | Output |
 | :--- | :--- |
-<img src="../Assets/Samples/Input_Depth.png" width="256"/> | <img src="../Assets/Samples/Output_ControlNet.png" width="256"/>
+<img src="../Assets/Samples/Input_Depth.png" width="256"/> | <img src="../Assets/Samples/Output_ControlNet.png" width="256"/>
+
+
+
+
+## Stable Diffusion VideoToVideo Example
+Run Stable Diffusion process on a video frame by frame
+```csharp
+//Model: 
+//https://huggingface.co/runwayml/stable-diffusion-v1-5 (onnx branch)
+
+ // Create Pipeline
+var pipeline = StableDiffusionPipeline.CreatePipeline("models\\stable-diffusion-v1-5");
+
+ // Preload Models (optional)
+ await pipeline.LoadAsync();
+
+ // Load Video
+ var targetFPS = 15;
+ var videoInput = await VideoInput.FromFileAsync("Input.gif", targetFPS);
+
+ // Add text and video to prompt
+ var promptOptions = new PromptOptions
+ {
+     Prompt = "Elon Musk",
+     DiffuserType = DiffuserType.ImageToImage,
+     InputVideo = videoInput
+ };
+
+ // Run pipeline
+ var result = await pipeline.RunAsync(promptOptions, progressCallback: OutputHelpers.FrameProgressCallback);
+
+ // Save Video File
+ var outputFilename = Path.Combine(_outputDirectory, "Output_VideoToVideo.mp4");
+ await VideoInput.SaveFileAsync(result, outputFilename, targetFPS);
+
+// Unload Pipleine
+await pipeline.UnloadAsync();
+```
+
+| Input  | Output |
+| :--- | :--- |
+<img src="../Assets/Samples/Input.gif" width="256"/> | <img src="../Assets/Samples/Output_VideoToVideo.gif" width="256"/>
+_converted to gif for github readme_ | _converted to gif for github readme_