Skip to content
This repository was archived by the owner on Nov 27, 2024. It is now read-only.

Commit b69fbf3

Browse files
saddam213saddam213
authored
Merge pull request #132 from saddam213/FeatureExtractor
Feature Extractor Improvments
2 parents 651d55e + c642250 commit b69fbf3

File tree

12 files changed

+150
-372
lines changed

12 files changed

+150
-372
lines changed

‎OnnxStack.Console/Examples/BackgroundRemovalImageExample.cs‎

Lines changed: 0 additions & 51 deletions
This file was deleted.

‎OnnxStack.Console/Examples/BackgroundRemovalVideoExample.cs‎

Lines changed: 0 additions & 54 deletions
This file was deleted.

‎OnnxStack.Console/Examples/ControlNetFeatureExample.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ public async Task RunAsync()
3535
var inputImage = await OnnxImage.FromFileAsync("D:\\Repositories\\OnnxStack\\Assets\\Samples\\Img2Img_Start.bmp");
3636

3737
// Create Annotation pipeline
38-
var annotationPipeline = FeatureExtractorPipeline.CreatePipeline("D:\\Repositories\\controlnet_onnx\\annotators\\depth.onnx", true);
38+
var annotationPipeline = FeatureExtractorPipeline.CreatePipeline("D:\\Repositories\\controlnet_onnx\\annotators\\depth.onnx", sampleSize: 512, normalizeOutputTensor: true);
3939

4040
// Create Depth Image
4141
var controlImage = await annotationPipeline.RunAsync(inputImage);

‎OnnxStack.Console/Examples/FeatureExtractorExample.cs‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -37,10 +37,8 @@ public async Task RunAsync()
3737
{
3838
FeatureExtractorPipeline.CreatePipeline("D:\\Repositories\\controlnet_onnx\\annotators\\canny.onnx"),
3939
FeatureExtractorPipeline.CreatePipeline("D:\\Repositories\\controlnet_onnx\\annotators\\hed.onnx"),
40-
FeatureExtractorPipeline.CreatePipeline("D:\\Repositories\\controlnet_onnx\\annotators\\depth.onnx", true),
41-
42-
// FeatureExtractorPipeline.CreatePipeline("D:\\Repositories\\depth-anything-large-hf\\onnx\\model.onnx", normalize: true, sampleSize: 504),
43-
// FeatureExtractorPipeline.CreatePipeline("D:\\Repositories\\sentis-MiDaS\\dpt_beit_large_512.onnx", normalize: true, sampleSize: 384),
40+
FeatureExtractorPipeline.CreatePipeline("D:\\Repositories\\controlnet_onnx\\annotators\\depth.onnx", sampleSize: 512, normalizeOutputTensor: true, inputResizeMode: ImageResizeMode.Stretch),
41+
FeatureExtractorPipeline.CreatePipeline("D:\\Repositories\\RMBG-1.4\\onnx\\model.onnx", sampleSize: 1024, setOutputToInputAlpha: true, inputResizeMode: ImageResizeMode.Stretch)
4442
};
4543

4644
foreach (var pipeline in pipelines)
@@ -49,7 +47,7 @@ public async Task RunAsync()
4947
OutputHelpers.WriteConsole($"Load pipeline`{pipeline.Name}`", ConsoleColor.Cyan);
5048

5149
// Run Image Pipeline
52-
var imageFeature = await pipeline.RunAsync(inputImage);
50+
var imageFeature = await pipeline.RunAsync(inputImage.Clone());
5351

5452
OutputHelpers.WriteConsole($"Generating image", ConsoleColor.Cyan);
5553

‎OnnxStack.Core/Extensions/Extensions.cs‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using Microsoft.ML.OnnxRuntime;
2+
using Microsoft.ML.OnnxRuntime.Tensors;
23
using OnnxStack.Core.Config;
34
using System;
45
using System.Collections.Concurrent;
@@ -244,5 +245,26 @@ public static long[] ToLong(this int[] array)
244245
{
245246
return Array.ConvertAll(array, Convert.ToInt64);
246247
}
248+
249+
250+
/// <summary>
251+
/// Normalize the data using Min-Max scaling to ensure all values are in the range [0, 1].
252+
/// </summary>
253+
/// <param name="values">The values.</param>
254+
public static void NormalizeMinMax(this Span<float> values)
255+
{
256+
float min = float.PositiveInfinity, max = float.NegativeInfinity;
257+
foreach (var val in values)
258+
{
259+
if (min > val) min = val;
260+
if (max < val) max = val;
261+
}
262+
263+
var range = max - min;
264+
for (var i = 0; i < values.Length; i++)
265+
{
266+
values[i] = (values[i] - min) / range;
267+
}
268+
}
247269
}
248270
}

‎OnnxStack.Core/Extensions/TensorExtension.cs‎

Lines changed: 1 addition & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -286,19 +286,7 @@ public static DenseTensor<float> Repeat(this DenseTensor<float> tensor1, int cou
286286
/// <param name="tensor">The tensor.</param>
287287
public static void NormalizeMinMax(this DenseTensor<float> tensor)
288288
{
289-
var values = tensor.Buffer.Span;
290-
float min = float.PositiveInfinity, max = float.NegativeInfinity;
291-
foreach (var val in values)
292-
{
293-
if (min > val) min = val;
294-
if (max < val) max = val;
295-
}
296-
297-
var range = max - min;
298-
for (var i = 0; i < values.Length; i++)
299-
{
300-
values[i] = (values[i] - min) / range;
301-
}
289+
tensor.Buffer.Span.NormalizeMinMax();
302290
}
303291

304292

‎OnnxStack.Core/Image/Extensions.cs‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
using Microsoft.ML.OnnxRuntime.Tensors;
22
using SixLabors.ImageSharp;
33
using SixLabors.ImageSharp.PixelFormats;
4+
using SixLabors.ImageSharp.Processing;
45

56
namespace OnnxStack.Core.Image
67
{
@@ -29,11 +30,27 @@ public static OnnxImage ToImageMask(this DenseTensor<float> imageTensor)
2930
}
3031
}
3132

33+
34+
public static ResizeMode ToResizeMode(this ImageResizeMode resizeMode)
35+
{
36+
return resizeMode switch
37+
{
38+
ImageResizeMode.Stretch => ResizeMode.Stretch,
39+
_ => ResizeMode.Crop
40+
};
41+
}
42+
3243
}
3344

3445
public enum ImageNormalizeType
3546
{
3647
ZeroToOne = 0,
37-
OneToOne = 1,
48+
OneToOne = 1
49+
}
50+
51+
public enum ImageResizeMode
52+
{
53+
Crop = 0,
54+
Stretch = 1
3855
}
3956
}

‎OnnxStack.Core/Image/OnnxImage.cs‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -230,15 +230,27 @@ public DenseTensor<float> GetImageTensor(ImageNormalizeType normalizeType = Imag
230230
/// <param name="normalizeType">Type of the normalize.</param>
231231
/// <param name="channels">The channels.</param>
232232
/// <returns></returns>
233-
public DenseTensor<float> GetImageTensor(int height, int width, ImageNormalizeType normalizeType = ImageNormalizeType.OneToOne, int channels = 3)
233+
public DenseTensor<float> GetImageTensor(int height, int width, ImageNormalizeType normalizeType = ImageNormalizeType.OneToOne, int channels = 3, ImageResizeMode resizeMode = ImageResizeMode.Crop)
234234
{
235235
if (height > 0 && width > 0)
236-
Resize(height, width);
236+
Resize(height, width, resizeMode);
237237

238238
return GetImageTensor(normalizeType, channels);
239239
}
240240

241241

242+
/// <summary>
243+
/// Gets the image as tensor asynchronously.
244+
/// </summary>
245+
/// <param name="normalizeType">Type of the normalize.</param>
246+
/// <param name="channels">The channels.</param>
247+
/// <returns></returns>
248+
public Task<DenseTensor<float>> GetImageTensorAsync(ImageNormalizeType normalizeType = ImageNormalizeType.OneToOne, int channels = 3)
249+
{
250+
return Task.Run(() => GetImageTensor(normalizeType, channels));
251+
}
252+
253+
242254
/// <summary>
243255
/// Gets the image as tensor asynchronously.
244256
/// </summary>
@@ -247,9 +259,9 @@ public DenseTensor<float> GetImageTensor(int height, int width, ImageNormalizeTy
247259
/// <param name="normalizeType">Type of the normalize.</param>
248260
/// <param name="channels">The channels.</param>
249261
/// <returns></returns>
250-
public Task<DenseTensor<float>> GetImageTensorAsync(int height, int width, ImageNormalizeType normalizeType = ImageNormalizeType.OneToOne, int channels = 3)
262+
public Task<DenseTensor<float>> GetImageTensorAsync(int height, int width, ImageNormalizeType normalizeType = ImageNormalizeType.OneToOne, int channels = 3, ImageResizeMode resizeMode = ImageResizeMode.Crop)
251263
{
252-
return Task.Run(() => GetImageTensor(height, width, normalizeType, channels));
264+
return Task.Run(() => GetImageTensor(height, width, normalizeType, channels, resizeMode));
253265
}
254266

255267

@@ -259,20 +271,21 @@ public Task<DenseTensor<float>> GetImageTensorAsync(int height, int width, Image
259271
/// <param name="height">The height.</param>
260272
/// <param name="width">The width.</param>
261273
/// <param name="resizeMode">The resize mode.</param>
262-
public void Resize(int height, int width, ResizeMode resizeMode = ResizeMode.Crop)
274+
public void Resize(int height, int width, ImageResizeMode resizeMode = ImageResizeMode.Crop)
263275
{
264276
_imageData.Mutate(x =>
265277
{
266278
x.Resize(new ResizeOptions
267279
{
268280
Size = new Size(width, height),
269-
Mode = resizeMode,
281+
Mode = resizeMode.ToResizeMode(),
270282
Sampler = KnownResamplers.Lanczos8,
271283
Compand = true
272284
});
273285
});
274286
}
275287

288+
276289
public OnnxImage Clone()
277290
{
278291
return new OnnxImage(_imageData);

‎OnnxStack.FeatureExtractor/Common/FeatureExtractorModel.cs‎

Lines changed: 19 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,56 +1,52 @@
11
using Microsoft.ML.OnnxRuntime;
22
using OnnxStack.Core.Config;
3+
using OnnxStack.Core.Image;
34
using OnnxStack.Core.Model;
45

56
namespace OnnxStack.FeatureExtractor.Common
67
{
78
public class FeatureExtractorModel : OnnxModelSession
89
{
9-
private readonly int _sampleSize;
10-
private readonly bool _normalize;
11-
private readonly int _channels;
10+
private readonly FeatureExtractorModelConfig _configuration;
1211

1312
public FeatureExtractorModel(FeatureExtractorModelConfig configuration)
1413
: base(configuration)
1514
{
16-
_sampleSize = configuration.SampleSize;
17-
_normalize = configuration.Normalize;
18-
_channels = configuration.Channels;
15+
_configuration = configuration;
1916
}
2017

21-
public int SampleSize => _sampleSize;
22-
23-
public bool Normalize => _normalize;
24-
25-
public int Channels => _channels;
18+
public int OutputChannels => _configuration.OutputChannels;
19+
public int SampleSize => _configuration.SampleSize;
20+
public bool NormalizeOutputTensor => _configuration.NormalizeOutputTensor;
21+
public bool SetOutputToInputAlpha => _configuration.SetOutputToInputAlpha;
22+
public ImageResizeMode InputResizeMode => _configuration.InputResizeMode;
23+
public ImageNormalizeType InputNormalization => _configuration.NormalizeInputTensor;
2624

2725
public static FeatureExtractorModel Create(FeatureExtractorModelConfig configuration)
2826
{
2927
return new FeatureExtractorModel(configuration);
3028
}
3129

32-
public static FeatureExtractorModel Create(string modelFile, bool normalize = false, int sampleSize = 512, int channels = 3, int deviceId = 0, ExecutionProvider executionProvider = ExecutionProvider.DirectML)
30+
public static FeatureExtractorModel Create(string modelFile, int sampleSize = 0, int outputChannels = 1, bool normalizeOutputTensor = false, ImageNormalizeType normalizeInputTensor = ImageNormalizeType.ZeroToOne, ImageResizeMode inputResizeMode = ImageResizeMode.Crop, bool setOutputToInputAlpha = false, int deviceId = 0, ExecutionProvider executionProvider = ExecutionProvider.DirectML)
3331
{
3432
var configuration = new FeatureExtractorModelConfig
3533
{
36-
SampleSize = sampleSize,
37-
Normalize = normalize,
38-
Channels = channels,
3934
DeviceId = deviceId,
4035
ExecutionProvider = executionProvider,
4136
ExecutionMode = ExecutionMode.ORT_SEQUENTIAL,
4237
InterOpNumThreads = 0,
4338
IntraOpNumThreads = 0,
44-
OnnxModelPath = modelFile
39+
OnnxModelPath = modelFile,
40+
41+
42+
SampleSize = sampleSize,
43+
OutputChannels = outputChannels,
44+
NormalizeOutputTensor = normalizeOutputTensor,
45+
SetOutputToInputAlpha = setOutputToInputAlpha,
46+
NormalizeInputTensor = normalizeInputTensor,
47+
InputResizeMode = inputResizeMode
4548
};
4649
return new FeatureExtractorModel(configuration);
4750
}
4851
}
49-
50-
public record FeatureExtractorModelConfig : OnnxModelConfig
51-
{
52-
public int SampleSize { get; set; }
53-
public bool Normalize { get; set; }
54-
public int Channels { get; set; }
55-
}
5652
}

‎OnnxStack.FeatureExtractor/Common/FeatureExtractorModelConfig.cs‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
using OnnxStack.Core.Config;
2+
using OnnxStack.Core.Image;
3+
4+
namespace OnnxStack.FeatureExtractor.Common
5+
{
6+
public record FeatureExtractorModelConfig : OnnxModelConfig
7+
{
8+
public int SampleSize { get; set; }
9+
public int OutputChannels { get; set; }
10+
public bool NormalizeOutputTensor { get; set; }
11+
public bool SetOutputToInputAlpha { get; set; }
12+
public ImageResizeMode InputResizeMode { get; set; }
13+
public ImageNormalizeType NormalizeInputTensor { get; set; }
14+
}
15+
}

0 commit comments

Comments
 (0)