Skip to content
This repository was archived by the owner on Nov 27, 2024. It is now read-only.

Commit 4837ae4

Browse files
saddam213saddam213
committed
SDXL TextToImage prototype
1 parent e4f86a2 commit 4837ae4

File tree

7 files changed

+103
-8
lines changed

7 files changed

+103
-8
lines changed

‎OnnxStack.Console/appsettings.json‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
"TokenizerLimit": 77,
1717
"EmbeddingsLength": 768,
1818
"ScaleFactor": 0.18215,
19+
"SampleSize": 512,
1920
"PipelineType": "StableDiffusion",
2021
"Diffusers": [
2122
"TextToImage",
@@ -58,6 +59,7 @@
5859
"TokenizerLimit": 77,
5960
"EmbeddingsLength": 768,
6061
"ScaleFactor": 0.18215,
62+
"SampleSize": 512,
6163
"PipelineType": "LatentConsistency",
6264
"Diffusers": [
6365
"TextToImage",
@@ -99,6 +101,7 @@
99101
"TokenizerLimit": 77,
100102
"EmbeddingsLength": 768,
101103
"ScaleFactor": 0.18215,
104+
"SampleSize": 512,
102105
"PipelineType": "StableDiffusion",
103106
"Diffusers": [
104107
"TextToImage",
@@ -141,6 +144,7 @@
141144
"TokenizerLimit": 77,
142145
"EmbeddingsLength": 768,
143146
"ScaleFactor": 0.18215,
147+
"SampleSize": 512,
144148
"PipelineType": "InstaFlow",
145149
"Diffusers": [
146150
"TextToImage"
@@ -176,11 +180,14 @@
176180
{
177181
"Name": "DreamShaper XL",
178182
"IsEnabled": true,
179-
"PadTokenId": 49407,
183+
"PadTokenId": 1,
180184
"BlankTokenId": 49407,
181185
"TokenizerLimit": 77,
182-
"EmbeddingsLength": 2816,
186+
"EmbeddingsLength": 768,
187+
"DualEmbeddingsLength": 1280,
188+
"IsDualTokenizer": true,
183189
"ScaleFactor": 0.13025,
190+
"SampleSize": 1024,
184191
"PipelineType": "StableDiffusionXL",
185192
"Diffusers": [
186193
"TextToImage"
@@ -197,7 +204,7 @@
197204
},
198205
{
199206
"Type": "Tokenizer2",
200-
"OnnxModelPath": "D:\\Repositories\\dreamshaper-xl-1-0-Olive-Onnx\\tokenizer2\\model.onnx"
207+
"OnnxModelPath": "D:\\Repositories\\dreamshaper-xl-1-0-Olive-Onnx\\tokenizer_2\\model.onnx"
201208
},
202209
{
203210
"Type": "Unet",
@@ -209,7 +216,7 @@
209216
},
210217
{
211218
"Type": "TextEncoder2",
212-
"OnnxModelPath": "D:\\Repositories\\dreamshaper-xl-1-0-Olive-Onnx\\text_encoder2\\model.onnx"
219+
"OnnxModelPath": "D:\\Repositories\\dreamshaper-xl-1-0-Olive-Onnx\\text_encoder_2\\model.onnx"
213220
},
214221
{
215222
"Type": "VaeEncoder",

‎OnnxStack.Core/Extensions/OrtValueExtensions.cs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,17 @@ public static OrtValue ToOrtValue(this DenseTensor<int> tensor, OnnxNamedMetadat
4949
}
5050

5151

52+
/// <summary>
53+
/// Converts DenseTensor<int> to OrtValue.
54+
/// </summary>
55+
/// <param name="tensor">The tensor.</param>
56+
/// <returns></returns>
57+
public static OrtValue ToOrtValue(this DenseTensor<long> tensor, OnnxNamedMetadata metadata)
58+
{
59+
return OrtValue.CreateTensorValueFromMemory(OrtMemoryInfo.DefaultInstance, tensor.Buffer, tensor.Dimensions.ToLong());
60+
}
61+
62+
5263
/// <summary>
5364
/// Creates and allocates the output tensors buffer.
5465
/// </summary>

‎OnnxStack.Core/Model/OnnxInferenceParameters.cs‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,16 @@ public void AddInputTensor(DenseTensor<int> value)
6767
_inputs.Add(metaData, value.ToOrtValue(metaData));
6868
}
6969

70+
/// <summary>
71+
/// Adds the input tensor.
72+
/// </summary>
73+
/// <param name="value">The value.</param>
74+
public void AddInputTensor(DenseTensor<long> value)
75+
{
76+
var metaData = GetNextInputMetadata();
77+
_inputs.Add(metaData, value.ToOrtValue(metaData));
78+
}
79+
7080

7181
/// <summary>
7282
/// Adds an output parameter with known output size.
@@ -90,6 +100,19 @@ public void AddOutputBuffer(ReadOnlySpan<int> bufferDimension)
90100
}
91101

92102

103+
/// <summary>
104+
/// Adds the output buffer.
105+
/// </summary>
106+
/// <param name="index">The index.</param>
107+
/// <param name="bufferDimension">The buffer dimension.</param>
108+
public void AddOutputBuffer(int index, ReadOnlySpan<int> bufferDimension)
109+
{
110+
var metadata = _metadata.Outputs[index];
111+
_outputs.Add(metadata, metadata.CreateOutputBuffer(bufferDimension));
112+
}
113+
114+
115+
93116
/// <summary>
94117
/// Adds an output parameter with unknown output size.
95118
/// </summary>

‎OnnxStack.StableDiffusion/Common/IModelOptions.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,12 @@ public interface IModelOptions : IOnnxModel
1010
bool IsEnabled { get; set; }
1111
int PadTokenId { get; set; }
1212
int BlankTokenId { get; set; }
13+
int SampleSize { get; set; }
1314
float ScaleFactor { get; set; }
1415
int TokenizerLimit { get; set; }
1516
int EmbeddingsLength { get; set; }
17+
int DualEmbeddingsLength { get; set; }
18+
bool IsDualTokenizer { get; set; }
1619
DiffuserPipelineType PipelineType { get; set; }
1720
List<DiffuserType> Diffusers { get; set; }
1821
ImmutableArray<int> BlankTokenValueArray { get; set; }

‎OnnxStack.StableDiffusion/Config/ModelOptions.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@ public class ModelOptions : IModelOptions, IOnnxModelSetConfig
1717
public int BlankTokenId { get; set; }
1818
public int TokenizerLimit { get; set; }
1919
public int EmbeddingsLength { get; set; }
20+
public int DualEmbeddingsLength { get; set; }
21+
public bool IsDualTokenizer { get; set; }
22+
public int SampleSize { get; set; } = 512;
2023
public float ScaleFactor { get; set; }
2124
public DiffuserPipelineType PipelineType { get; set; }
2225
public List<DiffuserType> Diffusers { get; set; } = new List<DiffuserType>();

‎OnnxStack.StableDiffusion/Diffusers/StableDiffusionXL/StableDiffusionXLDiffuser.cs‎

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ public StableDiffusionXLDiffuser(IOnnxModelService onnxModelService, IPromptServ
4545
/// <param name="progressCallback">The progress callback.</param>
4646
/// <param name="cancellationToken">The cancellation token.</param>
4747
/// <returns></returns>
48-
protected override async Task<DenseTensor<float>> SchedulerStepAsync(IModelOptions modelOptions, PromptOptions promptOptions, SchedulerOptions schedulerOptions, DenseTensor<float> promptEmbeddings, bool performGuidance, Action<int, int> progressCallback = null, CancellationToken cancellationToken = default)
48+
protected override async Task<DenseTensor<float>> SchedulerStepAsync(IModelOptions modelOptions, PromptOptions promptOptions, SchedulerOptions schedulerOptions, PromptEmbeddingsResult promptEmbeddings, bool performGuidance, Action<int, int> progressCallback = null, CancellationToken cancellationToken = default)
4949
{
5050
// Get Scheduler
5151
using (var scheduler = GetScheduler(schedulerOptions))
@@ -71,14 +71,17 @@ protected override async Task<DenseTensor<float>> SchedulerStepAsync(IModelOptio
7171
var inputLatent = performGuidance ? latents.Repeat(2) : latents;
7272
var inputTensor = scheduler.ScaleInput(inputLatent, timestep);
7373
var timestepTensor = CreateTimestepTensor(timestep);
74+
var addTimeIds = GetAddTimeIds(schedulerOptions, performGuidance);
7475

7576
var outputChannels = performGuidance ? 2 : 1;
7677
var outputDimension = schedulerOptions.GetScaledDimension(outputChannels);
7778
using (var inferenceParameters = new OnnxInferenceParameters(metadata))
7879
{
7980
inferenceParameters.AddInputTensor(inputTensor);
8081
inferenceParameters.AddInputTensor(timestepTensor);
81-
inferenceParameters.AddInputTensor(promptEmbeddings);
82+
inferenceParameters.AddInputTensor(promptEmbeddings.PromptEmbeds);
83+
inferenceParameters.AddInputTensor(promptEmbeddings.PooledPromptEmbeds);
84+
inferenceParameters.AddInputTensor(addTimeIds);
8285
inferenceParameters.AddOutputBuffer(outputDimension);
8386

8487
var results = await _onnxModelService.RunInferenceAsync(modelOptions, OnnxModelType.Unet, inferenceParameters);
@@ -105,6 +108,27 @@ protected override async Task<DenseTensor<float>> SchedulerStepAsync(IModelOptio
105108
}
106109

107110

111+
/// <summary>
112+
/// Gets the add AddTimeIds.
113+
/// </summary>
114+
/// <param name="schedulerOptions">The scheduler options.</param>
115+
/// <returns></returns>
116+
private DenseTensor<float> GetAddTimeIds(SchedulerOptions schedulerOptions, bool performGuidance)
117+
{
118+
var addTimeIds = new float[]
119+
{
120+
schedulerOptions.Height, schedulerOptions.Width, //original_size
121+
0, 0, //crops_coords_top_left
122+
schedulerOptions.Height, schedulerOptions.Width //negative_target_size
123+
};
124+
var result = TensorHelper.CreateTensor(addTimeIds, new[] { 1, addTimeIds.Length });
125+
if (performGuidance)
126+
return result.Repeat(2);
127+
128+
return result;
129+
}
130+
131+
108132
/// <summary>
109133
/// Gets the scheduler.
110134
/// </summary>

‎OnnxStack.StableDiffusion/Helpers/TensorHelper.cs‎

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -268,8 +268,11 @@ public static DenseTensor<float> Divide(this DenseTensor<float> tensor, DenseTen
268268
/// <exception cref="System.NotImplementedException">Only axis 0 is supported</exception>
269269
public static DenseTensor<float> Concatenate(this DenseTensor<float> tensor1, DenseTensor<float> tensor2, int axis = 0)
270270
{
271-
if (axis != 0)
272-
throw new NotImplementedException("Only axis 0 is supported");
271+
if (axis != 0 && axis != 2)
272+
throw new NotImplementedException("Only axis 0, 2 is supported");
273+
274+
if (axis == 2)
275+
return Concatenate(tensor1, tensor2);
273276

274277
var dimensions = tensor1.Dimensions.ToArray();
275278
dimensions[0] += tensor2.Dimensions[0];
@@ -280,6 +283,27 @@ public static DenseTensor<float> Concatenate(this DenseTensor<float> tensor1, De
280283
return new DenseTensor<float>(buffer, dimensions);
281284
}
282285

286+
private static DenseTensor<float> Concatenate(DenseTensor<float> tensor1, DenseTensor<float> tensor2)
287+
{
288+
var dimensions = new int[] { tensor1.Dimensions[0], tensor1.Dimensions[1], tensor1.Dimensions[2] + tensor2.Dimensions[2] };
289+
DenseTensor<float> concatenatedTensor = new DenseTensor<float>(dimensions);
290+
291+
// Copy data from the first tensor
292+
for (int i = 0; i < dimensions[0]; i++)
293+
for (int j = 0; j < dimensions[1]; j++)
294+
for (int k = 0; k < tensor1.Dimensions[2]; k++)
295+
concatenatedTensor[i, j, k] = tensor1[i, j, k];
296+
297+
// Copy data from the second tensor
298+
for (int i = 0; i < dimensions[0]; i++)
299+
for (int j = 0; j < dimensions[1]; j++)
300+
for (int k = 0; k < tensor2.Dimensions[2]; k++)
301+
concatenatedTensor[i, j, k + tensor1.Dimensions[2]] = tensor2[i, j, k];
302+
303+
return concatenatedTensor;
304+
}
305+
306+
283307

284308
/// <summary>
285309
/// Repeats the specified Tensor along the 0 axis.

0 commit comments

Comments
 (0)