fix bug bring in diffuser 0.24.0

Author: SH

inference.sh

@@ -1,2 +1,2 @@
 
-python src/diffusers_sample.py --delta_ckpt logs/wooden_pot/delta.bin --ckpt "/data/home/chensh/data/huggingface_model/stable-diffusion-xl-base-1.0" --prompt "<new1> cat playing with a ball"
+CUDA_VISIBLE_DEVICES=0 python src/diffusers_sample.py --delta_ckpt logs/wooden_pot/delta.bin --sdxl --ckpt "/data/home/chensh/data/huggingface_model/stable-diffusion-xl-base-1.0" --prompt "<new1> cat playing with a ball"

src/diffusers_model_pipeline.py

@@ -212,7 +212,8 @@
 #    limitations under the License.
 from typing import Callable, Optional
 import torch
-from transformers import CLIPFeatureExtractor, CLIPTextModel, CLIPTokenizer, CLIPTextModelWithProjection
+from transformers import CLIPFeatureExtractor, CLIPTextModel, CLIPTokenizer, CLIPTextModelWithProjection, \
+    CLIPVisionModelWithProjection, CLIPImageProcessor
 from accelerate.logging import get_logger
 
 from diffusers.models import AutoencoderKL, UNet2DConditionModel
@@ -551,22 +552,36 @@ def __init__(
         tokenizer_2: CLIPTokenizer,
         unet: UNet2DConditionModel,
         scheduler: KarrasDiffusionSchedulers,
+        image_encoder: CLIPVisionModelWithProjection = None,
+        feature_extractor: CLIPImageProcessor = None,
         force_zeros_for_empty_prompt: bool = True,
         add_watermarker: Optional[bool] = None,
         modifier_token: list = [],
         modifier_token_id: list = [],
         modifier_token_id_2: list = []
     ):
-        super().__init__(vae=vae,
-                         text_encoder=text_encoder,
-                         text_encoder_2=text_encoder_2,
-                         tokenizer=tokenizer,
-                         tokenizer_2=tokenizer_2,
-                         unet=unet,
-                         scheduler=scheduler,
+        super().__init__(vae,
+                         text_encoder,
+                         text_encoder_2,
+                         tokenizer,
+                         tokenizer_2,
+                         unet,
+                         scheduler,
+                         image_encoder=image_encoder,
+                         feature_extractor=feature_extractor,
                          force_zeros_for_empty_prompt=force_zeros_for_empty_prompt,
                          add_watermarker=add_watermarker,
                          )
+        # super().__init__(vae,
+        #                  text_encoder,
+        #                  text_encoder_2,
+        #                  tokenizer,
+        #                  tokenizer_2,
+        #                  unet,
+        #                  scheduler,
+        #                  force_zeros_for_empty_prompt,
+        #                  add_watermarker,
+        #                  )
 
         # change attn class
         self.modifier_token = modifier_token

src/diffusers_sample.py

@@ -10,6 +10,7 @@
 import numpy as np
 import torch
 from PIL import Image
+from diffusers import StableDiffusionXLPipeline
 
 sys.path.append('./')
 from src.diffusers_model_pipeline import CustomDiffusionPipeline, CustomDiffusionXLPipeline
@@ -18,7 +19,10 @@
 def sample(ckpt, delta_ckpt, from_file, prompt, compress, batch_size, freeze_model, sdxl=False):
     model_id = ckpt
     if sdxl:
-        pipe = CustomDiffusionXLPipeline.from_pretrained(model_id, torch_dtype=torch.float16).to("cuda")
+        pipe = CustomDiffusionXLPipeline.from_pretrained(model_id, torch_dtype=torch.float16)
+        print(pipe.components)
+        # pipe = StableDiffusionXLPipeline.from_pretrained(model_id, torch_dtype=torch.float16)
+        pipe = pipe.to("cuda")
     else:
         pipe = CustomDiffusionPipeline.from_pretrained(model_id, torch_dtype=torch.float16).to("cuda")
     pipe.load_model(delta_ckpt, compress)