Fix lora tag check and add device maps and precision for diffusion pipelines (#1829)

Making lora tag check case insensitive.

CC: @linoytsaban @sayakpaul

---------

Co-authored-by: vb <vaibhavs10@gmail.com>
Co-authored-by: Lucain <lucain@huggingface.co>

Author: 3 people

packages/tasks/src/model-libraries-snippets.ts

@@ -348,7 +348,7 @@ dam = DescribeAnythingModel(
 )`,
 ];
 
-const diffusers_install = "pip install -U diffusers transformers";
+const diffusers_install = "pip install -U diffusers transformers accelerate";
 
 const diffusersDefaultPrompt = "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k";
 
@@ -357,19 +357,23 @@ const diffusersImg2ImgDefaultPrompt = "Turn this cat into a dog";
 const diffusersVideoDefaultPrompt = "A man with short gray hair plays a red electric guitar.";
 
 const diffusers_default = (model: ModelData) => [
-	`from diffusers import DiffusionPipeline
+	`import torch
+from diffusers import DiffusionPipeline
 
-pipe = DiffusionPipeline.from_pretrained("${model.id}")
+# switch to "mps" for apple devices
+pipe = DiffusionPipeline.from_pretrained("${model.id}", dtype=torch.bfloat16, device_map="cuda")
 
 prompt = "${get_prompt_from_diffusers_model(model) ?? diffusersDefaultPrompt}"
 image = pipe(prompt).images[0]`,
 ];
 
 const diffusers_image_to_image = (model: ModelData) => [
-	`from diffusers import DiffusionPipeline
+	`import torch
+from diffusers import DiffusionPipeline
 from diffusers.utils import load_image
 
-pipe = DiffusionPipeline.from_pretrained("${model.id}")
+# switch to "mps" for apple devices
+pipe = DiffusionPipeline.from_pretrained("${model.id}", dtype=torch.bfloat16, device_map="cuda")
 
 prompt = "${get_prompt_from_diffusers_model(model) ?? diffusersImg2ImgDefaultPrompt}"
 input_image = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cat.png")
@@ -382,7 +386,8 @@ const diffusers_image_to_video = (model: ModelData) => [
 from diffusers import DiffusionPipeline
 from diffusers.utils import load_image, export_to_video
 
-pipe = DiffusionPipeline.from_pretrained("${model.id}", torch_dtype=torch.float16)
+# switch to "mps" for apple devices
+pipe = DiffusionPipeline.from_pretrained("${model.id}", dtype=torch.bfloat16, device_map="cuda")
 pipe.to("cuda")
 
 prompt = "${get_prompt_from_diffusers_model(model) ?? diffusersVideoDefaultPrompt}"
@@ -404,20 +409,24 @@ pipe = StableDiffusionControlNetPipeline.from_pretrained(
 ];
 
 const diffusers_lora = (model: ModelData) => [
-	`from diffusers import DiffusionPipeline
+	`import torch
+from diffusers import DiffusionPipeline
 
-pipe = DiffusionPipeline.from_pretrained("${get_base_diffusers_model(model)}")
+# switch to "mps" for apple devices
+pipe = DiffusionPipeline.from_pretrained("${get_base_diffusers_model(model)}", dtype=torch.bfloat16, device_map="cuda")
 pipe.load_lora_weights("${model.id}")
 
 prompt = "${get_prompt_from_diffusers_model(model) ?? diffusersDefaultPrompt}"
 image = pipe(prompt).images[0]`,
 ];
 
 const diffusers_lora_image_to_image = (model: ModelData) => [
-	`from diffusers import DiffusionPipeline
+	`import torch
+from diffusers import DiffusionPipeline
 from diffusers.utils import load_image
 
-pipe = DiffusionPipeline.from_pretrained("${get_base_diffusers_model(model)}")
+# switch to "mps" for apple devices
+pipe = DiffusionPipeline.from_pretrained("${get_base_diffusers_model(model)}", dtype=torch.bfloat16, device_map="cuda")
 pipe.load_lora_weights("${model.id}")
 
 prompt = "${get_prompt_from_diffusers_model(model) ?? diffusersImg2ImgDefaultPrompt}"
@@ -427,10 +436,12 @@ image = pipe(image=input_image, prompt=prompt).images[0]`,
 ];
 
 const diffusers_lora_text_to_video = (model: ModelData) => [
-	`from diffusers import DiffusionPipeline
+	`import torch
+from diffusers import DiffusionPipeline
 from diffusers.utils import export_to_video
 
-pipe = DiffusionPipeline.from_pretrained("${get_base_diffusers_model(model)}")
+# switch to "mps" for apple devices
+pipe = DiffusionPipeline.from_pretrained("${get_base_diffusers_model(model)}", dtype=torch.bfloat16, device_map="cuda")
 pipe.load_lora_weights("${model.id}")
 
 prompt = "${get_prompt_from_diffusers_model(model) ?? diffusersVideoDefaultPrompt}"
@@ -440,10 +451,12 @@ export_to_video(output, "output.mp4")`,
 ];
 
 const diffusers_lora_image_to_video = (model: ModelData) => [
-	`from diffusers import DiffusionPipeline
+	`import torch
+from diffusers import DiffusionPipeline
 from diffusers.utils import load_image, export_to_video
 
-pipe = DiffusionPipeline.from_pretrained("${get_base_diffusers_model(model)}")
+# switch to "mps" for apple devices
+pipe = DiffusionPipeline.from_pretrained("${get_base_diffusers_model(model)}", dtype=torch.bfloat16, device_map="cuda")
 pipe.load_lora_weights("${model.id}")
 
 prompt = "${get_prompt_from_diffusers_model(model) ?? diffusersVideoDefaultPrompt}"
@@ -454,9 +467,11 @@ export_to_video(output, "output.mp4")`,
 ];
 
 const diffusers_textual_inversion = (model: ModelData) => [
-	`from diffusers import DiffusionPipeline
+	`import torch
+from diffusers import DiffusionPipeline
 
-pipe = DiffusionPipeline.from_pretrained("${get_base_diffusers_model(model)}")
+# switch to "mps" for apple devices
+pipe = DiffusionPipeline.from_pretrained("${get_base_diffusers_model(model)}", dtype=torch.bfloat16, device_map="cuda")
 pipe.load_textual_inversion("${model.id}")`,
 ];
 
@@ -468,7 +483,8 @@ from diffusers.utils import load_image
 image = load_image("https://huggingface.co/datasets/diffusers/diffusers-images-docs/resolve/main/cup.png")
 mask = load_image("https://huggingface.co/datasets/diffusers/diffusers-images-docs/resolve/main/cup_mask.png")
 
-pipe = FluxFillPipeline.from_pretrained("${model.id}", torch_dtype=torch.bfloat16).to("cuda")
+# switch to "mps" for apple devices
+pipe = FluxFillPipeline.from_pretrained("${model.id}", dtype=torch.bfloat16, device_map="cuda")
 image = pipe(
     prompt="a white paper cup",
     image=image,
@@ -488,7 +504,8 @@ const diffusers_inpainting = (model: ModelData) => [
 from diffusers import AutoPipelineForInpainting
 from diffusers.utils import load_image
 
-pipe = AutoPipelineForInpainting.from_pretrained("${model.id}", torch_dtype=torch.float16, variant="fp16").to("cuda")
+# switch to "mps" for apple devices
+pipe = AutoPipelineForInpainting.from_pretrained("${model.id}", dtype=torch.float16, variant="fp16", device_map="cuda")
 
 img_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png"
 mask_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png"
@@ -1615,7 +1632,7 @@ export const transformers = (model: ModelData): string[] => {
 		autoSnippet.push(
 			"# Load model directly",
 			`from transformers import ${info.auto_model}`,
-			`model = ${info.auto_model}.from_pretrained("${model.id}"` + remote_code_snippet + ', torch_dtype="auto")'
+			`model = ${info.auto_model}.from_pretrained("${model.id}"` + remote_code_snippet + ', dtype="auto")'
 		);
 	}