I can't load loras
Opened this issue · 0 comments
loboere commented
I try to load loras with
pipeline.pipe.load_lora_weights("/kaggle/input/lorass/acuarelac1400.safetensors")
I don't know if it is the correct way, it would be helpful if you told me how to load loras
but i get
---------------------------------------------------------------------
RuntimeError Traceback (most recent call last)
Cell In[175], line 23
21 seed_everything(seed)
22 #out=pipeline.inference(prompt, init_image, mask_image , 0.8, (1, H, W), neg_prompt, id_embeddings, id_scale, scale, steps )
---> 23 out=pipeline.pipe(prompt=prompt,
24 image=init_image,
25 mask_image=mask_image,
26 strength=0.8,
27 negative_prompt=neg_prompt,
28 num_images_per_prompt=1,
29 height=H,
30 width=W,
31 num_inference_steps=steps,
32 guidance_scale= scale,
33 cross_attention_kwargs={ 'id_embedding': id_embeddings, 'id_scale': id_scale},)
35 out[0]
File /opt/conda/lib/python3.10/site-packages/torch/utils/_contextlib.py:115, in context_decorator.<locals>.decorate_context(*args, **kwargs)
112 @functools.wraps(func)
113 def decorate_context(*args, **kwargs):
114 with ctx_factory():
--> 115 return func(*args, **kwargs)
File /opt/conda/lib/python3.10/site-packages/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py:1707, in StableDiffusionXLInpaintPipeline.__call__(self, prompt, prompt_2, image, mask_image, masked_image_latents, height, width, strength, num_inference_steps, timesteps, denoising_start, denoising_end, guidance_scale, negative_prompt, negative_prompt_2, num_images_per_prompt, eta, generator, latents, prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds, ip_adapter_image, output_type, return_dict, cross_attention_kwargs, guidance_rescale, original_size, crops_coords_top_left, target_size, negative_original_size, negative_crops_coords_top_left, negative_target_size, aesthetic_score, negative_aesthetic_score, clip_skip, callback_on_step_end, callback_on_step_end_tensor_inputs, **kwargs)
1705 if ip_adapter_image is not None:
1706 added_cond_kwargs["image_embeds"] = image_embeds
-> 1707 noise_pred = self.unet(
1708 latent_model_input,
1709 t,
1710 encoder_hidden_states=prompt_embeds,
1711 timestep_cond=timestep_cond,
1712 cross_attention_kwargs=self.cross_attention_kwargs,
1713 added_cond_kwargs=added_cond_kwargs,
1714 return_dict=False,
1715 )[0]
1717 # perform guidance
1718 if self.do_classifier_free_guidance:
File /opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py:1501, in Module._call_impl(self, *args, **kwargs)
1496 # If we don't have any hooks, we want to skip the rest of the logic in
1497 # this function, and just call forward.
1498 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
1499 or _global_backward_pre_hooks or _global_backward_hooks
1500 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1501 return forward_call(*args, **kwargs)
1502 # Do not call functions when jit is used
1503 full_backward_hooks, non_full_backward_hooks = [], []
File /opt/conda/lib/python3.10/site-packages/diffusers/models/unet_2d_condition.py:1112, in UNet2DConditionModel.forward(self, sample, timestep, encoder_hidden_states, class_labels, timestep_cond, attention_mask, cross_attention_kwargs, added_cond_kwargs, down_block_additional_residuals, mid_block_additional_residual, down_intrablock_additional_residuals, encoder_attention_mask, return_dict)
1109 if is_adapter and len(down_intrablock_additional_residuals) > 0:
1110 additional_residuals["additional_residuals"] = down_intrablock_additional_residuals.pop(0)
-> 1112 sample, res_samples = downsample_block(
1113 hidden_states=sample,
1114 temb=emb,
1115 encoder_hidden_states=encoder_hidden_states,
1116 attention_mask=attention_mask,
1117 cross_attention_kwargs=cross_attention_kwargs,
1118 encoder_attention_mask=encoder_attention_mask,
1119 **additional_residuals,
1120 )
1121 else:
1122 sample, res_samples = downsample_block(hidden_states=sample, temb=emb, scale=lora_scale)
File /opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py:1501, in Module._call_impl(self, *args, **kwargs)
1496 # If we don't have any hooks, we want to skip the rest of the logic in
1497 # this function, and just call forward.
1498 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
1499 or _global_backward_pre_hooks or _global_backward_hooks
1500 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1501 return forward_call(*args, **kwargs)
1502 # Do not call functions when jit is used
1503 full_backward_hooks, non_full_backward_hooks = [], []
File /opt/conda/lib/python3.10/site-packages/diffusers/models/unet_2d_blocks.py:1160, in CrossAttnDownBlock2D.forward(self, hidden_states, temb, encoder_hidden_states, attention_mask, cross_attention_kwargs, encoder_attention_mask, additional_residuals)
1158 else:
1159 hidden_states = resnet(hidden_states, temb, scale=lora_scale)
-> 1160 hidden_states = attn(
1161 hidden_states,
1162 encoder_hidden_states=encoder_hidden_states,
1163 cross_attention_kwargs=cross_attention_kwargs,
1164 attention_mask=attention_mask,
1165 encoder_attention_mask=encoder_attention_mask,
1166 return_dict=False,
1167 )[0]
1169 # apply additional residuals to the output of the last pair of resnet and attention blocks
1170 if i == len(blocks) - 1 and additional_residuals is not None:
File /opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py:1501, in Module._call_impl(self, *args, **kwargs)
1496 # If we don't have any hooks, we want to skip the rest of the logic in
1497 # this function, and just call forward.
1498 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
1499 or _global_backward_pre_hooks or _global_backward_hooks
1500 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1501 return forward_call(*args, **kwargs)
1502 # Do not call functions when jit is used
1503 full_backward_hooks, non_full_backward_hooks = [], []
File /opt/conda/lib/python3.10/site-packages/diffusers/models/transformer_2d.py:392, in Transformer2DModel.forward(self, hidden_states, encoder_hidden_states, timestep, added_cond_kwargs, class_labels, cross_attention_kwargs, attention_mask, encoder_attention_mask, return_dict)
380 hidden_states = torch.utils.checkpoint.checkpoint(
381 create_custom_forward(block),
382 hidden_states,
(...)
389 **ckpt_kwargs,
390 )
391 else:
--> 392 hidden_states = block(
393 hidden_states,
394 attention_mask=attention_mask,
395 encoder_hidden_states=encoder_hidden_states,
396 encoder_attention_mask=encoder_attention_mask,
397 timestep=timestep,
398 cross_attention_kwargs=cross_attention_kwargs,
399 class_labels=class_labels,
400 )
402 # 3. Output
403 if self.is_input_continuous:
File /opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py:1501, in Module._call_impl(self, *args, **kwargs)
1496 # If we don't have any hooks, we want to skip the rest of the logic in
1497 # this function, and just call forward.
1498 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
1499 or _global_backward_pre_hooks or _global_backward_hooks
1500 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1501 return forward_call(*args, **kwargs)
1502 # Do not call functions when jit is used
1503 full_backward_hooks, non_full_backward_hooks = [], []
File /opt/conda/lib/python3.10/site-packages/diffusers/models/attention.py:366, in BasicTransformerBlock.forward(self, hidden_states, attention_mask, encoder_hidden_states, encoder_attention_mask, timestep, cross_attention_kwargs, class_labels, added_cond_kwargs)
363 if self.pos_embed is not None and self.use_ada_layer_norm_single is False:
364 norm_hidden_states = self.pos_embed(norm_hidden_states)
--> 366 attn_output = self.attn2(
367 norm_hidden_states,
368 encoder_hidden_states=encoder_hidden_states,
369 attention_mask=encoder_attention_mask,
370 **cross_attention_kwargs,
371 )
372 hidden_states = attn_output + hidden_states
374 # 4. Feed-forward
File /opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py:1501, in Module._call_impl(self, *args, **kwargs)
1496 # If we don't have any hooks, we want to skip the rest of the logic in
1497 # this function, and just call forward.
1498 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
1499 or _global_backward_pre_hooks or _global_backward_hooks
1500 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1501 return forward_call(*args, **kwargs)
1502 # Do not call functions when jit is used
1503 full_backward_hooks, non_full_backward_hooks = [], []
File /opt/conda/lib/python3.10/site-packages/diffusers/models/attention_processor.py:527, in Attention.forward(self, hidden_states, encoder_hidden_states, attention_mask, **cross_attention_kwargs)
508 r"""
509 The forward method of the `Attention` class.
510
(...)
522 `torch.Tensor`: The output of the attention layer.
523 """
524 # The `Attention` class can call different attention processors / attention functions
525 # here we simply pass along all tensors to the selected processor class
526 # For standard processors that are defined here, `**cross_attention_kwargs` is empty
--> 527 return self.processor(
528 self,
529 hidden_states,
530 encoder_hidden_states=encoder_hidden_states,
531 attention_mask=attention_mask,
532 **cross_attention_kwargs,
533 )
File /kaggle/working/PuLID/pulid/attention_processor.py:365, in IDAttnProcessor2_0.__call__(self, attn, hidden_states, encoder_hidden_states, attention_mask, temb, id_embedding, id_scale)
359 else:
360 zero_tensor = torch.zeros(
361 (id_embedding.size(0), NUM_ZERO, id_embedding.size(-1)),
362 dtype=id_embedding.dtype,
363 device=id_embedding.device,
364 )
--> 365 id_key = self.id_to_k(torch.cat((id_embedding, zero_tensor), dim=1)).to(query.dtype)
366 id_value = self.id_to_v(torch.cat((id_embedding, zero_tensor), dim=1)).to(query.dtype)
368 id_key = id_key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
File /opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py:1501, in Module._call_impl(self, *args, **kwargs)
1496 # If we don't have any hooks, we want to skip the rest of the logic in
1497 # this function, and just call forward.
1498 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
1499 or _global_backward_pre_hooks or _global_backward_hooks
1500 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1501 return forward_call(*args, **kwargs)
1502 # Do not call functions when jit is used
1503 full_backward_hooks, non_full_backward_hooks = [], []
File /opt/conda/lib/python3.10/site-packages/torch/nn/modules/linear.py:114, in Linear.forward(self, input)
113 def forward(self, input: Tensor) -> Tensor:
--> 114 return F.linear(input, self.weight, self.bias)
RuntimeError: expected scalar type Float but found Half
I don't know if it is the correct way, it would be helpful if you told me how to load loras