Every time I start the training, I always get this.

Question

Every time I start the training, I always get this.

Closed this issue 7 months ago · 5 comments

╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮
│ /content/kohya-trainer/train_network.py:17 in
│
│ 14 from accelerate.utils import set_seed
│ 15 from diffusers import DDPMScheduler
│ 16
│ ❱ 17 import library.train_util as train_util
│ 18 from library.train_util import (
│ 19 │ DreamBoothDataset,
│ 20 )
│
│ /content/kohya-trainer/library/train_util.py:1861 in
│
│ 1858 │ │ return dq, dk, dv, None, None, None, None
│ 1859
│ 1860
│ ❱ 1861 def replace_unet_modules(unet: diffusers.models.unet_2d_condition.UNet2DConditionModel,
│ 1862 │ # unet is not used currently, but it is here for future use
│ 1863 │ if mem_eff_attn:
│ 1864 │ │ replace_unet_cross_attn_to_memory_efficient()
│
│ /usr/local/lib/python3.10/dist-packages/diffusers/utils/import_utils.py:711 in getattr
│
│ 708 │ │ │ module = self._get_module(self._class_to_module[name])
│ 709 │ │ │ value = getattr(module, name)
│ 710 │ │ else:
│ ❱ 711 │ │ │ raise AttributeError(f"module {self.name} has no attribute {name}")
│ 712 │ │
│ 713 │ │ setattr(self, name, value)
│ 714 │ │ return value
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
AttributeError: module diffusers.models has no attribute unet_2d_condition
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮
│ /usr/local/bin/accelerate:8 in
│
│ 5 from accelerate.commands.accelerate_cli import main
│ 6 if name == 'main':
│ 7 │ sys.argv[0] = re.sub(r'(-script.pyw|.exe)?$', '', sys.argv[0])
│ ❱ 8 │ sys.exit(main())
│ 9
│
│ /usr/local/lib/python3.10/dist-packages/accelerate/commands/accelerate_cli.py:45 in main
│
│ 42 │ │ exit(1)
│ 43 │
│ 44 │ # Run
│ ❱ 45 │ args.func(args)
│ 46
│ 47
│ 48 if name == "main":
│
│ /usr/local/lib/python3.10/dist-packages/accelerate/commands/launch.py:918 in launch_command
│
│ 915 │ elif defaults is not None and defaults.compute_environment == ComputeEnvironment.AMA
│ 916 │ │ sagemaker_launcher(defaults, args)
│ 917 │ else:
│ ❱ 918 │ │ simple_launcher(args)
│ 919
│ 920
│ 921 def main():
│
│ /usr/local/lib/python3.10/dist-packages/accelerate/commands/launch.py:580 in simple_launcher
│
│ 577 │ process.wait()
│ 578 │ if process.returncode != 0:
│ 579 │ │ if not args.quiet:
│ ❱ 580 │ │ │ raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd)
│ 581 │ │ else:
│ 582 │ │ │ sys.exit(1)
│ 583

Answer 1 · 2024-03-16T09:55:05.000Z

Hi, i just made it ok. they changed folders in diffusers. : use diffusers.models.unets.unet_2d_condition.UNet2DConditionModel instead of diffusers.models.unet_2d_condition.UNet2DConditionModel. however, after this i got another error:
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮
│ /content/kohya-trainer/train_network.py:873 in │
│ │
│ 870 │ args = parser.parse_args() │
│ 871 │ args = train_util.read_config_from_file(args, parser) │
│ 872 │ │
│ ❱ 873 │ train(args) │
│ 874 │
│ │
│ /content/kohya-trainer/train_network.py:168 in train │
│ │
│ 165 │ weight_dtype, save_dtype = train_util.prepare_dtype(args) │
│ 166 │ │
│ 167 │ # モデルを読み込む │
│ ❱ 168 │ text_encoder, vae, unet, _ = train_util.load_target_model(args, weight_dtype, accele │
│ 169 │ │
│ 170 │ # モデルに xformers とか memory efficient attention を組み込む │
│ 171 │ train_util.replace_unet_modules(unet, args.mem_eff_attn, args.xformers) │
│ │
│ /content/kohya-trainer/library/train_util.py:3150 in load_target_model │
│ │
│ 3147 │ │ if pi == accelerator.state.local_process_index: │
│ 3148 │ │ │ print(f"loading model for process {accelerator.state.local_process_index}/{a │
│ 3149 │ │ │ │
│ ❱ 3150 │ │ │ text_encoder, vae, unet, load_stable_diffusion_format = _load_target_model( │
│ 3151 │ │ │ │ args, weight_dtype, accelerator.device if args.lowram else "cpu" │
│ 3152 │ │ │ ) │
│ 3153 │
│ │
│ /content/kohya-trainer/library/train_util.py:3116 in _load_target_model │
│ │
│ 3113 │ load_stable_diffusion_format = os.path.isfile(name_or_path) # determine SD or Diffu │
│ 3114 │ if load_stable_diffusion_format: │
│ 3115 │ │ print(f"load StableDiffusion checkpoint: {name_or_path}") │
│ ❱ 3116 │ │ text_encoder, vae, unet = model_util.load_models_from_stable_diffusion_checkpoin │
│ 3117 │ else: │
│ 3118 │ │ # Diffusers model is loaded to CPU │
│ 3119 │ │ print(f"load Diffusers pretrained models: {name_or_path}") │
│ │
│ /content/kohya-trainer/library/model_util.py:871 in load_models_from_stable_diffusion_checkpoint │
│ │
│ 868 │ converted_vae_checkpoint = convert_ldm_vae_checkpoint(state_dict, vae_config) │
│ 869 │ │
│ 870 │ vae = AutoencoderKL(**vae_config).to(device) │
│ ❱ 871 │ info = vae.load_state_dict(converted_vae_checkpoint) │
│ 872 │ print("loading vae:", info) │
│ 873 │ │
│ 874 │ # convert text_model │
│ │
│ /usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py:2153 in load_state_dict │
│ │
│ 2150 │ │ │ │ │ │ ', '.join(f'"{k}"' for k in missing_keys))) │
│ 2151 │ │ │
│ 2152 │ │ if len(error_msgs) > 0: │
│ ❱ 2153 │ │ │ raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format( │
│ 2154 │ │ │ │ │ │ │ self.class.name, "\n\t".join(error_msgs))) │
│ 2155 │ │ return _IncompatibleKeys(missing_keys, unexpected_keys) │
│ 2156 │
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
RuntimeError: Error(s) in loading state_dict for AutoencoderKL:
Missing key(s) in state_dict: "encoder.mid_block.attentions.0.to_q.weight",
"encoder.mid_block.attentions.0.to_q.bias", "encoder.mid_block.attentions.0.to_k.weight",
"encoder.mid_block.attentions.0.to_k.bias", "encoder.mid_block.attentions.0.to_v.weight",
"encoder.mid_block.attentions.0.to_v.bias", "encoder.mid_block.attentions.0.to_out.0.weight",
"encoder.mid_block.attentions.0.to_out.0.bias", "decoder.mid_block.attentions.0.to_q.weight",
"decoder.mid_block.attentions.0.to_q.bias", "decoder.mid_block.attentions.0.to_k.weight",
"decoder.mid_block.attentions.0.to_k.bias", "decoder.mid_block.attentions.0.to_v.weight",
"decoder.mid_block.attentions.0.to_v.bias", "decoder.mid_block.attentions.0.to_out.0.weight",
"decoder.mid_block.attentions.0.to_out.0.bias".
Unexpected key(s) in state_dict: "encoder.mid_block.attentions.0.key.bias",
"encoder.mid_block.attentions.0.key.weight", "encoder.mid_block.attentions.0.proj_attn.bias",
"encoder.mid_block.attentions.0.proj_attn.weight", "encoder.mid_block.attentions.0.query.bias",
"encoder.mid_block.attentions.0.query.weight", "encoder.mid_block.attentions.0.value.bias",
"encoder.mid_block.attentions.0.value.weight", "decoder.mid_block.attentions.0.key.bias",
"decoder.mid_block.attentions.0.key.weight", "decoder.mid_block.attentions.0.proj_attn.bias",
"decoder.mid_block.attentions.0.proj_attn.weight", "decoder.mid_block.attentions.0.query.bias",
"decoder.mid_block.attentions.0.query.weight", "decoder.mid_block.attentions.0.value.bias",
"decoder.mid_block.attentions.0.value.weight".

Answer 2 · 2024-03-16T10:17:21.000Z

I'm having the same error, if someone knows how to fix it pls reply to the post

Answer 3 · 2024-03-16T15:41:48.000Z

I'm having the same error.

⭐ Starting trainer...

╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮
│ /content/kohya-trainer/train_network.py:17 in │
│ │
│ 14 from accelerate.utils import set_seed │
│ 15 from diffusers import DDPMScheduler │
│ 16 │
│ ❱ 17 import library.train_util as train_util │
│ 18 from library.train_util import ( │
│ 19 │ DreamBoothDataset, │
│ 20 ) │
│ │
│ /content/kohya-trainer/library/train_util.py:1861 in │
│ │
│ 1858 │ │ return dq, dk, dv, None, None, None, None │
│ 1859 │
│ 1860 │
│ ❱ 1861 def replace_unet_modules(unet: diffusers.models.unet_2d_condition.UNet2DConditionModel, │
│ 1862 │ # unet is not used currently, but it is here for future use │
│ 1863 │ if mem_eff_attn: │
│ 1864 │ │ replace_unet_cross_attn_to_memory_efficient() │
│ │
│ /usr/local/lib/python3.10/dist-packages/diffusers/utils/import_utils.py:711 in getattr │
│ │
│ 708 │ │ │ module = self._get_module(self._class_to_module[name]) │
│ 709 │ │ │ value = getattr(module, name) │
│ 710 │ │ else: │
│ ❱ 711 │ │ │ raise AttributeError(f"module {self.name} has no attribute {name}") │
│ 712 │ │ │
│ 713 │ │ setattr(self, name, value) │
│ 714 │ │ return value │
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
AttributeError: module diffusers.models has no attribute unet_2d_condition
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮
│ /usr/local/bin/accelerate:8 in │
│ │
│ 5 from accelerate.commands.accelerate_cli import main │
│ 6 if name == 'main': │
│ 7 │ sys.argv[0] = re.sub(r'(-script.pyw|.exe)?$', '', sys.argv[0]) │
│ ❱ 8 │ sys.exit(main()) │
│ 9 │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/commands/accelerate_cli.py:45 in main │
│ │
│ 42 │ │ exit(1) │
│ 43 │ │
│ 44 │ # Run │
│ ❱ 45 │ args.func(args) │
│ 46 │
│ 47 │
│ 48 if name == "main": │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/commands/launch.py:918 in launch_command │
│ │
│ 915 │ elif defaults is not None and defaults.compute_environment == ComputeEnvironment.AMA │
│ 916 │ │ sagemaker_launcher(defaults, args) │
│ 917 │ else: │
│ ❱ 918 │ │ simple_launcher(args) │
│ 919 │
│ 920 │
│ 921 def main(): │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/commands/launch.py:580 in simple_launcher │
│ │
│ 577 │ process.wait() │
│ 578 │ if process.returncode != 0: │
│ 579 │ │ if not args.quiet: │
│ ❱ 580 │ │ │ raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd) │
│ 581 │ │ else: │
│ 582 │ │ │ sys.exit(1) │
│ 583 │
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯

Answer 4 · 2024-03-16T19:40:35.000Z

Should be fixed

Answer 5 · 2024-03-17T10:06:55.000Z

Hi again, I have tried to train the Lora but i continue having the error:
preparing accelerator
loading model for process 0/1
load StableDiffusion checkpoint: /content/downloaded_model.safetensors
UNet2DConditionModel: 64, 8, 768, False, False
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮
│ /content/kohya-trainer/train_network.py:991 in │
│ │
│ 988 │ args = train_util.read_config_from_file(args, parser) │
│ 989 │ │
│ 990 │ trainer = NetworkTrainer() │
│ ❱ 991 │ trainer.train(args) │
│ 992 │
│ │
│ /content/kohya-trainer/train_network.py:213 in train │
│ │
│ 210 │ │ vae_dtype = torch.float32 if args.no_half_vae else weight_dtype │
│ 211 │ │ │
│ 212 │ │ # モデルを読み込む │
│ ❱ 213 │ │ model_version, text_encoder, vae, unet = self.load_target_model(args, weight_dty │
│ 214 │ │ │
│ 215 │ │ # text_encoder is List[CLIPTextModel] or CLIPTextModel │
│ 216 │ │ text_encoders = text_encoder if isinstance(text_encoder, list) else [text_encode │
│ │
│ /content/kohya-trainer/train_network.py:90 in load_target_model │
│ │
│ 87 │ │ pass │
│ 88 │ │
│ 89 │ def load_target_model(self, args, weight_dtype, accelerator): │
│ ❱ 90 │ │ text_encoder, vae, unet, _ = train_util.load_target_model(args, weight_dtype, ac │
│ 91 │ │ return model_util.get_model_version_str_for_sd1_sd2(args.v2, args.v_parameteriza │
│ 92 │ │
│ 93 │ def load_tokenizer(self, args): │
│ │
│ /content/kohya-trainer/library/train_util.py:3659 in load_target_model │
│ │
│ 3656 │ │ if pi == accelerator.state.local_process_index: │
│ 3657 │ │ │ print(f"loading model for process {accelerator.state.local_process_index}/{a │
│ 3658 │ │ │ │
│ ❱ 3659 │ │ │ text_encoder, vae, unet, load_stable_diffusion_format = _load_target_model( │
│ 3660 │ │ │ │ args, │
│ 3661 │ │ │ │ weight_dtype, │
│ 3662 │ │ │ │ accelerator.device if args.lowram else "cpu", │
│ │
│ /content/kohya-trainer/library/train_util.py:3602 in _load_target_model │
│ │
│ 3599 │ load_stable_diffusion_format = os.path.isfile(name_or_path) # determine SD or Diffu │
│ 3600 │ if load_stable_diffusion_format: │
│ 3601 │ │ print(f"load StableDiffusion checkpoint: {name_or_path}") │
│ ❱ 3602 │ │ text_encoder, vae, unet = model_util.load_models_from_stable_diffusion_checkpoin │
│ 3603 │ │ │ args.v2, name_or_path, device, unet_use_linear_projection_in_v2=unet_use_lin │
│ 3604 │ │ ) │
│ 3605 │ else: │
│ │
│ /content/kohya-trainer/library/model_util.py:999 in load_models_from_stable_diffusion_checkpoint │
│ │
│ 996 │ converted_unet_checkpoint = convert_ldm_unet_checkpoint(v2, state_dict, unet_config) │
│ 997 │ │
│ 998 │ unet = UNet2DConditionModel(**unet_config).to(device) │
│ ❱ 999 │ info = unet.load_state_dict(converted_unet_checkpoint) │
│ 1000 │ print("loading u-net:", info) │
│ 1001 │ │
│ 1002 │ # Convert the VAE model. │
│ │
│ /usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py:2153 in load_state_dict │
│ │
│ 2150 │ │ │ │ │ │ ', '.join(f'"{k}"' for k in missing_keys))) │
│ 2151 │ │ │
│ 2152 │ │ if len(error_msgs) > 0: │
│ ❱ 2153 │ │ │ raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format( │
│ 2154 │ │ │ │ │ │ │ self.class.name, "\n\t".join(error_msgs))) │
│ 2155 │ │ return _IncompatibleKeys(missing_keys, unexpected_keys) │
│ 2156 │
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
RuntimeError: Error(s) in loading state_dict for UNet2DConditionModel:
Missing key(s) in state_dict: "down_blocks.0.attentions.0.norm.weight",
"down_blocks.0.attentions.0.norm.bias", "down_blocks.0.attentions.0.proj_in.weight",
"down_blocks.0.attentions.0.proj_in.bias",
"down_blocks.0.attentions.0.transformer_blocks.0.attn1.to_q.weight",
"down_blocks.0.attentions.0.transformer_blocks.0.attn1.to_k.weight",
"down_blocks.0.attentions.0.transformer_blocks.0.attn1.to_v.weight",
....

param with shape torch.Size([1280, 2048]) from checkpoint, the shape in current model is
torch.Size([1280, 768]).
size mismatch for mid_block.attentions.0.proj_out.weight: copying a param with shape
torch.Size([1280, 1280]) from checkpoint, the shape in current model is torch.Size([1280, 1280, 1,
1]).
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮
│ /usr/local/bin/accelerate:8 in │
│ │
│ 5 from accelerate.commands.accelerate_cli import main │
│ 6 if name == 'main': │
│ 7 │ sys.argv[0] = re.sub(r'(-script.pyw|.exe)?$', '', sys.argv[0]) │
│ ❱ 8 │ sys.exit(main()) │
│ 9 │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/commands/accelerate_cli.py:45 in main │
│ │
│ 42 │ │ exit(1) │
│ 43 │ │
│ 44 │ # Run │
│ ❱ 45 │ args.func(args) │
│ 46 │
│ 47 │
│ 48 if name == "main": │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/commands/launch.py:918 in launch_command │
│ │
│ 915 │ elif defaults is not None and defaults.compute_environment == ComputeEnvironment.AMA │
│ 916 │ │ sagemaker_launcher(defaults, args) │
│ 917 │ else: │
│ ❱ 918 │ │ simple_launcher(args) │
│ 919 │
│ 920 │
│ 921 def main(): │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/commands/launch.py:580 in simple_launcher │
│ │
│ 577 │ process.wait() │
│ 578 │ if process.returncode != 0: │
│ 579 │ │ if not args.quiet: │
│ ❱ 580 │ │ │ raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd) │
│ 581 │ │ else: │
│ 582 │ │ │ sys.exit(1) │
│ 583 │
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯

Has anyone any idea? Thanks