diff --git a/fine_tune.py b/fine_tune.py index 38b1962f6..08afedc20 100644 --- a/fine_tune.py +++ b/fine_tune.py @@ -255,7 +255,7 @@ def fn_recursive_set_mem_eff(module: torch.nn.Module): ds_model = deepspeed_utils.prepare_deepspeed_model(args, unet=unet, text_encoder=text_encoder) else: ds_model = deepspeed_utils.prepare_deepspeed_model(args, unet=unet) - if args.optimizer_type.lower().endswith("scheduleFree"): + if args.optimizer_type.lower().endswith("schedulefree"): ds_model, optimizer, train_dataloader = accelerator.prepare( ds_model, optimizer, train_dataloader ) @@ -267,7 +267,7 @@ def fn_recursive_set_mem_eff(module: torch.nn.Module): else: # acceleratorがなんかよろしくやってくれるらしい if args.train_text_encoder: - if args.optimizer_type.lower().endswith("scheduleFree"): + if args.optimizer_type.lower().endswith("schedulefree"): unet, text_encoder, optimizer, train_dataloader = accelerator.prepare( unet, text_encoder, optimizer, train_dataloader ) @@ -276,7 +276,7 @@ def fn_recursive_set_mem_eff(module: torch.nn.Module): unet, text_encoder, optimizer, train_dataloader, lr_scheduler ) else: - if args.optimizer_type.lower().endswith("scheduleFree"): + if args.optimizer_type.lower().endswith("schedulefree"): unet, optimizer, train_dataloader = accelerator.prepare(unet, optimizer, train_dataloader) else: unet, optimizer, train_dataloader, lr_scheduler = accelerator.prepare(unet, optimizer, train_dataloader, lr_scheduler) @@ -405,7 +405,7 @@ def fn_recursive_set_mem_eff(module: torch.nn.Module): accelerator.clip_grad_norm_(params_to_clip, args.max_grad_norm) optimizer.step() - if not args.optimizer_type.lower().endswith("scheduleFree"): + if not args.optimizer_type.lower().endswith("schedulefree"): lr_scheduler.step() optimizer.zero_grad(set_to_none=True) diff --git a/sdxl_train.py b/sdxl_train.py index 09ca438f9..a78687fd8 100644 --- a/sdxl_train.py +++ b/sdxl_train.py @@ -415,7 +415,7 @@ def fn_recursive_set_mem_eff(module: torch.nn.Module): text_encoder2=text_encoder2 if train_text_encoder2 else None, ) # most of ZeRO stage uses optimizer partitioning, so we have to prepare optimizer and ds_model at the same time. # pull/1139#issuecomment-1986790007 - if args.optimizer_type.lower().endswith("scheduleFree"): + if args.optimizer_type.lower().endswith("schedulefree"): ds_model, optimizer, train_dataloader = accelerator.prepare( ds_model, optimizer, train_dataloader ) @@ -433,7 +433,7 @@ def fn_recursive_set_mem_eff(module: torch.nn.Module): text_encoder1 = accelerator.prepare(text_encoder1) if train_text_encoder2: text_encoder2 = accelerator.prepare(text_encoder2) - if args.optimizer_type.lower().endswith("scheduleFree"): + if args.optimizer_type.lower().endswith("schedulefree"): optimizer, train_dataloader = accelerator.prepare(optimizer, train_dataloader) else: optimizer, train_dataloader, lr_scheduler = accelerator.prepare(optimizer, train_dataloader, lr_scheduler) @@ -636,7 +636,7 @@ def fn_recursive_set_mem_eff(module: torch.nn.Module): accelerator.clip_grad_norm_(params_to_clip, args.max_grad_norm) optimizer.step() - if not args.optimizer_type.lower().endswith("scheduleFree"): + if not args.optimizer_type.lower().endswith("schedulefree"): lr_scheduler.step() optimizer.zero_grad(set_to_none=True) diff --git a/sdxl_train_control_net_lllite.py b/sdxl_train_control_net_lllite.py index 056f197f4..d2b578c92 100644 --- a/sdxl_train_control_net_lllite.py +++ b/sdxl_train_control_net_lllite.py @@ -286,7 +286,7 @@ def train(args): unet.to(weight_dtype) # acceleratorがなんかよろしくやってくれるらしい - if args.optimizer_type.lower().endswith("scheduleFree"): + if args.optimizer_type.lower().endswith("schedulefree"): unet, optimizer, train_dataloader = accelerator.prepare(unet, optimizer, train_dataloader) else: unet, optimizer, train_dataloader, lr_scheduler = accelerator.prepare(unet, optimizer, train_dataloader, lr_scheduler) @@ -491,7 +491,7 @@ def remove_model(old_ckpt_name): accelerator.clip_grad_norm_(params_to_clip, args.max_grad_norm) optimizer.step() - if not args.optimizer_type.lower().endswith("scheduleFree"): + if not args.optimizer_type.lower().endswith("schedulefree"): lr_scheduler.step() optimizer.zero_grad(set_to_none=True) diff --git a/sdxl_train_control_net_lllite_old.py b/sdxl_train_control_net_lllite_old.py index 30240dd38..ee0ef930b 100644 --- a/sdxl_train_control_net_lllite_old.py +++ b/sdxl_train_control_net_lllite_old.py @@ -254,7 +254,7 @@ def train(args): network.to(weight_dtype) # acceleratorがなんかよろしくやってくれるらしい - if args.optimizer_type.lower().endswith("scheduleFree"): + if args.optimizer_type.lower().endswith("schedulefree"): unet, network, optimizer, train_dataloader = accelerator.prepare( unet, network, optimizer, train_dataloader ) @@ -460,7 +460,7 @@ def remove_model(old_ckpt_name): accelerator.clip_grad_norm_(params_to_clip, args.max_grad_norm) optimizer.step() - if not args.optimizer_type.lower().endswith("scheduleFree"): + if not args.optimizer_type.lower().endswith("schedulefree"): lr_scheduler.step() optimizer.zero_grad(set_to_none=True) diff --git a/train_controlnet.py b/train_controlnet.py index 849e467b7..38cfb7f2d 100644 --- a/train_controlnet.py +++ b/train_controlnet.py @@ -276,7 +276,7 @@ def train(args): controlnet.to(weight_dtype) # acceleratorがなんかよろしくやってくれるらしい - if args.optimizer_type.lower().endswith("scheduleFree"): + if args.optimizer_type.lower().endswith("schedulefree"): controlnet, optimizer, train_dataloader = accelerator.prepare( controlnet, optimizer, train_dataloader ) diff --git a/train_db.py b/train_db.py index 1f3ffd2cb..6a946aaab 100644 --- a/train_db.py +++ b/train_db.py @@ -229,7 +229,7 @@ def train(args): ds_model = deepspeed_utils.prepare_deepspeed_model(args, unet=unet, text_encoder=text_encoder) else: ds_model = deepspeed_utils.prepare_deepspeed_model(args, unet=unet) - if args.optimizer_type.lower().endswith("scheduleFree"): + if args.optimizer_type.lower().endswith("schedulefree"): ds_model, optimizer, train_dataloader = accelerator.prepare( ds_model, optimizer, train_dataloader ) @@ -241,7 +241,7 @@ def train(args): else: if train_text_encoder: - if args.optimizer_type.lower().endswith("scheduleFree"): + if args.optimizer_type.lower().endswith("schedulefree"): unet, text_encoder, optimizer, train_dataloader = accelerator.prepare( unet, text_encoder, optimizer, train_dataloader ) @@ -251,7 +251,7 @@ def train(args): ) training_models = [unet, text_encoder] else: - if args.optimizer_type.lower().endswith("scheduleFree"): + if args.optimizer_type.lower().endswith("schedulefree"): unet, optimizer, train_dataloader = accelerator.prepare(unet, optimizer, train_dataloader) else: unet, optimizer, train_dataloader, lr_scheduler = accelerator.prepare(unet, optimizer, train_dataloader, lr_scheduler) @@ -399,7 +399,7 @@ def train(args): accelerator.clip_grad_norm_(params_to_clip, args.max_grad_norm) optimizer.step() - if not args.optimizer_type.lower().endswith("scheduleFree"): + if not args.optimizer_type.lower().endswith("schedulefree"): lr_scheduler.step() optimizer.zero_grad(set_to_none=True) diff --git a/train_network.py b/train_network.py index fe491d880..335209e23 100644 --- a/train_network.py +++ b/train_network.py @@ -420,7 +420,7 @@ def train(self, args): text_encoder2=text_encoders[1] if train_text_encoder and len(text_encoders) > 1 else None, network=network, ) - if args.optimizer_type.lower().endswith("scheduleFree"): + if args.optimizer_type.lower().endswith("schedulefree"): ds_model, optimizer, train_dataloader = accelerator.prepare( ds_model, optimizer, train_dataloader ) @@ -443,7 +443,7 @@ def train(self, args): else: pass # if text_encoder is not trained, no need to prepare. and device and dtype are already set - if args.optimizer_type.lower().endswith("scheduleFree"): + if args.optimizer_type.lower().endswith("schedulefree"): network, optimizer, train_dataloader = accelerator.prepare( network, optimizer, train_dataloader ) @@ -924,7 +924,7 @@ def remove_model(old_ckpt_name): accelerator.clip_grad_norm_(params_to_clip, args.max_grad_norm) optimizer.step() - if not args.optimizer_type.lower().endswith("scheduleFree"): + if not args.optimizer_type.lower().endswith("schedulefree"): lr_scheduler.step() optimizer.zero_grad(set_to_none=True) diff --git a/train_textual_inversion.py b/train_textual_inversion.py index bd90a4e5b..fa1c24e06 100644 --- a/train_textual_inversion.py +++ b/train_textual_inversion.py @@ -416,7 +416,7 @@ def train(self, args): # acceleratorがなんかよろしくやってくれるらしい if len(text_encoders) == 1: - if args.optimizer_type.lower().endswith("scheduleFree"): + if args.optimizer_type.lower().endswith("schedulefree"): text_encoder_or_list, optimizer, train_dataloader = accelerator.preparet( text_encoder_or_list, optimizer, train_dataloader ) @@ -426,7 +426,7 @@ def train(self, args): ) elif len(text_encoders) == 2: - if args.optimizer_type.lower().endswith("scheduleFree"): + if args.optimizer_type.lower().endswith("schedulefree"): text_encoder1, text_encoder2, optimizer, train_dataloader = accelerator.prepare( text_encoders[0], text_encoders[1], optimizer, train_dataloader ) diff --git a/train_textual_inversion_XTI.py b/train_textual_inversion_XTI.py index 712af33ec..4e95197f7 100644 --- a/train_textual_inversion_XTI.py +++ b/train_textual_inversion_XTI.py @@ -335,7 +335,7 @@ def train(args): lr_scheduler = train_util.get_scheduler_fix(args, optimizer, accelerator.num_processes) # acceleratorがなんかよろしくやってくれるらしい - if args.optimizer_type.lower().endswith("scheduleFree"): + if args.optimizer_type.lower().endswith("schedulefree"): text_encoder, optimizer, train_dataloader = accelerator.prepare( text_encoder, optimizer, train_dataloader ) @@ -507,7 +507,7 @@ def remove_model(old_ckpt_name): accelerator.clip_grad_norm_(params_to_clip, args.max_grad_norm) optimizer.step() - if not args.optimizer_type.lower().endswith("scheduleFree"): + if not args.optimizer_type.lower().endswith("schedulefree"): lr_scheduler.step() optimizer.zero_grad(set_to_none=True)