From 6f4fe294783997353d59889d492fa42859a6a047 Mon Sep 17 00:00:00 2001 From: zhangting2020 Date: Tue, 10 Oct 2023 19:28:21 +0800 Subject: [PATCH] add fuse_ln args --- paddlemix/examples/evaclip/run_pretrain_dist.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/paddlemix/examples/evaclip/run_pretrain_dist.py b/paddlemix/examples/evaclip/run_pretrain_dist.py index c77eea01f..032233f4d 100644 --- a/paddlemix/examples/evaclip/run_pretrain_dist.py +++ b/paddlemix/examples/evaclip/run_pretrain_dist.py @@ -114,6 +114,10 @@ class PreTrainingArguments(TrainingArguments): default=False, metadata={"help": "Whether to use tensorboard to record loss."}, ) + fuse_ln: bool = field( + default=True, + metadata={"help": "Whether to use fused layer norm."}, + ) pretrained_text_model: str = field(default="openclip", metadata={"help": "the model to pre-extract text feats"}) @@ -189,6 +193,8 @@ def main_worker(training_args, model_args, data_args): paddle.set_default_dtype("bfloat16") config = EVACLIPConfig.from_pretrained(model_args.model) + config["text_config"]["fusedLN"] = training_args.fuse_ln + config["vision_config"]["fusedLN"] = training_args.fuse_ln model = EVACLIP( config, disable_text=False,