delete model file for converting ckpt

FlagOpen · Dec 27, 2024 · 126a754 · 126a754
1 parent efab98e
commit 126a754
Show file tree

Hide file tree

Showing 6 changed files with 6 additions and 1,635 deletions.
diff --git a/megatron/megatron/core/transformer/attention.py b/megatron/megatron/core/transformer/attention.py
@@ -521,7 +521,7 @@ def __init__(
                 # nums_head_cur_rank = divide(self.config.num_attention_heads, tp_world_size)
                 self.q_layernorm = build_module(
                     submodules.q_layernorm,
-                    hidden_size=self.config.num_attention_heads * self.hidden_size_per_attention_head,
+                    hidden_size=self.query_projection_size,
                     config=self.config,
                     eps=self.config.layernorm_epsilon,
                 )
@@ -542,7 +542,7 @@ def __init__(
                 # nums_head_cur_rank = divide(self.config.num_attention_heads, tp_world_size)
                 self.k_layernorm = build_module(
                     submodules.k_layernorm,
-                    hidden_size=self.config.num_query_groups * self.hidden_size_per_attention_head,
+                    hidden_size=self.kv_projection_size,
                     config=self.config,
                     eps=self.config.layernorm_epsilon,
                 )

diff --git a/tools/checkpoint/aquila/llama_model/__init__.py b/tools/checkpoint/aquila/llama_model/__init__.py
diff --git a/tools/checkpoint/aquila/llama_model/config.json b/tools/checkpoint/aquila/llama_model/config.json
diff --git a/tools/checkpoint/aquila/llama_model/configuration_llama.py b/tools/checkpoint/aquila/llama_model/configuration_llama.py