tuning embedding for qwen-7b

InternLM · Oct 11, 2023 · 34dc327 · 34dc327
1 parent 5b543b3
commit 34dc327
Show file tree

Hide file tree

Showing 18 changed files with 18 additions and 0 deletions.
diff --git a/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_alpaca_e3.py b/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_alpaca_e3.py
@@ -76,6 +76,7 @@
         type=LoraConfig,
         r=64,
         lora_alpha=16,
+        target_modules=['c_attn', 'c_proj', 'w1', 'w2', 'wte'],
         lora_dropout=0.1,
         bias='none',
         task_type='CAUSAL_LM'))

diff --git a/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_alpaca_enzh_e3.py b/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_alpaca_enzh_e3.py
@@ -78,6 +78,7 @@
         type=LoraConfig,
         r=64,
         lora_alpha=16,
+        target_modules=['c_attn', 'c_proj', 'w1', 'w2', 'wte'],
         lora_dropout=0.1,
         bias='none',
         task_type='CAUSAL_LM'))

diff --git a/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_alpaca_enzh_oasst1_e3.py b/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_alpaca_enzh_oasst1_e3.py
@@ -79,6 +79,7 @@
         type=LoraConfig,
         r=64,
         lora_alpha=16,
+        target_modules=['c_attn', 'c_proj', 'w1', 'w2', 'wte'],
         lora_dropout=0.1,
         bias='none',
         task_type='CAUSAL_LM'))

diff --git a/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_alpaca_zh_e3.py b/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_alpaca_zh_e3.py
@@ -76,6 +76,7 @@
         type=LoraConfig,
         r=64,
         lora_alpha=16,
+        target_modules=['c_attn', 'c_proj', 'w1', 'w2', 'wte'],
         lora_dropout=0.1,
         bias='none',
         task_type='CAUSAL_LM'))

diff --git a/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_arxiv_gentitle_e3.py b/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_arxiv_gentitle_e3.py
@@ -110,6 +110,7 @@
         type=LoraConfig,
         r=64,
         lora_alpha=16,
+        target_modules=['c_attn', 'c_proj', 'w1', 'w2', 'wte'],
         lora_dropout=0.1,
         bias='none',
         task_type='CAUSAL_LM'))

diff --git a/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_code_alpaca_e3.py b/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_code_alpaca_e3.py
@@ -80,6 +80,7 @@
         type=LoraConfig,
         r=64,
         lora_alpha=16,
+        target_modules=['c_attn', 'c_proj', 'w1', 'w2', 'wte'],
         lora_dropout=0.1,
         bias='none',
         task_type='CAUSAL_LM'))

diff --git a/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_colorist_e5.py b/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_colorist_e5.py
@@ -76,6 +76,7 @@
         type=LoraConfig,
         r=64,
         lora_alpha=16,
+        target_modules=['c_attn', 'c_proj', 'w1', 'w2', 'wte'],
         lora_dropout=0.1,
         bias='none',
         task_type='CAUSAL_LM'))

diff --git a/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_lawyer_e3.py b/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_lawyer_e3.py
@@ -78,6 +78,7 @@
         type=LoraConfig,
         r=64,
         lora_alpha=16,
+        target_modules=['c_attn', 'c_proj', 'w1', 'w2', 'wte'],
         lora_dropout=0.1,
         bias='none',
         task_type='CAUSAL_LM'))

diff --git a/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_medical_e1.py b/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_medical_e1.py
@@ -78,6 +78,7 @@
         type=LoraConfig,
         r=64,
         lora_alpha=16,
+        target_modules=['c_attn', 'c_proj', 'w1', 'w2', 'wte'],
         lora_dropout=0.1,
         bias='none',
         task_type='CAUSAL_LM'))

diff --git a/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_moss_sft_all_e1.py b/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_moss_sft_all_e1.py
@@ -76,6 +76,7 @@
         type=LoraConfig,
         r=64,
         lora_alpha=16,
+        target_modules=['c_attn', 'c_proj', 'w1', 'w2', 'wte'],
         lora_dropout=0.1,
         bias='none',
         task_type='CAUSAL_LM'))

diff --git a/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_moss_sft_all_e2_gpu8.py b/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_moss_sft_all_e2_gpu8.py
@@ -76,6 +76,7 @@
         type=LoraConfig,
         r=64,
         lora_alpha=16,
+        target_modules=['c_attn', 'c_proj', 'w1', 'w2', 'wte'],
         lora_dropout=0.1,
         bias='none',
         task_type='CAUSAL_LM'))

diff --git a/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_moss_sft_plugins_e1.py b/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_moss_sft_plugins_e1.py
@@ -75,6 +75,7 @@
         type=LoraConfig,
         r=64,
         lora_alpha=16,
+        target_modules=['c_attn', 'c_proj', 'w1', 'w2', 'wte'],
         lora_dropout=0.1,
         bias='none',
         task_type='CAUSAL_LM'))

diff --git a/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_oasst1_512_e3.py b/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_oasst1_512_e3.py
@@ -76,6 +76,7 @@
         type=LoraConfig,
         r=64,
         lora_alpha=16,
+        target_modules=['c_attn', 'c_proj', 'w1', 'w2', 'wte'],
         lora_dropout=0.1,
         bias='none',
         task_type='CAUSAL_LM'))

diff --git a/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_oasst1_e3.py b/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_oasst1_e3.py
@@ -76,6 +76,7 @@
         type=LoraConfig,
         r=64,
         lora_alpha=16,
+        target_modules=['c_attn', 'c_proj', 'w1', 'w2', 'wte'],
         lora_dropout=0.1,
         bias='none',
         task_type='CAUSAL_LM'))

diff --git a/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_open_platypus_e3.py b/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_open_platypus_e3.py
@@ -76,6 +76,7 @@
         type=LoraConfig,
         r=64,
         lora_alpha=16,
+        target_modules=['c_attn', 'c_proj', 'w1', 'w2', 'wte'],
         lora_dropout=0.1,
         bias='none',
         task_type='CAUSAL_LM'))

diff --git a/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_openorca_e1.py b/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_openorca_e1.py
@@ -76,6 +76,7 @@
         type=LoraConfig,
         r=64,
         lora_alpha=16,
+        target_modules=['c_attn', 'c_proj', 'w1', 'w2', 'wte'],
         lora_dropout=0.1,
         bias='none',
         task_type='CAUSAL_LM'))

diff --git a/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_sql_e3.py b/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_sql_e3.py
@@ -80,6 +80,7 @@
         type=LoraConfig,
         r=64,
         lora_alpha=16,
+        target_modules=['c_attn', 'c_proj', 'w1', 'w2', 'wte'],
         lora_dropout=0.1,
         bias='none',
         task_type='CAUSAL_LM'))

diff --git a/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_tiny_codes_e1.py b/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_tiny_codes_e1.py
@@ -80,6 +80,7 @@
         type=LoraConfig,
         r=64,
         lora_alpha=16,
+        target_modules=['c_attn', 'c_proj', 'w1', 'w2', 'wte'],
         lora_dropout=0.1,
         bias='none',
         task_type='CAUSAL_LM'))