Improve Arabic QA assets for Jais (#261)

* Added Jais-Zero-Shot Asset for the XQUAD dataset * added jais-zero-shot-asset for the tydiqa dataset * added jais-zero-shot asset for the arcd dataset * Added Zero Shot Asset for the MLQA dataset * added zero-shot-jais asset for anercorp dataset * added zero-shot-jais asset for the aqmar dataset * Added zero-shot-jais asset for the mgb words dataset * formatted written assets for jais * Fix casing in QA asset names * Add scores for QA assets * Remove NER assets as Jais cannot reliably do the task --------- Co-authored-by: Fahim Imaduddin Dalvi <[email protected]>
qcri · Jan 31, 2024 · 6ecd798 · 6ecd798
1 parent 7fde120
commit 6ecd798
Show file tree

Hide file tree

Showing 4 changed files with 15 additions and 4 deletions.
diff --git a/assets/ar/QA/ARCD_JAIS13b_ZeroShot.py b/assets/ar/QA/ARCD_JAIS13b_ZeroShot.py
@@ -8,6 +8,7 @@ def metadata():
         "author": "Arabic Language Technologies, QCRI, HBKU",
         "model": "Jais-13b-chat",
         "description": "Locally hosted Jais-13b-chat model using FastChat.",
+        "scores": {"F1": "0.546"},
     }
 
 
@@ -16,11 +17,15 @@ def config():
         "dataset": ARCDDataset,
         "task": QATask,
         "model": FastChatModel,
+        "model_args": {
+            "max_tries": 3,
+        },
     }
 
 
 def prompt(input_sample):
-    base_prompt = f"مهمتك هي الإجابة على الأسئلة باللغة العربية بناءً على سياق معين.\nملاحظة: يجب أن تكون إجاباتك مستخرجة من السياق المحدد دون أي اضافات.\nلست بحاجة إلى تقديم إجابة كاملة.\nالسياق: {input_sample['context']}\n السؤال: {input_sample['question']}\n الجواب:"
+    base_prompt = f"Your task is to answer questions in Arabic based on a given context.\nNote: Your answers should be spans extracted from the given context without any illustrations.\nYou don't need to provide a complete answer\nContext:{input_sample['context']}\nQuestion:{input_sample['question']}\nAnswer:"
+
     return [
         {
             "role": "user",

diff --git a/assets/ar/QA/MLQA_JAIS13b_ZeroShot.py b/assets/ar/QA/MLQA_JAIS13b_ZeroShot.py
@@ -8,6 +8,7 @@ def metadata():
         "author": "Arabic Language Technologies, QCRI, HBKU",
         "model": "Jais-13b-chat",
         "description": "Locally hosted Jais-13b-chat model using FastChat.",
+        "scores": {"F1": "0.540"},
     }
 
 
@@ -17,13 +18,14 @@ def config():
         "task": QATask,
         "model": FastChatModel,
         "model_args": {
-            "max_tries": 50,
+            "max_tries": 3,
         },
     }
 
 
 def prompt(input_sample):
     base_prompt = f"Your task is to answer questions in Arabic based on a given context.\nNote: Your answers should be spans extracted from the given context without any illustrations.\nYou don't need to provide a complete answer\nContext:{input_sample['context']}\nQuestion:{input_sample['question']}\nAnswer:"
+
     return [
         {
             "role": "user",

diff --git a/assets/ar/QA/TyDiQA_JAIS13b_ZeroShot.py b/assets/ar/QA/TyDiQA_JAIS13b_ZeroShot.py
@@ -8,6 +8,7 @@ def metadata():
         "author": "Arabic Language Technologies, QCRI, HBKU",
         "model": "Jais-13b-chat",
         "description": "Locally hosted Jais-13b-chat model using FastChat.",
+        "scores": {"F1": "0.724"},
     }
 
 
@@ -17,14 +18,15 @@ def config():
         "task": QATask,
         "model": FastChatModel,
         "model_args": {
-            "max_tries": 50,
+            "max_tries": 3,
         },
         "general_args": {"test_split": "dev"},
     }
 
 
 def prompt(input_sample):
     base_prompt = f"Your task is to answer questions in Arabic based on a given context.\nNote: Your answers should be spans extracted from the given context without any illustrations.\nYou don't need to provide a complete answer\nContext:{input_sample['context']}\nQuestion:{input_sample['question']}\nAnswer:"
+
     return [
         {
             "role": "user",

diff --git a/assets/ar/QA/XQuAD_JAIS13b_ZeroShot.py b/assets/ar/QA/XQuAD_JAIS13b_ZeroShot.py
@@ -8,6 +8,7 @@ def metadata():
         "author": "Arabic Language Technologies, QCRI, HBKU",
         "model": "Jais-13b-chat",
         "description": "Locally hosted Jais-13b-chat model using FastChat.",
+        "scores": {"F1": "0.636"},
     }
 
 
@@ -17,13 +18,14 @@ def config():
         "task": QATask,
         "model": FastChatModel,
         "model_args": {
-            "max_tries": 50,
+            "max_tries": 3,
         },
     }
 
 
 def prompt(input_sample):
     base_prompt = f"Your task is to answer questions in Arabic based on a given context.\nNote: Your answers should be spans extracted from the given context without any illustrations.\nYou don't need to provide a complete answer\nContext:{input_sample['context']}\nQuestion:{input_sample['question']}\nAnswer:"
+
     return [
         {
             "role": "user",