From 6ecd798f14010607a9dc52a3b201df0f17821ebd Mon Sep 17 00:00:00 2001
From: Basel Mousi <59998313+baselmousi@users.noreply.github.com>
Date: Wed, 31 Jan 2024 15:36:17 +0300
Subject: [PATCH] Improve Arabic QA assets for Jais (#261)

* Added Jais-Zero-Shot Asset for the XQUAD dataset

* added jais-zero-shot-asset for the tydiqa dataset

* added jais-zero-shot asset for the arcd dataset

* Added Zero Shot Asset for the MLQA dataset

* added zero-shot-jais asset for anercorp dataset

* added zero-shot-jais asset for the aqmar dataset

* Added zero-shot-jais asset for the mgb words dataset

* formatted written assets for jais

* Fix casing in QA asset names

* Add scores for QA assets

* Remove NER assets as Jais cannot reliably do the task

---------

Co-authored-by: Fahim Imaduddin Dalvi <faimaduddin@hbku.edu.qa>
---
 assets/ar/QA/ARCD_JAIS13b_ZeroShot.py   | 7 ++++++-
 assets/ar/QA/MLQA_JAIS13b_ZeroShot.py   | 4 +++-
 assets/ar/QA/TyDiQA_JAIS13b_ZeroShot.py | 4 +++-
 assets/ar/QA/XQuAD_JAIS13b_ZeroShot.py  | 4 +++-
 4 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/assets/ar/QA/ARCD_JAIS13b_ZeroShot.py b/assets/ar/QA/ARCD_JAIS13b_ZeroShot.py
index 5df663e3..aeccf7d1 100644
--- a/assets/ar/QA/ARCD_JAIS13b_ZeroShot.py
+++ b/assets/ar/QA/ARCD_JAIS13b_ZeroShot.py
@@ -8,6 +8,7 @@ def metadata():
         "author": "Arabic Language Technologies, QCRI, HBKU",
         "model": "Jais-13b-chat",
         "description": "Locally hosted Jais-13b-chat model using FastChat.",
+        "scores": {"F1": "0.546"},
     }
 
 
@@ -16,11 +17,15 @@ def config():
         "dataset": ARCDDataset,
         "task": QATask,
         "model": FastChatModel,
+        "model_args": {
+            "max_tries": 3,
+        },
     }
 
 
 def prompt(input_sample):
-    base_prompt = f"مهمتك هي الإجابة على الأسئلة باللغة العربية بناءً على سياق معين.\nملاحظة: يجب أن تكون إجاباتك مستخرجة من السياق المحدد دون أي اضافات.\nلست بحاجة إلى تقديم إجابة كاملة.\nالسياق: {input_sample['context']}\n السؤال: {input_sample['question']}\n الجواب:"
+    base_prompt = f"Your task is to answer questions in Arabic based on a given context.\nNote: Your answers should be spans extracted from the given context without any illustrations.\nYou don't need to provide a complete answer\nContext:{input_sample['context']}\nQuestion:{input_sample['question']}\nAnswer:"
+
     return [
         {
             "role": "user",
diff --git a/assets/ar/QA/MLQA_JAIS13b_ZeroShot.py b/assets/ar/QA/MLQA_JAIS13b_ZeroShot.py
index 603eafe5..6868a88c 100644
--- a/assets/ar/QA/MLQA_JAIS13b_ZeroShot.py
+++ b/assets/ar/QA/MLQA_JAIS13b_ZeroShot.py
@@ -8,6 +8,7 @@ def metadata():
         "author": "Arabic Language Technologies, QCRI, HBKU",
         "model": "Jais-13b-chat",
         "description": "Locally hosted Jais-13b-chat model using FastChat.",
+        "scores": {"F1": "0.540"},
     }
 
 
@@ -17,13 +18,14 @@ def config():
         "task": QATask,
         "model": FastChatModel,
         "model_args": {
-            "max_tries": 50,
+            "max_tries": 3,
         },
     }
 
 
 def prompt(input_sample):
     base_prompt = f"Your task is to answer questions in Arabic based on a given context.\nNote: Your answers should be spans extracted from the given context without any illustrations.\nYou don't need to provide a complete answer\nContext:{input_sample['context']}\nQuestion:{input_sample['question']}\nAnswer:"
+
     return [
         {
             "role": "user",
diff --git a/assets/ar/QA/TyDiQA_JAIS13b_ZeroShot.py b/assets/ar/QA/TyDiQA_JAIS13b_ZeroShot.py
index 896a800d..7bad8cce 100644
--- a/assets/ar/QA/TyDiQA_JAIS13b_ZeroShot.py
+++ b/assets/ar/QA/TyDiQA_JAIS13b_ZeroShot.py
@@ -8,6 +8,7 @@ def metadata():
         "author": "Arabic Language Technologies, QCRI, HBKU",
         "model": "Jais-13b-chat",
         "description": "Locally hosted Jais-13b-chat model using FastChat.",
+        "scores": {"F1": "0.724"},
     }
 
 
@@ -17,7 +18,7 @@ def config():
         "task": QATask,
         "model": FastChatModel,
         "model_args": {
-            "max_tries": 50,
+            "max_tries": 3,
         },
         "general_args": {"test_split": "dev"},
     }
@@ -25,6 +26,7 @@ def config():
 
 def prompt(input_sample):
     base_prompt = f"Your task is to answer questions in Arabic based on a given context.\nNote: Your answers should be spans extracted from the given context without any illustrations.\nYou don't need to provide a complete answer\nContext:{input_sample['context']}\nQuestion:{input_sample['question']}\nAnswer:"
+
     return [
         {
             "role": "user",
diff --git a/assets/ar/QA/XQuAD_JAIS13b_ZeroShot.py b/assets/ar/QA/XQuAD_JAIS13b_ZeroShot.py
index 66bbeacd..16ba431a 100644
--- a/assets/ar/QA/XQuAD_JAIS13b_ZeroShot.py
+++ b/assets/ar/QA/XQuAD_JAIS13b_ZeroShot.py
@@ -8,6 +8,7 @@ def metadata():
         "author": "Arabic Language Technologies, QCRI, HBKU",
         "model": "Jais-13b-chat",
         "description": "Locally hosted Jais-13b-chat model using FastChat.",
+        "scores": {"F1": "0.636"},
     }
 
 
@@ -17,13 +18,14 @@ def config():
         "task": QATask,
         "model": FastChatModel,
         "model_args": {
-            "max_tries": 50,
+            "max_tries": 3,
         },
     }
 
 
 def prompt(input_sample):
     base_prompt = f"Your task is to answer questions in Arabic based on a given context.\nNote: Your answers should be spans extracted from the given context without any illustrations.\nYou don't need to provide a complete answer\nContext:{input_sample['context']}\nQuestion:{input_sample['question']}\nAnswer:"
+
     return [
         {
             "role": "user",