Feat/jais evaluation on NER and QA

assets/ar/QA/ARCD_JAIS13b_ZeroShot.py

@@ -8,6 +8,7 @@ def metadata():
         "author": "Arabic Language Technologies, QCRI, HBKU",
         "model": "Jais-13b-chat",
         "description": "Locally hosted Jais-13b-chat model using FastChat.",
+        "scores": {"F1": "0.546"},
     }
 
 
@@ -16,11 +17,15 @@ def config():
         "dataset": ARCDDataset,
         "task": QATask,
         "model": FastChatModel,
+        "model_args": {
+            "max_tries": 3,
+        },
     }
 
 
 def prompt(input_sample):
-    base_prompt = f"مهمتك هي الإجابة على الأسئلة باللغة العربية بناءً على سياق معين.\nملاحظة: يجب أن تكون إجاباتك مستخرجة من السياق المحدد دون أي اضافات.\nلست بحاجة إلى تقديم إجابة كاملة.\nالسياق: {input_sample['context']}\n السؤال: {input_sample['question']}\n الجواب:"
+    base_prompt = f"Your task is to answer questions in Arabic based on a given context.\nNote: Your answers should be spans extracted from the given context without any illustrations.\nYou don't need to provide a complete answer\nContext:{input_sample['context']}\nQuestion:{input_sample['question']}\nAnswer:"
+
     return [
         {
             "role": "user",

assets/ar/QA/MLQA_JAIS13b_ZeroShot.py

@@ -8,6 +8,7 @@ def metadata():
         "author": "Arabic Language Technologies, QCRI, HBKU",
         "model": "Jais-13b-chat",
         "description": "Locally hosted Jais-13b-chat model using FastChat.",
+        "scores": {"F1": "0.540"},
     }
 
 
@@ -17,13 +18,14 @@ def config():
         "task": QATask,
         "model": FastChatModel,
         "model_args": {
-            "max_tries": 50,
+            "max_tries": 3,
         },
     }
 
 
 def prompt(input_sample):
     base_prompt = f"Your task is to answer questions in Arabic based on a given context.\nNote: Your answers should be spans extracted from the given context without any illustrations.\nYou don't need to provide a complete answer\nContext:{input_sample['context']}\nQuestion:{input_sample['question']}\nAnswer:"
+
     return [
         {
             "role": "user",

assets/ar/QA/TyDiQA_JAIS13b_ZeroShot.py

@@ -8,6 +8,7 @@ def metadata():
         "author": "Arabic Language Technologies, QCRI, HBKU",
         "model": "Jais-13b-chat",
         "description": "Locally hosted Jais-13b-chat model using FastChat.",
+        "scores": {"F1": "0.724"},
     }
 
 
@@ -17,14 +18,15 @@ def config():
         "task": QATask,
         "model": FastChatModel,
         "model_args": {
-            "max_tries": 50,
+            "max_tries": 3,
         },
         "general_args": {"test_split": "dev"},
     }
 
 
 def prompt(input_sample):
     base_prompt = f"Your task is to answer questions in Arabic based on a given context.\nNote: Your answers should be spans extracted from the given context without any illustrations.\nYou don't need to provide a complete answer\nContext:{input_sample['context']}\nQuestion:{input_sample['question']}\nAnswer:"
+
     return [
         {
             "role": "user",

assets/ar/QA/XQuAD_JAIS13b_ZeroShot.py

@@ -8,6 +8,7 @@ def metadata():
         "author": "Arabic Language Technologies, QCRI, HBKU",
         "model": "Jais-13b-chat",
         "description": "Locally hosted Jais-13b-chat model using FastChat.",
+        "scores": {"F1": "0.636"},
     }
 
 
@@ -17,13 +18,14 @@ def config():
         "task": QATask,
         "model": FastChatModel,
         "model_args": {
-            "max_tries": 50,
+            "max_tries": 3,
         },
     }
 
 
 def prompt(input_sample):
     base_prompt = f"Your task is to answer questions in Arabic based on a given context.\nNote: Your answers should be spans extracted from the given context without any illustrations.\nYou don't need to provide a complete answer\nContext:{input_sample['context']}\nQuestion:{input_sample['question']}\nAnswer:"
+
     return [
         {
             "role": "user",