Merge branch 'main' into fix/datasets-bib-update

qcri · Sep 10, 2023 · 87cf3bf · 87cf3bf
2 parents 60fd18d + 07f4bf6
commit 87cf3bf
Show file tree

Hide file tree

Showing 237 changed files with 1,271 additions and 1,362 deletions.
diff --git a/.gitignore b/.gitignore
@@ -21,3 +21,6 @@ data
 
 # Temporary
 tmp
+
+# Model configs
+envs
diff --git a/assets/ar/MT/AraBench_Ara2Eng_Helsinki_NLP_Opus_MT_ZeroShot.py b/assets/ar/MT/AraBench_Ara2Eng_Helsinki_NLP_Opus_MT_ZeroShot.py
@@ -0,0 +1,79 @@
+from llmebench.datasets import AraBenchDataset
+from llmebench.models import HuggingFaceInferenceAPIModel, HuggingFaceTaskTypes
+from llmebench.tasks import MachineTranslationTask
+
+
+def config():
+    sets = [
+        "bible.test.mgr.0.ma",
+        "bible.test.mgr.0.tn",
+        "bible.test.msa.0.ms",
+        "bible.test.msa.1.ms",
+        "ldc_web_eg.test.lev.0.jo",
+        "ldc_web_eg.test.lev.0.ps",
+        "ldc_web_eg.test.lev.0.sy",
+        "ldc_web_eg.test.mgr.0.tn",
+        "ldc_web_eg.test.msa.0.ms",
+        "ldc_web_eg.test.nil.0.eg",
+        "ldc_web_lv.test.lev.0.lv",
+        "madar.test.glf.0.iq",
+        "madar.test.glf.0.om",
+        "madar.test.glf.0.qa",
+        "madar.test.glf.0.sa",
+        "madar.test.glf.0.ye",
+        "madar.test.glf.1.iq",
+        "madar.test.glf.1.sa",
+        "madar.test.glf.2.iq",
+        "madar.test.lev.0.jo",
+        "madar.test.lev.0.lb",
+        "madar.test.lev.0.pa",
+        "madar.test.lev.0.sy",
+        "madar.test.lev.1.jo",
+        "madar.test.lev.1.sy",
+        "madar.test.mgr.0.dz",
+        "madar.test.mgr.0.ly",
+        "madar.test.mgr.0.ma",
+        "madar.test.mgr.0.tn",
+        "madar.test.mgr.1.ly",
+        "madar.test.mgr.1.ma",
+        "madar.test.mgr.1.tn",
+        "madar.test.msa.0.ms",
+        "madar.test.nil.0.eg",
+        "madar.test.nil.0.sd",
+        "madar.test.nil.1.eg",
+        "madar.test.nil.2.eg",
+    ]
+
+    configs = []
+    for testset in sets:
+        configs.append(
+            {
+                "name": testset,
+                "config": {
+                    "dataset": AraBenchDataset,
+                    "dataset_args": {
+                        "src": f"{testset}.ar",
+                        "tgt": f"{testset}.en",
+                    },
+                    "task": MachineTranslationTask,
+                    "task_args": {},
+                    "model": HuggingFaceInferenceAPIModel,
+                    "model_args": {
+                        "task_type": HuggingFaceTaskTypes.Translation,
+                        "inference_api_url": "https://api-inference.huggingface.co/models/Helsinki-NLP/opus-mt-ar-en",
+                        "max_tries": 5,
+                    },
+                    "general_args": {"data_path": "data/MT/"},
+                },
+            }
+        )
+
+    return configs
+
+
+def prompt(input_sample):
+    return {"inputs": input_sample}
+
+
+def post_process(response):
+    return response[0]["translation_text"]
diff --git a/assets/ar/MT/AraBench_ar2en_BLOOMZ_ZeroShot.py b/assets/ar/MT/AraBench_ar2en_BLOOMZ_ZeroShot.py
@@ -1,5 +1,3 @@
-import os
-
 from llmebench.datasets import AraBenchDataset
 from llmebench.models import PetalsModel
 from llmebench.tasks import MachineTranslationTask
@@ -61,7 +59,6 @@ def config():
                     "task_args": {},
                     "model": PetalsModel,
                     "model_args": {
-                        "api_url": os.environ["API_URL"],
                         "max_tries": 3,
                     },
                     "general_args": {"data_path": "data/MT/"},

diff --git a/assets/ar/MT/AraBench_ar2en_GPT35_ZeroShot.py b/assets/ar/MT/AraBench_ar2en_GPT35_ZeroShot.py
@@ -1,5 +1,3 @@
-import os
-
 from llmebench.datasets import AraBenchDataset
 from llmebench.models import LegacyOpenAIModel
 from llmebench.tasks import MachineTranslationTask
@@ -61,11 +59,6 @@ def config():
                     "task_args": {},
                     "model": LegacyOpenAIModel,
                     "model_args": {
-                        "api_type": "azure",
-                        "api_version": "2023-03-15-preview",
-                        "api_base": os.environ["AZURE_API_URL"],
-                        "api_key": os.environ["AZURE_API_KEY"],
-                        "engine_name": os.environ["ENGINE_NAME"],
                         "max_tries": 5,
                     },
                     "general_args": {"data_path": "data/MT/"},

diff --git a/assets/ar/MT/AraBench_ar2en_GPT4_ZeroShot.py b/assets/ar/MT/AraBench_ar2en_GPT4_ZeroShot.py
@@ -1,5 +1,3 @@
-import os
-
 from llmebench.datasets import AraBenchDataset
 from llmebench.models import OpenAIModel
 from llmebench.tasks import MachineTranslationTask
@@ -60,11 +58,6 @@ def config():
                     "task_args": {},
                     "model": OpenAIModel,
                     "model_args": {
-                        "api_type": "azure",
-                        "api_version": "2023-03-15-preview",
-                        "api_base": os.environ["AZURE_API_URL"],
-                        "api_key": os.environ["AZURE_API_KEY"],
-                        "engine_name": os.environ["ENGINE_NAME"],
                         "max_tries": 5,
                     },
                     "general_args": {"data_path": "data/MT/"},

diff --git a/assets/ar/QA/ARCD_BLOOMZ_ZeroShot.py b/assets/ar/QA/ARCD_BLOOMZ_ZeroShot.py
@@ -1,5 +1,3 @@
-import os
-
 from llmebench.datasets import ARCDDataset
 from llmebench.models import PetalsModel
 from llmebench.tasks import QATask
@@ -13,7 +11,6 @@ def config():
         "task_args": {},
         "model": PetalsModel,
         "model_args": {
-            "api_url": os.environ["API_URL"],
             "max_tries": 5,
         },
         "general_args": {"data_path": "data/QA/ARCD/arcd-test.json"},

diff --git a/assets/ar/QA/ARCD_GPT35_ZeroShot.py b/assets/ar/QA/ARCD_GPT35_ZeroShot.py
@@ -1,5 +1,3 @@
-import os
-
 from llmebench.datasets import ARCDDataset
 from llmebench.models import LegacyOpenAIModel
 from llmebench.tasks import QATask
@@ -13,11 +11,6 @@ def config():
         "task_args": {},
         "model": LegacyOpenAIModel,
         "model_args": {
-            "api_type": "azure",
-            "api_version": "2023-03-15-preview",
-            "api_base": os.environ["AZURE_API_URL"],
-            "api_key": os.environ["AZURE_API_KEY"],
-            "engine_name": os.environ["ENGINE_NAME"],
             "max_tries": 3,
         },
         "general_args": {"data_path": "data/QA/ARCD/arcd-test.json"},

diff --git a/assets/ar/QA/ARCD_GPT4_FewShot.py b/assets/ar/QA/ARCD_GPT4_FewShot.py
@@ -1,4 +1,3 @@
-import os
 import random
 
 from llmebench.datasets import ARCDDataset
@@ -16,12 +15,6 @@ def config():
         "task_args": {},
         "model": OpenAIModel,
         "model_args": {
-            "api_type": "azure",
-            "api_version": "2023-03-15-preview",
-            "api_base": os.environ["AZURE_API_URL"],
-            "api_key": os.environ["AZURE_API_KEY"],
-            "engine_name": os.environ["ENGINE_NAME"],
-            "class_labels": "NA",
             "max_tries": 30,
         },
         "general_args": {

diff --git a/assets/ar/QA/ARCD_GPT4_ZeroShot.py b/assets/ar/QA/ARCD_GPT4_ZeroShot.py
@@ -1,5 +1,3 @@
-import os
-
 from llmebench.datasets import ARCDDataset
 from llmebench.models import OpenAIModel
 from llmebench.tasks import QATask
@@ -13,12 +11,6 @@ def config():
         "task_args": {},
         "model": OpenAIModel,
         "model_args": {
-            "api_type": "azure",
-            "api_version": "2023-03-15-preview",
-            "api_base": os.environ["AZURE_API_URL"],
-            "api_key": os.environ["AZURE_API_KEY"],
-            "engine_name": os.environ["ENGINE_NAME"],
-            "class_labels": "NA",
             "max_tries": 50,
         },
         "general_args": {"data_path": "data/QA/arcd/arcd-test.json"},

diff --git a/assets/ar/QA/MLQA_BLOOMZ_ZeroShot.py b/assets/ar/QA/MLQA_BLOOMZ_ZeroShot.py
@@ -1,5 +1,3 @@
-import os
-
 from llmebench.datasets import MLQADataset
 from llmebench.models import PetalsModel
 from llmebench.tasks import QATask
@@ -13,7 +11,6 @@ def config():
         "task_args": {},
         "model": PetalsModel,
         "model_args": {
-            "api_url": os.environ["API_URL"],
             "max_tries": 5,
         },
         "general_args": {

diff --git a/assets/ar/QA/MLQA_GPT35_ZeroShot.py b/assets/ar/QA/MLQA_GPT35_ZeroShot.py
@@ -1,5 +1,3 @@
-import os
-
 from llmebench.datasets import MLQADataset
 from llmebench.models import LegacyOpenAIModel
 from llmebench.tasks import QATask
@@ -13,11 +11,6 @@ def config():
         "task_args": {},
         "model": LegacyOpenAIModel,
         "model_args": {
-            "api_type": "azure",
-            "api_version": "2023-03-15-preview",
-            "api_base": os.environ["AZURE_API_URL"],
-            "api_key": os.environ["AZURE_API_KEY"],
-            "engine_name": os.environ["ENGINE_NAME"],
             "max_tries": 3,
         },
         "general_args": {

diff --git a/assets/ar/QA/MLQA_GPT4_FewShot.py b/assets/ar/QA/MLQA_GPT4_FewShot.py
@@ -1,4 +1,3 @@
-import os
 import random
 
 from llmebench.datasets import MLQADataset
@@ -16,12 +15,6 @@ def config():
         "task_args": {},
         "model": OpenAIModel,
         "model_args": {
-            "api_type": "azure",
-            "api_version": "2023-03-15-preview",
-            "api_base": os.environ["AZURE_API_URL"],
-            "api_key": os.environ["AZURE_API_KEY"],
-            "engine_name": os.environ["ENGINE_NAME"],
-            "class_labels": "NA",
             "max_tries": 30,
         },
         "general_args": {

diff --git a/assets/ar/QA/MLQA_GPT4_ZeroShot.py b/assets/ar/QA/MLQA_GPT4_ZeroShot.py
@@ -1,5 +1,3 @@
-import os
-
 from llmebench.datasets import MLQADataset
 from llmebench.models import OpenAIModel
 from llmebench.tasks import QATask
@@ -13,12 +11,6 @@ def config():
         "task_args": {},
         "model": OpenAIModel,
         "model_args": {
-            "api_type": "azure",
-            "api_version": "2023-03-15-preview",
-            "api_base": os.environ["AZURE_API_URL"],
-            "api_key": os.environ["AZURE_API_KEY"],
-            "engine_name": os.environ["ENGINE_NAME"],
-            "class_labels": "NA",
             "max_tries": 50,
         },
         "general_args": {

diff --git a/assets/ar/QA/MLQA_mdeberta_v3_base_squad2_ZeroShot.py b/assets/ar/QA/MLQA_mdeberta_v3_base_squad2_ZeroShot.py
@@ -0,0 +1,34 @@
+from llmebench.datasets import MLQADataset
+from llmebench.models import HuggingFaceInferenceAPIModel, HuggingFaceTaskTypes
+from llmebench.tasks import QATask
+
+
+def config():
+    return {
+        "dataset": MLQADataset,
+        "dataset_args": {},
+        "task": QATask,
+        "task_args": {},
+        "model": HuggingFaceInferenceAPIModel,
+        "model_args": {
+            "task_type": HuggingFaceTaskTypes.Question_Answering,
+            "inference_api_url": "https://api-inference.huggingface.co/models/timpal0l/mdeberta-v3-base-squad2",
+            "max_tries": 5,
+        },
+        "general_args": {
+            "data_path": "data/QA/MLQA/test/test-context-ar-question-ar.json"
+        },
+    }
+
+
+def prompt(input_sample):
+    return {
+        "inputs": {
+            "context": input_sample["context"],
+            "question": input_sample["question"],
+        }
+    }
+
+
+def post_process(response):
+    return response["answer"].strip()
diff --git a/assets/ar/QA/TyDiQA_BLOOMZ_ZeroShot.py b/assets/ar/QA/TyDiQA_BLOOMZ_ZeroShot.py
@@ -1,5 +1,3 @@
-import os
-
 from llmebench.datasets import TyDiQADataset
 from llmebench.models import PetalsModel
 from llmebench.tasks import QATask
@@ -13,7 +11,6 @@ def config():
         "task_args": {},
         "model": PetalsModel,
         "model_args": {
-            "api_url": os.environ["API_URL"],
             "max_tries": 5,
         },
         "general_args": {"data_path": "data/QA/tydiqa/tydiqa-goldp-dev-arabic.json"},

diff --git a/assets/ar/QA/TyDiQA_GPT35_ZeroShot.py b/assets/ar/QA/TyDiQA_GPT35_ZeroShot.py
@@ -1,5 +1,3 @@
-import os
-
 from llmebench.datasets import TyDiQADataset
 from llmebench.models import LegacyOpenAIModel
 from llmebench.tasks import QATask
@@ -13,11 +11,6 @@ def config():
         "task_args": {},
         "model": LegacyOpenAIModel,
         "model_args": {
-            "api_type": "azure",
-            "api_version": "2023-03-15-preview",
-            "api_base": os.environ["AZURE_API_URL"],
-            "api_key": os.environ["AZURE_API_KEY"],
-            "engine_name": os.environ["ENGINE_NAME"],
             "max_tries": 3,
         },
         "general_args": {"data_path": "data/QA/tydiqa/tydiqa-goldp-dev-arabic.json"},

diff --git a/assets/ar/QA/TyDiQA_GPT4_FewShot.py b/assets/ar/QA/TyDiQA_GPT4_FewShot.py
@@ -1,4 +1,3 @@
-import os
 import random
 
 from llmebench.datasets import TyDiQADataset
@@ -16,12 +15,6 @@ def config():
         "task_args": {},
         "model": OpenAIModel,
         "model_args": {
-            "api_type": "azure",
-            "api_version": "2023-03-15-preview",
-            "api_base": os.environ["AZURE_API_URL"],
-            "api_key": os.environ["AZURE_API_KEY"],
-            "engine_name": os.environ["ENGINE_NAME"],
-            "class_labels": "NA",
             "max_tries": 30,
         },
         "general_args": {

diff --git a/assets/ar/QA/TydiQA_GPT4_ZeroShot.py b/assets/ar/QA/TydiQA_GPT4_ZeroShot.py
@@ -1,5 +1,3 @@
-import os
-
 from llmebench.datasets import TyDiQADataset
 from llmebench.models import OpenAIModel
 from llmebench.tasks import QATask
@@ -13,12 +11,6 @@ def config():
         "task_args": {},
         "model": OpenAIModel,
         "model_args": {
-            "api_type": "azure",
-            "api_version": "2023-03-15-preview",
-            "api_base": os.environ["AZURE_API_URL"],
-            "api_key": os.environ["AZURE_API_KEY"],
-            "engine_name": os.environ["ENGINE_NAME"],
-            "class_labels": "NA",
             "max_tries": 50,
         },
         "general_args": {"data_path": "data/QA/tydiqa/tydiqa-goldp-dev-arabic.json"},

diff --git a/assets/ar/QA/XQuAD_BLOOMZ_ZeroShot.py b/assets/ar/QA/XQuAD_BLOOMZ_ZeroShot.py
@@ -1,5 +1,3 @@
-import os
-
 from llmebench.datasets import XQuADDataset
 from llmebench.models import PetalsModel
 from llmebench.tasks import QATask
@@ -13,7 +11,6 @@ def config():
         "task_args": {},
         "model": PetalsModel,
         "model_args": {
-            "api_url": os.environ["API_URL"],
             "max_tries": 5,
         },
         "general_args": {"data_path": "data/QA/xquad/xquad.ar.json"},
-Original file line number
+Diff line change
@@ Expand Up / @@ -21,3 +21,6 @@ data @@
     # Temporary
     tmp
+    # Model configs
+    envs