fix: reverting changes and setting simplest config since harness impl…

…ementation is bugged
22-hours · Jul 5, 2024 · 28a67dc · 28a67dc
1 parent e7d1d3f
commit 28a67dc
Show file tree

Hide file tree

Showing 5 changed files with 8 additions and 10 deletions.
diff --git a/lm_eval/tasks/__init__.py b/lm_eval/tasks/__init__.py
@@ -323,7 +323,7 @@ def _get_task_and_group(self, task_dir: str):
                 if f.endswith(".yaml"):
                     yaml_path = os.path.join(root, f)
                     config = utils.load_yaml_config(yaml_path, mode="simple")
-                    if self._config_is_group(config) and not self._config_is_python_task(config):
+                    if self._config_is_group(config):
                         # This is a group config
                         tasks_and_groups[config["group"]] = {
                             "type": "group",

diff --git a/lm_eval/tasks/portuguese_benchmark/assin2/assin2rte.yaml b/lm_eval/tasks/portuguese_benchmark/assin2/assin2rte.yaml
@@ -0,0 +1,3 @@
+group: portuguese_benchmark
+task: assin2rte
+class: !function task.Assin2RTE
diff --git a/lm_eval/tasks/portuguese_benchmark/assin2/assin2sts.yaml b/lm_eval/tasks/portuguese_benchmark/assin2/assin2sts.yaml
@@ -0,0 +1,3 @@
+group: portuguese_benchmark
+task: assin2sts
+class: !function task.Assin2STS
diff --git a/lm_eval/tasks/portuguese_benchmark/assin2/default.yaml b/lm_eval/tasks/portuguese_benchmark/assin2/default.yaml
diff --git a/lm_eval/tasks/portuguese_benchmark/faquad/default.yaml b/lm_eval/tasks/portuguese_benchmark/faquad/default.yaml
@@ -1,4 +1,3 @@
-group:
-  - portuguese_benchmark
+group: portuguese_benchmark
 task: faquad
 class: !function task.FaQuAD