Fix: change task list to include python tasks into group

22-hours · Jul 5, 2024 · c24baaa · c24baaa
1 parent 9103e51
commit c24baaa
Show file tree

Hide file tree

Showing 5 changed files with 22 additions and 16 deletions.
diff --git a/lm_eval/tasks/__init__.py b/lm_eval/tasks/__init__.py
@@ -323,13 +323,7 @@ def _get_task_and_group(self, task_dir: str):
                 if f.endswith(".yaml"):
                     yaml_path = os.path.join(root, f)
                     config = utils.load_yaml_config(yaml_path, mode="simple")
-                    if self._config_is_python_task(config):
-                        # This is a python class config
-                        tasks_and_groups[config["task"]] = {
-                            "type": "python_task",
-                            "yaml_path": yaml_path,
-                        }
-                    elif self._config_is_group(config):
+                    if self._config_is_group(config):
                         # This is a group config
                         tasks_and_groups[config["group"]] = {
                             "type": "group",
@@ -351,12 +345,19 @@ def _get_task_and_group(self, task_dir: str):
                         #             }
 
                     elif self._config_is_task(config):
-                        # This is a task config
                         task = config["task"]
-                        tasks_and_groups[task] = {
-                            "type": "task",
-                            "yaml_path": yaml_path,
-                        }
+                        if self._config_is_python_task(config):
+                            # This is a python class config
+                            tasks_and_groups[task] = {
+                                "type": "python_task",
+                                "yaml_path": yaml_path,
+                            }
+                        else:
+                            # This is a task config
+                            tasks_and_groups[task] = {
+                                "type": "task",
+                                "yaml_path": yaml_path,
+                            }
 
                         if "group" in config:
                             groups = config["group"]

diff --git a/lm_eval/tasks/portuguese_benchmark/assin2/assin2rte.yaml b/lm_eval/tasks/portuguese_benchmark/assin2/assin2rte.yaml
diff --git a/lm_eval/tasks/portuguese_benchmark/assin2/assin2sts.yaml b/lm_eval/tasks/portuguese_benchmark/assin2/assin2sts.yaml
diff --git a/lm_eval/tasks/portuguese_benchmark/assin2/default.yaml b/lm_eval/tasks/portuguese_benchmark/assin2/default.yaml
@@ -0,0 +1,7 @@
+group:
+  - portuguese_benchmark
+task:
+  - task: assin2rte
+    class: !function task.Assin2RTE
+  - task: assin2sts
+    class: !function task.Assin2STS
diff --git a/lm_eval/tasks/portuguese_benchmark/faquad/default.yaml b/lm_eval/tasks/portuguese_benchmark/faquad/default.yaml
@@ -1,2 +1,4 @@
+group:
+  - portuguese_benchmark
 task: faquad
 class: !function task.FaQuAD