From 28a67dcc83b509b6d99fffa81392f5ead6cd3ae3 Mon Sep 17 00:00:00 2001 From: celiolarcher Date: Fri, 5 Jul 2024 18:53:46 -0300 Subject: [PATCH] fix: reverting changes and setting simplest config since harness implementation is bugged --- lm_eval/tasks/__init__.py | 2 +- lm_eval/tasks/portuguese_benchmark/assin2/assin2rte.yaml | 3 +++ lm_eval/tasks/portuguese_benchmark/assin2/assin2sts.yaml | 3 +++ lm_eval/tasks/portuguese_benchmark/assin2/default.yaml | 7 ------- lm_eval/tasks/portuguese_benchmark/faquad/default.yaml | 3 +-- 5 files changed, 8 insertions(+), 10 deletions(-) create mode 100644 lm_eval/tasks/portuguese_benchmark/assin2/assin2rte.yaml create mode 100644 lm_eval/tasks/portuguese_benchmark/assin2/assin2sts.yaml delete mode 100644 lm_eval/tasks/portuguese_benchmark/assin2/default.yaml diff --git a/lm_eval/tasks/__init__.py b/lm_eval/tasks/__init__.py index cda769fa29..cfd56aa66d 100644 --- a/lm_eval/tasks/__init__.py +++ b/lm_eval/tasks/__init__.py @@ -323,7 +323,7 @@ def _get_task_and_group(self, task_dir: str): if f.endswith(".yaml"): yaml_path = os.path.join(root, f) config = utils.load_yaml_config(yaml_path, mode="simple") - if self._config_is_group(config) and not self._config_is_python_task(config): + if self._config_is_group(config): # This is a group config tasks_and_groups[config["group"]] = { "type": "group", diff --git a/lm_eval/tasks/portuguese_benchmark/assin2/assin2rte.yaml b/lm_eval/tasks/portuguese_benchmark/assin2/assin2rte.yaml new file mode 100644 index 0000000000..cf1b705926 --- /dev/null +++ b/lm_eval/tasks/portuguese_benchmark/assin2/assin2rte.yaml @@ -0,0 +1,3 @@ +group: portuguese_benchmark +task: assin2rte +class: !function task.Assin2RTE \ No newline at end of file diff --git a/lm_eval/tasks/portuguese_benchmark/assin2/assin2sts.yaml b/lm_eval/tasks/portuguese_benchmark/assin2/assin2sts.yaml new file mode 100644 index 0000000000..4d66d2a977 --- /dev/null +++ b/lm_eval/tasks/portuguese_benchmark/assin2/assin2sts.yaml @@ -0,0 +1,3 @@ +group: portuguese_benchmark +task: assin2sts +class: !function task.Assin2STS diff --git a/lm_eval/tasks/portuguese_benchmark/assin2/default.yaml b/lm_eval/tasks/portuguese_benchmark/assin2/default.yaml deleted file mode 100644 index 74cc6e3db3..0000000000 --- a/lm_eval/tasks/portuguese_benchmark/assin2/default.yaml +++ /dev/null @@ -1,7 +0,0 @@ -group: - - portuguese_benchmark -task: - - task: assin2rte - class: !function task.Assin2RTE - - task: assin2sts - class: !function task.Assin2STS diff --git a/lm_eval/tasks/portuguese_benchmark/faquad/default.yaml b/lm_eval/tasks/portuguese_benchmark/faquad/default.yaml index b1b6a1951c..b3b3222aa4 100644 --- a/lm_eval/tasks/portuguese_benchmark/faquad/default.yaml +++ b/lm_eval/tasks/portuguese_benchmark/faquad/default.yaml @@ -1,4 +1,3 @@ -group: - - portuguese_benchmark +group: portuguese_benchmark task: faquad class: !function task.FaQuAD