Skip to content
This repository has been archived by the owner on Jul 23, 2024. It is now read-only.

[Task Submission] OperationsResearchQA (operationsresearchqa) #22

Closed
wants to merge 11 commits into from
7 changes: 7 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -87,13 +87,20 @@ celerybeat-schedule
*.sage.py

# Environments
.genbench_venv
.submission_venv
.env
.venv
submission_
genbench_venv/
submission_venv/
env/
venv/
ENV/
env.bak/
venv.bak/
genbench_venv.bak/
submission_venv.bak/

# Spyder project settings
.spyderproject
2 changes: 1 addition & 1 deletion src/genbench/tasks/operationsresearchqa/config.jsonnet
Original file line number Diff line number Diff line change
@@ -35,4 +35,4 @@
'cot',

],
}
}
43 changes: 43 additions & 0 deletions src/genbench/tasks/operationsresearchqa/task.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
from typing import Any, Dict

from genbench import Task


class OperationsresearchqaTask(Task):
def format_example(self, example: Dict[str, Any]) -> Dict[str, Any]:
"""
Perform preprocessing/formatting on an example-level.
Args:
example: A dictionary containing key-value pairs for an example from the source dataset.
Returns:
A dictionary containing key-value pairs for the preprocessed/formatted example.
The dictionary should contain keys `input` and `target`.
"""
context_NL = example["context"]
question_NL = example["question"]
option_0 = example["target_options"][0]
option_1 = example["target_options"][1]
option_2 = example["target_options"][2]
option_3 = example["target_options"][3]
target = example["target"]

input_str = (
"\nContext: "
+ str(context_NL)
+ "\nQuestion: "
+ str(question_NL)
+ "\nChoices:\n"
+ str(option_0)
+ "\n"
+ str(option_1)
+ "\n"
+ str(option_2)
+ "\n"
+ str(option_3)
)
target_str = str(target)

return {
"input": input_str,
"target": target_str,
}