Restore tasks namespace, create agent factory (#33)

Create an agent factory for evaluating tool usage tasks

langchain_benchmarks/registration.py

@@ -2,10 +2,10 @@
 
 from langchain_benchmarks.extraction import email_task
 from langchain_benchmarks.schema import Registry
-from langchain_benchmarks.tool_usage import (
-    type_writer_26_funcs,
+from langchain_benchmarks.tool_usage.tasks import (
     type_writer,
     relational_data,
+    type_writer_26_funcs,
     multiverse_math,
 )
 

langchain_benchmarks/schema.py

@@ -99,16 +99,18 @@ def _repr_html_(self) -> str:
         """Return an HTML representation of the registry."""
         headers = [
             "Name",
+            "Type",
             "Dataset ID",
             "Description",
         ]
         table = [
             [
-                env.name,
-                env.dataset_id,
-                env.description,
+                task.name,
+                task.__class__.__name__,
+                task.dataset_id,
+                task.description,
             ]
-            for env in self.tasks
+            for task in self.tasks
         ]
         return tabulate(table, headers=headers, tablefmt="html")
 

langchain_benchmarks/tool_usage/agents.py

@@ -0,0 +1,76 @@
+"""Code for creating an agent factory for evaluating tool usage tasks."""
+from langchain.agents import AgentExecutor
+from langchain.agents.format_scratchpad import format_to_openai_functions
+from langchain.agents.output_parsers import OpenAIFunctionsAgentOutputParser
+from langchain.chat_models import ChatOpenAI
+from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder
+from langchain.schema.runnable import Runnable
+from langchain.tools.render import format_tool_to_openai_function
+
+from langchain_benchmarks.schema import ToolUsageTask
+
+
+def _ensure_output_exists(inputs: dict) -> dict:
+    """Make sure that the output key is always present."""
+    if "output" not in inputs:
+        return {"output": "", **inputs}
+    return inputs
+
+
+class OpenAIAgentFactory:
+    def __init__(
+        self, task: ToolUsageTask, *, model: str = "gpt-3.5-turbo-16k"
+    ) -> None:
+        """Create an OpenAI agent factory for the given task.
+
+        Args:
+            task: The task to create an agent factory for.
+            model: The model to use -- this must be an open AI model.
+        """
+        self.task = task
+        self.model = model
+
+    def create(self) -> Runnable:
+        """Agent Executor"""
+        llm = ChatOpenAI(
+            model=self.model,
+            temperature=0,
+        )
+
+        env = self.task.create_environment()
+
+        llm_with_tools = llm.bind(
+            functions=[format_tool_to_openai_function(t) for t in env.tools]
+        )
+        prompt = ChatPromptTemplate.from_messages(
+            [
+                (
+                    "system",
+                    self.task.instructions,
+                ),
+                MessagesPlaceholder(variable_name="agent_scratchpad"),
+                ("user", "{input}"),
+            ]
+        )
+
+        runnable_agent = (
+            {
+                "input": lambda x: x["question"],
+                "agent_scratchpad": lambda x: format_to_openai_functions(
+                    x["intermediate_steps"]
+                ),
+            }
+            | prompt
+            | llm_with_tools
+            | OpenAIFunctionsAgentOutputParser()
+        )
+
+        return (
+            AgentExecutor(
+                agent=runnable_agent,
+                tools=env.tools,
+                handle_parsing_errors=True,
+                return_intermediate_steps=True,
+            )
+            | _ensure_output_exists
+        )

tests/unit_tests/tool_usage/test_tool_usage.py

@@ -1,3 +1,2 @@
 def test_import_tool_usage() -> None:
     """Test that tool_usage can be imported"""
-    from langchain_benchmarks.tool_usage import evaluators  # noqa: F401