From 8403e92fac657794abe07be92d1808080223f76a Mon Sep 17 00:00:00 2001
From: OliBomby <olivierschipper@gmail.com>
Date: Mon, 20 May 2024 21:23:59 +0800
Subject: [PATCH] add human and gpt-4o llms

---
 agentboard/llm/__init__.py               |  4 +++-
 agentboard/llm/human.py                  | 19 +++++++++++++++++++
 eval_configs/main_results_all_tasks.yaml | 16 ++++++++++++++++
 3 files changed, 38 insertions(+), 1 deletion(-)
 create mode 100644 agentboard/llm/human.py

diff --git a/agentboard/llm/__init__.py b/agentboard/llm/__init__.py
index 36ecc833..a52e3a6b 100644
--- a/agentboard/llm/__init__.py
+++ b/agentboard/llm/__init__.py
@@ -4,13 +4,15 @@
 from .vllm import VLLM
 from common.registry import registry
 from .huggingface import HgModels
+from .human import HUMAN
 
 __all__ = [
     "OPENAI_GPT",
     "OPENAI_GPT_AZURE",
     "VLLM",
     "CLAUDE",
-    "HgModels"
+    "HgModels",
+    "HUMAN"
 ]
 
 
diff --git a/agentboard/llm/human.py b/agentboard/llm/human.py
new file mode 100644
index 00000000..2d61a0b5
--- /dev/null
+++ b/agentboard/llm/human.py
@@ -0,0 +1,19 @@
+from common.registry import registry
+
+@registry.register_llm("human")
+class HUMAN:
+    def __init__(self):
+        self.engine = "human"
+        self.context_length = 100000000
+        self.max_tokens = 10000
+
+    def generate(self, system_message, prompt):
+        return True, input("")
+
+    def num_tokens_from_messages(self, messages, model="gpt-3.5-turbo-0613"):
+        """Return the number of tokens used by a list of messages."""
+        return 0
+
+    @classmethod
+    def from_config(cls, config):
+        return cls()
diff --git a/eval_configs/main_results_all_tasks.yaml b/eval_configs/main_results_all_tasks.yaml
index 44ea01f2..a4f05678 100644
--- a/eval_configs/main_results_all_tasks.yaml
+++ b/eval_configs/main_results_all_tasks.yaml
@@ -13,6 +13,22 @@ agent:
   need_goal: True
 
 llm:
+  human:
+      name: human
+      engine: human
+      context_length: 128000
+      use_parser: False
+  gpt-4o:
+      name: gpt
+      engine: gpt-4o
+      context_length: 128000 # 4096
+      use_azure: False
+      temperature: 0.
+      top_p: 1
+      retry_delays: 20
+      max_retry_iters: 15
+      stop: "\n"
+      use_parser: False
   gpt-3.5-turbo-16k:
       name: gpt
       engine: gpt-3.5-turbo-16k