From 4182bc4241c8fe0090b6ace2f90b451f9fc732ac Mon Sep 17 00:00:00 2001 From: bsabri Date: Thu, 31 Aug 2023 14:36:07 +0300 Subject: [PATCH 01/15] Added FastChat models API which is compatible with openai api. In this implementation ENGINE_NAME=jais-13b-chat selectes the model among the loaded ones and AZURE_API_URL='http://10.4.64.46:5004/v10' defines the API base. It can be tested with curl http://10.4.64.46:5004/v1/models --- .../MT/AraBench_Ara2Eng_FastChat_ZeroShot.py | 95 ++++++++++++ llmebench/models/FastChat.py | 141 ++++++++++++++++++ llmebench/models/__init__.py | 1 + 3 files changed, 237 insertions(+) create mode 100644 assets/benchmark_v1/MT/AraBench_Ara2Eng_FastChat_ZeroShot.py create mode 100644 llmebench/models/FastChat.py diff --git a/assets/benchmark_v1/MT/AraBench_Ara2Eng_FastChat_ZeroShot.py b/assets/benchmark_v1/MT/AraBench_Ara2Eng_FastChat_ZeroShot.py new file mode 100644 index 00000000..f4f241b9 --- /dev/null +++ b/assets/benchmark_v1/MT/AraBench_Ara2Eng_FastChat_ZeroShot.py @@ -0,0 +1,95 @@ +import os + +from llmebench.datasets import AraBenchDataset +from llmebench.models import ChatCompletionModel +from llmebench.tasks import MachineTranslationTask + + +def config(): + sets = [ + "bible.test.mgr.0.ma", + "bible.test.mgr.0.tn", + "bible.test.msa.0.ms", + "bible.test.msa.1.ms", + "ldc_web_eg.test.lev.0.jo", + "ldc_web_eg.test.lev.0.ps", + "ldc_web_eg.test.lev.0.sy", + "ldc_web_eg.test.mgr.0.tn", + "ldc_web_eg.test.msa.0.ms", + "ldc_web_eg.test.nil.0.eg", + "ldc_web_lv.test.lev.0.lv", + "madar.test.glf.0.iq", + "madar.test.glf.0.om", + "madar.test.glf.0.qa", + "madar.test.glf.0.sa", + "madar.test.glf.0.ye", + "madar.test.glf.1.iq", + "madar.test.glf.1.sa", + "madar.test.glf.2.iq", + "madar.test.lev.0.jo", + "madar.test.lev.0.lb", + "madar.test.lev.0.pa", + "madar.test.lev.0.sy", + "madar.test.lev.1.jo", + "madar.test.lev.1.sy", + "madar.test.mgr.0.dz", + "madar.test.mgr.0.ly", + "madar.test.mgr.0.ma", + "madar.test.mgr.0.tn", + "madar.test.mgr.1.ly", + "madar.test.mgr.1.ma", + "madar.test.mgr.1.tn", + "madar.test.msa.0.ms", + "madar.test.nil.0.eg", + "madar.test.nil.0.sd", + "madar.test.nil.1.eg", + "madar.test.nil.2.eg", + ] + configs = [] + for testset in sets: + configs.append( + { + "name": testset, + "config": { + "dataset": AraBenchDataset, + "dataset_args": { + "src": f"{testset}.ar", + "tgt": f"{testset}.en", + }, + "task": MachineTranslationTask, + "task_args": {}, + "model": ChatCompletionModel, + "model_args": { + "api_type": "azure", + "api_version": "2023-03-15-preview", + "api_base": os.environ["AZURE_API_URL"], + "api_key": os.environ["AZURE_API_KEY"], + "engine_name": os.environ["ENGINE_NAME"], + "max_tries": 5, + }, + "general_args": {"data_path": "data/MT/"}, + }, + } + ) + + return configs + + +def prompt(input_sample): + return [ + { + "role": "system", + "content": "You are an expert translator specialized in translating texts from Arabic to English. You are concise as you only output the translation of the text without any illustrations or extra details", + }, + { + "role": "user", + "content": f"Translate the following text to English.\nText: {input_sample}\nTranslation: ", + }, + ] + + +def post_process(response): + response = response["choices"][0]["message"]["content"] + response = response.replace('"', "") + response = response.strip() + return response diff --git a/llmebench/models/FastChat.py b/llmebench/models/FastChat.py new file mode 100644 index 00000000..4931e583 --- /dev/null +++ b/llmebench/models/FastChat.py @@ -0,0 +1,141 @@ +import openai + +from llmebench.models.model_base import ModelBase + + +class FastChatModel(ModelBase): + def __init__( + self, + api_type, + api_base, + api_version, + api_key, + engine_name, + temperature=0, + top_p=0.95, + max_tokens=800, + frequency_penalty=0, + presence_penalty=0, + **kwargs + ): + # API parameters + # openai.api_type = api_type + openai.api_base = api_base + openai.api_version = api_version + openai.api_key = api_key + self.engine_name = engine_name + + # GPT parameters + self.temperature = temperature + self.top_p = top_p + self.max_tokens = max_tokens + self.frequency_penalty = frequency_penalty + self.presence_penalty = presence_penalty + + self.system_message_template = "<|im_start|>system\n{}\n<|im_end|>" + self.message_template = "\n<|im_start|>{}\n{}\n<|im_end|>" + + super(FastChatModel, self).__init__( + retry_exceptions=(openai.error.Timeout, openai.error.RateLimitError), + **kwargs + ) + + # defining a function to create the prompt from the system and user messages + def create_prompt(self, system_message, messages): + prompt = self.system_message_template.format(system_message) + + for message in messages: + prompt += self.message_template.format(message["sender"], message["text"]) + prompt += "\n<|im_start|>assistant\n" + return prompt + + def summarize_response(self, response): + if ( + "choices" in response + and isinstance(response["choices"], list) + and len(response["choices"]) > 0 + and "text" in response["choices"][0] + ): + return response["choices"][0]["text"] + + return None + + def prompt(self, processed_input): + system_message = processed_input["system_message"] + messages = processed_input["messages"] + prompt = self.create_prompt(system_message, messages) + response = openai.Completion.create( + engine=self.engine_name, + prompt=prompt, + temperature=self.temperature, + max_tokens=self.max_tokens, + top_p=self.top_p, + frequency_penalty=self.frequency_penalty, + presence_penalty=self.presence_penalty, + stop=["<|im_end|>"], + ) + + return response + + +class ChatCompletionModel(ModelBase): + def __init__( + self, + api_type, + api_base, + api_version, + api_key, + engine_name, + temperature=0, + top_p=0.95, + max_tokens=800, + frequency_penalty=0, + presence_penalty=0, + **kwargs + ): + # API parameters + # openai.api_type = api_type + openai.api_base = api_base + openai.api_version = api_version + openai.api_key = api_key + self.engine_name = engine_name + + # GPT parameters + self.temperature = temperature + self.top_p = top_p + self.max_tokens = max_tokens + self.frequency_penalty = frequency_penalty + self.presence_penalty = presence_penalty + + super(ChatCompletionModel, self).__init__( + retry_exceptions=(openai.error.Timeout, openai.error.RateLimitError), + **kwargs + ) + + def summarize_response(self, response): + if ( + "choices" in response + and isinstance(response["choices"], list) + and len(response["choices"]) > 0 + and "message" in response["choices"][0] + and "content" in response["choices"][0]["message"] + and response["choices"][0]["message"]["role"] == "assistant" + ): + return response["choices"][0]["message"]["content"] + + return None + + def prompt(self, processed_input): + response = openai.ChatCompletion.create( + # engine=self.engine_name, + model = self.engine_name, + messages=processed_input, + temperature=self.temperature, + max_tokens=self.max_tokens, + top_p=self.top_p, + frequency_penalty=self.frequency_penalty, + presence_penalty=self.presence_penalty, + stop=None, + ) + + return response diff --git a/llmebench/models/__init__.py b/llmebench/models/__init__.py index 5ea24565..97906b3a 100644 --- a/llmebench/models/__init__.py +++ b/llmebench/models/__init__.py @@ -1,3 +1,4 @@ from .BLOOMPetal import BLOOMPetalModel from .GPT import GPTChatCompletionModel, GPTModel +from .FastChat import ChatCompletionModel, FastChatModel from .RandomGPT import RandomGPTModel From 060609eab936364ee9563c364ec4468002e59fdd Mon Sep 17 00:00:00 2001 From: bsabri Date: Mon, 4 Sep 2023 10:17:51 +0300 Subject: [PATCH 02/15] code formatting --- assets/benchmark_v1/MT/AraBench_Ara2Eng_FastChat_ZeroShot.py | 4 ++-- llmebench/models/FastChat.py | 2 +- llmebench/models/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/assets/benchmark_v1/MT/AraBench_Ara2Eng_FastChat_ZeroShot.py b/assets/benchmark_v1/MT/AraBench_Ara2Eng_FastChat_ZeroShot.py index f4f241b9..f7b9cc04 100644 --- a/assets/benchmark_v1/MT/AraBench_Ara2Eng_FastChat_ZeroShot.py +++ b/assets/benchmark_v1/MT/AraBench_Ara2Eng_FastChat_ZeroShot.py @@ -60,8 +60,8 @@ def config(): "task_args": {}, "model": ChatCompletionModel, "model_args": { - "api_type": "azure", - "api_version": "2023-03-15-preview", + # "api_type": "azure", + # "api_version": "2023-03-15-preview", "api_base": os.environ["AZURE_API_URL"], "api_key": os.environ["AZURE_API_KEY"], "engine_name": os.environ["ENGINE_NAME"], diff --git a/llmebench/models/FastChat.py b/llmebench/models/FastChat.py index 4931e583..00b52caa 100644 --- a/llmebench/models/FastChat.py +++ b/llmebench/models/FastChat.py @@ -128,7 +128,7 @@ def summarize_response(self, response): def prompt(self, processed_input): response = openai.ChatCompletion.create( # engine=self.engine_name, - model = self.engine_name, + model=self.engine_name, messages=processed_input, temperature=self.temperature, max_tokens=self.max_tokens, diff --git a/llmebench/models/__init__.py b/llmebench/models/__init__.py index 97906b3a..9e9a7801 100644 --- a/llmebench/models/__init__.py +++ b/llmebench/models/__init__.py @@ -1,4 +1,4 @@ from .BLOOMPetal import BLOOMPetalModel -from .GPT import GPTChatCompletionModel, GPTModel from .FastChat import ChatCompletionModel, FastChatModel +from .GPT import GPTChatCompletionModel, GPTModel from .RandomGPT import RandomGPTModel From f07319d4c83c8d753d878de393b3fabd546fbe20 Mon Sep 17 00:00:00 2001 From: bsabri Date: Thu, 14 Sep 2023 13:17:24 +0300 Subject: [PATCH 03/15] fixed a few issues --- ...y => AraBench_ara2en_FastChat_ZeroShot.py} | 9 +- llmebench/models/FastChat.py | 145 ++---------------- llmebench/models/__init__.py | 8 +- 3 files changed, 12 insertions(+), 150 deletions(-) rename assets/ar/MT/{AraBench_Ara2Eng_FastChat_ZeroShot.py => AraBench_ara2en_FastChat_ZeroShot.py} (86%) diff --git a/assets/ar/MT/AraBench_Ara2Eng_FastChat_ZeroShot.py b/assets/ar/MT/AraBench_ara2en_FastChat_ZeroShot.py similarity index 86% rename from assets/ar/MT/AraBench_Ara2Eng_FastChat_ZeroShot.py rename to assets/ar/MT/AraBench_ara2en_FastChat_ZeroShot.py index f7b9cc04..638015a4 100644 --- a/assets/ar/MT/AraBench_Ara2Eng_FastChat_ZeroShot.py +++ b/assets/ar/MT/AraBench_ara2en_FastChat_ZeroShot.py @@ -1,7 +1,7 @@ import os from llmebench.datasets import AraBenchDataset -from llmebench.models import ChatCompletionModel +from llmebench.models import FastChatModel from llmebench.tasks import MachineTranslationTask @@ -58,13 +58,8 @@ def config(): }, "task": MachineTranslationTask, "task_args": {}, - "model": ChatCompletionModel, + "model": FastChatModel, "model_args": { - # "api_type": "azure", - # "api_version": "2023-03-15-preview", - "api_base": os.environ["AZURE_API_URL"], - "api_key": os.environ["AZURE_API_KEY"], - "engine_name": os.environ["ENGINE_NAME"], "max_tries": 5, }, "general_args": {"data_path": "data/MT/"}, diff --git a/llmebench/models/FastChat.py b/llmebench/models/FastChat.py index 00b52caa..f92d0ccb 100644 --- a/llmebench/models/FastChat.py +++ b/llmebench/models/FastChat.py @@ -1,141 +1,14 @@ -import openai +import os -from llmebench.models.model_base import ModelBase +from llmebench.models.OpenAI import OpenAIModel -class FastChatModel(ModelBase): - def __init__( - self, - api_type, - api_base, - api_version, - api_key, - engine_name, - temperature=0, - top_p=0.95, - max_tokens=800, - frequency_penalty=0, - presence_penalty=0, - **kwargs - ): - # API parameters - # openai.api_type = api_type - openai.api_base = api_base - openai.api_version = api_version - openai.api_key = api_key - self.engine_name = engine_name - - # GPT parameters - self.temperature = temperature - self.top_p = top_p - self.max_tokens = max_tokens - self.frequency_penalty = frequency_penalty - self.presence_penalty = presence_penalty - - self.system_message_template = "<|im_start|>system\n{}\n<|im_end|>" - self.message_template = "\n<|im_start|>{}\n{}\n<|im_end|>" - +class FastChatModel(OpenAIModel): + def __init__(self, api_base, api_key, model_name, **kwargs): + api_base = api_base or os.getenv("FASTCHAT_API_BASE") + api_key = api_key or os.getenv("FASTCHAT_API_KEY") + model_name = model_name or os.getenv("FASTCHAT_MODEL_NAME") + # checks for valid config settings) super(FastChatModel, self).__init__( - retry_exceptions=(openai.error.Timeout, openai.error.RateLimitError), - **kwargs - ) - - # defining a function to create the prompt from the system and user messages - def create_prompt(self, system_message, messages): - prompt = self.system_message_template.format(system_message) - - for message in messages: - prompt += self.message_template.format(message["sender"], message["text"]) - prompt += "\n<|im_start|>assistant\n" - return prompt - - def summarize_response(self, response): - if ( - "choices" in response - and isinstance(response["choices"], list) - and len(response["choices"]) > 0 - and "text" in response["choices"][0] - ): - return response["choices"][0]["text"] - - return None - - def prompt(self, processed_input): - system_message = processed_input["system_message"] - messages = processed_input["messages"] - prompt = self.create_prompt(system_message, messages) - response = openai.Completion.create( - engine=self.engine_name, - prompt=prompt, - temperature=self.temperature, - max_tokens=self.max_tokens, - top_p=self.top_p, - frequency_penalty=self.frequency_penalty, - presence_penalty=self.presence_penalty, - stop=["<|im_end|>"], - ) - - return response - - -class ChatCompletionModel(ModelBase): - def __init__( - self, - api_type, - api_base, - api_version, - api_key, - engine_name, - temperature=0, - top_p=0.95, - max_tokens=800, - frequency_penalty=0, - presence_penalty=0, - **kwargs - ): - # API parameters - # openai.api_type = api_type - openai.api_base = api_base - openai.api_version = api_version - openai.api_key = api_key - self.engine_name = engine_name - - # GPT parameters - self.temperature = temperature - self.top_p = top_p - self.max_tokens = max_tokens - self.frequency_penalty = frequency_penalty - self.presence_penalty = presence_penalty - - super(ChatCompletionModel, self).__init__( - retry_exceptions=(openai.error.Timeout, openai.error.RateLimitError), - **kwargs + api_base=api_base, api_key=api_key, model_name=model_name, **kwargs ) - - def summarize_response(self, response): - if ( - "choices" in response - and isinstance(response["choices"], list) - and len(response["choices"]) > 0 - and "message" in response["choices"][0] - and "content" in response["choices"][0]["message"] - and response["choices"][0]["message"]["role"] == "assistant" - ): - return response["choices"][0]["message"]["content"] - - return None - - def prompt(self, processed_input): - response = openai.ChatCompletion.create( - # engine=self.engine_name, - model=self.engine_name, - messages=processed_input, - temperature=self.temperature, - max_tokens=self.max_tokens, - top_p=self.top_p, - frequency_penalty=self.frequency_penalty, - presence_penalty=self.presence_penalty, - stop=None, - ) - - return response diff --git a/llmebench/models/__init__.py b/llmebench/models/__init__.py index 71633623..d8f10f7b 100644 --- a/llmebench/models/__init__.py +++ b/llmebench/models/__init__.py @@ -1,9 +1,3 @@ -<<<<<<< HEAD -from .BLOOMPetal import BLOOMPetalModel -from .FastChat import ChatCompletionModel, FastChatModel -from .GPT import GPTChatCompletionModel, GPTModel -from .RandomGPT import RandomGPTModel -======= +from .FastChat import FastChatModel from .OpenAI import LegacyOpenAIModel, OpenAIModel from .Petals import PetalsModel ->>>>>>> origin/main From 4800bdf4ee68276a0711f3a122b91ab6da3c73b9 Mon Sep 17 00:00:00 2001 From: bsabri Date: Sun, 17 Sep 2023 10:17:19 +0300 Subject: [PATCH 04/15] fixed env variables --- assets/ar/MT/AraBench_ara2en_FastChat_ZeroShot.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/assets/ar/MT/AraBench_ara2en_FastChat_ZeroShot.py b/assets/ar/MT/AraBench_ara2en_FastChat_ZeroShot.py index 638015a4..02cf13e6 100644 --- a/assets/ar/MT/AraBench_ara2en_FastChat_ZeroShot.py +++ b/assets/ar/MT/AraBench_ara2en_FastChat_ZeroShot.py @@ -60,6 +60,10 @@ def config(): "task_args": {}, "model": FastChatModel, "model_args": { + "api_type": "openai", + "api_key": os.environ["OPENAI_API_KEY"], + "api_base": os.environ["OPENAI_API_URL"], + "model_name": os.environ["OPENAI_MODEL"], "max_tries": 5, }, "general_args": {"data_path": "data/MT/"}, From 575900ef41d8e2b3740d027be6cbebdda38321da Mon Sep 17 00:00:00 2001 From: bsabri Date: Sun, 17 Sep 2023 10:57:09 +0300 Subject: [PATCH 05/15] fixed env variable issues --- ...py => AraBench_ar2en_FastChat_ZeroShot.py} | 4 ---- llmebench/models/FastChat.py | 20 +++++++++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) rename assets/ar/MT/{AraBench_ara2en_FastChat_ZeroShot.py => AraBench_ar2en_FastChat_ZeroShot.py} (91%) diff --git a/assets/ar/MT/AraBench_ara2en_FastChat_ZeroShot.py b/assets/ar/MT/AraBench_ar2en_FastChat_ZeroShot.py similarity index 91% rename from assets/ar/MT/AraBench_ara2en_FastChat_ZeroShot.py rename to assets/ar/MT/AraBench_ar2en_FastChat_ZeroShot.py index 02cf13e6..638015a4 100644 --- a/assets/ar/MT/AraBench_ara2en_FastChat_ZeroShot.py +++ b/assets/ar/MT/AraBench_ar2en_FastChat_ZeroShot.py @@ -60,10 +60,6 @@ def config(): "task_args": {}, "model": FastChatModel, "model_args": { - "api_type": "openai", - "api_key": os.environ["OPENAI_API_KEY"], - "api_base": os.environ["OPENAI_API_URL"], - "model_name": os.environ["OPENAI_MODEL"], "max_tries": 5, }, "general_args": {"data_path": "data/MT/"}, diff --git a/llmebench/models/FastChat.py b/llmebench/models/FastChat.py index f92d0ccb..7aa45668 100644 --- a/llmebench/models/FastChat.py +++ b/llmebench/models/FastChat.py @@ -4,10 +4,22 @@ class FastChatModel(OpenAIModel): - def __init__(self, api_base, api_key, model_name, **kwargs): - api_base = api_base or os.getenv("FASTCHAT_API_BASE") - api_key = api_key or os.getenv("FASTCHAT_API_KEY") - model_name = model_name or os.getenv("FASTCHAT_MODEL_NAME") + def __init__(self, api_base=None, api_key=None, model_name=None, **kwargs): + self.api_base = api_base or os.getenv("OPENAI_API_BASE") + self.api_key = api_key or os.getenv("OPENAI_API_KEY") + self.model_name = model_name or os.getenv("OPENAI_MODEL") + if self.api_base is None: + raise Exception( + "API url must be provided as model config or environment variable (`OPENAI_API_BASE`)" + ) + if self.api_key is None: + raise Exception( + "API url must be provided as model config or environment variable (`OPENAI_API_KEY`)" + ) + if self.model_name is None: + raise Exception( + "API url must be provided as model config or environment variable (`OPENAI_MODEL`)" + ) # checks for valid config settings) super(FastChatModel, self).__init__( api_base=api_base, api_key=api_key, model_name=model_name, **kwargs From 698ca51eb681b8ab83e77b5fb9ea0961fa72f31f Mon Sep 17 00:00:00 2001 From: bsabri Date: Sun, 17 Sep 2023 14:01:51 +0300 Subject: [PATCH 06/15] resetting OPENAI environment variables during Azure model testing. --- tests/models/test_OpenAIModel.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/models/test_OpenAIModel.py b/tests/models/test_OpenAIModel.py index 7f2ee1ba..29074f06 100644 --- a/tests/models/test_OpenAIModel.py +++ b/tests/models/test_OpenAIModel.py @@ -80,6 +80,9 @@ def test_openai_config_azure(self): "AZURE_API_URL": "url", "AZURE_API_KEY": "secret-key", "AZURE_ENGINE_NAME": "private-model", + "OPENAI_API_BASE": "", + "OPENAI_API_KEY": "", + "OPENAI_MODEL": "", }, ) def test_openai_config_env_var_azure(self): From ff163cb53fbec09e7495dfb5f277ec6ae18b964f Mon Sep 17 00:00:00 2001 From: bsabri Date: Sun, 17 Sep 2023 15:57:29 +0300 Subject: [PATCH 07/15] code cleaning --- assets/ar/MT/AraBench_ar2en_FastChat_ZeroShot.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/assets/ar/MT/AraBench_ar2en_FastChat_ZeroShot.py b/assets/ar/MT/AraBench_ar2en_FastChat_ZeroShot.py index 638015a4..07ed5bba 100644 --- a/assets/ar/MT/AraBench_ar2en_FastChat_ZeroShot.py +++ b/assets/ar/MT/AraBench_ar2en_FastChat_ZeroShot.py @@ -1,5 +1,3 @@ -import os - from llmebench.datasets import AraBenchDataset from llmebench.models import FastChatModel from llmebench.tasks import MachineTranslationTask @@ -45,6 +43,7 @@ def config(): "madar.test.nil.1.eg", "madar.test.nil.2.eg", ] + configs = [] for testset in sets: configs.append( @@ -60,7 +59,7 @@ def config(): "task_args": {}, "model": FastChatModel, "model_args": { - "max_tries": 5, + "max_tries": 3, }, "general_args": {"data_path": "data/MT/"}, }, From a74f75a428a667dc1e2fb11d59fb5495af0d8bba Mon Sep 17 00:00:00 2001 From: bsabri Date: Mon, 25 Sep 2023 10:18:56 +0300 Subject: [PATCH 08/15] minor corrections --- llmebench/models/FastChat.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/llmebench/models/FastChat.py b/llmebench/models/FastChat.py index 7aa45668..52579352 100644 --- a/llmebench/models/FastChat.py +++ b/llmebench/models/FastChat.py @@ -5,18 +5,18 @@ class FastChatModel(OpenAIModel): def __init__(self, api_base=None, api_key=None, model_name=None, **kwargs): - self.api_base = api_base or os.getenv("OPENAI_API_BASE") - self.api_key = api_key or os.getenv("OPENAI_API_KEY") - self.model_name = model_name or os.getenv("OPENAI_MODEL") - if self.api_base is None: + api_base = api_base or os.getenv("OPENAI_API_BASE") + api_key = api_key or os.getenv("OPENAI_API_KEY") + model_name = model_name or os.getenv("OPENAI_MODEL") + if api_base is None: raise Exception( "API url must be provided as model config or environment variable (`OPENAI_API_BASE`)" ) - if self.api_key is None: + if api_key is None: raise Exception( "API url must be provided as model config or environment variable (`OPENAI_API_KEY`)" ) - if self.model_name is None: + if model_name is None: raise Exception( "API url must be provided as model config or environment variable (`OPENAI_MODEL`)" ) From fd260a682ff7731a9fbf8e31b7dfe070b965ba17 Mon Sep 17 00:00:00 2001 From: Fahim Imaduddin Dalvi Date: Mon, 25 Sep 2023 10:41:32 +0300 Subject: [PATCH 09/15] Add FASTCHAT* environment variables for model config --- llmebench/models/FastChat.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/llmebench/models/FastChat.py b/llmebench/models/FastChat.py index 52579352..47803996 100644 --- a/llmebench/models/FastChat.py +++ b/llmebench/models/FastChat.py @@ -5,20 +5,20 @@ class FastChatModel(OpenAIModel): def __init__(self, api_base=None, api_key=None, model_name=None, **kwargs): - api_base = api_base or os.getenv("OPENAI_API_BASE") - api_key = api_key or os.getenv("OPENAI_API_KEY") - model_name = model_name or os.getenv("OPENAI_MODEL") + api_base = api_base or os.getenv("FASTCHAT_API_BASE") + api_key = api_key or os.getenv("FASTCHAT_API_KEY") + model_name = model_name or os.getenv("FASTCHAT_MODEL") if api_base is None: raise Exception( - "API url must be provided as model config or environment variable (`OPENAI_API_BASE`)" + "API url must be provided as model config or environment variable (`FASTCHAT_API_BASE`)" ) if api_key is None: raise Exception( - "API url must be provided as model config or environment variable (`OPENAI_API_KEY`)" + "API url must be provided as model config or environment variable (`FASTCHAT_API_KEY`)" ) if model_name is None: raise Exception( - "API url must be provided as model config or environment variable (`OPENAI_MODEL`)" + "API url must be provided as model config or environment variable (`FASTCHAT_MODEL`)" ) # checks for valid config settings) super(FastChatModel, self).__init__( From b681f1eb4624af7416ab20b8dcddd6b1c35de5fc Mon Sep 17 00:00:00 2001 From: Fahim Imaduddin Dalvi Date: Mon, 25 Sep 2023 10:42:50 +0300 Subject: [PATCH 10/15] Add docstring for model --- llmebench/models/FastChat.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/llmebench/models/FastChat.py b/llmebench/models/FastChat.py index 47803996..4dbe8abc 100644 --- a/llmebench/models/FastChat.py +++ b/llmebench/models/FastChat.py @@ -4,6 +4,26 @@ class FastChatModel(OpenAIModel): + """ + FastChat Model interface. Can be used for models hosted using FastChat + https://github.com/lm-sys/FastChat + + Accepts all arguments used by `OpenAIModel`, and overrides the arguments listed + below with FastChat-specific variables. + + Arguments + --------- + api_base : str + URL where the model is hosted. If not provided, the implementation will look at + environment variable `FASTCHAT_API_BASE` + api_key : str + Authentication token for the API. If not provided, the implementation will derive it + from environment variable `FASTCHAT_API_KEY` + model_name : str + Name of the model to use. If not provided, the implementation will derive it from + environment variable `FASTCHAT_MODEL` + """ + def __init__(self, api_base=None, api_key=None, model_name=None, **kwargs): api_base = api_base or os.getenv("FASTCHAT_API_BASE") api_key = api_key or os.getenv("FASTCHAT_API_KEY") From 715ddcde630665524e9d97e9265369b5e5f8dd17 Mon Sep 17 00:00:00 2001 From: Fahim Imaduddin Dalvi Date: Mon, 25 Sep 2023 10:43:00 +0300 Subject: [PATCH 11/15] Add tests for FastChat --- tests/models/test_FastChatModel.py | 76 ++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 tests/models/test_FastChatModel.py diff --git a/tests/models/test_FastChatModel.py b/tests/models/test_FastChatModel.py new file mode 100644 index 00000000..bde0f2fe --- /dev/null +++ b/tests/models/test_FastChatModel.py @@ -0,0 +1,76 @@ +import unittest +from unittest.mock import patch + +import openai + +from llmebench import Benchmark +from llmebench.models import FastChatModel + +from tests.models.test_OpenAIModel import TestAssetsForOpenAIPrompts + + +class TestAssetsForFastChatPrompts(TestAssetsForOpenAIPrompts): + @classmethod + def setUpClass(cls): + # Load the benchmark assets + benchmark = Benchmark(benchmark_dir="assets") + all_assets = benchmark.find_assets() + + # Filter out assets not using the GPT model + cls.assets = [ + asset for asset in all_assets if asset["config"]["model"] in [FastChatModel] + ] + + def test_fastchat_prompts(self): + "Test if all assets using this model return data in an appropriate format for prompting" + + self.test_openai_prompts() + + +class TestFastChatConfig(unittest.TestCase): + def test_fastchat_config(self): + "Test if model config parameters passed as arguments are used" + model = FastChatModel( + api_base="llmebench.qcri.org", + api_key="secret-key", + model_name="private-model", + ) + + self.assertEqual(openai.api_type, "openai") + self.assertEqual(openai.api_base, "llmebench.qcri.org") + self.assertEqual(openai.api_key, "secret-key") + self.assertEqual(model.model_params["model"], "private-model") + + @patch.dict( + "os.environ", + { + "FASTCHAT_API_BASE": "llmebench.qcri.org", + "FASTCHAT_API_KEY": "secret-key", + "FASTCHAT_MODEL": "private-model", + }, + ) + def test_fastchat_config_env_var(self): + "Test if model config parameters passed as environment variables are used" + model = FastChatModel() + + self.assertEqual(openai.api_type, "openai") + self.assertEqual(openai.api_base, "llmebench.qcri.org") + self.assertEqual(openai.api_key, "secret-key") + self.assertEqual(model.model_params["model"], "private-model") + + @patch.dict( + "os.environ", + { + "FASTCHAT_API_BASE": "llmebench.qcri.org", + "FASTCHAT_API_KEY": "secret-key", + "FASTCHAT_MODEL": "private-model", + }, + ) + def test_fastchat_config_priority(self): + "Test if model config parameters override environment variables" + model = FastChatModel(model_name="another-model") + + self.assertEqual(openai.api_type, "openai") + self.assertEqual(openai.api_base, "llmebench.qcri.org") + self.assertEqual(openai.api_key, "secret-key") + self.assertEqual(model.model_params["model"], "another-model") From 9c3b653a4c208abfb0fd192be918da6bbea74118 Mon Sep 17 00:00:00 2001 From: Fahim Imaduddin Dalvi Date: Mon, 25 Sep 2023 10:43:28 +0300 Subject: [PATCH 12/15] Update Asset to match new Dataset format --- .../ar/MT/AraBench_ar2en_FastChat_ZeroShot.py | 81 +++++-------------- 1 file changed, 19 insertions(+), 62 deletions(-) diff --git a/assets/ar/MT/AraBench_ar2en_FastChat_ZeroShot.py b/assets/ar/MT/AraBench_ar2en_FastChat_ZeroShot.py index 07ed5bba..1daf04a6 100644 --- a/assets/ar/MT/AraBench_ar2en_FastChat_ZeroShot.py +++ b/assets/ar/MT/AraBench_ar2en_FastChat_ZeroShot.py @@ -3,70 +3,27 @@ from llmebench.tasks import MachineTranslationTask -def config(): - sets = [ - "bible.test.mgr.0.ma", - "bible.test.mgr.0.tn", - "bible.test.msa.0.ms", - "bible.test.msa.1.ms", - "ldc_web_eg.test.lev.0.jo", - "ldc_web_eg.test.lev.0.ps", - "ldc_web_eg.test.lev.0.sy", - "ldc_web_eg.test.mgr.0.tn", - "ldc_web_eg.test.msa.0.ms", - "ldc_web_eg.test.nil.0.eg", - "ldc_web_lv.test.lev.0.lv", - "madar.test.glf.0.iq", - "madar.test.glf.0.om", - "madar.test.glf.0.qa", - "madar.test.glf.0.sa", - "madar.test.glf.0.ye", - "madar.test.glf.1.iq", - "madar.test.glf.1.sa", - "madar.test.glf.2.iq", - "madar.test.lev.0.jo", - "madar.test.lev.0.lb", - "madar.test.lev.0.pa", - "madar.test.lev.0.sy", - "madar.test.lev.1.jo", - "madar.test.lev.1.sy", - "madar.test.mgr.0.dz", - "madar.test.mgr.0.ly", - "madar.test.mgr.0.ma", - "madar.test.mgr.0.tn", - "madar.test.mgr.1.ly", - "madar.test.mgr.1.ma", - "madar.test.mgr.1.tn", - "madar.test.msa.0.ms", - "madar.test.nil.0.eg", - "madar.test.nil.0.sd", - "madar.test.nil.1.eg", - "madar.test.nil.2.eg", - ] +def metadata(): + return { + "author": "Arabic Language Technologies, QCRI, HBKU", + "model": "bloomz-176b (8bit quantized)", + "description": "Locally hosted BLOOMZ 176b model (8 bit quantized version) using FastChat.", + } - configs = [] - for testset in sets: - configs.append( - { - "name": testset, - "config": { - "dataset": AraBenchDataset, - "dataset_args": { - "src": f"{testset}.ar", - "tgt": f"{testset}.en", - }, - "task": MachineTranslationTask, - "task_args": {}, - "model": FastChatModel, - "model_args": { - "max_tries": 3, - }, - "general_args": {"data_path": "data/MT/"}, - }, - } - ) - return configs +def config(): + return { + "dataset": AraBenchDataset, + "dataset_args": { + "src_lang": "ar", + "tgt_lang": "en", + }, + "task": MachineTranslationTask, + "model": FastChatModel, + "model_args": { + "max_tries": 3, + }, + } def prompt(input_sample): From b67d3c0383e1c764d730afcb8a619090eb2b6709 Mon Sep 17 00:00:00 2001 From: Fahim Imaduddin Dalvi Date: Mon, 25 Sep 2023 11:53:15 +0300 Subject: [PATCH 13/15] Add sample env file --- envs/fastchat.env | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 envs/fastchat.env diff --git a/envs/fastchat.env b/envs/fastchat.env new file mode 100644 index 00000000..6fe9ec30 --- /dev/null +++ b/envs/fastchat.env @@ -0,0 +1,4 @@ +# Sample env file for using a model hosted using FastChat +FASTCHAT_MODEL="..." +FASTCHAT_API_BASE="..." +FASTCHAT_API_KEY="..." \ No newline at end of file From d2d8a475debb48e6671b893c279322d898e4a5bf Mon Sep 17 00:00:00 2001 From: Fahim Imaduddin Dalvi Date: Mon, 25 Sep 2023 11:55:35 +0300 Subject: [PATCH 14/15] Rename asset and update metadata --- ...hot.py => AraBench_ar2en_Helsinki_NLP_Opus_MT_ZeroShot.py} | 0 ...n_FastChat_ZeroShot.py => AraBench_ar2en_Jais_ZeroShot.py} | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename assets/ar/MT/{AraBench_Ara2Eng_Helsinki_NLP_Opus_MT_ZeroShot.py => AraBench_ar2en_Helsinki_NLP_Opus_MT_ZeroShot.py} (100%) rename assets/ar/MT/{AraBench_ar2en_FastChat_ZeroShot.py => AraBench_ar2en_Jais_ZeroShot.py} (88%) diff --git a/assets/ar/MT/AraBench_Ara2Eng_Helsinki_NLP_Opus_MT_ZeroShot.py b/assets/ar/MT/AraBench_ar2en_Helsinki_NLP_Opus_MT_ZeroShot.py similarity index 100% rename from assets/ar/MT/AraBench_Ara2Eng_Helsinki_NLP_Opus_MT_ZeroShot.py rename to assets/ar/MT/AraBench_ar2en_Helsinki_NLP_Opus_MT_ZeroShot.py diff --git a/assets/ar/MT/AraBench_ar2en_FastChat_ZeroShot.py b/assets/ar/MT/AraBench_ar2en_Jais_ZeroShot.py similarity index 88% rename from assets/ar/MT/AraBench_ar2en_FastChat_ZeroShot.py rename to assets/ar/MT/AraBench_ar2en_Jais_ZeroShot.py index 1daf04a6..9cb974fd 100644 --- a/assets/ar/MT/AraBench_ar2en_FastChat_ZeroShot.py +++ b/assets/ar/MT/AraBench_ar2en_Jais_ZeroShot.py @@ -6,8 +6,8 @@ def metadata(): return { "author": "Arabic Language Technologies, QCRI, HBKU", - "model": "bloomz-176b (8bit quantized)", - "description": "Locally hosted BLOOMZ 176b model (8 bit quantized version) using FastChat.", + "model": "jais-13b-chat", + "description": "Locally hosted Jais Chat 13b model using FastChat.", } From ab784bb1e50ee5e5073766fcebe168271fe6fa8e Mon Sep 17 00:00:00 2001 From: Fahim Imaduddin Dalvi Date: Mon, 25 Sep 2023 13:15:59 +0300 Subject: [PATCH 15/15] Update doc with link to adapters --- llmebench/models/FastChat.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/llmebench/models/FastChat.py b/llmebench/models/FastChat.py index 4dbe8abc..ebd75b3b 100644 --- a/llmebench/models/FastChat.py +++ b/llmebench/models/FastChat.py @@ -11,6 +11,10 @@ class FastChatModel(OpenAIModel): Accepts all arguments used by `OpenAIModel`, and overrides the arguments listed below with FastChat-specific variables. + See the [https://github.com/lm-sys/FastChat/blob/main/docs/model_support.md](model_support) + page in FastChat's documentation for supported models and instructions on extending + to custom models. + Arguments --------- api_base : str