diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..f67fcef --- /dev/null +++ b/.env.example @@ -0,0 +1,2 @@ +# Path to the gorilla-cli executable +GORILLA_CLI_PATH=/path/to/gorilla-cli diff --git a/.gitignore b/.gitignore index d9005f2..03d0190 100644 --- a/.gitignore +++ b/.gitignore @@ -150,3 +150,4 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ +.aider* diff --git a/OAI_CONFIG_LIST.json b/OAI_CONFIG_LIST.json new file mode 100644 index 0000000..ec899a8 --- /dev/null +++ b/OAI_CONFIG_LIST.json @@ -0,0 +1,28 @@ +[ + { + "model": "gpt-4", + "api_key": "${OPENAI_API_KEY}" + }, + { + "model": "gpt-4-1106-preview", + "api_key": "${OPENAI_API_KEY}" + }, + { + "model": "gpt-3.5-turbo-16k", + "api_key": "${OPENAI_API_KEY}" + }, + { + "model": "gpt-4", + "api_key": "${AZURE_OPENAI_API_KEY}", + "api_base": "${AZURE_OPENAI_API_BASE}", + "api_type": "azure", + "api_version": "2023-07-01-preview" + }, + { + "model": "gpt-3.5-turbo", + "api_key": "${AZURE_OPENAI_API_KEY}", + "api_base": "${AZURE_OPENAI_API_BASE}", + "api_type": "azure", + "api_version": "2023-07-01-preview" + } +] diff --git a/README.md b/README.md index eaf4091..5c093ca 100644 --- a/README.md +++ b/README.md @@ -10,23 +10,87 @@ pinned: false license: mit --- -## Use and Install on the Command Line +**Check the plugins folder for new Semantic Kernel Plugins** -```bashh -git clone https://github.com/Josephrp/LablabAutogen.git -``` +## Before You Install and Use + +- sign up and get an api key for open ai +- sign up and set up a project in [zilliz cloud](https://cloud.zilliz.com/) + +======= +- sign up and get an api key for Bing! Search + +## Zilliz Plugin + +This plugin allows users to plug in their existing zilliz account to a multiagent framework using autogen and semantic-kernel. + +#### Set Up Zilliz +![Screenshot 2023-12-11 131536](https://github.com/Josephrp/semantic-kernel-v1.0-hackathon/assets/18212928/d1b42e9c-8fa0-4145-bf60-c975277c6f27) +![Screenshot 2023-12-11 131608](https://github.com/Josephrp/semantic-kernel-v1.0-hackathon/assets/18212928/5b6b1510-631a-43bb-a647-ea892793e821) + +#### Create an Account + +1. Navigate to cloud.zilliz.com +2. sign-up + + +![Screenshot 2023-12-11 131518](https://github.com/Josephrp/semantic-kernel-v1.0-hackathon/assets/18212928/5d657875-dc31-4f16-a36f-77f8f2391add) +![Screenshot 2023-12-11 131237](https://github.com/Josephrp/semantic-kernel-v1.0-hackathon/assets/18212928/4747afcf-8e34-40ae-9cd4-47d70a6fb908) +![Screenshot 2023-12-11 131243](https://github.com/Josephrp/semantic-kernel-v1.0-hackathon/assets/18212928/d90029c5-869b-444d-adc1-6a997cac0976) + +#### Create a Cluster +![Screenshot 2023-12-11 131633](https://github.com/Josephrp/semantic-kernel-v1.0-hackathon/assets/18212928/01af90cd-22d8-4813-b677-c13714c3b79c) +![Screenshot 2023-12-11 131710](https://github.com/Josephrp/semantic-kernel-v1.0-hackathon/assets/18212928/918eaa0a-cb67-4835-a302-2666193de29c) +![Screenshot 2023-12-11 131739](https://github.com/Josephrp/semantic-kernel-v1.0-hackathon/assets/18212928/515855a8-1ff8-407f-9184-972848f8b0af) +![Screenshot 2023-12-11 131744](https://github.com/Josephrp/semantic-kernel-v1.0-hackathon/assets/18212928/c728e6dc-b02d-476b-8b6a-8f5f7c6f8072) + +#### AutoCreate a Pipeline +![Screenshot 2023-12-11 131824](https://github.com/Josephrp/semantic-kernel-v1.0-hackathon/assets/18212928/0b9de3e2-74c2-428f-960a-bf7f2e901904) +![Screenshot 2023-12-11 131913](https://github.com/Josephrp/semantic-kernel-v1.0-hackathon/assets/18212928/73550d75-9a6d-4454-a12c-1935584cfc92) +![Screenshot 2023-12-11 132006](https://github.com/Josephrp/semantic-kernel-v1.0-hackathon/assets/18212928/3fd90763-d64d-4194-bd96-cda996921425) + +#### AutoCreate all the Pipeline +![Screenshot 2023-12-11 132023](https://github.com/Josephrp/semantic-kernel-v1.0-hackathon/assets/18212928/7f5a9910-fad7-45c9-9f18-af9e2b876699) +![Screenshot 2023-12-11 132035](https://github.com/Josephrp/semantic-kernel-v1.0-hackathon/assets/18212928/69b23ec3-ecb8-494d-bb69-c7665d9e31e8) + +#### Use Curl to Upload a Document +![Screenshot 2023-12-11 135943](https://github.com/Josephrp/semantic-kernel-v1.0-hackathon/assets/18212928/21bdfac4-99bf-413a-9cf8-a2fafeb9c837) +![Screenshot 2023-12-11 140115](https://github.com/Josephrp/semantic-kernel-v1.0-hackathon/assets/18212928/b89f3c69-258f-4311-962f-10f7f5bc0096) +![Screenshot 2023-12-11 132130](https://github.com/Josephrp/semantic-kernel-v1.0-hackathon/assets/18212928/66a17880-699b-4dde-bc8a-d3e37b04e69e) + +#### Use Your Credentials + Existing Zilliz Cloud With Semantic Kernel ! + +## Get Your Bing! Search API Key + +1. visit this weblink [https://aka.ms/bingapisignup](https://portal.azure.com/#create/microsoft.bingsearch) +2. open your portal : [https://portal.azure.com/#create/microsoft.bingsearch](https://portal.azure.com/#create/microsoft.bingsearch) + +# **Check the plugins folder for new Semantic Kernel Plugins** +>>>>>>> 8ff1c563cb0c715e932c267d2fe967f50b8aee0d + +## Use and Install + +on the command line : ```bash -cd LablabAutogen +git clone https://github.com/Tonic-AI/EasyAGI ``` ```bash -nano app.py +cd EasyAGI ``` -edit line 17 " ```"openai_api_key": "YOUR_KEY_HERE", # OpenAI API Key``` with your key +If you're on Windows run the following command and edit the files below using notepad or VSCode and save them accordingly. + +```bash +set PATH=%PATH% +``` +then edit the OAI_CONFIG_LIST file or on the command line: -then press: +```bash +nano OAI_CONFIG_LIST.json +``` +on the command line , press: ```nano control + x @@ -38,7 +102,38 @@ Write : Y ``` -to save then type : +to save then run + +```bash +nano app.py +``` + +and edit lines 25-27 of app.py + +```python + "host": "your_milvus_host", + "port": "your_milvus_port", + "collection_name": "your_collection_name" +``` + +with your zilliz cloud credentials. + +and line 15 with your Bing! api key then save. + +or if you're on the command line press: + +```nano + control + x +``` +Write : + +```nano +y +``` + +to save. + +then type the following in your command line ```bash pip install -r requirements.txt @@ -49,4 +144,4 @@ and finally : ```bash python app.py ``` -to run. +to run. or install and run the application inside your compiler - like VS Code. diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app.py b/app.py index 352dbd5..db1f4b6 100644 --- a/app.py +++ b/app.py @@ -1,4 +1,5 @@ import gradio as gr +import semantic_kernel , autogen from pydantic import BaseModel, ValidationError from plugins.sk_bing_plugin import BingPlugin from plugins.sk_web_pages_plugin import WebPagesPlugin @@ -8,28 +9,47 @@ from azure.core.credentials import AzureKeyCredential from semantic_kernel.core_skills.text_skill import TextSkill from semantic_kernel.planning.basic_planner import BasicPlanner +from semantic_kernel import Kernel # Configure your credentials here bing_api_key = "ArXXXXdpJ" # Replace with your Bing API key -llm_config = { - "type": "openai", # "azure" or "openai" - "openai_api_key": "sk-rR5XXXXm", # OpenAI API Key - "azure_deployment": "", # Azure OpenAI deployment name - "azure_api_key": "", # Azure OpenAI API key in the Azure portal - "azure_endpoint": "" # Endpoint URL for Azure OpenAI, e.g. https://contoso.openai.azure.com/ +# llm_config = { +# "type": "openai", # "azure" or "openai" +# "openai_api_key": "sk-rR5XXXXm", # OpenAI API Key +# "azure_deployment": "", # Azure OpenAI deployment name +# "azure_api_key": "", # Azure OpenAI API key in the Azure portal +# "azure_endpoint": "" # Endpoint URL for Azure OpenAI, e.g. https://contoso.openai.azure.com/ +#} +llm_config = autogen.config_list_from_json( + env_or_file="OAI_CONFIG_LIST.json", + filter_dict={"model": {"gpt-4", "gpt-3.5-turbo-16k", "gpt-4-1106-preview"}} +) + +builder_config_path = autogen.config_list_from_json( + env_or_file="OAI_CONFIG_LIST.json", + filter_dict={"model": {"gpt-4-1106-preview"}} +) + +Zilliz_config = { + "host": "your_milvus_host", # use Zilliz Cloud + "port": "your_milvus_port", # use Zilliz Cloud + "collection_name": "your_collection_name" # use Zilliz Cloud } -import semantic_kernel kernel = semantic_kernel.Kernel() kernel.import_skill(BingPlugin(bing_api_key)) kernel.import_skill(WebPagesPlugin()) -sk_planner = AutoGenPlanner(kernel, llm_config) -assistant = sk_planner.create_assistant_agent("Assistant") +sk_planner = AutoGenPlanner(kernel, llm_config, builder_config_path) +assistant = sk_planner.create_assistant_agent("Assistant") def get_response(question, max_auto_reply): worker = sk_planner.create_user_agent("Worker", max_auto_reply=max_auto_reply, human_input="NEVER") + assistant = sk_planner.create_assistant_agent("Assistant") worker.initiate_chat(assistant, message=question) return worker.get_response() -iface = gr.Interface(fn=get_response, inputs=["text", "number"], outputs="text", inputs_label=["Question", "Max Auto Reply"]) -iface.launch() +if __name__ == "__main__": + question = input("Tonic's EasyAGI builds multi-agent systems that use Semantic-Kernel Plugins to automate your business operations ! Describe your problem in detail, then optionally bullet point a brief step by step way to solve it, then (or optionally) give a clear command or instruction to solve the issues above:") + max_auto_reply = int(input("Set a maximum number of autoreplies by entering a number with minimum 10: ")) + response = get_response(question, max_auto_reply) + print("Response:", response) \ No newline at end of file diff --git a/chainlitdemo.py b/chainlitdemo.py new file mode 100644 index 0000000..db1d22e --- /dev/null +++ b/chainlitdemo.py @@ -0,0 +1,78 @@ +import gradio as gr +from pydantic import BaseModel, ValidationError +from plugins.sk_bing_plugin import BingPlugin +from plugins.sk_web_pages_plugin import WebPagesPlugin +from planning.autogen_planner import AutoGenPlanner +from web_search_client import WebSearchClient +from web_search_client.models import SafeSearch +from azure.core.credentials import AzureKeyCredential +from semantic_kernel.core_skills.text_skill import TextSkill +from semantic_kernel.planning.basic_planner import BasicPlanner +from semantic_kernel import Kernel + +# Configure your credentials here +bing_api_key = "ArXXXXdpJ" # Replace with your Bing API key + +llm_config = { + "type": "openai", # "azure" or "openai" + "openai_api_key": "sk-rR5XXXXm", # OpenAI API Key + "azure_deployment": "", # Azure OpenAI deployment name + "azure_api_key": "", # Azure OpenAI API key in the Azure portal + "azure_endpoint": "" # Endpoint URL for Azure OpenAI, e.g. https://contoso.openai.azure.com/ +} +import semantic_kernel +kernel = semantic_kernel.Kernel() +kernel.import_skill(BingPlugin(bing_api_key)) +kernel.import_skill(WebPagesPlugin()) +sk_planner = AutoGenPlanner(kernel, llm_config) +assistant = sk_planner.create_assistant_agent("Assistant") + +def get_response(question, max_auto_reply): + worker = sk_planner.create_user_agent("Worker", max_auto_reply=max_auto_reply, human_input="NEVER") + assistant = sk_planner.create_assistant_agent("Assistant") + worker.initiate_chat(assistant, message=question) + return worker.get_response() + +class ChainlitAssistantAgent(AssistantAgent): + def __init__(self, name, sk_planner): + super().__init__(name) + self.sk_planner = sk_planner + + async def process_message(self, message): + # Use sk_planner to process the message and generate a response + response = self.sk_planner.create_assistant_agent("Assistant") + response.initiate_chat(self, message=message) + return response.get_response() + +class ChainlitUserProxyAgent(UserProxyAgent): + def __init__(self, name, assistant_agent): + super().__init__(name) + self.assistant_agent = assistant_agent + + async def get_human_input(self, prompt): + # Get input from the user via Chainlit interface + reply = await cl.ask_user_message(content=prompt) + return reply["content"].strip() + + async def send(self, message): + # Send the message to the assistant agent and get the response + response = await self.assistant_agent.process_message(message) + # Display the response in the Chainlit interface + cl.message(content=response, author=self.assistant_agent.name).send() + +# Initialize the agents +assistant_agent = ChainlitAssistantAgent("Assistant", sk_planner) +user_proxy_agent = ChainlitUserProxyAgent("User_Proxy", assistant_agent) + +# Chainlit Web Interface +@cl.page("/") +def main_page(): + with cl.form("user_input_form"): + question = cl.text_input("Describe your problem:") + submit_button = cl.button("Submit") + + if submit_button: + cl.run_async(user_proxy_agent.send(question)) + +if __name__ == "__main__": + cl.run(main_page) \ No newline at end of file diff --git a/planning/agent_builder.py b/planning/agent_builder.py new file mode 100644 index 0000000..f320dbf --- /dev/null +++ b/planning/agent_builder.py @@ -0,0 +1,408 @@ +import autogen +import time +import subprocess as sp +import socket +import os +import json +import hashlib +from typing import Optional, List, Dict, Tuple, Union + + +class AgentBuilder: + """ + AgentBuilder can help user build an automatic task solving process powered by multi-agent system. + Specifically, our building pipeline includes initialize and build. + In build(), we prompt a gpt-4 model to create multiple participant agents, and specify whether + this task need programming to solve. + User can save the built agents' config by calling save(), and load the saved configs by load(), which can skip the + building process. + """ + + openai_server_name = "openai" + max_tokens = 945 + max_agents = 5 # maximum number of agents build manager can create. + + CODING_PROMPT = """Does the following task need programming (i.e., access external API or tool by coding) to solve, + or use program may help the following task become easier? + + TASK: {task} + + Hint: + # Answer only YES or NO. + """ + + AGENT_NAME_PROMPT = """To complete the following task, what positions/jobs should be set to maximize the efficiency? + + TASK: {task} + + Hint: + # Considering the effort, the position in this task should be no more then {max_agents}, less is better. + # Answer the name of those positions/jobs, separated by comma and use "_" instead of space. For example: Product_manager,Programmer + # Only return the list of positions. + """ + + AGENT_SYS_MSG_PROMPT = """Considering the following position and corresponding task: + + TASK: {task} + POSITION: {position} + + Modify the following position requirement, let it more suitable for the above task and position: + + REQUIREMENT: {default_sys_msg} + + Hint: + # The modified requirement should not contain the code interpreter skill. + # Coding skill is limited to Python. + # Your answer should omit the word "REQUIREMENT". + # Your should let them reply "TERMINATE" in the end when the task complete (user's need has been satisfied). + """ + + def __init__( + self, + config_path: Optional[str] = "OAI_CONFIG_LIST.json", + builder_model: Optional[str] = "gpt-4", + agent_model: Optional[str] = "gpt-4", + host: Optional[str] = "localhost", + endpoint_building_timeout: Optional[int] = 600, + ): + """ + Args: + config_path: path of the OpenAI api configs. + builder_model: specify a model as the backbone of build manager. + host: endpoint host. + endpoint_building_timeout: timeout for building up an endpoint server. + """ + self.host = host + self.builder_model = builder_model + self.agent_model = agent_model + self.config_path = config_path + self.endpoint_building_timeout = endpoint_building_timeout + + self.building_task: str = None + self.agent_configs: List[Dict] = [] + self.open_ports: List[str] = [] + self.agent_procs: Dict[str, Tuple[sp.Popen, str]] = {} + self.agent_procs_assign: Dict[str, Tuple[autogen.ConversableAgent, str]] = {} + self.cached_configs: Dict = {} + + for port in range(8000, 65535): + if self._is_port_open(host, port): + self.open_ports.append(str(port)) + + @staticmethod + def _is_port_open(host, port): + """Check if a tcp port is open.""" + try: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(10) + s.bind((host, int(port))) + s.close() + return True + except OSError: + return False + + def _create_agent( + self, + agent_name: str, + model_name_or_hf_repo: str, + llm_config: dict, + system_message: Optional[str] = autogen.AssistantAgent.DEFAULT_SYSTEM_MESSAGE, + use_oai_assistant: Optional[bool] = False, + world_size: Optional[int] = 1, + ) -> autogen.AssistantAgent: + """ + Create a group chat participant agent. + + If the agent rely on an open-source model, this function will automatically set up an endpoint for that agent. + The API address of that endpoint will be "localhost:{free port}". + + Args: + agent_name: the name that identify the function of the agent (e.g., Coder, Product Manager,...) + model_name_or_hf_repo: + llm_config: specific configs for LLM (e.g., config_list, seed, temperature, ...). + system_message: system prompt use to format an agent's behavior. + use_oai_assistant: use OpenAI assistant api instead of self-constructed agent. + world_size: the max size of parallel tensors (in most of the cases, this is identical to the amount of GPUs). + + Returns: + agent: a set-up agent. + """ + config_list = autogen.config_list_from_json(self.config_path, filter_dict={"model": [model_name_or_hf_repo]}) + if "gpt-" in model_name_or_hf_repo: + server_id = self.openai_server_name + else: + model_name = model_name_or_hf_repo.split("/")[-1] + server_id = f"{model_name}_{self.host}" + if self.agent_procs.get(server_id, None) is None: + while True: + port = self.open_ports.pop() + if self._is_port_open(self.host, port): + break + + # Use vLLM to set up a server with OpenAI API support. + agent_proc = sp.Popen( + [ + "python", + "-m", + "vllm.entrypoints.openai.api_server", + "--host", + f"{self.host}", + "--port", + f"{port}", + "--model", + f"{model_name_or_hf_repo}", + "--tensor-parallel-size", + f"{world_size}", + ], + stdout=sp.PIPE, + stderr=sp.STDOUT, + ) + timeout_start = time.time() + + while True: + server_stdout = agent_proc.stdout.readline() + if server_stdout != b"": + print(server_stdout) + timeout_end = time.time() + if b"running" in server_stdout: + print( + f"Running {model_name_or_hf_repo} on http://{self.host}:{port} " + f"with tensor parallel size {world_size}." + ) + break + elif b"address already in use" in server_stdout: + raise RuntimeError( + f"{self.host}:{port} already in use. Fail to set up the endpoint for " + f"{model_name_or_hf_repo} on {self.host}:{port}." + ) + elif timeout_end - timeout_start > self.endpoint_building_timeout: + raise RuntimeError( + f"Timeout exceed. Fail to set up the endpoint for " + f"{model_name_or_hf_repo} on {self.host}:{port}." + ) + self.agent_procs[server_id] = (agent_proc, port) + else: + port = self.agent_procs[server_id][1] + + config_list[0]["base_url"] = f"http://{self.host}:{port}/v1" + + current_config = llm_config.copy() + current_config.update( + {"config_list": config_list, "model": model_name_or_hf_repo, "max_tokens": self.max_tokens} + ) + if use_oai_assistant: + from autogen.agentchat.contrib.gpt_assistant_agent import GPTAssistantAgent + + agent = GPTAssistantAgent( + name=agent_name, + llm_config={**current_config, "assistant_id": None}, + instructions=system_message, + overwrite_instructions=False, + ) + else: + agent = autogen.AssistantAgent( + name=agent_name, llm_config=current_config.copy(), system_message=system_message + ) + self.agent_procs_assign[agent_name] = (agent, server_id) + return agent + + def clear_agent(self, agent_name: str, recycle_endpoint: Optional[bool] = True): + """ + Clear a specific agent by name. + + Args: + agent_name: the name of agent. + recycle_endpoint: trigger for recycle the endpoint server. If true, the endpoint will be recycled + when there is no agent depending on. + """ + _, server_id = self.agent_procs_assign[agent_name] + del self.agent_procs_assign[agent_name] + if recycle_endpoint: + if server_id == self.openai_server_name: + return + else: + for _, iter_sid in self.agent_procs_assign.values(): + if server_id == iter_sid: + return + self.agent_procs[server_id][0].terminate() + self.open_ports.append(server_id.split("_")[-1]) + print(f"Agent {agent_name} has been cleared.") + + def clear_all_agents(self, recycle_endpoint: Optional[bool] = True): + """ + Clear all cached agents. + """ + for agent_name in [agent_name for agent_name in self.agent_procs_assign.keys()]: + self.clear_agent(agent_name, recycle_endpoint) + print("All agents have been cleared.") + + def build( + self, + building_task: Optional[str] = None, + default_llm_config: Optional[Dict] = None, + coding: Optional[bool] = None, + cached_configs: Optional[Dict] = None, + use_oai_assistant: Optional[bool] = False, + code_execution_config: Optional[Dict] = None, + **kwargs, + ): + """ + Auto build agents based on the building task. + + Args: + building_task: instruction that helps build manager (gpt-4) to decide what agent should be built. + default_llm_config: specific configs for LLM (e.g., config_list, seed, temperature, ...). + coding: use to identify if the user proxy (a code interpreter) should be added. + cached_configs: previously saved agent configs. + use_oai_assistant: use OpenAI assistant api instead of self-constructed agent. + code_execution_config: specific configs for user proxy (e.g., last_n_messages, work_dir, ...). + """ + use_api = False + + if code_execution_config is None: + code_execution_config = { + "last_n_messages": 2, + "work_dir": "groupchat", + "use_docker": False, + "timeout": 60, + } + + if cached_configs is None: + use_api = True + agent_configs = [] + self.building_task = building_task + else: + self.building_task = building_task = cached_configs["building_task"] + default_llm_config = cached_configs["default_llm_config"] + coding = cached_configs["coding"] + agent_configs = cached_configs["agent_configs"] + + if use_api: + config_list = autogen.config_list_from_json(self.config_path, filter_dict={"model": [self.builder_model]}) + build_manager = autogen.OpenAIWrapper(config_list=config_list) + + print("Generating agents...") + resp_agent_name = ( + build_manager.create( + messages=[ + { + "role": "user", + "content": self.AGENT_NAME_PROMPT.format(task=building_task, max_agents=self.max_agents), + } + ] + ) + .choices[0] + .message.content + ) + agent_name_list = resp_agent_name.split(",") + print(f"{resp_agent_name} are generated.") + + agent_sys_msg_list = [] + for name in agent_name_list: + print(f"Preparing configuration for {name}...") + resp_agent_sys_msg = ( + build_manager.create( + messages=[ + { + "role": "user", + "content": self.AGENT_SYS_MSG_PROMPT.format( + task=building_task, + position=name, + default_sys_msg=autogen.AssistantAgent.DEFAULT_SYSTEM_MESSAGE, + ), + } + ] + ) + .choices[0] + .message.content + ) + agent_sys_msg_list.append(resp_agent_sys_msg) + + for i in range(len(agent_name_list)): + agent_configs.append( + {"name": agent_name_list[i], "model": self.agent_model, "system_message": agent_sys_msg_list[i]} + ) + + if coding is None: + resp = ( + build_manager.create( + messages=[{"role": "user", "content": self.CODING_PROMPT.format(task=building_task)}] + ) + .choices[0] + .message.content + ) + coding = True if resp == "YES" else False + + for config in agent_configs: + print(f"Creating agent {config['name']} with backbone {config['model']}...") + self._create_agent( + config["name"], + config["model"], + default_llm_config, + system_message=config["system_message"], + use_oai_assistant=use_oai_assistant, + **kwargs, + ) + agent_list = [agent_config[0] for agent_config in self.agent_procs_assign.values()] + + if coding is True: + print("Adding user console proxy...") + agent_list = [ + autogen.UserProxyAgent( + name="User_console_and_Python_code_interpreter", + is_termination_msg=lambda x: "TERMINATE" in x.get("content"), + system_message="User console with a python code interpreter interface.", + code_execution_config=code_execution_config, + human_input_mode="NEVER", + ) + ] + agent_list + + self.cached_configs.update( + { + "building_task": building_task, + "agent_configs": agent_configs, + "coding": coding, + "default_llm_config": default_llm_config, + "code_execution_config": code_execution_config, + } + ) + + return agent_list, self.cached_configs.copy() + + def save(self, filepath: Optional[str] = None) -> str: + """ + Save building configs. If the filepath is not specific, this function will create a filename by encrypt the + building_task string by md5 with "save_config_" prefix, and save config to the local path. + + Args: + filepath: save path. + + Return: + filepath: path save. + """ + if filepath is None: + filepath = f'./save_config_{hashlib.md5(self.building_task.encode("utf-8")).hexdigest()}.json' + with open(filepath, "w") as save_file: + json.dump(self.cached_configs, save_file, indent=4) + print(f"Building config saved to {filepath}") + + return filepath + + def load( + self, + filepath: str, + **kwargs, + ): + """ + Load building configs and call the build function to complete building without calling online LLMs' api. + + Args: + filepath: filepath for the save config. + """ + try: + print(f"Loding config from {filepath}") + cached_configs = json.load(open(filepath)) + except FileNotFoundError: + raise FileNotFoundError(f"Config file {filepath} does not exist.") + + return self.build(cached_configs=cached_configs, **kwargs) \ No newline at end of file diff --git a/planning/autogen_planner.py b/planning/autogen_planner.py index 6160d05..6c91396 100644 --- a/planning/autogen_planner.py +++ b/planning/autogen_planner.py @@ -1,6 +1,8 @@ from typing import Optional, Dict, List import semantic_kernel, autogen import datetime +import autogen +from .agent_builder import AgentBuilder class AutoGenPlanner: """ @@ -17,21 +19,39 @@ class AutoGenPlanner: f"Reply TERMINATE when the task is done." ) - def __init__(self, kernel: semantic_kernel.Kernel, llm_config: Dict = None, builder_config_path: str = None): + def __init__(self, kernel: semantic_kernel.Kernel, llm_config: Dict = None, builder_config_path: str = None, zilliz_config: Dict = None): self.kernel = kernel self.llm_config = llm_config or {} self.builder_config_path = builder_config_path - self.validate_llm_config() +# self.validate_llm_config() self.builder = self.create_builder() + self.Zilliz_agent = ZillizRetrievalAgent(**zilliz_config) if zilliz_config else None - def create_builder(self) -> autogen.agentchat.contrib.agent_builder.AgentBuilder: + def execute_code(self, code: str) -> str: + """ + Execute a Python code snippet and return the result. + Args: + code (str): The Python code to execute. + Returns: + str: The output of the executed code. + """ + ipython = get_ipython() + result = ipython.run_cell(code) + output = str(result.result) + if result.error_before_exec is not None: + output += f"\n{result.error_before_exec}" + if result.error_in_exec is not None: + output += f"\n{result.error_in_exec}" + return output + + def create_builder(self) -> AgentBuilder: """ Create an instance of AgentBuilder. """ if not self.builder_config_path: raise ValueError("Builder config path is required to create AgentBuilder.") - return autogen.agentchat.contrib.agent_builder.AgentBuilder( - config_path=self.builder_config_path, + return AgentBuilder( + config_path= '', builder_model='gpt-4-1106-preview', agent_model='gpt-4-1106-preview' ) @@ -61,17 +81,22 @@ def create_user_agent( max_consecutive_auto_reply=max_auto_reply, function_map=self.__get_function_map(), ) + + def perform_retrieval(self, query_vector: List[float], top_k: int = 10): + if not self.Zilliz_agent: + raise ValueError("Zilliz agent is not configured.") + return self.Zilliz_agent.search([query_vector], top_k, {"nprobe": 16}) - def validate_llm_config(self): - if self.llm_config.get("type") == "openai": - if not self.llm_config.get("openai_api_key"): - raise ValueError("OpenAI API key is required for OpenAI LLM.") - elif self.llm_config.get("type") == "azure": - required_keys = ["azure_api_key", "azure_deployment", "azure_endpoint"] - if any(key not in self.llm_config for key in required_keys): - raise ValueError("Azure OpenAI API configuration is incomplete.") - else: - raise ValueError("LLM type not provided, must be 'openai' or 'azure'.") +# def validate_llm_config(self): +# if self.llm_config.get("type") == "openai": +# if not self.llm_config.get("openai_api_key"): +# raise ValueError("OpenAI API key is required for OpenAI LLM.") +# elif self.llm_config.get("type") == "azure": +# required_keys = ["azure_api_key", "azure_deployment", "azure_endpoint"] +# if any(key not in self.llm_config for key in required_keys): +# raise ValueError("Azure OpenAI API configuration is incomplete.") +# else: +# raise ValueError("LLM type not provided, must be 'openai' or 'azure'.") def update_llm_config(self, new_config: Dict): self.llm_config = new_config @@ -91,22 +116,16 @@ def load_semantic_kernel_plugins(self, plugins: List[str]): print(f"Error loading plugin '{plugin}': {e}") def __get_autogen_config(self) -> Dict: - if self.llm_config["type"] == "openai": - return { - "functions": self.__get_function_definitions(), - "config_list": [{"model": "gpt-3.5-turbo", "api_key": self.llm_config["openai_api_key"]}] - } - elif self.llm_config["type"] == "azure": - return { - "functions": self.__get_function_definitions(), - "config_list": [{ - "model": self.llm_config["azure_deployment"], - "api_type": "azure", - "api_key": self.llm_config["azure_api_key"], - "api_base": self.llm_config["azure_endpoint"], - "api_version": "2023-08-01-preview" - }] - } + + desired_model = "gpt-4-1106-preview" # Update this with your choice of model + + for config in self.llm_config: + if config.get("model") == desired_model: + return { + "functions": self.__get_function_definitions(), + "config_list": [config] + } + raise ValueError(f"No suitable LLM configuration found for model '{desired_model}'.") def __get_function_definitions(self) -> List: functions = [] diff --git a/plugins/gorilla_plugin.py b/plugins/gorilla_plugin.py new file mode 100644 index 0000000..1de370a --- /dev/null +++ b/plugins/gorilla_plugin.py @@ -0,0 +1,141 @@ +import asyncio +import asyncio.subprocess as subprocess +from typing import List, Dict + +class GorillaPlugin: + _cli_path: str + _env_info: Dict[str, str] + + """A plugin that uses the Gorilla CLI to perform a series of executions based on a natural language query or high level overview of the user's problem.""" + + + async def collect_environment_info(self) -> None: + """ + Collects information about the environment where the commands are executed. + """ + uname_command = "uname -a" # This is for Unix-like systems, for Windows use 'systeminfo' + try: + process = await subprocess.create_subprocess_shell( + uname_command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + stdout, stderr = await process.communicate() + + if process.returncode == 0: + self._env_info['uname'] = stdout.decode().strip() + else: + self._env_info['uname'] = f"Error collecting environment info: {stderr.decode().strip()}" + except Exception as e: + self._env_info['uname'] = f"Exception collecting environment info: {str(e)}" + + def compare_environment_info(self, initial_env_info: Dict[str, str], updated_env_info: Dict[str, str]) -> Dict[str, str]: + """ + Compares the initial and updated environment information and returns the differences. + """ + return { + key: { + 'initial': initial_env_info[key], + 'updated': updated_env_info.get(key), + } + for key, value in initial_env_info.items() + if value != updated_env_info.get(key) + } + + async def queue_commands(self, natural_language_commands: List[str]) -> List[str]: + """ + Processes natural language commands and queues them for execution after user confirmation. + """ + queued_commands = [] + for nl_command in natural_language_commands: + # Pass the natural language command to the Gorilla CLI and get the CLI command + try: + process = await subprocess.create_subprocess_shell( + f"{self._cli_path} \"{nl_command}\"", + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + stdout, stderr = await process.communicate() + + if process.returncode != 0: + print(f"Failed to get CLI command for: {nl_command}") + print(f"Error: {stderr.decode().strip()}") + continue + except Exception as e: + print(f"Exception while processing command '{nl_command}': {str(e)}") + continue + + cli_command = stdout.decode().strip() + queued_commands.append(cli_command) + return queued_commands + + async def execute_commands(self, cli_commands: List[str]): + """ + Executes a list of CLI commands after user confirmation. + """ + # Ask for user confirmation before executing commands + user_confirmation = input("Do you want to execute the queued commands? (yes/no): ") + if user_confirmation.lower() != 'yes': + print("Execution cancelled by the user.") + return + + # Collect initial environment info + await self.collect_environment_info() + initial_env_info = self._env_info.copy() + + for cli_command in cli_commands: + # Execute the CLI command using subprocess + process = await subprocess.create_subprocess_shell( + cli_command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + stdout, stderr = await process.communicate() + + if process.returncode == 0: + print(f"Command executed successfully: {cli_command}") + print(f"Output: {stdout.decode().strip()}") + else: + print(f"Command failed: {cli_command}") + print(f"Error: {stderr.decode().strip()}") + + # Collect updated environment info + await self.collect_environment_info() + updated_env_info = self._env_info.copy() + + if env_changes := self.compare_environment_info( + initial_env_info, updated_env_info + ): + print("Environment changes detected:") + for key, change in env_changes.items(): + print(f"{key}: from '{change['initial']}' to '{change['updated']}'") + +async def confirm_and_execute_commands(gorilla_plugin: GorillaPlugin, queued_commands: List[str]): + """ + Confirms with the user before executing queued commands. + """ + # Ask for user confirmation before executing commands + user_confirmation = input("Do you want to execute the queued commands? (yes/no): ") + if user_confirmation.lower() != 'yes': + print("Execution cancelled by the user.") + return + + # If confirmed, execute the commands + await gorilla_plugin.execute_commands(queued_commands) + +async def main(): + # Example user input + user_input = "Generate a report from yesterday's logs and email it to the team" + + # Initialize GorillaPlugin with the path to the Gorilla CLI + import os + gorilla_plugin = GorillaPlugin(cli_path=os.getenv('GORILLA_CLI_PATH')) + + # Process the input and queue CLI commands + queued_commands = await gorilla_plugin.queue_commands([user_input]) + + # Confirm and execute commands + await confirm_and_execute_commands(gorilla_plugin, queued_commands) + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/plugins/sk_bing_plugin.py b/plugins/sk_bing_plugin.py index 13850cb..14075a6 100644 --- a/plugins/sk_bing_plugin.py +++ b/plugins/sk_bing_plugin.py @@ -1,9 +1,8 @@ # Copyright (c) Microsoft. All rights reserved. -from semantic_kernel.skill_definition import sk_function +from semantic_kernel.skill_definition import sk_function, sk_function_context_parameter +from semantic_kernel.orchestration.sk_context import SKContext from plugins.bing_connector import BingConnector - - class BingPlugin: """ A plugin to search Bing. @@ -15,24 +14,26 @@ def __init__(self, bing_api_key: str): raise Exception("Bing API key is not set") @sk_function( - description="Use Bing to find a page about a topic. The return is a URL of the page found.", + description="Use Bing to find a page about a topic. The return is a URL of the page found", name="find_web_page_about", - input_description="Two comma separated values: #1 Offset from the first result (default zero), #2 The topic to search, e.g. '0,who won the F1 title in 2023?'.", + input_description="The topic to search, e.g. 'who won the F1 title in 2023?'", + ) + @sk_function_context_parameter( + name="limit", description="How many results to return", default_value="1" ) - async def find_web_page_about(self, input: str) -> str: + @sk_function_context_parameter( + name="offset", description="How many results to skip", default_value="0" + ) + async def find_web_page_about(self, input: str, context: "SKContext") -> str: """ A native function that uses Bing to find a page URL about a topic. - To simplify the integration with Autogen, the input parameter is a string with two comma separated - values, rather than the usual context dictionary. """ - - # Input validation, the error message can help self-correct the input - if "," not in input: - raise ValueError("The input argument must contain a comma, e.g. '0,who won the F1 title in 2023?'") - - parts = input.split(",", 1) - result = await self.bing.search_url_async(query=parts[1], num_results=1, offset=parts[0]) + result = await self.bing.search_url_async( + query=input, + num_results=context.variables.get("limit", 1), + offset=context.variables.get("offset", 0), + ) if result: return result[0] else: - return f"Nothing found, try again or try to adjust the topic." + return f"Nothing found, try again or try to adjust the topic." \ No newline at end of file diff --git a/plugins/sk_web_pages_plugin.py b/plugins/sk_web_pages_plugin.py index df690de..81cfd29 100644 --- a/plugins/sk_web_pages_plugin.py +++ b/plugins/sk_web_pages_plugin.py @@ -3,8 +3,6 @@ from semantic_kernel.skill_definition import sk_function from bs4 import BeautifulSoup import re, aiohttp - - class WebPagesPlugin: """ A plugin to interact with web pages, e.g. download the text content of a page. @@ -32,4 +30,8 @@ async def fetch_webpage(self, input: str) -> str: # get text and compact empty lines text = soup.get_text() - return re.sub(r"[\r\n][\r\n]{2,}", "\n\n", text) + # remove multiple empty lines + text = re.sub(r"[\r\n][\r\n]{2,}", "\n\n", text) + # remove leading and trailing empty spaces, leaving max 1 empty space at the beginning of each line + text = re.sub(r"[\n] +", "\n ", text) + return text \ No newline at end of file diff --git a/plugins/zillizcloud.py b/plugins/zillizcloud.py new file mode 100644 index 0000000..1bb1749 --- /dev/null +++ b/plugins/zillizcloud.py @@ -0,0 +1,32 @@ +import milvus +from milvus import Milvus, connections, Collection +from typing import List, Dict, Any + +class ZillizRetrievalAgent: + def __init__(self, host: str, port: str, collection_name: str): + self.host = host + self.port = port + self.collection_name = collection_name + self.collection = None + self.connect() + + def connect(self): + connections.connect("default", host=self.host, port=self.port) + self.collection = Collection(name=self.collection_name) + + def search(self, vectors: List[List[float]], top_k: int, params: Dict[str, Any]): + """ + Search the Milvus collection for similar vectors. + Args: + vectors (List[List[float]]): The query vectors. + top_k (int): Number of top similar results to retrieve. + params (Dict[str, Any]): Additional search parameters. + Returns: + List of search results. + """ + search_params = { + "metric_type": "L2", + "params": params + } + results = self.collection.search(vectors, "embedding", search_params, top_k) + return results diff --git a/requirements.txt b/requirements.txt index aa36a33..946155a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ beautifulsoup4~=4.12 typing-extensions pydantic gradio +openai azure-common azure-core azure-search-documents @@ -20,3 +21,7 @@ microsoft-bing-customimagesearch microsoft-bing-customwebsearch msrest msrestazure +pymilvus +bs4 +gorilla-cli +python-dotenv \ No newline at end of file