From ff2900a168dc52c9c27e29aa371e30c53d99104d Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Fri, 20 Dec 2024 10:32:04 +0100 Subject: [PATCH 01/13] Blog polishing WIP --- website/README.md | 2 +- .../img/one_to_rule_them_all.png | 3 ++ .../index.mdx | 15 ++++-- website/mint.json | 48 ++++++++++++++++++- 4 files changed, 63 insertions(+), 5 deletions(-) create mode 100644 website/blog/2024-12-18-Tools-interoperability/img/one_to_rule_them_all.png diff --git a/website/README.md b/website/README.md index 22a4e10d6a..e1996afcec 100644 --- a/website/README.md +++ b/website/README.md @@ -25,7 +25,7 @@ Install it [here](https://github.com/quarto-dev/quarto-cli/releases). Navigate to the `website` folder and run: ```console -pydoc-markdown +python ./process_api_reference.py python ./process_notebooks.py render npm install ``` diff --git a/website/blog/2024-12-18-Tools-interoperability/img/one_to_rule_them_all.png b/website/blog/2024-12-18-Tools-interoperability/img/one_to_rule_them_all.png new file mode 100644 index 0000000000..45cf075e36 --- /dev/null +++ b/website/blog/2024-12-18-Tools-interoperability/img/one_to_rule_them_all.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f41e305d4525e167b62d0c769599ec2667ef282918e835372ad20a1b3b3c03c +size 310123 diff --git a/website/blog/2024-12-18-Tools-interoperability/index.mdx b/website/blog/2024-12-18-Tools-interoperability/index.mdx index 3f6ae0962d..4b3a23d8bb 100644 --- a/website/blog/2024-12-18-Tools-interoperability/index.mdx +++ b/website/blog/2024-12-18-Tools-interoperability/index.mdx @@ -14,9 +14,18 @@ AG2 lets you bring in tools from different frameworks like **LangChain**, **Crew With AG2, you can combine these tools and enhance your agents' capabilities. + +![One to rule them all img](img/one_to_rule_them_all.png) + In this post, we’ll walk through how to integrate tools from various frameworks—like [LangChain Tools](https://python.langchain.com/v0.1/docs/modules/tools), [CrewAI Tools](https://github.com/crewAIInc/crewAI-tools/tree/main), and [PydanticAI Tools](https://ai.pydantic.dev/tools/)—into AG2. -This allows you to use tools from different frameworks within AG2, giving your agents more power and flexibility. You’ll see how to set up agents, adapt the tools, and validate the integration through examples.his post, you will understand how to configure agents, adapt these tools for use in AG2, and validate the integration through practical examples. + +And let’s be honest: + +Because, really, the magic happens when you combine them all. This allows you to use tools from different frameworks within AG2, giving your agents more power and flexibility. + +In this post, you will understand how to configure agents, adapt these tools for use in AG2, and validate the integration through practical examples. + ## LangChain Tools Integration @@ -75,8 +84,8 @@ chatbot = AssistantAgent( ### Tool Integration - Initialize and register the LangChain tool with AG2. -- `WikipediaAPIWrapper`: Configured to fetch the top 1 result from Wikipedia with a maximum of 1000 characters per document. -- `WikipediaQueryRun`: A LangChain tool that executes Wikipedia queries. +- [WikipediaAPIWrapper](https://python.langchain.com/api_reference/community/utilities/langchain_community.utilities.wikipedia.WikipediaAPIWrapper.html): Configured to fetch the top 1 result from Wikipedia with a maximum of 1000 characters per document. +- [WikipediaQueryRun](https://python.langchain.com/api_reference/community/tools/langchain_community.tools.wikipedia.tool.WikipediaQueryRun.html): A LangChain tool that executes Wikipedia queries. - `LangchainInteroperability`: Converts the LangChain tool into a format compatible with the AG2 framework. - `ag2_tool.register_for_execution(user_proxy)`: Registers the tool for use by the user_proxy agent. - `ag2_tool.register_for_llm(chatbot)`: Registers the tool for integration with the chatbot agent. diff --git a/website/mint.json b/website/mint.json index 5ee80c8390..798f7e7f22 100644 --- a/website/mint.json +++ b/website/mint.json @@ -300,6 +300,17 @@ "docs/reference/agentchat/contrib/web_surfer" ] }, + { + "group": "agentchat.realtime_agent", + "pages": [ + "docs/reference/agentchat/realtime_agent/client", + "docs/reference/agentchat/realtime_agent/function_observer", + "docs/reference/agentchat/realtime_agent/realtime_agent", + "docs/reference/agentchat/realtime_agent/realtime_observer", + "docs/reference/agentchat/realtime_agent/twilio_observer", + "docs/reference/agentchat/realtime_agent/websocket_observer" + ] + }, "docs/reference/agentchat/agent", "docs/reference/agentchat/assistant_agent", "docs/reference/agentchat/chat", @@ -344,6 +355,33 @@ "docs/reference/coding/utils" ] }, + { + "group": "interop", + "pages": [ + { + "group": "interop.crewai", + "pages": [ + "docs/reference/interop/crewai/crewai" + ] + }, + { + "group": "interop.langchain", + "pages": [ + "docs/reference/interop/langchain/langchain" + ] + }, + { + "group": "interop.pydantic_ai", + "pages": [ + "docs/reference/interop/pydantic_ai/pydantic_ai", + "docs/reference/interop/pydantic_ai/pydantic_ai_tool" + ] + }, + "docs/reference/interop/interoperability", + "docs/reference/interop/interoperable", + "docs/reference/interop/registry" + ] + }, { "group": "io", "pages": [ @@ -377,6 +415,13 @@ "docs/reference/oai/together" ] }, + { + "group": "tools", + "pages": [ + "docs/reference/tools/pydantic_ai_tool", + "docs/reference/tools/tool" + ] + }, "docs/reference/browser_utils", "docs/reference/code_utils", "docs/reference/exception_utils", @@ -450,6 +495,7 @@ { "group": "Recent posts", "pages": [ + "blog/2024-12-18-Tools-interoperability/index", "blog/2024-12-06-FalkorDB-Structured/index", "blog/2024-12-02-ReasoningAgent2/index", "blog/2024-11-27-Prompt-Leakage-Probing/index", @@ -585,4 +631,4 @@ "discord": "https://discord.gg/pAbnFJrkgZ", "youtube": "https://www.youtube.com/@ag2ai" } -} +} \ No newline at end of file From 3b6c90e9f8e8538307351a8a9a5d50bd587534ba Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Fri, 20 Dec 2024 11:14:48 +0100 Subject: [PATCH 02/13] Update interoperability blog --- .../index.mdx | 108 +++++++++++------- website/mint.json | 6 +- 2 files changed, 68 insertions(+), 46 deletions(-) diff --git a/website/blog/2024-12-18-Tools-interoperability/index.mdx b/website/blog/2024-12-18-Tools-interoperability/index.mdx index 4b3a23d8bb..6c3f43da1c 100644 --- a/website/blog/2024-12-18-Tools-interoperability/index.mdx +++ b/website/blog/2024-12-18-Tools-interoperability/index.mdx @@ -29,16 +29,16 @@ In this post, you will understand how to configure agents, adapt these tools for ## LangChain Tools Integration -LangChain is a popular framework that offers a wide range of tools to work with LLMs. LangChain has already implemented a variety of tools that can be easily integrated into AG2. You can explore the available tools in the [LangChain Community Tools](https://github.com/langchain-ai/langchain/tree/master/libs/community/langchain_community/tools) folder. These tools, such as those for querying APIs, web scraping, and text generation, can be quickly incorporated into AG2, providing powerful functionality for your agents. +LangChain is a popular framework with lots of tools for working with LLMs. It's got a range of tools that can be easily integrated into AG2. If you want to see the full list, check out the [LangChain Community Tools](https://github.com/langchain-ai/langchain/tree/master/libs/community/langchain_community/tools). You can quickly add things like API queries, web scraping, and text generation to your AG2 setup. ### Installation -To integrate LangChain tools into the AG2 framework, install the required dependencies: +To get LangChain tools working with AG2, you’ll need to install a couple of dependencies: ```bash pip install ag2[interop-langchain] ``` -Additionally, this notebook uses LangChain's [Wikipedia Tool](https://api.python.langchain.com/en/latest/tools/langchain_community.tools.wikipedia.tool.WikipediaQueryRun.html), which requires the `wikipedia` package. Install it with: +Also, we’ll use LangChain’s [Wikipedia Tool](https://python.langchain.com/docs/integrations/tools/wikipedia/), which needs the wikipedia package. Install it like this: ```bash pip install wikipedia @@ -46,10 +46,11 @@ pip install wikipedia ### Imports -Import necessary modules and tools. -- `WikipediaQueryRun` and `WikipediaAPIWrapper`: Tools for querying Wikipedia. -- `AssistantAgent` and `UserProxyAgent`: Agents that facilitate communication in the AG2 framework. -- `Interoperability`: This module acts as a bridge, making it easier to integrate LangChain tools with AG2’s architecture. +Now, let’s import the necessary modules and tools. + +- [WikipediaQueryRun](https://api.python.langchain.com/en/latest/tools/langchain_community.tools.wikipedia.tool.WikipediaQueryRun.html) and [WikipediaAPIWrapper](https://python.langchain.com/api_reference/community/utilities/langchain_community.utilities.wikipedia.WikipediaAPIWrapper.html) are the tools for querying Wikipedia. +- `AssistantAgent` and `UserProxyAgent` are the agents for interaction within AG2. +- `Interoperability` is what helps connect LangChain tools with AG2. ```python import os @@ -63,8 +64,8 @@ from autogen.interop import Interoperability ### Agent Configuration -Configure the agents for the interaction. -- `config_list` defines the LLM configurations, including the model and API key. +Let’s set up the agents for interaction. +- `config_list` is where you define the LLM configuration, like the model and API key. - `UserProxyAgent` simulates user inputs without requiring actual human interaction (set to `NEVER`). - `AssistantAgent` represents the AI agent, configured with the LLM settings. @@ -82,13 +83,11 @@ chatbot = AssistantAgent( ``` ### Tool Integration - -- Initialize and register the LangChain tool with AG2. -- [WikipediaAPIWrapper](https://python.langchain.com/api_reference/community/utilities/langchain_community.utilities.wikipedia.WikipediaAPIWrapper.html): Configured to fetch the top 1 result from Wikipedia with a maximum of 1000 characters per document. -- [WikipediaQueryRun](https://python.langchain.com/api_reference/community/tools/langchain_community.tools.wikipedia.tool.WikipediaQueryRun.html): A LangChain tool that executes Wikipedia queries. -- `LangchainInteroperability`: Converts the LangChain tool into a format compatible with the AG2 framework. -- `ag2_tool.register_for_execution(user_proxy)`: Registers the tool for use by the user_proxy agent. -- `ag2_tool.register_for_llm(chatbot)`: Registers the tool for integration with the chatbot agent. +Here’s where we connect everything. +- First, we set up [WikipediaAPIWrapper](https://python.langchain.com/api_reference/community/utilities/langchain_community.utilities.wikipedia.WikipediaAPIWrapper.html), which fetches the top Wikipedia result (with a character limit). +- Then, we use [WikipediaQueryRun](https://api.python.langchain.com/en/latest/tools/langchain_community.tools.wikipedia.tool.WikipediaQueryRun.html) to perform Wikipedia queries. +- `Interoperability` helps convert the LangChain tool to AG2’s format. +- Finally, we register the tool for use with both the `user_proxy` and `chatbot`. ```python api_wrapper = WikipediaAPIWrapper(top_k_results=1, doc_content_chars_max=1000) @@ -101,13 +100,18 @@ ag2_tool.register_for_execution(user_proxy) ag2_tool.register_for_llm(chatbot) ``` -### Initiate Chat +### Initiating the Chat + +Once everything’s set up, we can send a message to the chatbot, and it’ll use the Wikipedia tool to fetch the relevant information. + ```python message = "Tell me about the history of the United States" user_proxy.initiate_chat(recipient=chatbot, message=message, max_turns=2) ``` -Output: +### Output + +When the chat is initiated, here’s the output you’ll see: ```console User (to chatbot): @@ -162,8 +166,8 @@ pip install ag2[interop-crewai] ### Imports Import necessary modules and tools. -- `ScrapeWebsiteTool` are the CrewAI tools for web scraping -- `AssistantAgent` and `UserProxyAgent` are core AG2 classes. +- [ScrapeWebsiteTool](https://docs.crewai.com/tools/scrapewebsitetool): A CrewAI tool for web scraping. +- `AssistantAgent` and `UserProxyAgent`: Core AG2 classes. - `Interoperability`: This module acts as a bridge, making it easier to integrate CrewAI tools with AG2’s architecture. ```python @@ -197,10 +201,10 @@ chatbot = AssistantAgent( ### Tool Integration -Initialize and register the CrewAI tool with AG2. -- `crewai_tool` is an instance of the `ScrapeWebsiteTool` from CrewAI. -- `Interoperability` converts the CrewAI tool to make it usable in AG2. -- `register_for_execution` and `register_for_llm` allow the tool to work with the UserProxyAgent and AssistantAgent. +Integrate the CrewAI tool with AG2. +- [ScrapeWebsiteTool](https://docs.crewai.com/tools/scrapewebsitetool) is used for web scraping tasks. +- `Interoperability` converts the CrewAI tool to a format compatible with AG2. +- Register the tool for both execution and interaction with LLMs. ```python interop = Interoperability() @@ -209,13 +213,17 @@ ag2_tool = interop.convert_tool(tool=crewai_tool, type="crewai") ag2_tool.register_for_execution(user_proxy) ag2_tool.register_for_llm(chatbot) +``` +### Initiating the chat +Initiate the conversation between the `UserProxyAgent` and the `AssistantAgent` to utilize the CrewAI tool. +```python message = "Scrape the website https://ag2.ai/" - chat_result = user_proxy.initiate_chat(recipient=chatbot, message=message, max_turns=2) ``` -Output: +### Output +The `chatbot` provides results based on the web scraping operation: ```console User (to chatbot): @@ -271,12 +279,12 @@ This comprehensive platform seems to aim at both individual developers and enter -------------------------------------------------------------------------------- ``` +You can also access a detailed summary of the interaction: + ```python print(chat_result.summary) ``` -Output: - ```console The website "https://ag2.ai/" promotes a platform named AgentOS, which is designed for building multi-agent systems efficiently. Key highlights from the website are: @@ -303,10 +311,10 @@ This comprehensive platform seems to aim at both individual developers and enter ## PydanticAI Tools Integration -[PydanticAI](https://ai.pydantic.dev/) is a newer framework that offers powerful capabilities for working with LLMs. Although it currently does not have a repository with pre-built tools, it provides features like **dependency injection**, allowing you to inject a "Context" into a tool for better execution without relying on LLMs. This context can be used for passing parameters or managing state during the execution of a tool. While the framework is still growing, you can integrate its tools into AG2 to enhance agent capabilities, especially for tasks that involve structured data and context-driven logic. +[PydanticAI](https://ai.pydantic.dev/) is a newer framework that brings powerful features for working with LLMs. Although it doesn't yet have a collection of pre-built tools like other frameworks, it offers useful capabilities such as **dependency injection**. This feature allows you to inject a "Context" into tools, which can help pass parameters or manage state without relying on LLMs. Though it's still evolving, you can easily integrate PydanticAI tools into AG2 to boost agent capabilities, particularly for tasks that involve structured data and context-driven logic. ### Installation -To integrate LangChain tools into the AG2 framework, install the required dependencies: +To get PydanticAI tools working with AG2, install the necessary dependencies: ```bash pip install ag2[interop-pydantic-ai] @@ -317,8 +325,8 @@ pip install ag2[interop-pydantic-ai] Import necessary modules and tools. - `BaseModel`: Used to define data structures for tool inputs and outputs. -- `RunContext`: Provides context during the execution of tools. -- `PydanticAITool`: Represents a tool in the PydanticAI framework. +- [RunContext](https://ai.pydantic.dev/api/tools/#pydantic_ai.tools.RunContext): Provides context during the execution of tools. +- [PydanticAITool](https://ai.pydantic.dev/api/tools/#pydantic_ai.tools.Tool): Represents a tool in the PydanticAI framework. - `AssistantAgent` and `UserProxyAgent`: Agents that facilitate communication in the AG2 framework. - `Interoperability`: This module acts as a bridge, making it easier to integrate PydanticAI tools with AG2’s architecture. @@ -356,14 +364,13 @@ chatbot = AssistantAgent( ### Tool Integration -Integrate the PydanticAI tool with AG2. +To integrate a PydanticAI tool into AG2: -- Define a `Player` model using `BaseModel` to structure the input data. -- Use `RunContext` to securely inject dependencies (like the `Player` instance) into the tool function without exposing them to the LLM. -- Implement `get_player` to define the tool's functionality, accessing `ctx.deps` for injected data. -- Convert the tool to an AG2-compatible format with `Interoperability` and register it for execution and LLM communication. -- Convert the PydanticAI tool into an AG2-compatible format using `convert_tool`. -- Register the tool for both execution and communication with the LLM by associating it with the `user_proxy` and `chatbot`. +- First, define a `Player` model using `BaseModel` to structure the input data. +- Use [RunContext](https://ai.pydantic.dev/api/tools/#pydantic_ai.tools.RunContext) to inject dependencies (like the `Player` instance) securely into the tool. +- The `get_player` function defines the tool’s functionality, retrieving injected data through `ctx.deps`. +- Then, convert the tool into an AG2-compatible format with `Interoperability`. +- Register the tool for execution and interaction with both the `user_proxy` and `chatbot`. ```python class Player(BaseModel): @@ -391,11 +398,12 @@ ag2_tool.register_for_execution(user_proxy) ag2_tool.register_for_llm(chatbot) ``` -Initiate a conversation between the `UserProxyAgent` and the `AssistantAgent`. +### Initiating the chat +Now that everything is set up, you can initiate a chat between the `UserProxyAgent` and the `AssistantAgent`: -- Use the `initiate_chat` method to send a message from the `user_proxy` to the `chatbot`. -- In this example, the user requests the chatbot to retrieve player information, providing "goal keeper" as additional context. -- The `Player` instance is securely injected into the tool using `RunContext`, ensuring the chatbot can retrieve and use this data during the interaction. +- The `user_proxy` sends a message to the `chatbot`. +- The user requests player information, and includes "goal keeper" as additional context. +- The `Player` data is securely injected into the tool, and the chatbot can access and use it during the chat. ```python user_proxy.initiate_chat( @@ -403,7 +411,7 @@ user_proxy.initiate_chat( ) ``` -Output: +### Output ```console User (to chatbot): @@ -432,3 +440,15 @@ chatbot (to User): The player's name is Luka, who is a 25-year-old goalkeeper. TERMINATE ``` + +## Summary + +In this post, we've explored how to integrate tools from multiple frameworks (LangChain, CrewAI, and PydanticAI) into the AG2 framework, enabling cross-framework interoperability. By integrating these tools, you can enhance your agents with a variety of capabilities, such as API querying, web scraping, and structured data processing. + +- **LangChain** offers a wide range of pre-built tools for working with APIs and web scraping, making it easy to extend AG2's functionality. +- **CrewAI** brings diverse tools for search, web scraping, and more, allowing for robust agent interactions. +- **PydanticAI** introduces dependency injection and context-driven logic, enabling efficient data handling without relying on LLMs. + +With AG2's flexible architecture and the power of these frameworks, developers can create agents that are more capable and adaptable. By following the integration steps for each framework, you can enhance your agents' performance, expand their capabilities, and create more dynamic interactions. + +Now you should have a better understanding of how to integrate tools from different frameworks into AG2, and how to use these tools effectively within your own projects. diff --git a/website/mint.json b/website/mint.json index 798f7e7f22..9b5cdb5ae1 100644 --- a/website/mint.json +++ b/website/mint.json @@ -617,7 +617,9 @@ "notebooks/JSON_mode_example", "notebooks/agentchat_RetrieveChat", "notebooks/agentchat_graph_rag_neo4j", - "notebooks/agentchat_swarm_enhanced" + "notebooks/agentchat_swarm_enhanced", + "notebooks/agentchat_reasoning_agent", + "notebooks/agentchat_realtime_swarm" ] }, "notebooks/Gallery" @@ -631,4 +633,4 @@ "discord": "https://discord.gg/pAbnFJrkgZ", "youtube": "https://www.youtube.com/@ag2ai" } -} \ No newline at end of file +} From 3ff42544b03bc3511ff974dd5cc3dcccffad3952 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Fri, 20 Dec 2024 12:33:12 +0100 Subject: [PATCH 03/13] Fix notebook referencing in the blog --- notebook/tools_interoperability.ipynb | 18 +++++++- .../index.mdx | 41 +++++++++---------- website/mint.json | 1 + website/snippets/data/NotebooksMetadata.mdx | 8 ++++ 4 files changed, 46 insertions(+), 22 deletions(-) diff --git a/notebook/tools_interoperability.ipynb b/notebook/tools_interoperability.ipynb index 00469be0d4..170a6e21c0 100644 --- a/notebook/tools_interoperability.ipynb +++ b/notebook/tools_interoperability.ipynb @@ -38,6 +38,7 @@ "### Imports\n", "\n", "Import necessary modules and tools.\n", + "\n", "- `WikipediaQueryRun` and `WikipediaAPIWrapper`: Tools for querying Wikipedia.\n", "- `AssistantAgent` and `UserProxyAgent`: Agents that facilitate communication in the AG2 framework.\n", "- `Interoperability`: This module acts as a bridge, making it easier to integrate LangChain tools with AG2’s architecture." @@ -65,6 +66,7 @@ "### Agent Configuration\n", "\n", "Configure the agents for the interaction.\n", + "\n", "- `config_list` defines the LLM configurations, including the model and API key.\n", "- `UserProxyAgent` simulates user inputs without requiring actual human interaction (set to `NEVER`).\n", "- `AssistantAgent` represents the AI agent, configured with the LLM settings." @@ -152,6 +154,7 @@ "### Imports\n", "\n", "Import necessary modules and tools.\n", + "\n", "- `ScrapeWebsiteTool` are the CrewAI tools for web scraping\n", "- `AssistantAgent` and `UserProxyAgent` are core AG2 classes.\n", "- `Interoperability`: This module acts as a bridge, making it easier to integrate CrewAI tools with AG2’s architecture." @@ -178,6 +181,7 @@ "### Agent Configuration\n", "\n", "Configure the agents for the interaction.\n", + "\n", "- `config_list` defines the LLM configurations, including the model and API key.\n", "- `UserProxyAgent` simulates user inputs without requiring actual human interaction (set to `NEVER`).\n", "- `AssistantAgent` represents the AI agent, configured with the LLM settings." @@ -208,6 +212,7 @@ "### Tool Integration\n", "\n", "Initialize and register the CrewAI tool with AG2.\n", + "\n", "- `crewai_tool` is an instance of the `ScrapeWebsiteTool` from CrewAI.\n", "- `Interoperability` converts the CrewAI tool to make it usable in AG2.\n", "- `register_for_execution` and `register_for_llm` allow the tool to work with the UserProxyAgent and AssistantAgent." @@ -260,9 +265,10 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Imports\n", + "### Imports\n", "\n", "Import necessary modules and tools.\n", + "\n", "- `BaseModel`: Used to define data structures for tool inputs and outputs.\n", "- `RunContext`: Provides context during the execution of tools.\n", "- `PydanticAITool`: Represents a tool in the PydanticAI framework.\n", @@ -294,6 +300,7 @@ "### Agent Configuration\n", "\n", "Configure the agents for the interaction.\n", + "\n", "- `config_list` defines the LLM configurations, including the model and API key.\n", "- `UserProxyAgent` simulates user inputs without requiring actual human interaction (set to `NEVER`).\n", "- `AssistantAgent` represents the AI agent, configured with the LLM settings." @@ -395,6 +402,15 @@ } ], "metadata": { + "front_matter": { + "description": "Cross-Framework LLM Tool Integration with AG2", + "tags": [ + "tools", + "langchain", + "crewai", + "pydanticai" + ] + }, "kernelspec": { "display_name": ".venv", "language": "python", diff --git a/website/blog/2024-12-18-Tools-interoperability/index.mdx b/website/blog/2024-12-18-Tools-interoperability/index.mdx index 6c3f43da1c..07946bb9e0 100644 --- a/website/blog/2024-12-18-Tools-interoperability/index.mdx +++ b/website/blog/2024-12-18-Tools-interoperability/index.mdx @@ -20,9 +20,8 @@ With AG2, you can combine these tools and enhance your agents' capabilities. In this post, we’ll walk through how to integrate tools from various frameworks—like [LangChain Tools](https://python.langchain.com/v0.1/docs/modules/tools), [CrewAI Tools](https://github.com/crewAIInc/crewAI-tools/tree/main), and [PydanticAI Tools](https://ai.pydantic.dev/tools/)—into AG2. -And let’s be honest: - Because, really, the magic happens when you combine them all. This allows you to use tools from different frameworks within AG2, giving your agents more power and flexibility. +This blog builds upon the concepts covered in the [Tool Integration notebook](/notebooks/tools_interoperability). In this post, you will understand how to configure agents, adapt these tools for use in AG2, and validate the integration through practical examples. @@ -49,8 +48,8 @@ pip install wikipedia Now, let’s import the necessary modules and tools. - [WikipediaQueryRun](https://api.python.langchain.com/en/latest/tools/langchain_community.tools.wikipedia.tool.WikipediaQueryRun.html) and [WikipediaAPIWrapper](https://python.langchain.com/api_reference/community/utilities/langchain_community.utilities.wikipedia.WikipediaAPIWrapper.html) are the tools for querying Wikipedia. -- `AssistantAgent` and `UserProxyAgent` are the agents for interaction within AG2. -- `Interoperability` is what helps connect LangChain tools with AG2. +- [`AssistantAgent`](/docs/reference/agentchat/assistant_agent) and [`UserProxyAgent`](/docs/reference/agentchat/user_proxy_agent) are the agents for interaction within AG2. +- [`Interoperability`](/docs/reference/interop/interoperability) is what helps connect LangChain tools with AG2. ```python import os @@ -66,8 +65,8 @@ from autogen.interop import Interoperability Let’s set up the agents for interaction. - `config_list` is where you define the LLM configuration, like the model and API key. -- `UserProxyAgent` simulates user inputs without requiring actual human interaction (set to `NEVER`). -- `AssistantAgent` represents the AI agent, configured with the LLM settings. +- [`UserProxyAgent`](/docs/reference/agentchat/user_proxy_agent) simulates user inputs without requiring actual human interaction (set to `NEVER`). +- [`AssistantAgent`](/docs/reference/agentchat/assistant_agent) represents the AI agent, configured with the LLM settings. ```python config_list = [{"model": "gpt-4o", "api_key": os.environ["OPENAI_API_KEY"]}] @@ -86,7 +85,7 @@ chatbot = AssistantAgent( Here’s where we connect everything. - First, we set up [WikipediaAPIWrapper](https://python.langchain.com/api_reference/community/utilities/langchain_community.utilities.wikipedia.WikipediaAPIWrapper.html), which fetches the top Wikipedia result (with a character limit). - Then, we use [WikipediaQueryRun](https://api.python.langchain.com/en/latest/tools/langchain_community.tools.wikipedia.tool.WikipediaQueryRun.html) to perform Wikipedia queries. -- `Interoperability` helps convert the LangChain tool to AG2’s format. +- [`Interoperability`](/docs/reference/interop/interoperability) helps convert the LangChain tool to AG2’s format. - Finally, we register the tool for use with both the `user_proxy` and `chatbot`. ```python @@ -167,8 +166,8 @@ pip install ag2[interop-crewai] Import necessary modules and tools. - [ScrapeWebsiteTool](https://docs.crewai.com/tools/scrapewebsitetool): A CrewAI tool for web scraping. -- `AssistantAgent` and `UserProxyAgent`: Core AG2 classes. -- `Interoperability`: This module acts as a bridge, making it easier to integrate CrewAI tools with AG2’s architecture. +- [`AssistantAgent`](/docs/reference/agentchat/assistant_agent) and [`UserProxyAgent`](/docs/reference/agentchat/user_proxy_agent): Core AG2 classes. +- [`Interoperability`](/docs/reference/interop/interoperability): This module acts as a bridge, making it easier to integrate CrewAI tools with AG2’s architecture. ```python import os @@ -183,8 +182,8 @@ from autogen.interop import Interoperability Configure the agents for the interaction. - `config_list` defines the LLM configurations, including the model and API key. -- `UserProxyAgent` simulates user inputs without requiring actual human interaction (set to `NEVER`). -- `AssistantAgent` represents the AI agent, configured with the LLM settings. +- [`UserProxyAgent`](/docs/reference/agentchat/user_proxy_agent) simulates user inputs without requiring actual human interaction (set to `NEVER`). +- [`AssistantAgent`](/docs/reference/agentchat/assistant_agent) represents the AI agent, configured with the LLM settings. ```python config_list = [{"model": "gpt-4o", "api_key": os.environ["OPENAI_API_KEY"]}] @@ -202,8 +201,8 @@ chatbot = AssistantAgent( ### Tool Integration Integrate the CrewAI tool with AG2. +- [`Interoperability`](/docs/reference/interop/interoperability) converts the CrewAI tool to a format compatible with AG2. - [ScrapeWebsiteTool](https://docs.crewai.com/tools/scrapewebsitetool) is used for web scraping tasks. -- `Interoperability` converts the CrewAI tool to a format compatible with AG2. - Register the tool for both execution and interaction with LLMs. ```python @@ -216,7 +215,7 @@ ag2_tool.register_for_llm(chatbot) ``` ### Initiating the chat -Initiate the conversation between the `UserProxyAgent` and the `AssistantAgent` to utilize the CrewAI tool. +Initiate the conversation between the [`UserProxyAgent`](/docs/reference/agentchat/user_proxy_agent) and the [`AssistantAgent`](/docs/reference/agentchat/assistant_agent) to utilize the CrewAI tool. ```python message = "Scrape the website https://ag2.ai/" chat_result = user_proxy.initiate_chat(recipient=chatbot, message=message, max_turns=2) @@ -324,11 +323,11 @@ pip install ag2[interop-pydantic-ai] ### Imports Import necessary modules and tools. -- `BaseModel`: Used to define data structures for tool inputs and outputs. +- [BaseModel](https://docs.pydantic.dev/latest/api/base_model/): Used to define data structures for tool inputs and outputs. - [RunContext](https://ai.pydantic.dev/api/tools/#pydantic_ai.tools.RunContext): Provides context during the execution of tools. - [PydanticAITool](https://ai.pydantic.dev/api/tools/#pydantic_ai.tools.Tool): Represents a tool in the PydanticAI framework. -- `AssistantAgent` and `UserProxyAgent`: Agents that facilitate communication in the AG2 framework. -- `Interoperability`: This module acts as a bridge, making it easier to integrate PydanticAI tools with AG2’s architecture. +- [`AssistantAgent`](/docs/reference/agentchat/assistant_agent) and [`UserProxyAgent`](/docs/reference/agentchat/user_proxy_agent): Agents that facilitate communication in the AG2 framework. +- [`Interoperability`](/docs/reference/interop/interoperability): This module acts as a bridge, making it easier to integrate PydanticAI tools with AG2’s architecture. ```python import os @@ -346,8 +345,8 @@ from autogen.interop import Interoperability Configure the agents for the interaction. - `config_list` defines the LLM configurations, including the model and API key. -- `UserProxyAgent` simulates user inputs without requiring actual human interaction (set to `NEVER`). -- `AssistantAgent` represents the AI agent, configured with the LLM settings. +- [`UserProxyAgent`](/docs/reference/agentchat/user_proxy_agent) simulates user inputs without requiring actual human interaction (set to `NEVER`). +- [`AssistantAgent`](/docs/reference/agentchat/assistant_agent) represents the AI agent, configured with the LLM settings. ```python config_list = [{"model": "gpt-4o", "api_key": os.environ["OPENAI_API_KEY"]}] @@ -366,10 +365,10 @@ chatbot = AssistantAgent( To integrate a PydanticAI tool into AG2: -- First, define a `Player` model using `BaseModel` to structure the input data. +- First, define a `Player` model using [BaseModel](https://docs.pydantic.dev/latest/api/base_model/) to structure the input data. - Use [RunContext](https://ai.pydantic.dev/api/tools/#pydantic_ai.tools.RunContext) to inject dependencies (like the `Player` instance) securely into the tool. - The `get_player` function defines the tool’s functionality, retrieving injected data through `ctx.deps`. -- Then, convert the tool into an AG2-compatible format with `Interoperability`. +- Then, convert the tool into an AG2-compatible format with [`Interoperability`](/docs/reference/interop/interoperability). - Register the tool for execution and interaction with both the `user_proxy` and `chatbot`. ```python @@ -399,7 +398,7 @@ ag2_tool.register_for_llm(chatbot) ``` ### Initiating the chat -Now that everything is set up, you can initiate a chat between the `UserProxyAgent` and the `AssistantAgent`: +Now that everything is set up, you can initiate a chat between the [`UserProxyAgent`](/docs/reference/agentchat/user_proxy_agent) and the [`AssistantAgent`](/docs/reference/agentchat/assistant_agent): - The `user_proxy` sends a message to the `chatbot`. - The user requests player information, and includes "goal keeper" as additional context. diff --git a/website/mint.json b/website/mint.json index 9b5cdb5ae1..2aae976304 100644 --- a/website/mint.json +++ b/website/mint.json @@ -618,6 +618,7 @@ "notebooks/agentchat_RetrieveChat", "notebooks/agentchat_graph_rag_neo4j", "notebooks/agentchat_swarm_enhanced", + "notebooks/tools_interoperability", "notebooks/agentchat_reasoning_agent", "notebooks/agentchat_realtime_swarm" ] diff --git a/website/snippets/data/NotebooksMetadata.mdx b/website/snippets/data/NotebooksMetadata.mdx index d5c69dcf3a..ec6e9da249 100644 --- a/website/snippets/data/NotebooksMetadata.mdx +++ b/website/snippets/data/NotebooksMetadata.mdx @@ -1003,5 +1003,13 @@ export const notebooksMetadata = [ "image": null, "tags": [], "source": "/website/docs/topics/non-openai-models/cloud-cerebras.ipynb" + }, + { + "title": "Cross-Framework LLM Tool Integration with AG2", + "link": "/notebooks/tools_interoperability", + "description": "", + "image": null, + "tags": [], + "source": "/notebook/tools_interoperability.ipynb" } ]; From c59636ae48549ff599b70ff10eb3c78a6c6402d9 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Fri, 20 Dec 2024 12:41:15 +0100 Subject: [PATCH 04/13] Add youtube video placeholder to the blog --- .../blog/2024-12-18-Tools-interoperability/index.mdx | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/website/blog/2024-12-18-Tools-interoperability/index.mdx b/website/blog/2024-12-18-Tools-interoperability/index.mdx index 07946bb9e0..86bbc33070 100644 --- a/website/blog/2024-12-18-Tools-interoperability/index.mdx +++ b/website/blog/2024-12-18-Tools-interoperability/index.mdx @@ -5,14 +5,16 @@ authors: tags: [LLM, tools, langchain, crewai, pydanticai] --- + + **TL;DR** -AG2 lets you bring in tools from different frameworks like **LangChain**, **CrewAI**, and **PydanticAI**. +AG2 lets you bring in **Tools** from different frameworks like **LangChain**, **CrewAI**, and **PydanticAI**. -- **LangChain**: Useful for tasks like API querying and web scraping. -- **CrewAI**: Offers a variety of tools for web scraping, search, and more. -- **PydanticAI**: Adds context-driven tools and structured data processing. +- [LangChain Tools](https://python.langchain.com/v0.1/docs/modules/tools): Useful for tasks like API querying and web scraping. +- [CrewAI Tools](https://github.com/crewAIInc/crewAI-tools/tree/main): Offers a variety of tools for web scraping, search, and more. +- [PydanticAI Tools](https://ai.pydantic.dev/tools/): Adds context-driven tools and structured data processing. -With AG2, you can combine these tools and enhance your agents' capabilities. +**With AG2, you can combine these tools and enhance your agents' capabilities.** ![One to rule them all img](img/one_to_rule_them_all.png) From d55d278ddba105e2e0683f4f6133d9989731aa50 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Fri, 20 Dec 2024 13:15:09 +0100 Subject: [PATCH 05/13] Update references in tools_ineroperability.ipynb --- notebook/tools_interoperability.ipynb | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/notebook/tools_interoperability.ipynb b/notebook/tools_interoperability.ipynb index 170a6e21c0..99075818c3 100644 --- a/notebook/tools_interoperability.ipynb +++ b/notebook/tools_interoperability.ipynb @@ -39,7 +39,7 @@ "\n", "Import necessary modules and tools.\n", "\n", - "- `WikipediaQueryRun` and `WikipediaAPIWrapper`: Tools for querying Wikipedia.\n", + "- [WikipediaQueryRun](https://api.python.langchain.com/en/latest/tools/langchain_community.tools.wikipedia.tool.WikipediaQueryRun.html) and [WikipediaAPIWrapper](https://python.langchain.com/api_reference/community/utilities/langchain_community.utilities.wikipedia.WikipediaAPIWrapper.html): Tools for querying Wikipedia.\n", "- `AssistantAgent` and `UserProxyAgent`: Agents that facilitate communication in the AG2 framework.\n", "- `Interoperability`: This module acts as a bridge, making it easier to integrate LangChain tools with AG2’s architecture." ] @@ -97,9 +97,9 @@ "### Tool Integration\n", "\n", "- Initialize and register the LangChain tool with AG2.\n", - "- `WikipediaAPIWrapper`: Configured to fetch the top 1 result from Wikipedia with a maximum of 1000 characters per document.\n", - "- `WikipediaQueryRun`: A LangChain tool that executes Wikipedia queries.\n", - "- `LangchainInteroperability`: Converts the LangChain tool into a format compatible with the AG2 framework.\n", + "- [WikipediaAPIWrapper](https://python.langchain.com/api_reference/community/utilities/langchain_community.utilities.wikipedia.WikipediaAPIWrapper.html): Configured to fetch the top 1 result from Wikipedia with a maximum of 1000 characters per document.\n", + "- [WikipediaQueryRun](https://api.python.langchain.com/en/latest/tools/langchain_community.tools.wikipedia.tool.WikipediaQueryRun.html): A LangChain tool that executes Wikipedia queries.\n", + "- `Interoperability`: Converts the LangChain tool into a format compatible with the AG2 framework.\n", "- `ag2_tool.register_for_execution(user_proxy)`: Registers the tool for use by the user_proxy agent.\n", "- `ag2_tool.register_for_llm(chatbot)`: Registers the tool for integration with the chatbot agent.\n" ] @@ -155,7 +155,7 @@ "\n", "Import necessary modules and tools.\n", "\n", - "- `ScrapeWebsiteTool` are the CrewAI tools for web scraping\n", + "- [ScrapeWebsiteTool](https://docs.crewai.com/tools/scrapewebsitetool) are the CrewAI tools for web scraping\n", "- `AssistantAgent` and `UserProxyAgent` are core AG2 classes.\n", "- `Interoperability`: This module acts as a bridge, making it easier to integrate CrewAI tools with AG2’s architecture." ] @@ -213,7 +213,7 @@ "\n", "Initialize and register the CrewAI tool with AG2.\n", "\n", - "- `crewai_tool` is an instance of the `ScrapeWebsiteTool` from CrewAI.\n", + "- `crewai_tool` is an instance of the [ScrapeWebsiteTool](https://docs.crewai.com/tools/scrapewebsitetool) from CrewAI.\n", "- `Interoperability` converts the CrewAI tool to make it usable in AG2.\n", "- `register_for_execution` and `register_for_llm` allow the tool to work with the UserProxyAgent and AssistantAgent." ] @@ -269,9 +269,9 @@ "\n", "Import necessary modules and tools.\n", "\n", - "- `BaseModel`: Used to define data structures for tool inputs and outputs.\n", - "- `RunContext`: Provides context during the execution of tools.\n", - "- `PydanticAITool`: Represents a tool in the PydanticAI framework.\n", + "- [BaseModel](https://docs.pydantic.dev/latest/api/base_model/): Used to define data structures for tool inputs and outputs.\n", + "- [RunContext](https://ai.pydantic.dev/api/tools/#pydantic_ai.tools.RunContext): Provides context during the execution of tools.\n", + "- [PydanticAITool](https://ai.pydantic.dev/api/tools/#pydantic_ai.tools.Tool): Represents a tool in the PydanticAI framework.\n", "- `AssistantAgent` and `UserProxyAgent`: Agents that facilitate communication in the AG2 framework.\n", "- `Interoperability`: This module acts as a bridge, making it easier to integrate PydanticAI tools with AG2’s architecture." ] @@ -332,8 +332,8 @@ "\n", "Integrate the PydanticAI tool with AG2.\n", "\n", - "- Define a `Player` model using `BaseModel` to structure the input data.\n", - "- Use `RunContext` to securely inject dependencies (like the `Player` instance) into the tool function without exposing them to the LLM.\n", + "- Define a `Player` model using [BaseModel](https://docs.pydantic.dev/latest/api/base_model/) to structure the input data.\n", + "- Use [RunContext](https://ai.pydantic.dev/api/tools/#pydantic_ai.tools.RunContext) to securely inject dependencies (like the `Player` instance) into the tool function without exposing them to the LLM.\n", "- Implement `get_player` to define the tool's functionality, accessing `ctx.deps` for injected data.\n", "- Convert the tool to an AG2-compatible format with `Interoperability` and register it for execution and LLM communication.\n", "- Convert the PydanticAI tool into an AG2-compatible format using `convert_tool`.\n", @@ -379,7 +379,7 @@ "\n", "- Use the `initiate_chat` method to send a message from the `user_proxy` to the `chatbot`.\n", "- In this example, the user requests the chatbot to retrieve player information, providing \"goal keeper\" as additional context.\n", - "- The `Player` instance is securely injected into the tool using `RunContext`, ensuring the chatbot can retrieve and use this data during the interaction." + "- The `Player` instance is securely injected into the tool using [RunContext](https://ai.pydantic.dev/api/tools/#pydantic_ai.tools.RunContext), ensuring the chatbot can retrieve and use this data during the interaction." ] }, { From 9f37ecadfe460e43330e2fc7fd99b91567f6295d Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Fri, 20 Dec 2024 13:39:09 +0100 Subject: [PATCH 06/13] Hide iframe block (make it non visable) in the blog until youtube video is not ready --- website/blog/2024-12-18-Tools-interoperability/index.mdx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/website/blog/2024-12-18-Tools-interoperability/index.mdx b/website/blog/2024-12-18-Tools-interoperability/index.mdx index 86bbc33070..241317ea04 100644 --- a/website/blog/2024-12-18-Tools-interoperability/index.mdx +++ b/website/blog/2024-12-18-Tools-interoperability/index.mdx @@ -5,7 +5,11 @@ authors: tags: [LLM, tools, langchain, crewai, pydanticai] --- - +
+ This content will be hidden. + REMOVE THE iframe BLOCK OUTSIDE OF div BLOCK ONCE THE VIDEO IS READY + +
**TL;DR** AG2 lets you bring in **Tools** from different frameworks like **LangChain**, **CrewAI**, and **PydanticAI**. From 091e640f03f2b277564f7fa588f252decf64fc07 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Fri, 20 Dec 2024 14:00:12 +0100 Subject: [PATCH 07/13] Update comment iterop blog --- website/blog/2024-12-18-Tools-interoperability/index.mdx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/website/blog/2024-12-18-Tools-interoperability/index.mdx b/website/blog/2024-12-18-Tools-interoperability/index.mdx index 241317ea04..f1089c5bc1 100644 --- a/website/blog/2024-12-18-Tools-interoperability/index.mdx +++ b/website/blog/2024-12-18-Tools-interoperability/index.mdx @@ -7,7 +7,8 @@ tags: [LLM, tools, langchain, crewai, pydanticai]
This content will be hidden. - REMOVE THE iframe BLOCK OUTSIDE OF div BLOCK ONCE THE VIDEO IS READY + MOVE THE iframe BLOCK OUTSIDE OF div BLOCK ONCE THE VIDEO IS READY + ALSO DELETE THIS div BLOCK
From d2ace5299b67a53b03bfb6c0cc1b67335d901fc9 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Fri, 20 Dec 2024 15:46:16 +0100 Subject: [PATCH 08/13] Remove meme from inerop blog --- .../img/one_to_rule_them_all.png | 3 --- website/blog/2024-12-18-Tools-interoperability/index.mdx | 2 -- 2 files changed, 5 deletions(-) delete mode 100644 website/blog/2024-12-18-Tools-interoperability/img/one_to_rule_them_all.png diff --git a/website/blog/2024-12-18-Tools-interoperability/img/one_to_rule_them_all.png b/website/blog/2024-12-18-Tools-interoperability/img/one_to_rule_them_all.png deleted file mode 100644 index 45cf075e36..0000000000 --- a/website/blog/2024-12-18-Tools-interoperability/img/one_to_rule_them_all.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5f41e305d4525e167b62d0c769599ec2667ef282918e835372ad20a1b3b3c03c -size 310123 diff --git a/website/blog/2024-12-18-Tools-interoperability/index.mdx b/website/blog/2024-12-18-Tools-interoperability/index.mdx index f1089c5bc1..c57d6869f4 100644 --- a/website/blog/2024-12-18-Tools-interoperability/index.mdx +++ b/website/blog/2024-12-18-Tools-interoperability/index.mdx @@ -22,8 +22,6 @@ AG2 lets you bring in **Tools** from different frameworks like **LangChain**, ** **With AG2, you can combine these tools and enhance your agents' capabilities.** -![One to rule them all img](img/one_to_rule_them_all.png) - In this post, we’ll walk through how to integrate tools from various frameworks—like [LangChain Tools](https://python.langchain.com/v0.1/docs/modules/tools), [CrewAI Tools](https://github.com/crewAIInc/crewAI-tools/tree/main), and [PydanticAI Tools](https://ai.pydantic.dev/tools/)—into AG2. From f80cbe133c7ce775eb9b8b2886cef2961d0af0b0 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Fri, 20 Dec 2024 15:57:17 +0100 Subject: [PATCH 09/13] Update mint.json --- website/mint.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/website/mint.json b/website/mint.json index 68fdfded93..66a31501d2 100644 --- a/website/mint.json +++ b/website/mint.json @@ -418,6 +418,7 @@ { "group": "tools", "pages": [ + "docs/reference/tools/pydantic_ai_tool", "docs/reference/tools/tool" ] }, @@ -621,8 +622,8 @@ "notebooks/agentchat_graph_rag_neo4j", "notebooks/agentchat_swarm_enhanced", "notebooks/tools_interoperability", - "notebooks/agentchat_reasoning_agent", - "notebooks/agentchat_realtime_swarm" + "notebooks/agentchat_realtime_swarm", + "notebooks/agentchat_reasoning_agent" ] }, "notebooks/Gallery" From 5b62def318fd2d30214c370b74a292ad307ab481 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Fri, 20 Dec 2024 16:06:44 +0100 Subject: [PATCH 10/13] Remove duplicated blog/2024-12-20-RealtimeAgent/index --- website/mint.json | 1 - 1 file changed, 1 deletion(-) diff --git a/website/mint.json b/website/mint.json index 114bb60e6d..39b665c2cb 100644 --- a/website/mint.json +++ b/website/mint.json @@ -498,7 +498,6 @@ "blog/2024-12-20-RealtimeAgent/index", "blog/2024-12-20-Tools-interoperability/index", "blog/2024-12-20-Reasoning-Update/index", - "blog/2024-12-20-RealtimeAgent/index", "blog/2024-12-18-Tools-interoperability/index", "blog/2024-12-06-FalkorDB-Structured/index", "blog/2024-12-02-ReasoningAgent2/index", From 831b87a0bf0b7dfb95a3e0397e943103e8e5400a Mon Sep 17 00:00:00 2001 From: Davor Runje Date: Fri, 20 Dec 2024 16:20:01 +0100 Subject: [PATCH 11/13] Typing fixed --- autogen/agentchat/contrib/swarm_agent.py | 2 +- autogen/agentchat/realtime_agent/client.py | 123 ++++++++++++------ .../realtime_agent/function_observer.py | 36 +++-- .../realtime_agent/realtime_agent.py | 32 +++-- .../realtime_agent/realtime_observer.py | 26 +++- .../realtime_agent/twilio_observer.py | 26 ++-- .../realtime_agent/websocket_observer.py | 24 ++-- notebook/agentchat_realtime_swarm.ipynb | 4 +- pyproject.toml | 2 + setup.py | 1 + test/agentchat/realtime_agent/__init__.py | 6 + .../realtime_agent/test_submodule.py | 15 +++ 12 files changed, 202 insertions(+), 95 deletions(-) create mode 100644 test/agentchat/realtime_agent/__init__.py create mode 100644 test/agentchat/realtime_agent/test_submodule.py diff --git a/autogen/agentchat/contrib/swarm_agent.py b/autogen/agentchat/contrib/swarm_agent.py index f604c13a57..94a75baa0f 100644 --- a/autogen/agentchat/contrib/swarm_agent.py +++ b/autogen/agentchat/contrib/swarm_agent.py @@ -123,7 +123,7 @@ def initiate_swarm_chat( user_agent: Optional[UserProxyAgent] = None, max_rounds: int = 20, context_variables: Optional[dict[str, Any]] = None, - after_work: Optional[Union[AFTER_WORK, Callable]] = AFTER_WORK(AfterWorkOption.TERMINATE), + after_work: Optional[Union[AfterWorkOption, Callable]] = AFTER_WORK(AfterWorkOption.TERMINATE), ) -> tuple[ChatResult, dict[str, Any], "SwarmAgent"]: """Initialize and run a swarm chat diff --git a/autogen/agentchat/realtime_agent/client.py b/autogen/agentchat/realtime_agent/client.py index ac2ed1674a..ca337a5769 100644 --- a/autogen/agentchat/realtime_agent/client.py +++ b/autogen/agentchat/realtime_agent/client.py @@ -8,15 +8,18 @@ # import asyncio import json import logging -from typing import Any, Optional +from typing import TYPE_CHECKING, Any, Optional -import anyio -import websockets -from asyncer import TaskGroup, asyncify, create_task_group, syncify +from asyncer import TaskGroup, asyncify, create_task_group +from websockets import connect +from websockets.asyncio.client import ClientConnection -from autogen.agentchat.contrib.swarm_agent import AfterWorkOption, initiate_swarm_chat +from ..contrib.swarm_agent import AfterWorkOption, SwarmAgent, initiate_swarm_chat -from .function_observer import FunctionObserver +if TYPE_CHECKING: + from .function_observer import FunctionObserver + from .realtime_agent import RealtimeAgent + from .realtime_observer import RealtimeObserver logger = logging.getLogger(__name__) @@ -24,47 +27,64 @@ class OpenAIRealtimeClient: """(Experimental) Client for OpenAI Realtime API.""" - def __init__(self, agent, audio_adapter, function_observer: FunctionObserver): + def __init__( + self, agent: "RealtimeAgent", audio_adapter: "RealtimeObserver", function_observer: "FunctionObserver" + ) -> None: """(Experimental) Client for OpenAI Realtime API. - args: - agent: Agent instance - the agent to be used for the conversation - audio_adapter: RealtimeObserver - adapter for streaming the audio from the client - function_observer: FunctionObserver - observer for handling function calls + Args: + agent (RealtimeAgent): The agent that the client is associated with. + audio_adapter (RealtimeObserver): The audio adapter for the client. + function_observer (FunctionObserver): The function observer for the client. + """ self._agent = agent - self._observers = [] - self._openai_ws = None # todo factor out to OpenAIClient + self._observers: list["RealtimeObserver"] = [] + self._openai_ws: Optional[ClientConnection] = None # todo factor out to OpenAIClient self.register(audio_adapter) self.register(function_observer) # LLM config llm_config = self._agent.llm_config - config = llm_config["config_list"][0] + config: dict[str, Any] = llm_config["config_list"][0] # type: ignore[index] - self.model = config["model"] - self.temperature = llm_config["temperature"] - self.api_key = config["api_key"] + self.model: str = config["model"] + self.temperature: float = llm_config["temperature"] # type: ignore[index] + self.api_key: str = config["api_key"] # create a task group to manage the tasks self.tg: Optional[TaskGroup] = None - def register(self, observer): + @property + def openai_ws(self) -> ClientConnection: + """Get the OpenAI WebSocket connection.""" + if self._openai_ws is None: + raise RuntimeError("OpenAI WebSocket is not initialized") + return self._openai_ws + + def register(self, observer: "RealtimeObserver") -> None: """Register an observer to the client.""" observer.register_client(self) self._observers.append(observer) - async def notify_observers(self, message): - """Notify all observers of a message from the OpenAI Realtime API.""" + async def notify_observers(self, message: dict[str, Any]) -> None: + """Notify all observers of a message from the OpenAI Realtime API. + + Args: + message (dict[str, Any]): The message from the OpenAI Realtime API. + + """ for observer in self._observers: await observer.update(message) - async def function_result(self, call_id, result): - """Send the result of a function call to the OpenAI Realtime API.""" + async def function_result(self, call_id: str, result: str) -> None: + """Send the result of a function call to the OpenAI Realtime API. + + Args: + call_id (str): The ID of the function call. + result (str): The result of the function call. + """ result_item = { "type": "conversation.item.create", "item": { @@ -73,11 +93,23 @@ async def function_result(self, call_id, result): "output": result, }, } + if self._openai_ws is None: + raise RuntimeError("OpenAI WebSocket is not initialized") + await self._openai_ws.send(json.dumps(result_item)) await self._openai_ws.send(json.dumps({"type": "response.create"})) - async def send_text(self, *, role: str, text: str): - """Send a text message to the OpenAI Realtime API.""" + async def send_text(self, *, role: str, text: str) -> None: + """Send a text message to the OpenAI Realtime API. + + Args: + role (str): The role of the message. + text (str): The text of the message. + """ + + if self._openai_ws is None: + raise RuntimeError("OpenAI WebSocket is not initialized") + await self._openai_ws.send(json.dumps({"type": "response.cancel"})) text_item = { "type": "conversation.item.create", @@ -87,7 +119,7 @@ async def send_text(self, *, role: str, text: str): await self._openai_ws.send(json.dumps({"type": "response.create"})) # todo override in specific clients - async def initialize_session(self): + async def initialize_session(self) -> None: """Control initial session with OpenAI.""" session_update = { # todo: move to config @@ -100,15 +132,25 @@ async def initialize_session(self): await self.session_update(session_update) # todo override in specific clients - async def session_update(self, session_options): - """Send a session update to the OpenAI Realtime API.""" + async def session_update(self, session_options: dict[str, Any]) -> None: + """Send a session update to the OpenAI Realtime API. + + Args: + session_options (dict[str, Any]): The session options to update. + """ + if self._openai_ws is None: + raise RuntimeError("OpenAI WebSocket is not initialized") + update = {"type": "session.update", "session": session_options} logger.info("Sending session update:", json.dumps(update)) await self._openai_ws.send(json.dumps(update)) logger.info("Sending session update finished") - async def _read_from_client(self): + async def _read_from_client(self) -> None: """Read messages from the OpenAI Realtime API.""" + if self._openai_ws is None: + raise RuntimeError("OpenAI WebSocket is not initialized") + try: async for openai_message in self._openai_ws: response = json.loads(openai_message) @@ -116,9 +158,9 @@ async def _read_from_client(self): except Exception as e: logger.warning(f"Error in _read_from_client: {e}") - async def run(self): + async def run(self) -> None: """Run the client.""" - async with websockets.connect( + async with connect( f"wss://api.openai.com/v1/realtime?model={self.model}", additional_headers={ "Authorization": f"Bearer {self.api_key}", @@ -127,17 +169,24 @@ async def run(self): ) as openai_ws: self._openai_ws = openai_ws await self.initialize_session() - # await asyncio.gather(self._read_from_client(), *[observer.run() for observer in self._observers]) async with create_task_group() as tg: self.tg = tg self.tg.soonify(self._read_from_client)() for observer in self._observers: self.tg.soonify(observer.run)() + + initial_agent = self._agent._initial_agent + agents = self._agent._agents + user_agent = self._agent + + if not (initial_agent and agents): + raise RuntimeError("Swarm not registered.") + if self._agent._start_swarm_chat: self.tg.soonify(asyncify(initiate_swarm_chat))( - initial_agent=self._agent._initial_agent, - agents=self._agent._agents, - user_agent=self._agent, + initial_agent=initial_agent, + agents=agents, + user_agent=user_agent, # type: ignore[arg-type] messages="Find out what the user wants.", after_work=AfterWorkOption.REVERT_TO_USER, ) diff --git a/autogen/agentchat/realtime_agent/function_observer.py b/autogen/agentchat/realtime_agent/function_observer.py index 14c70bca62..9e4c8d2649 100644 --- a/autogen/agentchat/realtime_agent/function_observer.py +++ b/autogen/agentchat/realtime_agent/function_observer.py @@ -8,38 +8,52 @@ import asyncio import json import logging +from typing import TYPE_CHECKING, Any from asyncer import asyncify from pydantic import BaseModel from .realtime_observer import RealtimeObserver +if TYPE_CHECKING: + from .realtime_agent import RealtimeAgent + logger = logging.getLogger(__name__) class FunctionObserver(RealtimeObserver): """Observer for handling function calls from the OpenAI Realtime API.""" - def __init__(self, agent): + def __init__(self, agent: "RealtimeAgent") -> None: """Observer for handling function calls from the OpenAI Realtime API. Args: - agent: Agent instance - the agent to be used for the conversation + agent (RealtimeAgent): The realtime agent attached to the observer. """ super().__init__() self._agent = agent - async def update(self, response): - """Handle function call events from the OpenAI Realtime API.""" + async def update(self, response: dict[str, Any]) -> None: + """Handle function call events from the OpenAI Realtime API. + + Args: + response (dict[str, Any]): The response from the OpenAI Realtime API. + """ if response.get("type") == "response.function_call_arguments.done": logger.info(f"Received event: {response['type']}", response) await self.call_function( call_id=response["call_id"], name=response["name"], kwargs=json.loads(response["arguments"]) ) - async def call_function(self, call_id, name, kwargs): - """Call a function registered with the agent.""" + async def call_function(self, call_id: str, name: str, kwargs: dict[str, Any]) -> None: + """Call a function registered with the agent. + + Args: + call_id (str): The ID of the function call. + name (str): The name of the function to call. + kwargs (Any[str, Any]): The arguments to pass to the function. + """ + if name in self._agent.realtime_functions: _, func = self._agent.realtime_functions[name] func = func if asyncio.iscoroutinefunction(func) else asyncify(func) @@ -54,19 +68,19 @@ async def call_function(self, call_id, name, kwargs): elif not isinstance(result, str): result = json.dumps(result) - await self._client.function_result(call_id, result) + await self.client.function_result(call_id, result) - async def run(self): + async def run(self) -> None: """Run the observer. Initialize the session with the OpenAI Realtime API. """ await self.initialize_session() - async def initialize_session(self): + async def initialize_session(self) -> None: """Add registered tools to OpenAI with a session update.""" session_update = { "tools": [schema for schema, _ in self._agent.realtime_functions.values()], "tool_choice": "auto", } - await self._client.session_update(session_update) + await self.client.session_update(session_update) diff --git a/autogen/agentchat/realtime_agent/realtime_agent.py b/autogen/agentchat/realtime_agent/realtime_agent.py index aadbc1f283..b4456715bb 100644 --- a/autogen/agentchat/realtime_agent/realtime_agent.py +++ b/autogen/agentchat/realtime_agent/realtime_agent.py @@ -52,8 +52,8 @@ def __init__( *, name: str, audio_adapter: RealtimeObserver, - system_message: Optional[Union[str, list]] = "You are a helpful AI Assistant.", - llm_config: Optional[Union[dict, Literal[False]]] = None, + system_message: Optional[Union[str, list[str]]] = "You are a helpful AI Assistant.", + llm_config: Optional[Union[dict[str, Any], Literal[False]]] = None, voice: str = "alloy", ): """(Experimental) Agent for interacting with the Realtime Clients. @@ -83,10 +83,10 @@ def __init__( silent=None, context_variables=None, ) - self.llm_config = llm_config + self.llm_config = llm_config # type: ignore[assignment] self._client = OpenAIRealtimeClient(self, audio_adapter, FunctionObserver(self)) self.voice = voice - self.realtime_functions = {} + self.realtime_functions: dict[str, tuple[dict[str, Any], Callable[..., Any]]] = {} self._oai_system_message = [{"content": system_message, "role": "system"}] # todo still needed? self.register_reply( @@ -96,8 +96,8 @@ def __init__( self._answer_event: anyio.Event = anyio.Event() self._answer: str = "" self._start_swarm_chat = False - self._initial_agent = None - self._agents = None + self._initial_agent: Optional[SwarmAgent] = None + self._agents: Optional[list[SwarmAgent]] = None def register_swarm( self, @@ -133,7 +133,7 @@ def register_swarm( self.set_answer ) - async def run(self): + async def run(self) -> None: """Run the agent.""" await self._client.run() @@ -143,11 +143,12 @@ def register_realtime_function( description: str, name: Optional[str] = None, ) -> Callable[[F], F]: - def _decorator(func: F, name=name) -> F: + def _decorator(func: F, name: Optional[str] = name) -> F: """Decorator for registering a function to be used by an agent. Args: - func: the function to be registered. + func (callable[..., Any]): the function to be registered. + name (str): the name of the function. Returns: The function to be registered, with the _description attribute set to the function description. @@ -183,7 +184,7 @@ async def get_answer(self) -> str: await self._answer_event.wait() return self._answer - async def ask_question(self, question: str, question_timeout: int) -> str: + async def ask_question(self, question: str, question_timeout: int) -> None: """ Send a question for the user to the agent and wait for the answer. If the answer is not received within the timeout, the question is repeated. @@ -196,7 +197,7 @@ async def ask_question(self, question: str, question_timeout: int) -> str: self.reset_answer() await self._client.send_text(role=QUESTION_ROLE, text=question) - async def _check_event_set(timeout: int = question_timeout) -> None: + async def _check_event_set(timeout: int = question_timeout) -> bool: for _ in range(timeout): if self._answer_event.is_set(): return True @@ -208,7 +209,7 @@ async def _check_event_set(timeout: int = question_timeout) -> None: def check_termination_and_human_reply( self, - messages: Optional[list[dict]] = None, + messages: Optional[list[dict[str, Any]]] = None, sender: Optional[Agent] = None, config: Optional[Any] = None, ) -> tuple[bool, Union[str, None]]: @@ -225,7 +226,10 @@ def check_termination_and_human_reply( the config for the agent """ - async def get_input(): + if not messages: + return False, None + + async def get_input() -> None: async with create_task_group() as tg: tg.soonify(self.ask_question)( QUESTION_MESSAGE.format(messages[-1]["content"]), @@ -234,4 +238,4 @@ async def get_input(): syncify(get_input)() - return True, {"role": "user", "content": self._answer} + return True, {"role": "user", "content": self._answer} # type: ignore[return-value] diff --git a/autogen/agentchat/realtime_agent/realtime_observer.py b/autogen/agentchat/realtime_agent/realtime_observer.py index 80d59de95c..6061efb230 100644 --- a/autogen/agentchat/realtime_agent/realtime_observer.py +++ b/autogen/agentchat/realtime_agent/realtime_observer.py @@ -6,24 +6,36 @@ # SPDX-License-Identifier: MIT from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any, Optional + +if TYPE_CHECKING: + from .client import OpenAIRealtimeClient class RealtimeObserver(ABC): """Observer for the OpenAI Realtime API.""" - def __init__(self): - self._client = None + def __init__(self) -> None: + self._client: Optional["OpenAIRealtimeClient"] = None + + @property + def client(self) -> "OpenAIRealtimeClient": + """Get the client associated with the observer.""" + if self._client is None: + raise ValueError("Observer client is not registered.") + + return self._client - def register_client(self, client): + def register_client(self, client: "OpenAIRealtimeClient") -> None: """Register a client with the observer.""" self._client = client @abstractmethod - async def run(self, openai_ws): + async def run(self) -> None: """Run the observer.""" - pass + ... @abstractmethod - async def update(self, message): + async def update(self, message: dict[str, Any]) -> None: """Update the observer with a message from the OpenAI Realtime API.""" - pass + ... diff --git a/autogen/agentchat/realtime_agent/twilio_observer.py b/autogen/agentchat/realtime_agent/twilio_observer.py index 4d6c793898..7dff6d4bdd 100644 --- a/autogen/agentchat/realtime_agent/twilio_observer.py +++ b/autogen/agentchat/realtime_agent/twilio_observer.py @@ -8,11 +8,13 @@ import base64 import json import logging - -from fastapi import WebSocketDisconnect +from typing import TYPE_CHECKING, Any, Optional from .realtime_observer import RealtimeObserver +if TYPE_CHECKING: + from fastapi.websockets import WebSocket + LOG_EVENT_TYPES = [ "error", "response.content.done", @@ -31,7 +33,7 @@ class TwilioAudioAdapter(RealtimeObserver): """Adapter for streaming audio from Twilio to OpenAI Realtime API and vice versa.""" - def __init__(self, websocket): + def __init__(self, websocket: "WebSocket"): """Adapter for streaming audio from Twilio to OpenAI Realtime API and vice versa. Args: @@ -45,10 +47,10 @@ def __init__(self, websocket): self.stream_sid = None self.latest_media_timestamp = 0 self.last_assistant_item = None - self.mark_queue = [] - self.response_start_timestamp_twilio = None + self.mark_queue: list[str] = [] + self.response_start_timestamp_twilio: Optional[int] = None - async def update(self, response): + async def update(self, response: dict[str, Any]) -> None: """Receive events from the OpenAI Realtime API, send audio back to Twilio.""" if response["type"] in LOG_EVENT_TYPES: logger.info(f"Received event: {response['type']}", response) @@ -76,7 +78,7 @@ async def update(self, response): logger.info(f"Interrupting response with id: {self.last_assistant_item}") await self.handle_speech_started_event() - async def handle_speech_started_event(self): + async def handle_speech_started_event(self) -> None: """Handle interruption when the caller's speech starts.""" logger.info("Handling speech started event.") if self.mark_queue and self.response_start_timestamp_twilio is not None: @@ -104,19 +106,19 @@ async def handle_speech_started_event(self): self.last_assistant_item = None self.response_start_timestamp_twilio = None - async def send_mark(self): + async def send_mark(self) -> None: """Send a mark of audio interruption to the Twilio websocket.""" if self.stream_sid: mark_event = {"event": "mark", "streamSid": self.stream_sid, "mark": {"name": "responsePart"}} await self.websocket.send_json(mark_event) self.mark_queue.append("responsePart") - async def run(self): + async def run(self) -> None: """Run the adapter. Start reading messages from the Twilio websocket and send audio to OpenAI. """ - openai_ws = self._client._openai_ws + openai_ws = self.client.openai_ws await self.initialize_session() async for message in self.websocket.iter_text(): @@ -135,10 +137,10 @@ async def run(self): if self.mark_queue: self.mark_queue.pop(0) - async def initialize_session(self): + async def initialize_session(self) -> None: """Control initial session with OpenAI.""" session_update = { "input_audio_format": "g711_ulaw", "output_audio_format": "g711_ulaw", } - await self._client.session_update(session_update) + await self.client.session_update(session_update) diff --git a/autogen/agentchat/realtime_agent/websocket_observer.py b/autogen/agentchat/realtime_agent/websocket_observer.py index 9509e2b314..dd0b67a87d 100644 --- a/autogen/agentchat/realtime_agent/websocket_observer.py +++ b/autogen/agentchat/realtime_agent/websocket_observer.py @@ -7,8 +7,10 @@ import base64 import json +from typing import TYPE_CHECKING, Any, Optional -from fastapi import WebSocketDisconnect +if TYPE_CHECKING: + from fastapi.websockets import WebSocket from .realtime_observer import RealtimeObserver @@ -26,7 +28,7 @@ class WebsocketAudioAdapter(RealtimeObserver): - def __init__(self, websocket): + def __init__(self, websocket: "WebSocket"): super().__init__() self.websocket = websocket @@ -34,10 +36,10 @@ def __init__(self, websocket): self.stream_sid = None self.latest_media_timestamp = 0 self.last_assistant_item = None - self.mark_queue = [] - self.response_start_timestamp_socket = None + self.mark_queue: list[str] = [] + self.response_start_timestamp_socket: Optional[int] = None - async def update(self, response): + async def update(self, response: dict[str, Any]) -> None: """Receive events from the OpenAI Realtime API, send audio back to websocket.""" if response["type"] in LOG_EVENT_TYPES: print(f"Received event: {response['type']}", response) @@ -65,7 +67,7 @@ async def update(self, response): print(f"Interrupting response with id: {self.last_assistant_item}") await self.handle_speech_started_event() - async def handle_speech_started_event(self): + async def handle_speech_started_event(self) -> None: """Handle interruption when the caller's speech starts.""" print("Handling speech started event.") if self.mark_queue and self.response_start_timestamp_socket is not None: @@ -93,14 +95,14 @@ async def handle_speech_started_event(self): self.last_assistant_item = None self.response_start_timestamp_socket = None - async def send_mark(self): + async def send_mark(self) -> None: if self.stream_sid: mark_event = {"event": "mark", "streamSid": self.stream_sid, "mark": {"name": "responsePart"}} await self.websocket.send_json(mark_event) self.mark_queue.append("responsePart") - async def run(self): - openai_ws = self._client._openai_ws + async def run(self) -> None: + openai_ws = self.client.openai_ws await self.initialize_session() async for message in self.websocket.iter_text(): @@ -119,7 +121,7 @@ async def run(self): if self.mark_queue: self.mark_queue.pop(0) - async def initialize_session(self): + async def initialize_session(self) -> None: """Control initial session with OpenAI.""" session_update = {"input_audio_format": "pcm16", "output_audio_format": "pcm16"} # g711_ulaw # "g711_ulaw", - await self._client.session_update(session_update) + await self.client.session_update(session_update) diff --git a/notebook/agentchat_realtime_swarm.ipynb b/notebook/agentchat_realtime_swarm.ipynb index 909f5ad95d..a78598025a 100644 --- a/notebook/agentchat_realtime_swarm.ipynb +++ b/notebook/agentchat_realtime_swarm.ipynb @@ -18,7 +18,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Install AG2 with realtime-twilio dependencies\n", + "## Install AG2 with `twilio` \n", "\n", "To use the realtime agent we will connect it to twilio service, this tutorial was inspired by [twilio tutorial](https://www.twilio.com/en-us/blog/voice-ai-assistant-openai-realtime-api-node) for connecting to OpenAPI real-time agent.\n", "\n", @@ -461,7 +461,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.16" + "version": "3.10.13" } }, "nbformat": 4, diff --git a/pyproject.toml b/pyproject.toml index 8f1db523ec..20f110b2e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,11 +62,13 @@ files = [ "autogen/io", "autogen/tools", "autogen/interop", + "autogen/agentchat/realtime_agent", "test/test_pydantic.py", "test/test_function_utils.py", "test/io", "test/tools", "test/interop", + "test/agentchat/realtime_agent", ] exclude = [ "autogen/math_utils\\.py", diff --git a/setup.py b/setup.py index e8eda4d66f..748f1bcc2e 100644 --- a/setup.py +++ b/setup.py @@ -82,6 +82,7 @@ "llama-index-core==0.12.5", ] +# used for twilio = ["fastapi>=0.115.0,<1", "uvicorn>=0.30.6,<1", "twilio>=9.3.2"] interop_crewai = ["crewai[tools]>=0.86,<1; python_version>='3.10' and python_version<'3.13'"] diff --git a/test/agentchat/realtime_agent/__init__.py b/test/agentchat/realtime_agent/__init__.py new file mode 100644 index 0000000000..87ec7612a0 --- /dev/null +++ b/test/agentchat/realtime_agent/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) 2023 - 2024, Owners of https://github.com/ag2ai +# +# SPDX-License-Identifier: Apache-2.0 +# +# Portions derived from https://github.com/microsoft/autogen are under the MIT License. +# SPDX-License-Identifier: MIT diff --git a/test/agentchat/realtime_agent/test_submodule.py b/test/agentchat/realtime_agent/test_submodule.py new file mode 100644 index 0000000000..eff9f04964 --- /dev/null +++ b/test/agentchat/realtime_agent/test_submodule.py @@ -0,0 +1,15 @@ +# Copyright (c) 2023 - 2024, Owners of https://github.com/ag2ai +# +# SPDX-License-Identifier: Apache-2.0 +# +# Portions derived from https://github.com/microsoft/autogen are under the MIT License. +# SPDX-License-Identifier: MIT + + +def test_import() -> None: + from autogen.agentchat.realtime_agent import ( + FunctionObserver, + RealtimeAgent, + TwilioAudioAdapter, + WebsocketAudioAdapter, + ) From 7a36e71a99132a95a2bf22d41bdb066666c9f82c Mon Sep 17 00:00:00 2001 From: Tvrtko Sternak Date: Fri, 20 Dec 2024 17:11:23 +0100 Subject: [PATCH 12/13] Finish missing comment in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 748f1bcc2e..bc2dd0e1ce 100644 --- a/setup.py +++ b/setup.py @@ -82,7 +82,7 @@ "llama-index-core==0.12.5", ] -# used for +# used for agentchat_realtime_swarm notebook and realtime agent twilio demo twilio = ["fastapi>=0.115.0,<1", "uvicorn>=0.30.6,<1", "twilio>=9.3.2"] interop_crewai = ["crewai[tools]>=0.86,<1; python_version>='3.10' and python_version<'3.13'"] From 5b43da5e90044c59897de46fc7d15f323e0bef7d Mon Sep 17 00:00:00 2001 From: Tvrtko Sternak Date: Fri, 20 Dec 2024 17:32:01 +0100 Subject: [PATCH 13/13] Add fastapi to types extra --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bc2dd0e1ce..b382c66a41 100644 --- a/setup.py +++ b/setup.py @@ -90,7 +90,7 @@ interop_pydantic_ai = ["pydantic-ai==0.0.13"] interop = interop_crewai + interop_langchain + interop_pydantic_ai -types = ["mypy==1.9.0"] + test + jupyter_executor + interop +types = ["mypy==1.9.0"] + test + jupyter_executor + interop + ["fastapi>=0.115.0,<1"] if current_os in ["Windows", "Darwin"]: retrieve_chat_pgvector.extend(["psycopg[binary]>=3.1.18"])