From ff2900a168dc52c9c27e29aa371e30c53d99104d Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Fri, 20 Dec 2024 10:32:04 +0100 Subject: [PATCH 01/10] Blog polishing WIP --- website/README.md | 2 +- .../img/one_to_rule_them_all.png | 3 ++ .../index.mdx | 15 ++++-- website/mint.json | 48 ++++++++++++++++++- 4 files changed, 63 insertions(+), 5 deletions(-) create mode 100644 website/blog/2024-12-18-Tools-interoperability/img/one_to_rule_them_all.png diff --git a/website/README.md b/website/README.md index 22a4e10d6a..e1996afcec 100644 --- a/website/README.md +++ b/website/README.md @@ -25,7 +25,7 @@ Install it [here](https://github.com/quarto-dev/quarto-cli/releases). Navigate to the `website` folder and run: ```console -pydoc-markdown +python ./process_api_reference.py python ./process_notebooks.py render npm install ``` diff --git a/website/blog/2024-12-18-Tools-interoperability/img/one_to_rule_them_all.png b/website/blog/2024-12-18-Tools-interoperability/img/one_to_rule_them_all.png new file mode 100644 index 0000000000..45cf075e36 --- /dev/null +++ b/website/blog/2024-12-18-Tools-interoperability/img/one_to_rule_them_all.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f41e305d4525e167b62d0c769599ec2667ef282918e835372ad20a1b3b3c03c +size 310123 diff --git a/website/blog/2024-12-18-Tools-interoperability/index.mdx b/website/blog/2024-12-18-Tools-interoperability/index.mdx index 3f6ae0962d..4b3a23d8bb 100644 --- a/website/blog/2024-12-18-Tools-interoperability/index.mdx +++ b/website/blog/2024-12-18-Tools-interoperability/index.mdx @@ -14,9 +14,18 @@ AG2 lets you bring in tools from different frameworks like **LangChain**, **Crew With AG2, you can combine these tools and enhance your agents' capabilities. + +![One to rule them all img](img/one_to_rule_them_all.png) + In this post, we’ll walk through how to integrate tools from various frameworks—like [LangChain Tools](https://python.langchain.com/v0.1/docs/modules/tools), [CrewAI Tools](https://github.com/crewAIInc/crewAI-tools/tree/main), and [PydanticAI Tools](https://ai.pydantic.dev/tools/)—into AG2. -This allows you to use tools from different frameworks within AG2, giving your agents more power and flexibility. You’ll see how to set up agents, adapt the tools, and validate the integration through examples.his post, you will understand how to configure agents, adapt these tools for use in AG2, and validate the integration through practical examples. + +And let’s be honest: + +Because, really, the magic happens when you combine them all. This allows you to use tools from different frameworks within AG2, giving your agents more power and flexibility. + +In this post, you will understand how to configure agents, adapt these tools for use in AG2, and validate the integration through practical examples. + ## LangChain Tools Integration @@ -75,8 +84,8 @@ chatbot = AssistantAgent( ### Tool Integration - Initialize and register the LangChain tool with AG2. -- `WikipediaAPIWrapper`: Configured to fetch the top 1 result from Wikipedia with a maximum of 1000 characters per document. -- `WikipediaQueryRun`: A LangChain tool that executes Wikipedia queries. +- [WikipediaAPIWrapper](https://python.langchain.com/api_reference/community/utilities/langchain_community.utilities.wikipedia.WikipediaAPIWrapper.html): Configured to fetch the top 1 result from Wikipedia with a maximum of 1000 characters per document. +- [WikipediaQueryRun](https://python.langchain.com/api_reference/community/tools/langchain_community.tools.wikipedia.tool.WikipediaQueryRun.html): A LangChain tool that executes Wikipedia queries. - `LangchainInteroperability`: Converts the LangChain tool into a format compatible with the AG2 framework. - `ag2_tool.register_for_execution(user_proxy)`: Registers the tool for use by the user_proxy agent. - `ag2_tool.register_for_llm(chatbot)`: Registers the tool for integration with the chatbot agent. diff --git a/website/mint.json b/website/mint.json index 5ee80c8390..798f7e7f22 100644 --- a/website/mint.json +++ b/website/mint.json @@ -300,6 +300,17 @@ "docs/reference/agentchat/contrib/web_surfer" ] }, + { + "group": "agentchat.realtime_agent", + "pages": [ + "docs/reference/agentchat/realtime_agent/client", + "docs/reference/agentchat/realtime_agent/function_observer", + "docs/reference/agentchat/realtime_agent/realtime_agent", + "docs/reference/agentchat/realtime_agent/realtime_observer", + "docs/reference/agentchat/realtime_agent/twilio_observer", + "docs/reference/agentchat/realtime_agent/websocket_observer" + ] + }, "docs/reference/agentchat/agent", "docs/reference/agentchat/assistant_agent", "docs/reference/agentchat/chat", @@ -344,6 +355,33 @@ "docs/reference/coding/utils" ] }, + { + "group": "interop", + "pages": [ + { + "group": "interop.crewai", + "pages": [ + "docs/reference/interop/crewai/crewai" + ] + }, + { + "group": "interop.langchain", + "pages": [ + "docs/reference/interop/langchain/langchain" + ] + }, + { + "group": "interop.pydantic_ai", + "pages": [ + "docs/reference/interop/pydantic_ai/pydantic_ai", + "docs/reference/interop/pydantic_ai/pydantic_ai_tool" + ] + }, + "docs/reference/interop/interoperability", + "docs/reference/interop/interoperable", + "docs/reference/interop/registry" + ] + }, { "group": "io", "pages": [ @@ -377,6 +415,13 @@ "docs/reference/oai/together" ] }, + { + "group": "tools", + "pages": [ + "docs/reference/tools/pydantic_ai_tool", + "docs/reference/tools/tool" + ] + }, "docs/reference/browser_utils", "docs/reference/code_utils", "docs/reference/exception_utils", @@ -450,6 +495,7 @@ { "group": "Recent posts", "pages": [ + "blog/2024-12-18-Tools-interoperability/index", "blog/2024-12-06-FalkorDB-Structured/index", "blog/2024-12-02-ReasoningAgent2/index", "blog/2024-11-27-Prompt-Leakage-Probing/index", @@ -585,4 +631,4 @@ "discord": "https://discord.gg/pAbnFJrkgZ", "youtube": "https://www.youtube.com/@ag2ai" } -} +} \ No newline at end of file From 3b6c90e9f8e8538307351a8a9a5d50bd587534ba Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Fri, 20 Dec 2024 11:14:48 +0100 Subject: [PATCH 02/10] Update interoperability blog --- .../index.mdx | 108 +++++++++++------- website/mint.json | 6 +- 2 files changed, 68 insertions(+), 46 deletions(-) diff --git a/website/blog/2024-12-18-Tools-interoperability/index.mdx b/website/blog/2024-12-18-Tools-interoperability/index.mdx index 4b3a23d8bb..6c3f43da1c 100644 --- a/website/blog/2024-12-18-Tools-interoperability/index.mdx +++ b/website/blog/2024-12-18-Tools-interoperability/index.mdx @@ -29,16 +29,16 @@ In this post, you will understand how to configure agents, adapt these tools for ## LangChain Tools Integration -LangChain is a popular framework that offers a wide range of tools to work with LLMs. LangChain has already implemented a variety of tools that can be easily integrated into AG2. You can explore the available tools in the [LangChain Community Tools](https://github.com/langchain-ai/langchain/tree/master/libs/community/langchain_community/tools) folder. These tools, such as those for querying APIs, web scraping, and text generation, can be quickly incorporated into AG2, providing powerful functionality for your agents. +LangChain is a popular framework with lots of tools for working with LLMs. It's got a range of tools that can be easily integrated into AG2. If you want to see the full list, check out the [LangChain Community Tools](https://github.com/langchain-ai/langchain/tree/master/libs/community/langchain_community/tools). You can quickly add things like API queries, web scraping, and text generation to your AG2 setup. ### Installation -To integrate LangChain tools into the AG2 framework, install the required dependencies: +To get LangChain tools working with AG2, you’ll need to install a couple of dependencies: ```bash pip install ag2[interop-langchain] ``` -Additionally, this notebook uses LangChain's [Wikipedia Tool](https://api.python.langchain.com/en/latest/tools/langchain_community.tools.wikipedia.tool.WikipediaQueryRun.html), which requires the `wikipedia` package. Install it with: +Also, we’ll use LangChain’s [Wikipedia Tool](https://python.langchain.com/docs/integrations/tools/wikipedia/), which needs the wikipedia package. Install it like this: ```bash pip install wikipedia @@ -46,10 +46,11 @@ pip install wikipedia ### Imports -Import necessary modules and tools. -- `WikipediaQueryRun` and `WikipediaAPIWrapper`: Tools for querying Wikipedia. -- `AssistantAgent` and `UserProxyAgent`: Agents that facilitate communication in the AG2 framework. -- `Interoperability`: This module acts as a bridge, making it easier to integrate LangChain tools with AG2’s architecture. +Now, let’s import the necessary modules and tools. + +- [WikipediaQueryRun](https://api.python.langchain.com/en/latest/tools/langchain_community.tools.wikipedia.tool.WikipediaQueryRun.html) and [WikipediaAPIWrapper](https://python.langchain.com/api_reference/community/utilities/langchain_community.utilities.wikipedia.WikipediaAPIWrapper.html) are the tools for querying Wikipedia. +- `AssistantAgent` and `UserProxyAgent` are the agents for interaction within AG2. +- `Interoperability` is what helps connect LangChain tools with AG2. ```python import os @@ -63,8 +64,8 @@ from autogen.interop import Interoperability ### Agent Configuration -Configure the agents for the interaction. -- `config_list` defines the LLM configurations, including the model and API key. +Let’s set up the agents for interaction. +- `config_list` is where you define the LLM configuration, like the model and API key. - `UserProxyAgent` simulates user inputs without requiring actual human interaction (set to `NEVER`). - `AssistantAgent` represents the AI agent, configured with the LLM settings. @@ -82,13 +83,11 @@ chatbot = AssistantAgent( ``` ### Tool Integration - -- Initialize and register the LangChain tool with AG2. -- [WikipediaAPIWrapper](https://python.langchain.com/api_reference/community/utilities/langchain_community.utilities.wikipedia.WikipediaAPIWrapper.html): Configured to fetch the top 1 result from Wikipedia with a maximum of 1000 characters per document. -- [WikipediaQueryRun](https://python.langchain.com/api_reference/community/tools/langchain_community.tools.wikipedia.tool.WikipediaQueryRun.html): A LangChain tool that executes Wikipedia queries. -- `LangchainInteroperability`: Converts the LangChain tool into a format compatible with the AG2 framework. -- `ag2_tool.register_for_execution(user_proxy)`: Registers the tool for use by the user_proxy agent. -- `ag2_tool.register_for_llm(chatbot)`: Registers the tool for integration with the chatbot agent. +Here’s where we connect everything. +- First, we set up [WikipediaAPIWrapper](https://python.langchain.com/api_reference/community/utilities/langchain_community.utilities.wikipedia.WikipediaAPIWrapper.html), which fetches the top Wikipedia result (with a character limit). +- Then, we use [WikipediaQueryRun](https://api.python.langchain.com/en/latest/tools/langchain_community.tools.wikipedia.tool.WikipediaQueryRun.html) to perform Wikipedia queries. +- `Interoperability` helps convert the LangChain tool to AG2’s format. +- Finally, we register the tool for use with both the `user_proxy` and `chatbot`. ```python api_wrapper = WikipediaAPIWrapper(top_k_results=1, doc_content_chars_max=1000) @@ -101,13 +100,18 @@ ag2_tool.register_for_execution(user_proxy) ag2_tool.register_for_llm(chatbot) ``` -### Initiate Chat +### Initiating the Chat + +Once everything’s set up, we can send a message to the chatbot, and it’ll use the Wikipedia tool to fetch the relevant information. + ```python message = "Tell me about the history of the United States" user_proxy.initiate_chat(recipient=chatbot, message=message, max_turns=2) ``` -Output: +### Output + +When the chat is initiated, here’s the output you’ll see: ```console User (to chatbot): @@ -162,8 +166,8 @@ pip install ag2[interop-crewai] ### Imports Import necessary modules and tools. -- `ScrapeWebsiteTool` are the CrewAI tools for web scraping -- `AssistantAgent` and `UserProxyAgent` are core AG2 classes. +- [ScrapeWebsiteTool](https://docs.crewai.com/tools/scrapewebsitetool): A CrewAI tool for web scraping. +- `AssistantAgent` and `UserProxyAgent`: Core AG2 classes. - `Interoperability`: This module acts as a bridge, making it easier to integrate CrewAI tools with AG2’s architecture. ```python @@ -197,10 +201,10 @@ chatbot = AssistantAgent( ### Tool Integration -Initialize and register the CrewAI tool with AG2. -- `crewai_tool` is an instance of the `ScrapeWebsiteTool` from CrewAI. -- `Interoperability` converts the CrewAI tool to make it usable in AG2. -- `register_for_execution` and `register_for_llm` allow the tool to work with the UserProxyAgent and AssistantAgent. +Integrate the CrewAI tool with AG2. +- [ScrapeWebsiteTool](https://docs.crewai.com/tools/scrapewebsitetool) is used for web scraping tasks. +- `Interoperability` converts the CrewAI tool to a format compatible with AG2. +- Register the tool for both execution and interaction with LLMs. ```python interop = Interoperability() @@ -209,13 +213,17 @@ ag2_tool = interop.convert_tool(tool=crewai_tool, type="crewai") ag2_tool.register_for_execution(user_proxy) ag2_tool.register_for_llm(chatbot) +``` +### Initiating the chat +Initiate the conversation between the `UserProxyAgent` and the `AssistantAgent` to utilize the CrewAI tool. +```python message = "Scrape the website https://ag2.ai/" - chat_result = user_proxy.initiate_chat(recipient=chatbot, message=message, max_turns=2) ``` -Output: +### Output +The `chatbot` provides results based on the web scraping operation: ```console User (to chatbot): @@ -271,12 +279,12 @@ This comprehensive platform seems to aim at both individual developers and enter -------------------------------------------------------------------------------- ``` +You can also access a detailed summary of the interaction: + ```python print(chat_result.summary) ``` -Output: - ```console The website "https://ag2.ai/" promotes a platform named AgentOS, which is designed for building multi-agent systems efficiently. Key highlights from the website are: @@ -303,10 +311,10 @@ This comprehensive platform seems to aim at both individual developers and enter ## PydanticAI Tools Integration -[PydanticAI](https://ai.pydantic.dev/) is a newer framework that offers powerful capabilities for working with LLMs. Although it currently does not have a repository with pre-built tools, it provides features like **dependency injection**, allowing you to inject a "Context" into a tool for better execution without relying on LLMs. This context can be used for passing parameters or managing state during the execution of a tool. While the framework is still growing, you can integrate its tools into AG2 to enhance agent capabilities, especially for tasks that involve structured data and context-driven logic. +[PydanticAI](https://ai.pydantic.dev/) is a newer framework that brings powerful features for working with LLMs. Although it doesn't yet have a collection of pre-built tools like other frameworks, it offers useful capabilities such as **dependency injection**. This feature allows you to inject a "Context" into tools, which can help pass parameters or manage state without relying on LLMs. Though it's still evolving, you can easily integrate PydanticAI tools into AG2 to boost agent capabilities, particularly for tasks that involve structured data and context-driven logic. ### Installation -To integrate LangChain tools into the AG2 framework, install the required dependencies: +To get PydanticAI tools working with AG2, install the necessary dependencies: ```bash pip install ag2[interop-pydantic-ai] @@ -317,8 +325,8 @@ pip install ag2[interop-pydantic-ai] Import necessary modules and tools. - `BaseModel`: Used to define data structures for tool inputs and outputs. -- `RunContext`: Provides context during the execution of tools. -- `PydanticAITool`: Represents a tool in the PydanticAI framework. +- [RunContext](https://ai.pydantic.dev/api/tools/#pydantic_ai.tools.RunContext): Provides context during the execution of tools. +- [PydanticAITool](https://ai.pydantic.dev/api/tools/#pydantic_ai.tools.Tool): Represents a tool in the PydanticAI framework. - `AssistantAgent` and `UserProxyAgent`: Agents that facilitate communication in the AG2 framework. - `Interoperability`: This module acts as a bridge, making it easier to integrate PydanticAI tools with AG2’s architecture. @@ -356,14 +364,13 @@ chatbot = AssistantAgent( ### Tool Integration -Integrate the PydanticAI tool with AG2. +To integrate a PydanticAI tool into AG2: -- Define a `Player` model using `BaseModel` to structure the input data. -- Use `RunContext` to securely inject dependencies (like the `Player` instance) into the tool function without exposing them to the LLM. -- Implement `get_player` to define the tool's functionality, accessing `ctx.deps` for injected data. -- Convert the tool to an AG2-compatible format with `Interoperability` and register it for execution and LLM communication. -- Convert the PydanticAI tool into an AG2-compatible format using `convert_tool`. -- Register the tool for both execution and communication with the LLM by associating it with the `user_proxy` and `chatbot`. +- First, define a `Player` model using `BaseModel` to structure the input data. +- Use [RunContext](https://ai.pydantic.dev/api/tools/#pydantic_ai.tools.RunContext) to inject dependencies (like the `Player` instance) securely into the tool. +- The `get_player` function defines the tool’s functionality, retrieving injected data through `ctx.deps`. +- Then, convert the tool into an AG2-compatible format with `Interoperability`. +- Register the tool for execution and interaction with both the `user_proxy` and `chatbot`. ```python class Player(BaseModel): @@ -391,11 +398,12 @@ ag2_tool.register_for_execution(user_proxy) ag2_tool.register_for_llm(chatbot) ``` -Initiate a conversation between the `UserProxyAgent` and the `AssistantAgent`. +### Initiating the chat +Now that everything is set up, you can initiate a chat between the `UserProxyAgent` and the `AssistantAgent`: -- Use the `initiate_chat` method to send a message from the `user_proxy` to the `chatbot`. -- In this example, the user requests the chatbot to retrieve player information, providing "goal keeper" as additional context. -- The `Player` instance is securely injected into the tool using `RunContext`, ensuring the chatbot can retrieve and use this data during the interaction. +- The `user_proxy` sends a message to the `chatbot`. +- The user requests player information, and includes "goal keeper" as additional context. +- The `Player` data is securely injected into the tool, and the chatbot can access and use it during the chat. ```python user_proxy.initiate_chat( @@ -403,7 +411,7 @@ user_proxy.initiate_chat( ) ``` -Output: +### Output ```console User (to chatbot): @@ -432,3 +440,15 @@ chatbot (to User): The player's name is Luka, who is a 25-year-old goalkeeper. TERMINATE ``` + +## Summary + +In this post, we've explored how to integrate tools from multiple frameworks (LangChain, CrewAI, and PydanticAI) into the AG2 framework, enabling cross-framework interoperability. By integrating these tools, you can enhance your agents with a variety of capabilities, such as API querying, web scraping, and structured data processing. + +- **LangChain** offers a wide range of pre-built tools for working with APIs and web scraping, making it easy to extend AG2's functionality. +- **CrewAI** brings diverse tools for search, web scraping, and more, allowing for robust agent interactions. +- **PydanticAI** introduces dependency injection and context-driven logic, enabling efficient data handling without relying on LLMs. + +With AG2's flexible architecture and the power of these frameworks, developers can create agents that are more capable and adaptable. By following the integration steps for each framework, you can enhance your agents' performance, expand their capabilities, and create more dynamic interactions. + +Now you should have a better understanding of how to integrate tools from different frameworks into AG2, and how to use these tools effectively within your own projects. diff --git a/website/mint.json b/website/mint.json index 798f7e7f22..9b5cdb5ae1 100644 --- a/website/mint.json +++ b/website/mint.json @@ -617,7 +617,9 @@ "notebooks/JSON_mode_example", "notebooks/agentchat_RetrieveChat", "notebooks/agentchat_graph_rag_neo4j", - "notebooks/agentchat_swarm_enhanced" + "notebooks/agentchat_swarm_enhanced", + "notebooks/agentchat_reasoning_agent", + "notebooks/agentchat_realtime_swarm" ] }, "notebooks/Gallery" @@ -631,4 +633,4 @@ "discord": "https://discord.gg/pAbnFJrkgZ", "youtube": "https://www.youtube.com/@ag2ai" } -} \ No newline at end of file +} From 3ff42544b03bc3511ff974dd5cc3dcccffad3952 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Fri, 20 Dec 2024 12:33:12 +0100 Subject: [PATCH 03/10] Fix notebook referencing in the blog --- notebook/tools_interoperability.ipynb | 18 +++++++- .../index.mdx | 41 +++++++++---------- website/mint.json | 1 + website/snippets/data/NotebooksMetadata.mdx | 8 ++++ 4 files changed, 46 insertions(+), 22 deletions(-) diff --git a/notebook/tools_interoperability.ipynb b/notebook/tools_interoperability.ipynb index 00469be0d4..170a6e21c0 100644 --- a/notebook/tools_interoperability.ipynb +++ b/notebook/tools_interoperability.ipynb @@ -38,6 +38,7 @@ "### Imports\n", "\n", "Import necessary modules and tools.\n", + "\n", "- `WikipediaQueryRun` and `WikipediaAPIWrapper`: Tools for querying Wikipedia.\n", "- `AssistantAgent` and `UserProxyAgent`: Agents that facilitate communication in the AG2 framework.\n", "- `Interoperability`: This module acts as a bridge, making it easier to integrate LangChain tools with AG2’s architecture." @@ -65,6 +66,7 @@ "### Agent Configuration\n", "\n", "Configure the agents for the interaction.\n", + "\n", "- `config_list` defines the LLM configurations, including the model and API key.\n", "- `UserProxyAgent` simulates user inputs without requiring actual human interaction (set to `NEVER`).\n", "- `AssistantAgent` represents the AI agent, configured with the LLM settings." @@ -152,6 +154,7 @@ "### Imports\n", "\n", "Import necessary modules and tools.\n", + "\n", "- `ScrapeWebsiteTool` are the CrewAI tools for web scraping\n", "- `AssistantAgent` and `UserProxyAgent` are core AG2 classes.\n", "- `Interoperability`: This module acts as a bridge, making it easier to integrate CrewAI tools with AG2’s architecture." @@ -178,6 +181,7 @@ "### Agent Configuration\n", "\n", "Configure the agents for the interaction.\n", + "\n", "- `config_list` defines the LLM configurations, including the model and API key.\n", "- `UserProxyAgent` simulates user inputs without requiring actual human interaction (set to `NEVER`).\n", "- `AssistantAgent` represents the AI agent, configured with the LLM settings." @@ -208,6 +212,7 @@ "### Tool Integration\n", "\n", "Initialize and register the CrewAI tool with AG2.\n", + "\n", "- `crewai_tool` is an instance of the `ScrapeWebsiteTool` from CrewAI.\n", "- `Interoperability` converts the CrewAI tool to make it usable in AG2.\n", "- `register_for_execution` and `register_for_llm` allow the tool to work with the UserProxyAgent and AssistantAgent." @@ -260,9 +265,10 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Imports\n", + "### Imports\n", "\n", "Import necessary modules and tools.\n", + "\n", "- `BaseModel`: Used to define data structures for tool inputs and outputs.\n", "- `RunContext`: Provides context during the execution of tools.\n", "- `PydanticAITool`: Represents a tool in the PydanticAI framework.\n", @@ -294,6 +300,7 @@ "### Agent Configuration\n", "\n", "Configure the agents for the interaction.\n", + "\n", "- `config_list` defines the LLM configurations, including the model and API key.\n", "- `UserProxyAgent` simulates user inputs without requiring actual human interaction (set to `NEVER`).\n", "- `AssistantAgent` represents the AI agent, configured with the LLM settings." @@ -395,6 +402,15 @@ } ], "metadata": { + "front_matter": { + "description": "Cross-Framework LLM Tool Integration with AG2", + "tags": [ + "tools", + "langchain", + "crewai", + "pydanticai" + ] + }, "kernelspec": { "display_name": ".venv", "language": "python", diff --git a/website/blog/2024-12-18-Tools-interoperability/index.mdx b/website/blog/2024-12-18-Tools-interoperability/index.mdx index 6c3f43da1c..07946bb9e0 100644 --- a/website/blog/2024-12-18-Tools-interoperability/index.mdx +++ b/website/blog/2024-12-18-Tools-interoperability/index.mdx @@ -20,9 +20,8 @@ With AG2, you can combine these tools and enhance your agents' capabilities. In this post, we’ll walk through how to integrate tools from various frameworks—like [LangChain Tools](https://python.langchain.com/v0.1/docs/modules/tools), [CrewAI Tools](https://github.com/crewAIInc/crewAI-tools/tree/main), and [PydanticAI Tools](https://ai.pydantic.dev/tools/)—into AG2. -And let’s be honest: - Because, really, the magic happens when you combine them all. This allows you to use tools from different frameworks within AG2, giving your agents more power and flexibility. +This blog builds upon the concepts covered in the [Tool Integration notebook](/notebooks/tools_interoperability). In this post, you will understand how to configure agents, adapt these tools for use in AG2, and validate the integration through practical examples. @@ -49,8 +48,8 @@ pip install wikipedia Now, let’s import the necessary modules and tools. - [WikipediaQueryRun](https://api.python.langchain.com/en/latest/tools/langchain_community.tools.wikipedia.tool.WikipediaQueryRun.html) and [WikipediaAPIWrapper](https://python.langchain.com/api_reference/community/utilities/langchain_community.utilities.wikipedia.WikipediaAPIWrapper.html) are the tools for querying Wikipedia. -- `AssistantAgent` and `UserProxyAgent` are the agents for interaction within AG2. -- `Interoperability` is what helps connect LangChain tools with AG2. +- [`AssistantAgent`](/docs/reference/agentchat/assistant_agent) and [`UserProxyAgent`](/docs/reference/agentchat/user_proxy_agent) are the agents for interaction within AG2. +- [`Interoperability`](/docs/reference/interop/interoperability) is what helps connect LangChain tools with AG2. ```python import os @@ -66,8 +65,8 @@ from autogen.interop import Interoperability Let’s set up the agents for interaction. - `config_list` is where you define the LLM configuration, like the model and API key. -- `UserProxyAgent` simulates user inputs without requiring actual human interaction (set to `NEVER`). -- `AssistantAgent` represents the AI agent, configured with the LLM settings. +- [`UserProxyAgent`](/docs/reference/agentchat/user_proxy_agent) simulates user inputs without requiring actual human interaction (set to `NEVER`). +- [`AssistantAgent`](/docs/reference/agentchat/assistant_agent) represents the AI agent, configured with the LLM settings. ```python config_list = [{"model": "gpt-4o", "api_key": os.environ["OPENAI_API_KEY"]}] @@ -86,7 +85,7 @@ chatbot = AssistantAgent( Here’s where we connect everything. - First, we set up [WikipediaAPIWrapper](https://python.langchain.com/api_reference/community/utilities/langchain_community.utilities.wikipedia.WikipediaAPIWrapper.html), which fetches the top Wikipedia result (with a character limit). - Then, we use [WikipediaQueryRun](https://api.python.langchain.com/en/latest/tools/langchain_community.tools.wikipedia.tool.WikipediaQueryRun.html) to perform Wikipedia queries. -- `Interoperability` helps convert the LangChain tool to AG2’s format. +- [`Interoperability`](/docs/reference/interop/interoperability) helps convert the LangChain tool to AG2’s format. - Finally, we register the tool for use with both the `user_proxy` and `chatbot`. ```python @@ -167,8 +166,8 @@ pip install ag2[interop-crewai] Import necessary modules and tools. - [ScrapeWebsiteTool](https://docs.crewai.com/tools/scrapewebsitetool): A CrewAI tool for web scraping. -- `AssistantAgent` and `UserProxyAgent`: Core AG2 classes. -- `Interoperability`: This module acts as a bridge, making it easier to integrate CrewAI tools with AG2’s architecture. +- [`AssistantAgent`](/docs/reference/agentchat/assistant_agent) and [`UserProxyAgent`](/docs/reference/agentchat/user_proxy_agent): Core AG2 classes. +- [`Interoperability`](/docs/reference/interop/interoperability): This module acts as a bridge, making it easier to integrate CrewAI tools with AG2’s architecture. ```python import os @@ -183,8 +182,8 @@ from autogen.interop import Interoperability Configure the agents for the interaction. - `config_list` defines the LLM configurations, including the model and API key. -- `UserProxyAgent` simulates user inputs without requiring actual human interaction (set to `NEVER`). -- `AssistantAgent` represents the AI agent, configured with the LLM settings. +- [`UserProxyAgent`](/docs/reference/agentchat/user_proxy_agent) simulates user inputs without requiring actual human interaction (set to `NEVER`). +- [`AssistantAgent`](/docs/reference/agentchat/assistant_agent) represents the AI agent, configured with the LLM settings. ```python config_list = [{"model": "gpt-4o", "api_key": os.environ["OPENAI_API_KEY"]}] @@ -202,8 +201,8 @@ chatbot = AssistantAgent( ### Tool Integration Integrate the CrewAI tool with AG2. +- [`Interoperability`](/docs/reference/interop/interoperability) converts the CrewAI tool to a format compatible with AG2. - [ScrapeWebsiteTool](https://docs.crewai.com/tools/scrapewebsitetool) is used for web scraping tasks. -- `Interoperability` converts the CrewAI tool to a format compatible with AG2. - Register the tool for both execution and interaction with LLMs. ```python @@ -216,7 +215,7 @@ ag2_tool.register_for_llm(chatbot) ``` ### Initiating the chat -Initiate the conversation between the `UserProxyAgent` and the `AssistantAgent` to utilize the CrewAI tool. +Initiate the conversation between the [`UserProxyAgent`](/docs/reference/agentchat/user_proxy_agent) and the [`AssistantAgent`](/docs/reference/agentchat/assistant_agent) to utilize the CrewAI tool. ```python message = "Scrape the website https://ag2.ai/" chat_result = user_proxy.initiate_chat(recipient=chatbot, message=message, max_turns=2) @@ -324,11 +323,11 @@ pip install ag2[interop-pydantic-ai] ### Imports Import necessary modules and tools. -- `BaseModel`: Used to define data structures for tool inputs and outputs. +- [BaseModel](https://docs.pydantic.dev/latest/api/base_model/): Used to define data structures for tool inputs and outputs. - [RunContext](https://ai.pydantic.dev/api/tools/#pydantic_ai.tools.RunContext): Provides context during the execution of tools. - [PydanticAITool](https://ai.pydantic.dev/api/tools/#pydantic_ai.tools.Tool): Represents a tool in the PydanticAI framework. -- `AssistantAgent` and `UserProxyAgent`: Agents that facilitate communication in the AG2 framework. -- `Interoperability`: This module acts as a bridge, making it easier to integrate PydanticAI tools with AG2’s architecture. +- [`AssistantAgent`](/docs/reference/agentchat/assistant_agent) and [`UserProxyAgent`](/docs/reference/agentchat/user_proxy_agent): Agents that facilitate communication in the AG2 framework. +- [`Interoperability`](/docs/reference/interop/interoperability): This module acts as a bridge, making it easier to integrate PydanticAI tools with AG2’s architecture. ```python import os @@ -346,8 +345,8 @@ from autogen.interop import Interoperability Configure the agents for the interaction. - `config_list` defines the LLM configurations, including the model and API key. -- `UserProxyAgent` simulates user inputs without requiring actual human interaction (set to `NEVER`). -- `AssistantAgent` represents the AI agent, configured with the LLM settings. +- [`UserProxyAgent`](/docs/reference/agentchat/user_proxy_agent) simulates user inputs without requiring actual human interaction (set to `NEVER`). +- [`AssistantAgent`](/docs/reference/agentchat/assistant_agent) represents the AI agent, configured with the LLM settings. ```python config_list = [{"model": "gpt-4o", "api_key": os.environ["OPENAI_API_KEY"]}] @@ -366,10 +365,10 @@ chatbot = AssistantAgent( To integrate a PydanticAI tool into AG2: -- First, define a `Player` model using `BaseModel` to structure the input data. +- First, define a `Player` model using [BaseModel](https://docs.pydantic.dev/latest/api/base_model/) to structure the input data. - Use [RunContext](https://ai.pydantic.dev/api/tools/#pydantic_ai.tools.RunContext) to inject dependencies (like the `Player` instance) securely into the tool. - The `get_player` function defines the tool’s functionality, retrieving injected data through `ctx.deps`. -- Then, convert the tool into an AG2-compatible format with `Interoperability`. +- Then, convert the tool into an AG2-compatible format with [`Interoperability`](/docs/reference/interop/interoperability). - Register the tool for execution and interaction with both the `user_proxy` and `chatbot`. ```python @@ -399,7 +398,7 @@ ag2_tool.register_for_llm(chatbot) ``` ### Initiating the chat -Now that everything is set up, you can initiate a chat between the `UserProxyAgent` and the `AssistantAgent`: +Now that everything is set up, you can initiate a chat between the [`UserProxyAgent`](/docs/reference/agentchat/user_proxy_agent) and the [`AssistantAgent`](/docs/reference/agentchat/assistant_agent): - The `user_proxy` sends a message to the `chatbot`. - The user requests player information, and includes "goal keeper" as additional context. diff --git a/website/mint.json b/website/mint.json index 9b5cdb5ae1..2aae976304 100644 --- a/website/mint.json +++ b/website/mint.json @@ -618,6 +618,7 @@ "notebooks/agentchat_RetrieveChat", "notebooks/agentchat_graph_rag_neo4j", "notebooks/agentchat_swarm_enhanced", + "notebooks/tools_interoperability", "notebooks/agentchat_reasoning_agent", "notebooks/agentchat_realtime_swarm" ] diff --git a/website/snippets/data/NotebooksMetadata.mdx b/website/snippets/data/NotebooksMetadata.mdx index d5c69dcf3a..ec6e9da249 100644 --- a/website/snippets/data/NotebooksMetadata.mdx +++ b/website/snippets/data/NotebooksMetadata.mdx @@ -1003,5 +1003,13 @@ export const notebooksMetadata = [ "image": null, "tags": [], "source": "/website/docs/topics/non-openai-models/cloud-cerebras.ipynb" + }, + { + "title": "Cross-Framework LLM Tool Integration with AG2", + "link": "/notebooks/tools_interoperability", + "description": "", + "image": null, + "tags": [], + "source": "/notebook/tools_interoperability.ipynb" } ]; From c59636ae48549ff599b70ff10eb3c78a6c6402d9 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Fri, 20 Dec 2024 12:41:15 +0100 Subject: [PATCH 04/10] Add youtube video placeholder to the blog --- .../blog/2024-12-18-Tools-interoperability/index.mdx | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/website/blog/2024-12-18-Tools-interoperability/index.mdx b/website/blog/2024-12-18-Tools-interoperability/index.mdx index 07946bb9e0..86bbc33070 100644 --- a/website/blog/2024-12-18-Tools-interoperability/index.mdx +++ b/website/blog/2024-12-18-Tools-interoperability/index.mdx @@ -5,14 +5,16 @@ authors: tags: [LLM, tools, langchain, crewai, pydanticai] --- + + **TL;DR** -AG2 lets you bring in tools from different frameworks like **LangChain**, **CrewAI**, and **PydanticAI**. +AG2 lets you bring in **Tools** from different frameworks like **LangChain**, **CrewAI**, and **PydanticAI**. -- **LangChain**: Useful for tasks like API querying and web scraping. -- **CrewAI**: Offers a variety of tools for web scraping, search, and more. -- **PydanticAI**: Adds context-driven tools and structured data processing. +- [LangChain Tools](https://python.langchain.com/v0.1/docs/modules/tools): Useful for tasks like API querying and web scraping. +- [CrewAI Tools](https://github.com/crewAIInc/crewAI-tools/tree/main): Offers a variety of tools for web scraping, search, and more. +- [PydanticAI Tools](https://ai.pydantic.dev/tools/): Adds context-driven tools and structured data processing. -With AG2, you can combine these tools and enhance your agents' capabilities. +**With AG2, you can combine these tools and enhance your agents' capabilities.** ![One to rule them all img](img/one_to_rule_them_all.png) From d55d278ddba105e2e0683f4f6133d9989731aa50 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Fri, 20 Dec 2024 13:15:09 +0100 Subject: [PATCH 05/10] Update references in tools_ineroperability.ipynb --- notebook/tools_interoperability.ipynb | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/notebook/tools_interoperability.ipynb b/notebook/tools_interoperability.ipynb index 170a6e21c0..99075818c3 100644 --- a/notebook/tools_interoperability.ipynb +++ b/notebook/tools_interoperability.ipynb @@ -39,7 +39,7 @@ "\n", "Import necessary modules and tools.\n", "\n", - "- `WikipediaQueryRun` and `WikipediaAPIWrapper`: Tools for querying Wikipedia.\n", + "- [WikipediaQueryRun](https://api.python.langchain.com/en/latest/tools/langchain_community.tools.wikipedia.tool.WikipediaQueryRun.html) and [WikipediaAPIWrapper](https://python.langchain.com/api_reference/community/utilities/langchain_community.utilities.wikipedia.WikipediaAPIWrapper.html): Tools for querying Wikipedia.\n", "- `AssistantAgent` and `UserProxyAgent`: Agents that facilitate communication in the AG2 framework.\n", "- `Interoperability`: This module acts as a bridge, making it easier to integrate LangChain tools with AG2’s architecture." ] @@ -97,9 +97,9 @@ "### Tool Integration\n", "\n", "- Initialize and register the LangChain tool with AG2.\n", - "- `WikipediaAPIWrapper`: Configured to fetch the top 1 result from Wikipedia with a maximum of 1000 characters per document.\n", - "- `WikipediaQueryRun`: A LangChain tool that executes Wikipedia queries.\n", - "- `LangchainInteroperability`: Converts the LangChain tool into a format compatible with the AG2 framework.\n", + "- [WikipediaAPIWrapper](https://python.langchain.com/api_reference/community/utilities/langchain_community.utilities.wikipedia.WikipediaAPIWrapper.html): Configured to fetch the top 1 result from Wikipedia with a maximum of 1000 characters per document.\n", + "- [WikipediaQueryRun](https://api.python.langchain.com/en/latest/tools/langchain_community.tools.wikipedia.tool.WikipediaQueryRun.html): A LangChain tool that executes Wikipedia queries.\n", + "- `Interoperability`: Converts the LangChain tool into a format compatible with the AG2 framework.\n", "- `ag2_tool.register_for_execution(user_proxy)`: Registers the tool for use by the user_proxy agent.\n", "- `ag2_tool.register_for_llm(chatbot)`: Registers the tool for integration with the chatbot agent.\n" ] @@ -155,7 +155,7 @@ "\n", "Import necessary modules and tools.\n", "\n", - "- `ScrapeWebsiteTool` are the CrewAI tools for web scraping\n", + "- [ScrapeWebsiteTool](https://docs.crewai.com/tools/scrapewebsitetool) are the CrewAI tools for web scraping\n", "- `AssistantAgent` and `UserProxyAgent` are core AG2 classes.\n", "- `Interoperability`: This module acts as a bridge, making it easier to integrate CrewAI tools with AG2’s architecture." ] @@ -213,7 +213,7 @@ "\n", "Initialize and register the CrewAI tool with AG2.\n", "\n", - "- `crewai_tool` is an instance of the `ScrapeWebsiteTool` from CrewAI.\n", + "- `crewai_tool` is an instance of the [ScrapeWebsiteTool](https://docs.crewai.com/tools/scrapewebsitetool) from CrewAI.\n", "- `Interoperability` converts the CrewAI tool to make it usable in AG2.\n", "- `register_for_execution` and `register_for_llm` allow the tool to work with the UserProxyAgent and AssistantAgent." ] @@ -269,9 +269,9 @@ "\n", "Import necessary modules and tools.\n", "\n", - "- `BaseModel`: Used to define data structures for tool inputs and outputs.\n", - "- `RunContext`: Provides context during the execution of tools.\n", - "- `PydanticAITool`: Represents a tool in the PydanticAI framework.\n", + "- [BaseModel](https://docs.pydantic.dev/latest/api/base_model/): Used to define data structures for tool inputs and outputs.\n", + "- [RunContext](https://ai.pydantic.dev/api/tools/#pydantic_ai.tools.RunContext): Provides context during the execution of tools.\n", + "- [PydanticAITool](https://ai.pydantic.dev/api/tools/#pydantic_ai.tools.Tool): Represents a tool in the PydanticAI framework.\n", "- `AssistantAgent` and `UserProxyAgent`: Agents that facilitate communication in the AG2 framework.\n", "- `Interoperability`: This module acts as a bridge, making it easier to integrate PydanticAI tools with AG2’s architecture." ] @@ -332,8 +332,8 @@ "\n", "Integrate the PydanticAI tool with AG2.\n", "\n", - "- Define a `Player` model using `BaseModel` to structure the input data.\n", - "- Use `RunContext` to securely inject dependencies (like the `Player` instance) into the tool function without exposing them to the LLM.\n", + "- Define a `Player` model using [BaseModel](https://docs.pydantic.dev/latest/api/base_model/) to structure the input data.\n", + "- Use [RunContext](https://ai.pydantic.dev/api/tools/#pydantic_ai.tools.RunContext) to securely inject dependencies (like the `Player` instance) into the tool function without exposing them to the LLM.\n", "- Implement `get_player` to define the tool's functionality, accessing `ctx.deps` for injected data.\n", "- Convert the tool to an AG2-compatible format with `Interoperability` and register it for execution and LLM communication.\n", "- Convert the PydanticAI tool into an AG2-compatible format using `convert_tool`.\n", @@ -379,7 +379,7 @@ "\n", "- Use the `initiate_chat` method to send a message from the `user_proxy` to the `chatbot`.\n", "- In this example, the user requests the chatbot to retrieve player information, providing \"goal keeper\" as additional context.\n", - "- The `Player` instance is securely injected into the tool using `RunContext`, ensuring the chatbot can retrieve and use this data during the interaction." + "- The `Player` instance is securely injected into the tool using [RunContext](https://ai.pydantic.dev/api/tools/#pydantic_ai.tools.RunContext), ensuring the chatbot can retrieve and use this data during the interaction." ] }, { From 9f37ecadfe460e43330e2fc7fd99b91567f6295d Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Fri, 20 Dec 2024 13:39:09 +0100 Subject: [PATCH 06/10] Hide iframe block (make it non visable) in the blog until youtube video is not ready --- website/blog/2024-12-18-Tools-interoperability/index.mdx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/website/blog/2024-12-18-Tools-interoperability/index.mdx b/website/blog/2024-12-18-Tools-interoperability/index.mdx index 86bbc33070..241317ea04 100644 --- a/website/blog/2024-12-18-Tools-interoperability/index.mdx +++ b/website/blog/2024-12-18-Tools-interoperability/index.mdx @@ -5,7 +5,11 @@ authors: tags: [LLM, tools, langchain, crewai, pydanticai] --- - +
+ This content will be hidden. + REMOVE THE iframe BLOCK OUTSIDE OF div BLOCK ONCE THE VIDEO IS READY + +
**TL;DR** AG2 lets you bring in **Tools** from different frameworks like **LangChain**, **CrewAI**, and **PydanticAI**. From 091e640f03f2b277564f7fa588f252decf64fc07 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Fri, 20 Dec 2024 14:00:12 +0100 Subject: [PATCH 07/10] Update comment iterop blog --- website/blog/2024-12-18-Tools-interoperability/index.mdx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/website/blog/2024-12-18-Tools-interoperability/index.mdx b/website/blog/2024-12-18-Tools-interoperability/index.mdx index 241317ea04..f1089c5bc1 100644 --- a/website/blog/2024-12-18-Tools-interoperability/index.mdx +++ b/website/blog/2024-12-18-Tools-interoperability/index.mdx @@ -7,7 +7,8 @@ tags: [LLM, tools, langchain, crewai, pydanticai]
This content will be hidden. - REMOVE THE iframe BLOCK OUTSIDE OF div BLOCK ONCE THE VIDEO IS READY + MOVE THE iframe BLOCK OUTSIDE OF div BLOCK ONCE THE VIDEO IS READY + ALSO DELETE THIS div BLOCK
From d2ace5299b67a53b03bfb6c0cc1b67335d901fc9 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Fri, 20 Dec 2024 15:46:16 +0100 Subject: [PATCH 08/10] Remove meme from inerop blog --- .../img/one_to_rule_them_all.png | 3 --- website/blog/2024-12-18-Tools-interoperability/index.mdx | 2 -- 2 files changed, 5 deletions(-) delete mode 100644 website/blog/2024-12-18-Tools-interoperability/img/one_to_rule_them_all.png diff --git a/website/blog/2024-12-18-Tools-interoperability/img/one_to_rule_them_all.png b/website/blog/2024-12-18-Tools-interoperability/img/one_to_rule_them_all.png deleted file mode 100644 index 45cf075e36..0000000000 --- a/website/blog/2024-12-18-Tools-interoperability/img/one_to_rule_them_all.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5f41e305d4525e167b62d0c769599ec2667ef282918e835372ad20a1b3b3c03c -size 310123 diff --git a/website/blog/2024-12-18-Tools-interoperability/index.mdx b/website/blog/2024-12-18-Tools-interoperability/index.mdx index f1089c5bc1..c57d6869f4 100644 --- a/website/blog/2024-12-18-Tools-interoperability/index.mdx +++ b/website/blog/2024-12-18-Tools-interoperability/index.mdx @@ -22,8 +22,6 @@ AG2 lets you bring in **Tools** from different frameworks like **LangChain**, ** **With AG2, you can combine these tools and enhance your agents' capabilities.** -![One to rule them all img](img/one_to_rule_them_all.png) - In this post, we’ll walk through how to integrate tools from various frameworks—like [LangChain Tools](https://python.langchain.com/v0.1/docs/modules/tools), [CrewAI Tools](https://github.com/crewAIInc/crewAI-tools/tree/main), and [PydanticAI Tools](https://ai.pydantic.dev/tools/)—into AG2. From f80cbe133c7ce775eb9b8b2886cef2961d0af0b0 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Fri, 20 Dec 2024 15:57:17 +0100 Subject: [PATCH 09/10] Update mint.json --- website/mint.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/website/mint.json b/website/mint.json index 68fdfded93..66a31501d2 100644 --- a/website/mint.json +++ b/website/mint.json @@ -418,6 +418,7 @@ { "group": "tools", "pages": [ + "docs/reference/tools/pydantic_ai_tool", "docs/reference/tools/tool" ] }, @@ -621,8 +622,8 @@ "notebooks/agentchat_graph_rag_neo4j", "notebooks/agentchat_swarm_enhanced", "notebooks/tools_interoperability", - "notebooks/agentchat_reasoning_agent", - "notebooks/agentchat_realtime_swarm" + "notebooks/agentchat_realtime_swarm", + "notebooks/agentchat_reasoning_agent" ] }, "notebooks/Gallery" From 5b62def318fd2d30214c370b74a292ad307ab481 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Fri, 20 Dec 2024 16:06:44 +0100 Subject: [PATCH 10/10] Remove duplicated blog/2024-12-20-RealtimeAgent/index --- website/mint.json | 1 - 1 file changed, 1 deletion(-) diff --git a/website/mint.json b/website/mint.json index 114bb60e6d..39b665c2cb 100644 --- a/website/mint.json +++ b/website/mint.json @@ -498,7 +498,6 @@ "blog/2024-12-20-RealtimeAgent/index", "blog/2024-12-20-Tools-interoperability/index", "blog/2024-12-20-Reasoning-Update/index", - "blog/2024-12-20-RealtimeAgent/index", "blog/2024-12-18-Tools-interoperability/index", "blog/2024-12-06-FalkorDB-Structured/index", "blog/2024-12-02-ReasoningAgent2/index",