diff --git a/.github/workflows/contrib-tests.yml b/.github/workflows/contrib-tests.yml index 7d8a932b0254..f13bfdbb985c 100644 --- a/.github/workflows/contrib-tests.yml +++ b/.github/workflows/contrib-tests.yml @@ -9,6 +9,8 @@ on: paths: - "autogen/**" - "test/agentchat/contrib/**" + - "test/test_browser_utils.py" + - "test/test_retrieve_utils.py" - ".github/workflows/contrib-tests.yml" - "setup.py" @@ -598,3 +600,79 @@ jobs: with: file: ./coverage.xml flags: unittests + + GroqTest: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-2019] + python-version: ["3.9", "3.10", "3.11", "3.12"] + exclude: + - os: macos-latest + python-version: "3.9" + steps: + - uses: actions/checkout@v4 + with: + lfs: true + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install packages and dependencies for all tests + run: | + python -m pip install --upgrade pip wheel + pip install pytest-cov>=5 + - name: Install packages and dependencies for Groq + run: | + pip install -e .[groq,test] + - name: Set AUTOGEN_USE_DOCKER based on OS + shell: bash + run: | + if [[ ${{ matrix.os }} != ubuntu-latest ]]; then + echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV + fi + - name: Coverage + run: | + pytest test/oai/test_groq.py --skip-openai + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + file: ./coverage.xml + flags: unittests + + CohereTest: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python-version: ["3.9", "3.10", "3.11", "3.12"] + steps: + - uses: actions/checkout@v4 + with: + lfs: true + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install packages and dependencies for all tests + run: | + python -m pip install --upgrade pip wheel + pip install pytest-cov>=5 + - name: Install packages and dependencies for Cohere + run: | + pip install -e .[cohere,test] + - name: Set AUTOGEN_USE_DOCKER based on OS + shell: bash + run: | + if [[ ${{ matrix.os }} != ubuntu-latest ]]; then + echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV + fi + - name: Coverage + run: | + pytest test/oai/test_cohere.py --skip-openai + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + file: ./coverage.xml + flags: unittests diff --git a/.github/workflows/dotnet-build.yml b/.github/workflows/dotnet-build.yml index f4074b061693..7e50025917de 100644 --- a/.github/workflows/dotnet-build.yml +++ b/.github/workflows/dotnet-build.yml @@ -56,11 +56,16 @@ jobs: - name: Setup .NET uses: actions/setup-dotnet@v4 with: - global-json-file: dotnet/global.json + dotnet-version: '8.0.x' - name: Restore dependencies run: | # dotnet nuget add source --name dotnet-tool https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json --configfile NuGet.config dotnet restore -bl + - name: Format check + run: | + echo "Format check" + echo "If you see any error in this step, please run 'dotnet format' locally to format the code." + dotnet format --verify-no-changes -v diag --no-restore - name: Build run: | echo "Build AutoGen" diff --git a/.github/workflows/dotnet-release.yml b/.github/workflows/dotnet-release.yml index 2877d058377b..aacfd115bb7e 100644 --- a/.github/workflows/dotnet-release.yml +++ b/.github/workflows/dotnet-release.yml @@ -32,7 +32,7 @@ jobs: - name: Setup .NET uses: actions/setup-dotnet@v4 with: - global-json-file: dotnet/global.json + dotnet-version: '8.0.x' - name: Restore dependencies run: | dotnet restore -bl diff --git a/README.md b/README.md index 5bff3300a50e..7c7ac4b85c59 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,12 @@ ## What is AutoGen -AutoGen is a framework that enables the development of LLM applications using multiple agents that can converse with each other to solve tasks. AutoGen agents are customizable, conversable, and seamlessly allow human participation. They can operate in various modes that employ combinations of LLMs, human inputs, and tools. +AutoGen is an open-source programming framework for building AI agents and facilitating cooperation among multiple agents to solve tasks. AutoGen aims to streamline the development and research of agentic AI, much like PyTorch does for Deep Learning. It offers features such as agents capable of interacting with each other, facilitates the use of various large language models (LLMs) and tool use support, autonomous and human-in-the-loop workflows, and multi-agent conversation patterns. + +**Open Source Statement**: The project welcomes contributions from developers and organizations worldwide. Our goal is to foster a collaborative and inclusive community where diverse perspectives and expertise can drive innovation and enhance the project's capabilities. Whether you are an individual contributor or represent an organization, we invite you to join us in shaping the future of this project. Together, we can build something truly remarkable. + +The project is currently maintained by a [dynamic group of volunteers](https://butternut-swordtail-8a5.notion.site/410675be605442d3ada9a42eb4dfef30?v=fa5d0a79fd3d4c0f9c112951b2831cbb&pvs=4) from several different organizations. Contact project administrators Chi Wang and Qingyun Wu via auto-gen@outlook.com if you are interested in becoming a maintainer. + ![AutoGen Overview](https://github.com/microsoft/autogen/blob/main/website/static/img/autogen_agentchat.png) @@ -288,6 +293,16 @@ In addition, you can find: } ``` +[StateFlow](https://arxiv.org/abs/2403.11322) +``` +@article{wu2024stateflow, + title={StateFlow: Enhancing LLM Task-Solving through State-Driven Workflows}, + author={Wu, Yiran and Yue, Tianwei and Zhang, Shaokun and Wang, Chi and Wu, Qingyun}, + journal={arXiv preprint arXiv:2403.11322}, + year={2024} +} +``` +

↑ Back to Top ↑ diff --git a/autogen/agentchat/contrib/agent_eval/README.md b/autogen/agentchat/contrib/agent_eval/README.md index 6588a1ec6113..478f28fd74ec 100644 --- a/autogen/agentchat/contrib/agent_eval/README.md +++ b/autogen/agentchat/contrib/agent_eval/README.md @@ -1,7 +1,9 @@ -Agents for running the AgentEval pipeline. +Agents for running the [AgentEval](https://microsoft.github.io/autogen/blog/2023/11/20/AgentEval/) pipeline. AgentEval is a process for evaluating a LLM-based system's performance on a given task. When given a task to evaluate and a few example runs, the critic and subcritic agents create evaluation criteria for evaluating a system's solution. Once the criteria has been created, the quantifier agent can evaluate subsequent task solutions based on the generated criteria. For more information see: [AgentEval Integration Roadmap](https://github.com/microsoft/autogen/issues/2162) + +See our [blog post](https://microsoft.github.io/autogen/blog/2024/06/21/AgentEval) for usage examples and general explanations. diff --git a/autogen/agentchat/contrib/llamaindex_conversable_agent.py b/autogen/agentchat/contrib/llamaindex_conversable_agent.py index f7a9c3e615dc..dbf6f274ae87 100644 --- a/autogen/agentchat/contrib/llamaindex_conversable_agent.py +++ b/autogen/agentchat/contrib/llamaindex_conversable_agent.py @@ -8,15 +8,14 @@ try: from llama_index.core.agent.runner.base import AgentRunner + from llama_index.core.base.llms.types import ChatMessage from llama_index.core.chat_engine.types import AgentChatResponse - from llama_index_client import ChatMessage except ImportError as e: logger.fatal("Failed to import llama-index. Try running 'pip install llama-index'") raise e class LLamaIndexConversableAgent(ConversableAgent): - def __init__( self, name: str, diff --git a/autogen/agentchat/contrib/retrieve_user_proxy_agent.py b/autogen/agentchat/contrib/retrieve_user_proxy_agent.py index 59a4abccb1d6..4842bd4e9f53 100644 --- a/autogen/agentchat/contrib/retrieve_user_proxy_agent.py +++ b/autogen/agentchat/contrib/retrieve_user_proxy_agent.py @@ -1,6 +1,7 @@ import hashlib import os import re +import uuid from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Union from IPython import get_ipython @@ -135,7 +136,7 @@ def __init__( - `client` (Optional, chromadb.Client) - the chromadb client. If key not provided, a default client `chromadb.Client()` will be used. If you want to use other vector db, extend this class and override the `retrieve_docs` function. - **Deprecated**: use `vector_db` instead. + *[Deprecated]* use `vector_db` instead. - `docs_path` (Optional, Union[str, List[str]]) - the path to the docs directory. It can also be the path to a single file, the url to a single file or a list of directories, files and urls. Default is None, which works only if the @@ -149,7 +150,7 @@ def __init__( By default, "extra_docs" is set to false, starting document IDs from zero. This poses a risk as new documents might overwrite existing ones, potentially causing unintended loss or alteration of data in the collection. - **Deprecated**: use `new_docs` when use `vector_db` instead of `client`. + *[Deprecated]* use `new_docs` when use `vector_db` instead of `client`. - `new_docs` (Optional, bool) - when True, only adds new documents to the collection; when False, updates existing documents and adds new ones. Default is True. Document id is used to determine if a document is new or existing. By default, the @@ -172,7 +173,7 @@ def __init__( models can be found at `https://www.sbert.net/docs/pretrained_models.html`. The default model is a fast model. If you want to use a high performance model, `all-mpnet-base-v2` is recommended. - **Deprecated**: no need when use `vector_db` instead of `client`. + *[Deprecated]* no need when use `vector_db` instead of `client`. - `embedding_function` (Optional, Callable) - the embedding function for creating the vector db. Default is None, SentenceTransformer with the given `embedding_model` will be used. If you want to use OpenAI, Cohere, HuggingFace or other embedding @@ -219,7 +220,7 @@ def __init__( Example of overriding retrieve_docs - If you have set up a customized vector db, and it's not compatible with chromadb, you can easily plug in it with below code. - **Deprecated**: Use `vector_db` instead. You can extend VectorDB and pass it to the agent. + *[Deprecated]* use `vector_db` instead. You can extend VectorDB and pass it to the agent. ```python class MyRetrieveUserProxyAgent(RetrieveUserProxyAgent): def query_vector_db( @@ -365,7 +366,11 @@ def _init_db(self): else: all_docs_ids = set() - chunk_ids = [hashlib.blake2b(chunk.encode("utf-8")).hexdigest()[:HASH_LENGTH] for chunk in chunks] + chunk_ids = ( + [hashlib.blake2b(chunk.encode("utf-8")).hexdigest()[:HASH_LENGTH] for chunk in chunks] + if not self._vector_db.type == "qdrant" + else [str(uuid.UUID(hex=hashlib.md5(chunk.encode("utf-8")).hexdigest())) for chunk in chunks] + ) chunk_ids_set = set(chunk_ids) chunk_ids_set_idx = [chunk_ids.index(hash_value) for hash_value in chunk_ids_set] docs = [ diff --git a/autogen/agentchat/contrib/vectordb/base.py b/autogen/agentchat/contrib/vectordb/base.py index 29a080086193..20b6376d01d9 100644 --- a/autogen/agentchat/contrib/vectordb/base.py +++ b/autogen/agentchat/contrib/vectordb/base.py @@ -1,4 +1,16 @@ -from typing import Any, List, Mapping, Optional, Protocol, Sequence, Tuple, TypedDict, Union, runtime_checkable +from typing import ( + Any, + Callable, + List, + Mapping, + Optional, + Protocol, + Sequence, + Tuple, + TypedDict, + Union, + runtime_checkable, +) Metadata = Union[Mapping[str, Any], None] Vector = Union[Sequence[float], Sequence[int]] @@ -49,6 +61,9 @@ class VectorDB(Protocol): active_collection: Any = None type: str = "" + embedding_function: Optional[Callable[[List[str]], List[List[float]]]] = ( + None # embeddings = embedding_function(sentences) + ) def create_collection(self, collection_name: str, overwrite: bool = False, get_or_create: bool = True) -> Any: """ @@ -185,7 +200,7 @@ class VectorDBFactory: Factory class for creating vector databases. """ - PREDEFINED_VECTOR_DB = ["chroma", "pgvector"] + PREDEFINED_VECTOR_DB = ["chroma", "pgvector", "qdrant"] @staticmethod def create_vector_db(db_type: str, **kwargs) -> VectorDB: @@ -207,6 +222,10 @@ def create_vector_db(db_type: str, **kwargs) -> VectorDB: from .pgvectordb import PGVectorDB return PGVectorDB(**kwargs) + if db_type.lower() in ["qdrant", "qdrantdb"]: + from .qdrant import QdrantVectorDB + + return QdrantVectorDB(**kwargs) else: raise ValueError( f"Unsupported vector database type: {db_type}. Valid types are {VectorDBFactory.PREDEFINED_VECTOR_DB}." diff --git a/autogen/agentchat/contrib/vectordb/qdrant.py b/autogen/agentchat/contrib/vectordb/qdrant.py new file mode 100644 index 000000000000..398734eb0334 --- /dev/null +++ b/autogen/agentchat/contrib/vectordb/qdrant.py @@ -0,0 +1,328 @@ +import abc +import logging +import os +from typing import Callable, List, Optional, Sequence, Tuple, Union + +from .base import Document, ItemID, QueryResults, VectorDB +from .utils import get_logger + +try: + from qdrant_client import QdrantClient, models +except ImportError: + raise ImportError("Please install qdrant-client: `pip install qdrant-client`") + +logger = get_logger(__name__) + +Embeddings = Union[Sequence[float], Sequence[int]] + + +class EmbeddingFunction(abc.ABC): + @abc.abstractmethod + def __call__(self, inputs: List[str]) -> List[Embeddings]: + raise NotImplementedError + + +class FastEmbedEmbeddingFunction(EmbeddingFunction): + """Embedding function implementation using FastEmbed - https://qdrant.github.io/fastembed.""" + + def __init__( + self, + model_name: str = "BAAI/bge-small-en-v1.5", + batch_size: int = 256, + cache_dir: Optional[str] = None, + threads: Optional[int] = None, + parallel: Optional[int] = None, + **kwargs, + ): + """Initialize fastembed.TextEmbedding. + + Args: + model_name (str): The name of the model to use. Defaults to `"BAAI/bge-small-en-v1.5"`. + batch_size (int): Batch size for encoding. Higher values will use more memory, but be faster.\ + Defaults to 256. + cache_dir (str, optional): The path to the model cache directory.\ + Can also be set using the `FASTEMBED_CACHE_PATH` env variable. + threads (int, optional): The number of threads single onnxruntime session can use. + parallel (int, optional): If `>1`, data-parallel encoding will be used, recommended for large datasets.\ + If `0`, use all available cores.\ + If `None`, don't use data-parallel processing, use default onnxruntime threading.\ + Defaults to None. + **kwargs: Additional options to pass to fastembed.TextEmbedding + Raises: + ValueError: If the model_name is not in the format / e.g. BAAI/bge-small-en-v1.5. + """ + try: + from fastembed import TextEmbedding + except ImportError as e: + raise ValueError( + "The 'fastembed' package is not installed. Please install it with `pip install fastembed`", + ) from e + self._batch_size = batch_size + self._parallel = parallel + self._model = TextEmbedding(model_name=model_name, cache_dir=cache_dir, threads=threads, **kwargs) + + def __call__(self, inputs: List[str]) -> List[Embeddings]: + embeddings = self._model.embed(inputs, batch_size=self._batch_size, parallel=self._parallel) + + return [embedding.tolist() for embedding in embeddings] + + +class QdrantVectorDB(VectorDB): + """ + A vector database implementation that uses Qdrant as the backend. + """ + + def __init__( + self, + *, + client=None, + embedding_function: EmbeddingFunction = None, + content_payload_key: str = "_content", + metadata_payload_key: str = "_metadata", + collection_options: dict = {}, + **kwargs, + ) -> None: + """ + Initialize the vector database. + + Args: + client: qdrant_client.QdrantClient | An instance of QdrantClient. + embedding_function: Callable | The embedding function used to generate the vector representation + of the documents. Defaults to FastEmbedEmbeddingFunction. + collection_options: dict | The options for creating the collection. + kwargs: dict | Additional keyword arguments. + """ + self.client: QdrantClient = client or QdrantClient(location=":memory:") + self.embedding_function = FastEmbedEmbeddingFunction() or embedding_function + self.collection_options = collection_options + self.content_payload_key = content_payload_key + self.metadata_payload_key = metadata_payload_key + self.type = "qdrant" + + def create_collection(self, collection_name: str, overwrite: bool = False, get_or_create: bool = True) -> None: + """ + Create a collection in the vector database. + Case 1. if the collection does not exist, create the collection. + Case 2. the collection exists, if overwrite is True, it will overwrite the collection. + Case 3. the collection exists and overwrite is False, if get_or_create is True, it will get the collection, + otherwise it raise a ValueError. + + Args: + collection_name: str | The name of the collection. + overwrite: bool | Whether to overwrite the collection if it exists. Default is False. + get_or_create: bool | Whether to get the collection if it exists. Default is True. + + Returns: + Any | The collection object. + """ + embeddings_size = len(self.embedding_function(["test"])[0]) + + if self.client.collection_exists(collection_name) and overwrite: + self.client.delete_collection(collection_name) + + if not self.client.collection_exists(collection_name): + self.client.create_collection( + collection_name, + vectors_config=models.VectorParams(size=embeddings_size, distance=models.Distance.COSINE), + **self.collection_options, + ) + elif not get_or_create: + raise ValueError(f"Collection {collection_name} already exists.") + + def get_collection(self, collection_name: str = None): + """ + Get the collection from the vector database. + + Args: + collection_name: str | The name of the collection. + + Returns: + Any | The collection object. + """ + if collection_name is None: + raise ValueError("The collection name is required.") + + return self.client.get_collection(collection_name) + + def delete_collection(self, collection_name: str) -> None: + """Delete the collection from the vector database. + + Args: + collection_name: str | The name of the collection. + + Returns: + Any + """ + return self.client.delete_collection(collection_name) + + def insert_docs(self, docs: List[Document], collection_name: str = None, upsert: bool = False) -> None: + """ + Insert documents into the collection of the vector database. + + Args: + docs: List[Document] | A list of documents. Each document is a TypedDict `Document`. + collection_name: str | The name of the collection. Default is None. + upsert: bool | Whether to update the document if it exists. Default is False. + kwargs: Dict | Additional keyword arguments. + + Returns: + None + """ + if not docs: + return + if any(doc.get("content") is None for doc in docs): + raise ValueError("The document content is required.") + if any(doc.get("id") is None for doc in docs): + raise ValueError("The document id is required.") + + if not upsert and not self._validate_upsert_ids(collection_name, [doc["id"] for doc in docs]): + logger.log("Some IDs already exist. Skipping insert", level=logging.WARN) + + self.client.upsert(collection_name, points=self._documents_to_points(docs)) + + def update_docs(self, docs: List[Document], collection_name: str = None) -> None: + if not docs: + return + if any(doc.get("id") is None for doc in docs): + raise ValueError("The document id is required.") + if any(doc.get("content") is None for doc in docs): + raise ValueError("The document content is required.") + if self._validate_update_ids(collection_name, [doc["id"] for doc in docs]): + return self.client.upsert(collection_name, points=self._documents_to_points(docs)) + + raise ValueError("Some IDs do not exist. Skipping update") + + def delete_docs(self, ids: List[ItemID], collection_name: str = None, **kwargs) -> None: + """ + Delete documents from the collection of the vector database. + + Args: + ids: List[ItemID] | A list of document ids. Each id is a typed `ItemID`. + collection_name: str | The name of the collection. Default is None. + kwargs: Dict | Additional keyword arguments. + + Returns: + None + """ + self.client.delete(collection_name, ids) + + def retrieve_docs( + self, + queries: List[str], + collection_name: str = None, + n_results: int = 10, + distance_threshold: float = 0, + **kwargs, + ) -> QueryResults: + """ + Retrieve documents from the collection of the vector database based on the queries. + + Args: + queries: List[str] | A list of queries. Each query is a string. + collection_name: str | The name of the collection. Default is None. + n_results: int | The number of relevant documents to return. Default is 10. + distance_threshold: float | The threshold for the distance score, only distance smaller than it will be + returned. Don't filter with it if < 0. Default is 0. + kwargs: Dict | Additional keyword arguments. + + Returns: + QueryResults | The query results. Each query result is a list of list of tuples containing the document and + the distance. + """ + embeddings = self.embedding_function(queries) + requests = [ + models.SearchRequest( + vector=embedding, + limit=n_results, + score_threshold=distance_threshold, + with_payload=True, + with_vector=False, + ) + for embedding in embeddings + ] + + batch_results = self.client.search_batch(collection_name, requests) + return [self._scored_points_to_documents(results) for results in batch_results] + + def get_docs_by_ids( + self, ids: List[ItemID] = None, collection_name: str = None, include=True, **kwargs + ) -> List[Document]: + """ + Retrieve documents from the collection of the vector database based on the ids. + + Args: + ids: List[ItemID] | A list of document ids. If None, will return all the documents. Default is None. + collection_name: str | The name of the collection. Default is None. + include: List[str] | The fields to include. Default is True. + If None, will include ["metadatas", "documents"], ids will always be included. + kwargs: dict | Additional keyword arguments. + + Returns: + List[Document] | The results. + """ + if ids is None: + results = self.client.scroll(collection_name=collection_name, with_payload=include, with_vectors=True)[0] + else: + results = self.client.retrieve(collection_name, ids=ids, with_payload=include, with_vectors=True) + return [self._point_to_document(result) for result in results] + + def _point_to_document(self, point) -> Document: + return { + "id": point.id, + "content": point.payload.get(self.content_payload_key, ""), + "metadata": point.payload.get(self.metadata_payload_key, {}), + "embedding": point.vector, + } + + def _points_to_documents(self, points) -> List[Document]: + return [self._point_to_document(point) for point in points] + + def _scored_point_to_document(self, scored_point: models.ScoredPoint) -> Tuple[Document, float]: + return self._point_to_document(scored_point), scored_point.score + + def _documents_to_points(self, documents: List[Document]): + contents = [document["content"] for document in documents] + embeddings = self.embedding_function(contents) + points = [ + models.PointStruct( + id=documents[i]["id"], + vector=embeddings[i], + payload={ + self.content_payload_key: documents[i].get("content"), + self.metadata_payload_key: documents[i].get("metadata"), + }, + ) + for i in range(len(documents)) + ] + return points + + def _scored_points_to_documents(self, scored_points: List[models.ScoredPoint]) -> List[Tuple[Document, float]]: + return [self._scored_point_to_document(scored_point) for scored_point in scored_points] + + def _validate_update_ids(self, collection_name: str, ids: List[str]) -> bool: + """ + Validates all the IDs exist in the collection + """ + retrieved_ids = [ + point.id for point in self.client.retrieve(collection_name, ids=ids, with_payload=False, with_vectors=False) + ] + + if missing_ids := set(ids) - set(retrieved_ids): + logger.log(f"Missing IDs: {missing_ids}. Skipping update", level=logging.WARN) + return False + + return True + + def _validate_upsert_ids(self, collection_name: str, ids: List[str]) -> bool: + """ + Validate none of the IDs exist in the collection + """ + retrieved_ids = [ + point.id for point in self.client.retrieve(collection_name, ids=ids, with_payload=False, with_vectors=False) + ] + + if existing_ids := set(ids) & set(retrieved_ids): + logger.log(f"Existing IDs: {existing_ids}.", level=logging.WARN) + return False + + return True diff --git a/autogen/agentchat/conversable_agent.py b/autogen/agentchat/conversable_agent.py index b434fc648eb1..81c666de022c 100644 --- a/autogen/agentchat/conversable_agent.py +++ b/autogen/agentchat/conversable_agent.py @@ -2526,14 +2526,16 @@ def _wrap_function(self, func: F) -> F: @functools.wraps(func) def _wrapped_func(*args, **kwargs): retval = func(*args, **kwargs) - log_function_use(self, func, kwargs, retval) + if logging_enabled(): + log_function_use(self, func, kwargs, retval) return serialize_to_str(retval) @load_basemodels_if_needed @functools.wraps(func) async def _a_wrapped_func(*args, **kwargs): retval = await func(*args, **kwargs) - log_function_use(self, func, kwargs, retval) + if logging_enabled(): + log_function_use(self, func, kwargs, retval) return serialize_to_str(retval) wrapped_func = _a_wrapped_func if inspect.iscoroutinefunction(func) else _wrapped_func diff --git a/autogen/logger/file_logger.py b/autogen/logger/file_logger.py index af5583587f66..61a8a6335284 100644 --- a/autogen/logger/file_logger.py +++ b/autogen/logger/file_logger.py @@ -18,7 +18,9 @@ if TYPE_CHECKING: from autogen import Agent, ConversableAgent, OpenAIWrapper from autogen.oai.anthropic import AnthropicClient + from autogen.oai.cohere import CohereClient from autogen.oai.gemini import GeminiClient + from autogen.oai.groq import GroqClient from autogen.oai.mistral import MistralAIClient from autogen.oai.together import TogetherClient @@ -204,7 +206,16 @@ def log_new_wrapper( def log_new_client( self, - client: AzureOpenAI | OpenAI | GeminiClient | AnthropicClient | MistralAIClient | TogetherClient, + client: ( + AzureOpenAI + | OpenAI + | GeminiClient + | AnthropicClient + | MistralAIClient + | TogetherClient + | GroqClient + | CohereClient + ), wrapper: OpenAIWrapper, init_args: Dict[str, Any], ) -> None: diff --git a/autogen/logger/sqlite_logger.py b/autogen/logger/sqlite_logger.py index 969a943017e3..2cf176ebb8f2 100644 --- a/autogen/logger/sqlite_logger.py +++ b/autogen/logger/sqlite_logger.py @@ -19,7 +19,9 @@ if TYPE_CHECKING: from autogen import Agent, ConversableAgent, OpenAIWrapper from autogen.oai.anthropic import AnthropicClient + from autogen.oai.cohere import CohereClient from autogen.oai.gemini import GeminiClient + from autogen.oai.groq import GroqClient from autogen.oai.mistral import MistralAIClient from autogen.oai.together import TogetherClient @@ -391,7 +393,16 @@ def log_function_use(self, source: Union[str, Agent], function: F, args: Dict[st def log_new_client( self, - client: Union[AzureOpenAI, OpenAI, GeminiClient, AnthropicClient, MistralAIClient, TogetherClient], + client: Union[ + AzureOpenAI, + OpenAI, + GeminiClient, + AnthropicClient, + MistralAIClient, + TogetherClient, + GroqClient, + CohereClient, + ], wrapper: OpenAIWrapper, init_args: Dict[str, Any], ) -> None: diff --git a/autogen/oai/anthropic.py b/autogen/oai/anthropic.py index 9faa4e2cb808..62078d42631d 100644 --- a/autogen/oai/anthropic.py +++ b/autogen/oai/anthropic.py @@ -16,6 +16,27 @@ ] assistant = autogen.AssistantAgent("assistant", llm_config={"config_list": config_list}) + +Example usage for Anthropic Bedrock: + +Install the `anthropic` package by running `pip install --upgrade anthropic`. +- https://docs.anthropic.com/en/docs/quickstart-guide + +import autogen + +config_list = [ + { + "model": "anthropic.claude-3-5-sonnet-20240620-v1:0", + "aws_access_key":, + "aws_secret_key":, + "aws_session_token":, + "aws_region":"us-east-1", + "api_type": "anthropic", + } +] + +assistant = autogen.AssistantAgent("assistant", llm_config={"config_list": config_list}) + """ from __future__ import annotations @@ -28,7 +49,7 @@ import warnings from typing import Any, Dict, List, Tuple, Union -from anthropic import Anthropic +from anthropic import Anthropic, AnthropicBedrock from anthropic import __version__ as anthropic_version from anthropic.types import Completion, Message, TextBlock, ToolUseBlock from openai.types.chat import ChatCompletion, ChatCompletionMessageToolCall @@ -49,10 +70,10 @@ "claude-3-5-sonnet-20240620": (0.003, 0.015), "claude-3-sonnet-20240229": (0.003, 0.015), "claude-3-opus-20240229": (0.015, 0.075), - "claude-2.0": (0.008, 0.024), + "claude-3-haiku-20240307": (0.00025, 0.00125), "claude-2.1": (0.008, 0.024), - "claude-3.0-opus": (0.015, 0.075), - "claude-3.0-haiku": (0.00025, 0.00125), + "claude-2.0": (0.008, 0.024), + "claude-instant-1.2": (0.008, 0.024), } @@ -64,14 +85,44 @@ def __init__(self, **kwargs: Any): api_key (str): The API key for the Anthropic API or set the `ANTHROPIC_API_KEY` environment variable. """ self._api_key = kwargs.get("api_key", None) + self._aws_access_key = kwargs.get("aws_access_key", None) + self._aws_secret_key = kwargs.get("aws_secret_key", None) + self._aws_session_token = kwargs.get("aws_session_token", None) + self._aws_region = kwargs.get("aws_region", None) if not self._api_key: self._api_key = os.getenv("ANTHROPIC_API_KEY") - if self._api_key is None: - raise ValueError("API key is required to use the Anthropic API.") + if not self._aws_access_key: + self._aws_access_key = os.getenv("AWS_ACCESS_KEY") + + if not self._aws_secret_key: + self._aws_secret_key = os.getenv("AWS_SECRET_KEY") + + if not self._aws_session_token: + self._aws_session_token = os.getenv("AWS_SESSION_TOKEN") + + if not self._aws_region: + self._aws_region = os.getenv("AWS_REGION") + + if self._api_key is None and ( + self._aws_access_key is None + or self._aws_secret_key is None + or self._aws_session_token is None + or self._aws_region is None + ): + raise ValueError("API key or AWS credentials are required to use the Anthropic API.") + + if self._api_key is not None: + self._client = Anthropic(api_key=self._api_key) + else: + self._client = AnthropicBedrock( + aws_access_key=self._aws_access_key, + aws_secret_key=self._aws_secret_key, + aws_session_token=self._aws_session_token, + aws_region=self._aws_region, + ) - self._client = Anthropic(api_key=self._api_key) self._last_tooluse_status = {} def load_config(self, params: Dict[str, Any]): @@ -107,6 +158,22 @@ def cost(self, response) -> float: def api_key(self): return self._api_key + @property + def aws_access_key(self): + return self._aws_access_key + + @property + def aws_secret_key(self): + return self._aws_secret_key + + @property + def aws_session_token(self): + return self._aws_session_token + + @property + def aws_region(self): + return self._aws_region + def create(self, params: Dict[str, Any]) -> Completion: if "tools" in params: converted_functions = self.convert_tools_to_functions(params["tools"]) @@ -250,6 +317,7 @@ def oai_messages_to_anthropic_messages(params: Dict[str, Any]) -> list[dict[str, tool_use_messages = 0 tool_result_messages = 0 last_tool_use_index = -1 + last_tool_result_index = -1 for message in params["messages"]: if message["role"] == "system": params["system"] = message["content"] @@ -290,25 +358,26 @@ def oai_messages_to_anthropic_messages(params: Dict[str, Any]) -> list[dict[str, } ) elif "tool_call_id" in message: - - if expected_role == "assistant": - # Insert an extra assistant message as we will append a user message - processed_messages.append(assistant_continue_message) - if has_tools: # Map the tool usage call to tool_result for Anthropic - processed_messages.append( - { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": message["tool_call_id"], - "content": message["content"], - } - ], - } - ) + tool_result = { + "type": "tool_result", + "tool_use_id": message["tool_call_id"], + "content": message["content"], + } + + # If the previous message also had a tool_result, add it to that + # Otherwise append a new message + if last_tool_result_index == len(processed_messages) - 1: + processed_messages[-1]["content"].append(tool_result) + else: + if expected_role == "assistant": + # Insert an extra assistant message as we will append a user message + processed_messages.append(assistant_continue_message) + + processed_messages.append({"role": "user", "content": [tool_result]}) + last_tool_result_index = len(processed_messages) - 1 + tool_result_messages += 1 else: # Not using tools, so put in a plain text message diff --git a/autogen/oai/client.py b/autogen/oai/client.py index 2c14ca0d4a0c..4e9d794a1f75 100644 --- a/autogen/oai/client.py +++ b/autogen/oai/client.py @@ -70,6 +70,20 @@ except ImportError as e: together_import_exception = e +try: + from autogen.oai.groq import GroqClient + + groq_import_exception: Optional[ImportError] = None +except ImportError as e: + groq_import_exception = e + +try: + from autogen.oai.cohere import CohereClient + + cohere_import_exception: Optional[ImportError] = None +except ImportError as e: + cohere_import_exception = e + logger = logging.getLogger(__name__) if not logger.handlers: # Add the console handler. @@ -483,7 +497,18 @@ def _register_default_client(self, config: Dict[str, Any], openai_config: Dict[s elif api_type is not None and api_type.startswith("together"): if together_import_exception: raise ImportError("Please install `together` to use the Together.AI API.") - self._clients.append(TogetherClient(**config)) + client = TogetherClient(**openai_config) + self._clients.append(client) + elif api_type is not None and api_type.startswith("groq"): + if groq_import_exception: + raise ImportError("Please install `groq` to use the Groq API.") + client = GroqClient(**openai_config) + self._clients.append(client) + elif api_type is not None and api_type.startswith("cohere"): + if cohere_import_exception: + raise ImportError("Please install `cohere` to use the Groq API.") + client = CohereClient(**openai_config) + self._clients.append(client) else: client = OpenAI(**openai_config) self._clients.append(OpenAIClient(client)) @@ -770,7 +795,7 @@ def _cost_with_customized_price( n_output_tokens = response.usage.completion_tokens if response.usage is not None else 0 # type: ignore [union-attr] if n_output_tokens is None: n_output_tokens = 0 - return n_input_tokens * price_1k[0] + n_output_tokens * price_1k[1] + return (n_input_tokens * price_1k[0] + n_output_tokens * price_1k[1]) / 1000 @staticmethod def _update_dict_from_chunk(chunk: BaseModel, d: Dict[str, Any], field: str) -> int: diff --git a/autogen/oai/cohere.py b/autogen/oai/cohere.py new file mode 100644 index 000000000000..e04d07327203 --- /dev/null +++ b/autogen/oai/cohere.py @@ -0,0 +1,459 @@ +"""Create an OpenAI-compatible client using Cohere's API. + +Example: + llm_config={ + "config_list": [{ + "api_type": "cohere", + "model": "command-r-plus", + "api_key": os.environ.get("COHERE_API_KEY") + } + ]} + + agent = autogen.AssistantAgent("my_agent", llm_config=llm_config) + +Install Cohere's python library using: pip install --upgrade cohere + +Resources: +- https://docs.cohere.com/reference/chat +""" + +from __future__ import annotations + +import json +import logging +import os +import random +import sys +import time +import warnings +from typing import Any, Dict, List + +from cohere import Client as Cohere +from cohere.types import ToolParameterDefinitionsValue, ToolResult +from flaml.automl.logger import logger_formatter +from openai.types.chat import ChatCompletion, ChatCompletionMessageToolCall +from openai.types.chat.chat_completion import ChatCompletionMessage, Choice +from openai.types.completion_usage import CompletionUsage + +from autogen.oai.client_utils import validate_parameter + +logger = logging.getLogger(__name__) +if not logger.handlers: + # Add the console handler. + _ch = logging.StreamHandler(stream=sys.stdout) + _ch.setFormatter(logger_formatter) + logger.addHandler(_ch) + + +COHERE_PRICING_1K = { + "command-r-plus": (0.003, 0.015), + "command-r": (0.0005, 0.0015), + "command-nightly": (0.00025, 0.00125), + "command": (0.015, 0.075), + "command-light": (0.008, 0.024), + "command-light-nightly": (0.008, 0.024), +} + + +class CohereClient: + """Client for Cohere's API.""" + + def __init__(self, **kwargs): + """Requires api_key or environment variable to be set + + Args: + api_key (str): The API key for using Cohere (or environment variable COHERE_API_KEY needs to be set) + """ + # Ensure we have the api_key upon instantiation + self.api_key = kwargs.get("api_key", None) + if not self.api_key: + self.api_key = os.getenv("COHERE_API_KEY") + + assert ( + self.api_key + ), "Please include the api_key in your config list entry for Cohere or set the COHERE_API_KEY env variable." + + def message_retrieval(self, response) -> List: + """ + Retrieve and return a list of strings or a list of Choice.Message from the response. + + NOTE: if a list of Choice.Message is returned, it currently needs to contain the fields of OpenAI's ChatCompletion Message object, + since that is expected for function or tool calling in the rest of the codebase at the moment, unless a custom agent is being used. + """ + return [choice.message for choice in response.choices] + + def cost(self, response) -> float: + return response.cost + + @staticmethod + def get_usage(response) -> Dict: + """Return usage summary of the response using RESPONSE_USAGE_KEYS.""" + # ... # pragma: no cover + return { + "prompt_tokens": response.usage.prompt_tokens, + "completion_tokens": response.usage.completion_tokens, + "total_tokens": response.usage.total_tokens, + "cost": response.cost, + "model": response.model, + } + + def parse_params(self, params: Dict[str, Any]) -> Dict[str, Any]: + """Loads the parameters for Cohere API from the passed in parameters and returns a validated set. Checks types, ranges, and sets defaults""" + cohere_params = {} + + # Check that we have what we need to use Cohere's API + # We won't enforce the available models as they are likely to change + cohere_params["model"] = params.get("model", None) + assert cohere_params[ + "model" + ], "Please specify the 'model' in your config list entry to nominate the Cohere model to use." + + # Validate allowed Cohere parameters + # https://docs.cohere.com/reference/chat + cohere_params["temperature"] = validate_parameter( + params, "temperature", (int, float), False, 0.3, (0, None), None + ) + cohere_params["max_tokens"] = validate_parameter(params, "max_tokens", int, True, None, (0, None), None) + cohere_params["k"] = validate_parameter(params, "k", int, False, 0, (0, 500), None) + cohere_params["p"] = validate_parameter(params, "p", (int, float), False, 0.75, (0.01, 0.99), None) + cohere_params["seed"] = validate_parameter(params, "seed", int, True, None, None, None) + cohere_params["frequency_penalty"] = validate_parameter( + params, "frequency_penalty", (int, float), True, 0, (0, 1), None + ) + cohere_params["presence_penalty"] = validate_parameter( + params, "presence_penalty", (int, float), True, 0, (0, 1), None + ) + + # Cohere parameters we are ignoring: + # preamble - we will put the system prompt in here. + # parallel_tool_calls (defaults to True), perfect as is. + # conversation_id - allows resuming a previous conversation, we don't support this. + logging.info("Conversation ID: %s", params.get("conversation_id", "None")) + # connectors - allows web search or other custom connectors, not implementing for now but could be useful in the future. + logging.info("Connectors: %s", params.get("connectors", "None")) + # search_queries_only - to control whether only search queries are used, we're not using connectors so ignoring. + # documents - a list of documents that can be used to support the chat. Perhaps useful in the future for RAG. + # citation_quality - used for RAG flows and dependent on other parameters we're ignoring. + # max_input_tokens - limits input tokens, not needed. + logging.info("Max Input Tokens: %s", params.get("max_input_tokens", "None")) + # stop_sequences - used to stop generation, not needed. + logging.info("Stop Sequences: %s", params.get("stop_sequences", "None")) + + return cohere_params + + def create(self, params: Dict) -> ChatCompletion: + + messages = params.get("messages", []) + + # Parse parameters to the Cohere API's parameters + cohere_params = self.parse_params(params) + + # Convert AutoGen messages to Cohere messages + cohere_messages, preamble, final_message = oai_messages_to_cohere_messages(messages, params, cohere_params) + + cohere_params["chat_history"] = cohere_messages + cohere_params["message"] = final_message + cohere_params["preamble"] = preamble + + # We use chat model by default + client = Cohere(api_key=self.api_key) + + # Token counts will be returned + prompt_tokens = 0 + completion_tokens = 0 + total_tokens = 0 + + # Stream if in parameters + streaming = True if "stream" in params and params["stream"] else False + cohere_finish = "" + + max_retries = 5 + for attempt in range(max_retries): + ans = None + try: + if streaming: + response = client.chat_stream(**cohere_params) + else: + response = client.chat(**cohere_params) + except CohereRateLimitError as e: + raise RuntimeError(f"Cohere exception occurred: {e}") + else: + + if streaming: + # Streaming... + ans = "" + for event in response: + if event.event_type == "text-generation": + ans = ans + event.text + elif event.event_type == "tool-calls-generation": + # When streaming, tool calls are compiled at the end into a single event_type + ans = event.text + cohere_finish = "tool_calls" + tool_calls = [] + for tool_call in event.tool_calls: + tool_calls.append( + ChatCompletionMessageToolCall( + id=str(random.randint(0, 100000)), + function={ + "name": tool_call.name, + "arguments": ( + "" if tool_call.parameters is None else json.dumps(tool_call.parameters) + ), + }, + type="function", + ) + ) + + # Not using billed_units, but that may be better for cost purposes + prompt_tokens = event.response.meta.tokens.input_tokens + completion_tokens = event.response.meta.tokens.output_tokens + total_tokens = prompt_tokens + completion_tokens + + response_id = event.response.response_id + else: + # Non-streaming finished + ans: str = response.text + + # Not using billed_units, but that may be better for cost purposes + prompt_tokens = response.meta.tokens.input_tokens + completion_tokens = response.meta.tokens.output_tokens + total_tokens = prompt_tokens + completion_tokens + + response_id = response.response_id + break + + if response is not None: + + response_content = ans + + if streaming: + # Streaming response + if cohere_finish == "": + cohere_finish = "stop" + tool_calls = None + else: + # Non-streaming response + # If we have tool calls as the response, populate completed tool calls for our return OAI response + if response.tool_calls is not None: + cohere_finish = "tool_calls" + tool_calls = [] + for tool_call in response.tool_calls: + + # if parameters are null, clear them out (Cohere can return a string "null" if no parameter values) + + tool_calls.append( + ChatCompletionMessageToolCall( + id=str(random.randint(0, 100000)), + function={ + "name": tool_call.name, + "arguments": ( + "" if tool_call.parameters is None else json.dumps(tool_call.parameters) + ), + }, + type="function", + ) + ) + else: + cohere_finish = "stop" + tool_calls = None + else: + raise RuntimeError(f"Failed to get response from Cohere after retrying {attempt + 1} times.") + + # 3. convert output + message = ChatCompletionMessage( + role="assistant", + content=response_content, + function_call=None, + tool_calls=tool_calls, + ) + choices = [Choice(finish_reason=cohere_finish, index=0, message=message)] + + response_oai = ChatCompletion( + id=response_id, + model=cohere_params["model"], + created=int(time.time()), + object="chat.completion", + choices=choices, + usage=CompletionUsage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens, + ), + cost=calculate_cohere_cost(prompt_tokens, completion_tokens, cohere_params["model"]), + ) + + return response_oai + + +def oai_messages_to_cohere_messages( + messages: list[Dict[str, Any]], params: Dict[str, Any], cohere_params: Dict[str, Any] +) -> tuple[list[dict[str, Any]], str, str]: + """Convert messages from OAI format to Cohere's format. + We correct for any specific role orders and types. + + Parameters: + messages: list[Dict[str, Any]]: AutoGen messages + params: Dict[str, Any]: AutoGen parameters dictionary + cohere_params: Dict[str, Any]: Cohere parameters dictionary + + Returns: + List[Dict[str, Any]]: Chat History messages + str: Preamble (system message) + str: Message (the final user message) + """ + + cohere_messages = [] + preamble = "" + + # Tools + if "tools" in params: + cohere_tools = [] + for tool in params["tools"]: + + # build list of properties + parameters = {} + + for key, value in tool["function"]["parameters"]["properties"].items(): + type_str = value["type"] + required = True # Defaults to False, we could consider leaving it as default. + description = value["description"] + + # If we have an 'enum' key, add that to the description (as not allowed to pass in enum as a field) + if "enum" in value: + # Access the enum list + enum_values = value["enum"] + enum_strings = [str(value) for value in enum_values] + enum_string = ", ".join(enum_strings) + description = description + ". Possible values are " + enum_string + "." + + parameters[key] = ToolParameterDefinitionsValue( + description=description, type=type_str, required=required + ) + + cohere_tool = { + "name": tool["function"]["name"], + "description": tool["function"]["description"], + "parameter_definitions": parameters, + } + + cohere_tools.append(cohere_tool) + + if len(cohere_tools) > 0: + cohere_params["tools"] = cohere_tools + + tool_calls = [] + tool_results = [] + + # Rules for cohere messages: + # no 'name' field + # 'system' messages go into the preamble parameter + # user role = 'USER' + # assistant role = 'CHATBOT' + # 'content' field renamed to 'message' + # tools go into tools parameter + # tool_results go into tool_results parameter + for message in messages: + + if "role" in message and message["role"] == "system": + # System message + if preamble == "": + preamble = message["content"] + else: + preamble = preamble + "\n" + message["content"] + elif "tool_calls" in message: + # Suggested tool calls, build up the list before we put it into the tool_results + for tool_call in message["tool_calls"]: + tool_calls.append(tool_call) + + # We also add the suggested tool call as a message + new_message = { + "role": "CHATBOT", + "message": message["content"], + # Not including tools in this message, may need to. Testing required. + } + + cohere_messages.append(new_message) + elif "role" in message and message["role"] == "tool": + if "tool_call_id" in message: + # Convert the tool call to a result + + tool_call_id = message["tool_call_id"] + content_output = message["content"] + + # Find the original tool + for tool_call in tool_calls: + if tool_call["id"] == tool_call_id: + + call = { + "name": tool_call["function"]["name"], + "parameters": json.loads( + tool_call["function"]["arguments"] + if not tool_call["function"]["arguments"] == "" + else "{}" + ), + } + output = [{"value": content_output}] + + tool_results.append(ToolResult(call=call, outputs=output)) + + break + elif "content" in message and isinstance(message["content"], str): + # Standard text message + new_message = { + "role": "USER" if message["role"] == "user" else "CHATBOT", + "message": message["content"], + } + + cohere_messages.append(new_message) + + # Append any Tool Results + if len(tool_results) != 0: + cohere_params["tool_results"] = tool_results + + # Enable multi-step tool use: https://docs.cohere.com/docs/multi-step-tool-use + cohere_params["force_single_step"] = False + + # If we're adding tool_results, like we are, the last message can't be a USER message + # So, we add a CHATBOT 'continue' message, if so. + if cohere_messages[-1]["role"] == "USER": + cohere_messages.append({"role": "CHATBOT", "content": "Please continue."}) + + # We return a blank message when we have tool results + # TODO: Check what happens if tool_results aren't the latest message + return cohere_messages, preamble, "" + + else: + + # We need to get the last message to assign to the message field for Cohere, + # if the last message is a user message, use that, otherwise put in 'continue'. + if cohere_messages[-1]["role"] == "USER": + return cohere_messages[0:-1], preamble, cohere_messages[-1]["message"] + else: + return cohere_messages, preamble, "Please continue." + + +def calculate_cohere_cost(input_tokens: int, output_tokens: int, model: str) -> float: + """Calculate the cost of the completion using the Cohere pricing.""" + total = 0.0 + + if model in COHERE_PRICING_1K: + input_cost_per_k, output_cost_per_k = COHERE_PRICING_1K[model] + input_cost = (input_tokens / 1000) * input_cost_per_k + output_cost = (output_tokens / 1000) * output_cost_per_k + total = input_cost + output_cost + else: + warnings.warn(f"Cost calculation not available for {model} model", UserWarning) + + return total + + +class CohereError(Exception): + """Base class for other Cohere exceptions""" + + pass + + +class CohereRateLimitError(CohereError): + """Raised when rate limit is exceeded""" + + pass diff --git a/autogen/oai/gemini.py b/autogen/oai/gemini.py index 8babb8727e3c..73d41cddbf53 100644 --- a/autogen/oai/gemini.py +++ b/autogen/oai/gemini.py @@ -72,7 +72,7 @@ class GeminiClient: "max_output_tokens": "max_output_tokens", } - def _initialize_vartexai(self, **params): + def _initialize_vertexai(self, **params): if "google_application_credentials" in params: # Path to JSON Keyfile os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = params["google_application_credentials"] @@ -106,7 +106,7 @@ def __init__(self, **kwargs): self.api_key = os.getenv("GOOGLE_API_KEY") if self.api_key is None: self.use_vertexai = True - self._initialize_vartexai(**kwargs) + self._initialize_vertexai(**kwargs) else: self.use_vertexai = False else: @@ -142,7 +142,7 @@ def get_usage(response) -> Dict: def create(self, params: Dict) -> ChatCompletion: if self.use_vertexai: - self._initialize_vartexai(**params) + self._initialize_vertexai(**params) else: assert ("project_id" not in params) and ( "location" not in params diff --git a/autogen/oai/groq.py b/autogen/oai/groq.py new file mode 100644 index 000000000000..d2abe5116a25 --- /dev/null +++ b/autogen/oai/groq.py @@ -0,0 +1,282 @@ +"""Create an OpenAI-compatible client using Groq's API. + +Example: + llm_config={ + "config_list": [{ + "api_type": "groq", + "model": "mixtral-8x7b-32768", + "api_key": os.environ.get("GROQ_API_KEY") + } + ]} + + agent = autogen.AssistantAgent("my_agent", llm_config=llm_config) + +Install Groq's python library using: pip install --upgrade groq + +Resources: +- https://console.groq.com/docs/quickstart +""" + +from __future__ import annotations + +import copy +import os +import time +import warnings +from typing import Any, Dict, List + +from groq import Groq, Stream +from openai.types.chat import ChatCompletion, ChatCompletionMessageToolCall +from openai.types.chat.chat_completion import ChatCompletionMessage, Choice +from openai.types.completion_usage import CompletionUsage + +from autogen.oai.client_utils import should_hide_tools, validate_parameter + +# Cost per thousand tokens - Input / Output (NOTE: Convert $/Million to $/K) +GROQ_PRICING_1K = { + "llama3-70b-8192": (0.00059, 0.00079), + "mixtral-8x7b-32768": (0.00024, 0.00024), + "llama3-8b-8192": (0.00005, 0.00008), + "gemma-7b-it": (0.00007, 0.00007), +} + + +class GroqClient: + """Client for Groq's API.""" + + def __init__(self, **kwargs): + """Requires api_key or environment variable to be set + + Args: + api_key (str): The API key for using Groq (or environment variable GROQ_API_KEY needs to be set) + """ + # Ensure we have the api_key upon instantiation + self.api_key = kwargs.get("api_key", None) + if not self.api_key: + self.api_key = os.getenv("GROQ_API_KEY") + + assert ( + self.api_key + ), "Please include the api_key in your config list entry for Groq or set the GROQ_API_KEY env variable." + + def message_retrieval(self, response) -> List: + """ + Retrieve and return a list of strings or a list of Choice.Message from the response. + + NOTE: if a list of Choice.Message is returned, it currently needs to contain the fields of OpenAI's ChatCompletion Message object, + since that is expected for function or tool calling in the rest of the codebase at the moment, unless a custom agent is being used. + """ + return [choice.message for choice in response.choices] + + def cost(self, response) -> float: + return response.cost + + @staticmethod + def get_usage(response) -> Dict: + """Return usage summary of the response using RESPONSE_USAGE_KEYS.""" + # ... # pragma: no cover + return { + "prompt_tokens": response.usage.prompt_tokens, + "completion_tokens": response.usage.completion_tokens, + "total_tokens": response.usage.total_tokens, + "cost": response.cost, + "model": response.model, + } + + def parse_params(self, params: Dict[str, Any]) -> Dict[str, Any]: + """Loads the parameters for Groq API from the passed in parameters and returns a validated set. Checks types, ranges, and sets defaults""" + groq_params = {} + + # Check that we have what we need to use Groq's API + # We won't enforce the available models as they are likely to change + groq_params["model"] = params.get("model", None) + assert groq_params[ + "model" + ], "Please specify the 'model' in your config list entry to nominate the Groq model to use." + + # Validate allowed Groq parameters + # https://console.groq.com/docs/api-reference#chat + groq_params["frequency_penalty"] = validate_parameter( + params, "frequency_penalty", (int, float), True, None, (-2, 2), None + ) + groq_params["max_tokens"] = validate_parameter(params, "max_tokens", int, True, None, (0, None), None) + groq_params["presence_penalty"] = validate_parameter( + params, "presence_penalty", (int, float), True, None, (-2, 2), None + ) + groq_params["seed"] = validate_parameter(params, "seed", int, True, None, None, None) + groq_params["stream"] = validate_parameter(params, "stream", bool, True, False, None, None) + groq_params["temperature"] = validate_parameter(params, "temperature", (int, float), True, 1, (0, 2), None) + groq_params["top_p"] = validate_parameter(params, "top_p", (int, float), True, None, None, None) + + # Groq parameters not supported by their models yet, ignoring + # logit_bias, logprobs, top_logprobs + + # Groq parameters we are ignoring: + # n (must be 1), response_format (to enforce JSON but needs prompting as well), user, + # parallel_tool_calls (defaults to True), stop + # function_call (deprecated), functions (deprecated) + # tool_choice (none if no tools, auto if there are tools) + + return groq_params + + def create(self, params: Dict) -> ChatCompletion: + + messages = params.get("messages", []) + + # Convert AutoGen messages to Groq messages + groq_messages = oai_messages_to_groq_messages(messages) + + # Parse parameters to the Groq API's parameters + groq_params = self.parse_params(params) + + # Add tools to the call if we have them and aren't hiding them + if "tools" in params: + hide_tools = validate_parameter( + params, "hide_tools", str, False, "never", None, ["if_all_run", "if_any_run", "never"] + ) + if not should_hide_tools(groq_messages, params["tools"], hide_tools): + groq_params["tools"] = params["tools"] + + groq_params["messages"] = groq_messages + + # We use chat model by default, and set max_retries to 5 (in line with typical retries loop) + client = Groq(api_key=self.api_key, max_retries=5) + + # Token counts will be returned + prompt_tokens = 0 + completion_tokens = 0 + total_tokens = 0 + + # Streaming tool call recommendations + streaming_tool_calls = [] + + ans = None + try: + response = client.chat.completions.create(**groq_params) + except Exception as e: + raise RuntimeError(f"Groq exception occurred: {e}") + else: + + if groq_params["stream"]: + # Read in the chunks as they stream, taking in tool_calls which may be across + # multiple chunks if more than one suggested + ans = "" + for chunk in response: + ans = ans + (chunk.choices[0].delta.content or "") + + if chunk.choices[0].delta.tool_calls: + # We have a tool call recommendation + for tool_call in chunk.choices[0].delta.tool_calls: + streaming_tool_calls.append( + ChatCompletionMessageToolCall( + id=tool_call.id, + function={ + "name": tool_call.function.name, + "arguments": tool_call.function.arguments, + }, + type="function", + ) + ) + + if chunk.choices[0].finish_reason: + prompt_tokens = chunk.x_groq.usage.prompt_tokens + completion_tokens = chunk.x_groq.usage.completion_tokens + total_tokens = chunk.x_groq.usage.total_tokens + else: + # Non-streaming finished + ans: str = response.choices[0].message.content + + prompt_tokens = response.usage.prompt_tokens + completion_tokens = response.usage.completion_tokens + total_tokens = response.usage.total_tokens + + if response is not None: + + if isinstance(response, Stream): + # Streaming response + if chunk.choices[0].finish_reason == "tool_calls": + groq_finish = "tool_calls" + tool_calls = streaming_tool_calls + else: + groq_finish = "stop" + tool_calls = None + + response_content = ans + response_id = chunk.id + else: + # Non-streaming response + # If we have tool calls as the response, populate completed tool calls for our return OAI response + if response.choices[0].finish_reason == "tool_calls": + groq_finish = "tool_calls" + tool_calls = [] + for tool_call in response.choices[0].message.tool_calls: + tool_calls.append( + ChatCompletionMessageToolCall( + id=tool_call.id, + function={"name": tool_call.function.name, "arguments": tool_call.function.arguments}, + type="function", + ) + ) + else: + groq_finish = "stop" + tool_calls = None + + response_content = response.choices[0].message.content + response_id = response.id + else: + raise RuntimeError("Failed to get response from Groq after retrying 5 times.") + + # 3. convert output + message = ChatCompletionMessage( + role="assistant", + content=response_content, + function_call=None, + tool_calls=tool_calls, + ) + choices = [Choice(finish_reason=groq_finish, index=0, message=message)] + + response_oai = ChatCompletion( + id=response_id, + model=groq_params["model"], + created=int(time.time()), + object="chat.completion", + choices=choices, + usage=CompletionUsage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens, + ), + cost=calculate_groq_cost(prompt_tokens, completion_tokens, groq_params["model"]), + ) + + return response_oai + + +def oai_messages_to_groq_messages(messages: list[Dict[str, Any]]) -> list[dict[str, Any]]: + """Convert messages from OAI format to Groq's format. + We correct for any specific role orders and types. + """ + + groq_messages = copy.deepcopy(messages) + + # Remove the name field + for message in groq_messages: + if "name" in message: + message.pop("name", None) + + return groq_messages + + +def calculate_groq_cost(input_tokens: int, output_tokens: int, model: str) -> float: + """Calculate the cost of the completion using the Groq pricing.""" + total = 0.0 + + if model in GROQ_PRICING_1K: + input_cost_per_k, output_cost_per_k = GROQ_PRICING_1K[model] + input_cost = (input_tokens / 1000) * input_cost_per_k + output_cost = (output_tokens / 1000) * output_cost_per_k + total = input_cost + output_cost + else: + warnings.warn(f"Cost calculation not available for model {model}", UserWarning) + + return total diff --git a/autogen/oai/openai_utils.py b/autogen/oai/openai_utils.py index 0c8a0a413375..749727d952c0 100644 --- a/autogen/oai/openai_utils.py +++ b/autogen/oai/openai_utils.py @@ -96,7 +96,7 @@ def is_valid_api_key(api_key: str) -> bool: Returns: bool: A boolean that indicates if input is valid OpenAI API key. """ - api_key_re = re.compile(r"^sk-(proj-)?[A-Za-z0-9]{32,}$") + api_key_re = re.compile(r"^sk-([A-Za-z0-9]+(-+[A-Za-z0-9]+)*-)?[A-Za-z0-9]{32,}$") return bool(re.fullmatch(api_key_re, api_key)) diff --git a/autogen/runtime_logging.py b/autogen/runtime_logging.py index adb55ba63b4f..1ffc8b622f0a 100644 --- a/autogen/runtime_logging.py +++ b/autogen/runtime_logging.py @@ -14,7 +14,9 @@ if TYPE_CHECKING: from autogen import Agent, ConversableAgent, OpenAIWrapper from autogen.oai.anthropic import AnthropicClient + from autogen.oai.cohere import CohereClient from autogen.oai.gemini import GeminiClient + from autogen.oai.groq import GroqClient from autogen.oai.mistral import MistralAIClient from autogen.oai.together import TogetherClient @@ -110,7 +112,9 @@ def log_new_wrapper(wrapper: OpenAIWrapper, init_args: Dict[str, Union[LLMConfig def log_new_client( - client: Union[AzureOpenAI, OpenAI, GeminiClient, AnthropicClient, MistralAIClient, TogetherClient], + client: Union[ + AzureOpenAI, OpenAI, GeminiClient, AnthropicClient, MistralAIClient, TogetherClient, GroqClient, CohereClient + ], wrapper: OpenAIWrapper, init_args: Dict[str, Any], ) -> None: diff --git a/autogen/token_count_utils.py b/autogen/token_count_utils.py index 2842a7494536..365285e09551 100644 --- a/autogen/token_count_utils.py +++ b/autogen/token_count_utils.py @@ -95,7 +95,7 @@ def _num_token_from_messages(messages: Union[List, Dict], model="gpt-3.5-turbo-0 try: encoding = tiktoken.encoding_for_model(model) except KeyError: - print("Warning: model not found. Using cl100k_base encoding.") + logger.warning(f"Model {model} not found. Using cl100k_base encoding.") encoding = tiktoken.get_encoding("cl100k_base") if model in { "gpt-3.5-turbo-0613", @@ -166,7 +166,7 @@ def num_tokens_from_functions(functions, model="gpt-3.5-turbo-0613") -> int: try: encoding = tiktoken.encoding_for_model(model) except KeyError: - print("Warning: model not found. Using cl100k_base encoding.") + logger.warning(f"Model {model} not found. Using cl100k_base encoding.") encoding = tiktoken.get_encoding("cl100k_base") num_tokens = 0 @@ -193,7 +193,7 @@ def num_tokens_from_functions(functions, model="gpt-3.5-turbo-0613") -> int: function_tokens += 3 function_tokens += len(encoding.encode(o)) else: - print(f"Warning: not supported field {field}") + logger.warning(f"Not supported field {field}") function_tokens += 11 if len(parameters["properties"]) == 0: function_tokens -= 2 diff --git a/autogen/version.py b/autogen/version.py index 110d3e10d2f0..93824aa1f87c 100644 --- a/autogen/version.py +++ b/autogen/version.py @@ -1 +1 @@ -__version__ = "0.2.30" +__version__ = "0.2.32" diff --git a/dotnet/AutoGen.sln b/dotnet/AutoGen.sln index 5ecfe1938873..1218cf129821 100644 --- a/dotnet/AutoGen.sln +++ b/dotnet/AutoGen.sln @@ -1,4 +1,3 @@ - Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.8.34322.80 @@ -33,6 +32,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.Mistral", "src\Auto EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.Mistral.Tests", "test\AutoGen.Mistral.Tests\AutoGen.Mistral.Tests.csproj", "{15441693-3659-4868-B6C1-B106F52FF3BA}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.WebAPI", "src\AutoGen.WebAPI\AutoGen.WebAPI.csproj", "{257FFD71-08E5-40C7-AB04-6A81A78EB410}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.WebAPI.Tests", "test\AutoGen.WebAPI.Tests\AutoGen.WebAPI.Tests.csproj", "{E2EF5E66-683C-4DDC-8ADA-5F676502B9BA}" +EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.SemanticKernel.Tests", "test\AutoGen.SemanticKernel.Tests\AutoGen.SemanticKernel.Tests.csproj", "{1DFABC4A-8458-4875-8DCB-59F3802DAC65}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.OpenAI.Tests", "test\AutoGen.OpenAI.Tests\AutoGen.OpenAI.Tests.csproj", "{D36A85F9-C172-487D-8192-6BFE5D05B4A7}" @@ -61,6 +64,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.Gemini.Sample", "sa EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.AotCompatibility.Tests", "test\AutoGen.AotCompatibility.Tests\AutoGen.AotCompatibility.Tests.csproj", "{6B82F26D-5040-4453-B21B-C8D1F913CE4C}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.OpenAI.Sample", "sample\AutoGen.OpenAI.Sample\AutoGen.OpenAI.Sample.csproj", "{0E635268-351C-4A6B-A28D-593D868C2CA4}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.WebAPI.Sample", "sample\AutoGen.WebAPI.Sample\AutoGen.WebAPI.Sample.csproj", "{12079C18-A519-403F-BBFD-200A36A0C083}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -115,6 +122,14 @@ Global {15441693-3659-4868-B6C1-B106F52FF3BA}.Debug|Any CPU.Build.0 = Debug|Any CPU {15441693-3659-4868-B6C1-B106F52FF3BA}.Release|Any CPU.ActiveCfg = Release|Any CPU {15441693-3659-4868-B6C1-B106F52FF3BA}.Release|Any CPU.Build.0 = Release|Any CPU + {257FFD71-08E5-40C7-AB04-6A81A78EB410}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {257FFD71-08E5-40C7-AB04-6A81A78EB410}.Debug|Any CPU.Build.0 = Debug|Any CPU + {257FFD71-08E5-40C7-AB04-6A81A78EB410}.Release|Any CPU.ActiveCfg = Release|Any CPU + {257FFD71-08E5-40C7-AB04-6A81A78EB410}.Release|Any CPU.Build.0 = Release|Any CPU + {E2EF5E66-683C-4DDC-8ADA-5F676502B9BA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E2EF5E66-683C-4DDC-8ADA-5F676502B9BA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E2EF5E66-683C-4DDC-8ADA-5F676502B9BA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E2EF5E66-683C-4DDC-8ADA-5F676502B9BA}.Release|Any CPU.Build.0 = Release|Any CPU {1DFABC4A-8458-4875-8DCB-59F3802DAC65}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {1DFABC4A-8458-4875-8DCB-59F3802DAC65}.Debug|Any CPU.Build.0 = Debug|Any CPU {1DFABC4A-8458-4875-8DCB-59F3802DAC65}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -171,6 +186,14 @@ Global {6B82F26D-5040-4453-B21B-C8D1F913CE4C}.Debug|Any CPU.Build.0 = Debug|Any CPU {6B82F26D-5040-4453-B21B-C8D1F913CE4C}.Release|Any CPU.ActiveCfg = Release|Any CPU {6B82F26D-5040-4453-B21B-C8D1F913CE4C}.Release|Any CPU.Build.0 = Release|Any CPU + {0E635268-351C-4A6B-A28D-593D868C2CA4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0E635268-351C-4A6B-A28D-593D868C2CA4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0E635268-351C-4A6B-A28D-593D868C2CA4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0E635268-351C-4A6B-A28D-593D868C2CA4}.Release|Any CPU.Build.0 = Release|Any CPU + {12079C18-A519-403F-BBFD-200A36A0C083}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {12079C18-A519-403F-BBFD-200A36A0C083}.Debug|Any CPU.Build.0 = Debug|Any CPU + {12079C18-A519-403F-BBFD-200A36A0C083}.Release|Any CPU.ActiveCfg = Release|Any CPU + {12079C18-A519-403F-BBFD-200A36A0C083}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -188,6 +211,8 @@ Global {63445BB7-DBB9-4AEF-9D6F-98BBE75EE1EC} = {18BF8DD7-0585-48BF-8F97-AD333080CE06} {6585D1A4-3D97-4D76-A688-1933B61AEB19} = {18BF8DD7-0585-48BF-8F97-AD333080CE06} {15441693-3659-4868-B6C1-B106F52FF3BA} = {F823671B-3ECA-4AE6-86DA-25E920D3FE64} + {257FFD71-08E5-40C7-AB04-6A81A78EB410} = {18BF8DD7-0585-48BF-8F97-AD333080CE06} + {E2EF5E66-683C-4DDC-8ADA-5F676502B9BA} = {F823671B-3ECA-4AE6-86DA-25E920D3FE64} {1DFABC4A-8458-4875-8DCB-59F3802DAC65} = {F823671B-3ECA-4AE6-86DA-25E920D3FE64} {D36A85F9-C172-487D-8192-6BFE5D05B4A7} = {F823671B-3ECA-4AE6-86DA-25E920D3FE64} {B61388CA-DC73-4B7F-A7B2-7B9A86C9229E} = {F823671B-3ECA-4AE6-86DA-25E920D3FE64} @@ -202,6 +227,8 @@ Global {8EA16BAB-465A-4C07-ABC4-1070D40067E9} = {F823671B-3ECA-4AE6-86DA-25E920D3FE64} {19679B75-CE3A-4DF0-A3F0-CA369D2760A4} = {FBFEAD1F-29EB-4D99-A672-0CD8473E10B9} {6B82F26D-5040-4453-B21B-C8D1F913CE4C} = {F823671B-3ECA-4AE6-86DA-25E920D3FE64} + {0E635268-351C-4A6B-A28D-593D868C2CA4} = {FBFEAD1F-29EB-4D99-A672-0CD8473E10B9} + {12079C18-A519-403F-BBFD-200A36A0C083} = {FBFEAD1F-29EB-4D99-A672-0CD8473E10B9} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {93384647-528D-46C8-922C-8DB36A382F0B} diff --git a/dotnet/Directory.Build.props b/dotnet/Directory.Build.props index 4b3e9441f1ee..29e40fff384c 100644 --- a/dotnet/Directory.Build.props +++ b/dotnet/Directory.Build.props @@ -31,6 +31,7 @@ + diff --git a/dotnet/eng/MetaInfo.props b/dotnet/eng/MetaInfo.props index 041ee0ec6c97..f43a47c8ce27 100644 --- a/dotnet/eng/MetaInfo.props +++ b/dotnet/eng/MetaInfo.props @@ -1,7 +1,7 @@ - 0.0.15 + 0.0.16 AutoGen https://microsoft.github.io/autogen-for-net/ https://github.com/microsoft/autogen diff --git a/dotnet/eng/Version.props b/dotnet/eng/Version.props index 0b8dcaa565cb..20be183219e5 100644 --- a/dotnet/eng/Version.props +++ b/dotnet/eng/Version.props @@ -2,8 +2,8 @@ 1.0.0-beta.17 - 1.10.0 - 1.10.0-alpha + 1.15.1 + 1.15.1-alpha 5.0.0 4.3.0 6.0.0 @@ -12,6 +12,7 @@ 17.7.0 1.0.0-beta.24229.4 8.0.0 + 8.0.4 3.0.0 4.3.0.2 diff --git a/dotnet/sample/AutoGen.Anthropic.Samples/AutoGen.Anthropic.Samples.csproj b/dotnet/sample/AutoGen.Anthropic.Samples/AutoGen.Anthropic.Samples.csproj index 33a5aa7f16b6..2948c9bf283c 100644 --- a/dotnet/sample/AutoGen.Anthropic.Samples/AutoGen.Anthropic.Samples.csproj +++ b/dotnet/sample/AutoGen.Anthropic.Samples/AutoGen.Anthropic.Samples.csproj @@ -13,6 +13,7 @@ + diff --git a/dotnet/sample/AutoGen.Anthropic.Samples/AnthropicSamples.cs b/dotnet/sample/AutoGen.Anthropic.Samples/Create_Anthropic_Agent.cs similarity index 93% rename from dotnet/sample/AutoGen.Anthropic.Samples/AnthropicSamples.cs rename to dotnet/sample/AutoGen.Anthropic.Samples/Create_Anthropic_Agent.cs index 94b5f37511e6..6f32c3cb4a21 100644 --- a/dotnet/sample/AutoGen.Anthropic.Samples/AnthropicSamples.cs +++ b/dotnet/sample/AutoGen.Anthropic.Samples/Create_Anthropic_Agent.cs @@ -1,5 +1,5 @@ // Copyright (c) Microsoft Corporation. All rights reserved. -// AnthropicSamples.cs +// Create_Anthropic_Agent.cs using AutoGen.Anthropic.Extensions; using AutoGen.Anthropic.Utils; @@ -7,7 +7,7 @@ namespace AutoGen.Anthropic.Samples; -public static class AnthropicSamples +public static class Create_Anthropic_Agent { public static async Task RunAsync() { diff --git a/dotnet/sample/AutoGen.Anthropic.Samples/Create_Anthropic_Agent_With_Tool.cs b/dotnet/sample/AutoGen.Anthropic.Samples/Create_Anthropic_Agent_With_Tool.cs new file mode 100644 index 000000000000..0324a39ffa59 --- /dev/null +++ b/dotnet/sample/AutoGen.Anthropic.Samples/Create_Anthropic_Agent_With_Tool.cs @@ -0,0 +1,100 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Create_Anthropic_Agent_With_Tool.cs + +using AutoGen.Anthropic.DTO; +using AutoGen.Anthropic.Extensions; +using AutoGen.Anthropic.Utils; +using AutoGen.Core; +using FluentAssertions; + +namespace AutoGen.Anthropic.Samples; + +#region WeatherFunction + +public partial class WeatherFunction +{ + ///

+ /// Gets the weather based on the location and the unit + /// + /// + /// + /// + [Function] + public async Task GetWeather(string location, string unit) + { + // dummy implementation + return $"The weather in {location} is currently sunny with a tempature of {unit} (s)"; + } +} +#endregion +public class Create_Anthropic_Agent_With_Tool +{ + public static async Task RunAsync() + { + #region define_tool + var tool = new Tool + { + Name = "GetWeather", + Description = "Get the current weather in a given location", + InputSchema = new InputSchema + { + Type = "object", + Properties = new Dictionary + { + { "location", new SchemaProperty { Type = "string", Description = "The city and state, e.g. San Francisco, CA" } }, + { "unit", new SchemaProperty { Type = "string", Description = "The unit of temperature, either \"celsius\" or \"fahrenheit\"" } } + }, + Required = new List { "location" } + } + }; + + var weatherFunction = new WeatherFunction(); + var functionMiddleware = new FunctionCallMiddleware( + functions: [ + weatherFunction.GetWeatherFunctionContract, + ], + functionMap: new Dictionary>> + { + { weatherFunction.GetWeatherFunctionContract.Name!, weatherFunction.GetWeatherWrapper }, + }); + + #endregion + + #region create_anthropic_agent + + var apiKey = Environment.GetEnvironmentVariable("ANTHROPIC_API_KEY") ?? + throw new Exception("Missing ANTHROPIC_API_KEY environment variable."); + + var anthropicClient = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, apiKey); + var agent = new AnthropicClientAgent(anthropicClient, "assistant", AnthropicConstants.Claude3Haiku, + tools: [tool]); // Define tools for AnthropicClientAgent + #endregion + + #region register_middleware + + var agentWithConnector = agent + .RegisterMessageConnector() + .RegisterPrintMessage() + .RegisterStreamingMiddleware(functionMiddleware); + #endregion register_middleware + + #region single_turn + var question = new TextMessage(Role.Assistant, + "What is the weather like in San Francisco?", + from: "user"); + var functionCallReply = await agentWithConnector.SendAsync(question); + #endregion + + #region Single_turn_verify_reply + functionCallReply.Should().BeOfType(); + #endregion Single_turn_verify_reply + + #region Multi_turn + var finalReply = await agentWithConnector.SendAsync(chatHistory: [question, functionCallReply]); + #endregion Multi_turn + + #region Multi_turn_verify_reply + finalReply.Should().BeOfType(); + #endregion Multi_turn_verify_reply + } +} diff --git a/dotnet/sample/AutoGen.Anthropic.Samples/Program.cs b/dotnet/sample/AutoGen.Anthropic.Samples/Program.cs index f3c615088610..6d1e4e594b99 100644 --- a/dotnet/sample/AutoGen.Anthropic.Samples/Program.cs +++ b/dotnet/sample/AutoGen.Anthropic.Samples/Program.cs @@ -7,6 +7,6 @@ internal static class Program { public static async Task Main(string[] args) { - await AnthropicSamples.RunAsync(); + await Create_Anthropic_Agent_With_Tool.RunAsync(); } } diff --git a/dotnet/sample/AutoGen.BasicSamples/CodeSnippet/CreateAnAgent.cs b/dotnet/sample/AutoGen.BasicSamples/CodeSnippet/CreateAnAgent.cs index 4833c6195c9d..a103f4ec2d4d 100644 --- a/dotnet/sample/AutoGen.BasicSamples/CodeSnippet/CreateAnAgent.cs +++ b/dotnet/sample/AutoGen.BasicSamples/CodeSnippet/CreateAnAgent.cs @@ -129,7 +129,7 @@ public async Task CodeSnippet5() }, functionMap: new Dictionary>> { - { this.UpperCaseFunction.Name, this.UpperCaseWrapper }, // The wrapper function for the UpperCase function + { this.UpperCaseFunctionContract.Name, this.UpperCaseWrapper }, // The wrapper function for the UpperCase function }); var response = await assistantAgent.SendAsync("hello"); diff --git a/dotnet/sample/AutoGen.BasicSamples/CodeSnippet/MiddlewareAgentCodeSnippet.cs b/dotnet/sample/AutoGen.BasicSamples/CodeSnippet/MiddlewareAgentCodeSnippet.cs index 320afd0de679..1b5a9a903207 100644 --- a/dotnet/sample/AutoGen.BasicSamples/CodeSnippet/MiddlewareAgentCodeSnippet.cs +++ b/dotnet/sample/AutoGen.BasicSamples/CodeSnippet/MiddlewareAgentCodeSnippet.cs @@ -13,38 +13,46 @@ public class MiddlewareAgentCodeSnippet public async Task CreateMiddlewareAgentAsync() { #region create_middleware_agent_with_original_agent - // Create an agent that always replies "Hello World" - IAgent agent = new DefaultReplyAgent(name: "assistant", defaultReply: "Hello World"); + // Create an agent that always replies "Hi!" + IAgent agent = new DefaultReplyAgent(name: "assistant", defaultReply: "Hi!"); // Create a middleware agent on top of default reply agent var middlewareAgent = new MiddlewareAgent(innerAgent: agent); middlewareAgent.Use(async (messages, options, agent, ct) => { - var lastMessage = messages.Last() as TextMessage; - lastMessage.Content = $"[middleware 0] {lastMessage.Content}"; + if (messages.Last() is TextMessage lastMessage && lastMessage.Content.Contains("Hello World")) + { + lastMessage.Content = $"[middleware 0] {lastMessage.Content}"; + return lastMessage; + } + return await agent.GenerateReplyAsync(messages, options, ct); }); var reply = await middlewareAgent.SendAsync("Hello World"); reply.GetContent().Should().Be("[middleware 0] Hello World"); + reply = await middlewareAgent.SendAsync("Hello AI!"); + reply.GetContent().Should().Be("Hi!"); #endregion create_middleware_agent_with_original_agent #region register_middleware_agent middlewareAgent = agent.RegisterMiddleware(async (messages, options, agent, ct) => { - var lastMessage = messages.Last() as TextMessage; - lastMessage.Content = $"[middleware 0] {lastMessage.Content}"; + if (messages.Last() is TextMessage lastMessage && lastMessage.Content.Contains("Hello World")) + { + lastMessage.Content = $"[middleware 0] {lastMessage.Content}"; + return lastMessage; + } + return await agent.GenerateReplyAsync(messages, options, ct); }); #endregion register_middleware_agent #region short_circuit_middleware_agent - // This middleware will short circuit the agent and return the last message directly. + // This middleware will short circuit the agent and return a message directly. middlewareAgent.Use(async (messages, options, agent, ct) => { - var lastMessage = messages.Last() as TextMessage; - lastMessage.Content = $"[middleware shortcut]"; - return lastMessage; + return new TextMessage(Role.Assistant, $"[middleware shortcut]"); }); #endregion short_circuit_middleware_agent } diff --git a/dotnet/sample/AutoGen.BasicSamples/Example04_Dynamic_GroupChat_Coding_Task.cs b/dotnet/sample/AutoGen.BasicSamples/Example04_Dynamic_GroupChat_Coding_Task.cs index 47dd8ce66c90..216059928408 100644 --- a/dotnet/sample/AutoGen.BasicSamples/Example04_Dynamic_GroupChat_Coding_Task.cs +++ b/dotnet/sample/AutoGen.BasicSamples/Example04_Dynamic_GroupChat_Coding_Task.cs @@ -17,14 +17,18 @@ public static async Task RunAsync() // setup dotnet interactive var workDir = Path.Combine(Path.GetTempPath(), "InteractiveService"); if (!Directory.Exists(workDir)) + { Directory.CreateDirectory(workDir); + } using var service = new InteractiveService(workDir); var dotnetInteractiveFunctions = new DotnetInteractiveFunction(service); var result = Path.Combine(workDir, "result.txt"); if (File.Exists(result)) + { File.Delete(result); + } await service.StartAsync(workDir, default); diff --git a/dotnet/sample/AutoGen.BasicSamples/Example07_Dynamic_GroupChat_Calculate_Fibonacci.cs b/dotnet/sample/AutoGen.BasicSamples/Example07_Dynamic_GroupChat_Calculate_Fibonacci.cs index 6584baa5fae5..004e0f055449 100644 --- a/dotnet/sample/AutoGen.BasicSamples/Example07_Dynamic_GroupChat_Calculate_Fibonacci.cs +++ b/dotnet/sample/AutoGen.BasicSamples/Example07_Dynamic_GroupChat_Calculate_Fibonacci.cs @@ -8,6 +8,7 @@ using AutoGen.Core; using AutoGen.DotnetInteractive; using AutoGen.OpenAI; +using AutoGen.OpenAI.Extension; using FluentAssertions; public partial class Example07_Dynamic_GroupChat_Calculate_Fibonacci @@ -138,7 +139,7 @@ public static async Task CreateReviewerAgentAsync() name: "code_reviewer", systemMessage: @"You review code block from coder", config: gpt3Config, - functions: [functions.ReviewCodeBlockFunction], + functions: [functions.ReviewCodeBlockFunctionContract.ToOpenAIFunctionDefinition()], functionMap: new Dictionary>>() { { nameof(ReviewCodeBlock), functions.ReviewCodeBlockWrapper }, @@ -224,7 +225,9 @@ public static async Task RunWorkflowAsync() long the39thFibonacciNumber = 63245986; var workDir = Path.Combine(Path.GetTempPath(), "InteractiveService"); if (!Directory.Exists(workDir)) + { Directory.CreateDirectory(workDir); + } using var service = new InteractiveService(workDir); var dotnetInteractiveFunctions = new DotnetInteractiveFunction(service); @@ -328,7 +331,9 @@ public static async Task RunAsync() long the39thFibonacciNumber = 63245986; var workDir = Path.Combine(Path.GetTempPath(), "InteractiveService"); if (!Directory.Exists(workDir)) + { Directory.CreateDirectory(workDir); + } using var service = new InteractiveService(workDir); var dotnetInteractiveFunctions = new DotnetInteractiveFunction(service); diff --git a/dotnet/sample/AutoGen.BasicSamples/Example09_LMStudio_FunctionCall.cs b/dotnet/sample/AutoGen.BasicSamples/Example09_LMStudio_FunctionCall.cs index 9a62144df2bd..c9dda27d2e23 100644 --- a/dotnet/sample/AutoGen.BasicSamples/Example09_LMStudio_FunctionCall.cs +++ b/dotnet/sample/AutoGen.BasicSamples/Example09_LMStudio_FunctionCall.cs @@ -5,6 +5,7 @@ using System.Text.Json.Serialization; using AutoGen.Core; using AutoGen.LMStudio; +using AutoGen.OpenAI.Extension; using Azure.AI.OpenAI; namespace AutoGen.BasicSample; @@ -69,8 +70,8 @@ public static async Task RunAsync() // And ask agent to response in function call object format using few-shot example object[] functionList = [ - SerializeFunctionDefinition(instance.GetWeatherFunction), - SerializeFunctionDefinition(instance.GoogleSearchFunction) + SerializeFunctionDefinition(instance.GetWeatherFunctionContract.ToOpenAIFunctionDefinition()), + SerializeFunctionDefinition(instance.GetWeatherFunctionContract.ToOpenAIFunctionDefinition()) ]; var functionListString = JsonSerializer.Serialize(functionList, new JsonSerializerOptions { WriteIndented = true }); var lmAgent = new LMStudioAgent( @@ -98,12 +99,12 @@ You are a helpful AI assistant { var arguments = JsonSerializer.Serialize(functionCall.Arguments); // invoke function wrapper - if (functionCall.Name == instance.GetWeatherFunction.Name) + if (functionCall.Name == instance.GetWeatherFunctionContract.Name) { var result = await instance.GetWeatherWrapper(arguments); return new TextMessage(Role.Assistant, result); } - else if (functionCall.Name == instance.GoogleSearchFunction.Name) + else if (functionCall.Name == instance.GetWeatherFunctionContract.Name) { var result = await instance.GoogleSearchWrapper(arguments); return new TextMessage(Role.Assistant, result); diff --git a/dotnet/sample/AutoGen.BasicSamples/Example13_OpenAIAgent_JsonMode.cs b/dotnet/sample/AutoGen.BasicSamples/Example13_OpenAIAgent_JsonMode.cs index dadad7f00b99..596ab08d02a1 100644 --- a/dotnet/sample/AutoGen.BasicSamples/Example13_OpenAIAgent_JsonMode.cs +++ b/dotnet/sample/AutoGen.BasicSamples/Example13_OpenAIAgent_JsonMode.cs @@ -1,68 +1,5 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Example13_OpenAIAgent_JsonMode.cs -using System.Text.Json; -using System.Text.Json.Serialization; -using AutoGen.Core; -using AutoGen.OpenAI; -using AutoGen.OpenAI.Extension; -using Azure.AI.OpenAI; -using FluentAssertions; +// this example has been moved to https://github.com/microsoft/autogen/blob/main/dotnet/sample/AutoGen.OpenAI.Sample/Use_Json_Mode.cs -namespace AutoGen.BasicSample; - -public class Example13_OpenAIAgent_JsonMode -{ - public static async Task RunAsync() - { - #region create_agent - var config = LLMConfiguration.GetAzureOpenAIGPT3_5_Turbo(deployName: "gpt-35-turbo"); // json mode only works with 0125 and later model. - var apiKey = config.ApiKey; - var endPoint = new Uri(config.Endpoint); - - var openAIClient = new OpenAIClient(endPoint, new Azure.AzureKeyCredential(apiKey)); - var openAIClientAgent = new OpenAIChatAgent( - openAIClient: openAIClient, - name: "assistant", - modelName: config.DeploymentName, - systemMessage: "You are a helpful assistant designed to output JSON.", - seed: 0, // explicitly set a seed to enable deterministic output - responseFormat: ChatCompletionsResponseFormat.JsonObject) // set response format to JSON object to enable JSON mode - .RegisterMessageConnector() - .RegisterPrintMessage(); - #endregion create_agent - - #region chat_with_agent - var reply = await openAIClientAgent.SendAsync("My name is John, I am 25 years old, and I live in Seattle."); - - var person = JsonSerializer.Deserialize(reply.GetContent()); - Console.WriteLine($"Name: {person.Name}"); - Console.WriteLine($"Age: {person.Age}"); - - if (!string.IsNullOrEmpty(person.Address)) - { - Console.WriteLine($"Address: {person.Address}"); - } - - Console.WriteLine("Done."); - #endregion chat_with_agent - - person.Name.Should().Be("John"); - person.Age.Should().Be(25); - person.Address.Should().BeNullOrEmpty(); - } -} - -#region person_class -public class Person -{ - [JsonPropertyName("name")] - public string Name { get; set; } - - [JsonPropertyName("age")] - public int Age { get; set; } - - [JsonPropertyName("address")] - public string Address { get; set; } -} -#endregion person_class diff --git a/dotnet/sample/AutoGen.BasicSamples/Example15_GPT4V_BinaryDataImageMessage.cs b/dotnet/sample/AutoGen.BasicSamples/Example15_GPT4V_BinaryDataImageMessage.cs index 788122d3f383..dee9915511d6 100644 --- a/dotnet/sample/AutoGen.BasicSamples/Example15_GPT4V_BinaryDataImageMessage.cs +++ b/dotnet/sample/AutoGen.BasicSamples/Example15_GPT4V_BinaryDataImageMessage.cs @@ -50,7 +50,9 @@ private static void AddMessagesFromResource(string imageResourcePath, List SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) - { - request.RequestUri = new Uri($"{_modelServiceUrl}{request.RequestUri.PathAndQuery}"); - - return base.SendAsync(request, cancellationToken); - } -} -#endregion CustomHttpClientHandler - -public class Example16_OpenAIChatAgent_ConnectToThirdPartyBackend -{ - public static async Task RunAsync() - { - #region create_agent - using var client = new HttpClient(new CustomHttpClientHandler("http://localhost:11434")); - var option = new OpenAIClientOptions(OpenAIClientOptions.ServiceVersion.V2024_04_01_Preview) - { - Transport = new HttpClientTransport(client), - }; - - // api-key is not required for local server - // so you can use any string here - var openAIClient = new OpenAIClient("api-key", option); - var model = "llama3"; - - var agent = new OpenAIChatAgent( - openAIClient: openAIClient, - name: "assistant", - modelName: model, - systemMessage: "You are a helpful assistant designed to output JSON.", - seed: 0) - .RegisterMessageConnector() - .RegisterPrintMessage(); - #endregion create_agent - - #region send_message - await agent.SendAsync("Can you write a piece of C# code to calculate 100th of fibonacci?"); - #endregion send_message - } -} +// this example has been moved to https://github.com/microsoft/autogen/blob/main/dotnet/sample/AutoGen.OpenAI.Sample/Connect_To_Ollama.cs diff --git a/dotnet/sample/AutoGen.BasicSamples/GettingStart/Dynamic_Group_Chat.cs b/dotnet/sample/AutoGen.BasicSamples/GettingStart/Dynamic_Group_Chat.cs index 9d21bbde7d30..7acaae4b1f82 100644 --- a/dotnet/sample/AutoGen.BasicSamples/GettingStart/Dynamic_Group_Chat.cs +++ b/dotnet/sample/AutoGen.BasicSamples/GettingStart/Dynamic_Group_Chat.cs @@ -1,5 +1,5 @@ // Copyright (c) Microsoft Corporation. All rights reserved. -// Dynamic_GroupChat.cs +// Dynamic_Group_Chat.cs using AutoGen.Core; using AutoGen.OpenAI; diff --git a/dotnet/sample/AutoGen.BasicSamples/GettingStart/Image_Chat_With_Agent.cs b/dotnet/sample/AutoGen.BasicSamples/GettingStart/Image_Chat_With_Agent.cs index 3352f90d9211..5b94a238bbe8 100644 --- a/dotnet/sample/AutoGen.BasicSamples/GettingStart/Image_Chat_With_Agent.cs +++ b/dotnet/sample/AutoGen.BasicSamples/GettingStart/Image_Chat_With_Agent.cs @@ -1,10 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Image_Chat_With_Agent.cs +#region Using using AutoGen.Core; using AutoGen.OpenAI; using AutoGen.OpenAI.Extension; using Azure.AI.OpenAI; +#endregion Using using FluentAssertions; namespace AutoGen.BasicSample; @@ -33,16 +35,17 @@ public static async Task RunAsync() var imageMessage = new ImageMessage(Role.User, BinaryData.FromBytes(imageBytes, "image/png")); #endregion Prepare_Image_Input - #region Chat_With_Agent - var reply = await agent.SendAsync("what's in the picture", chatHistory: [imageMessage]); - #endregion Chat_With_Agent - #region Prepare_Multimodal_Input var textMessage = new TextMessage(Role.User, "what's in the picture"); var multimodalMessage = new MultiModalMessage(Role.User, [textMessage, imageMessage]); - reply = await agent.SendAsync(multimodalMessage); #endregion Prepare_Multimodal_Input + #region Chat_With_Agent + var reply = await agent.SendAsync("what's in the picture", chatHistory: [imageMessage]); + // or use multimodal message to generate reply + reply = await agent.SendAsync(multimodalMessage); + #endregion Chat_With_Agent + #region verify_reply reply.Should().BeOfType(); #endregion verify_reply diff --git a/dotnet/sample/AutoGen.BasicSamples/GettingStart/Streaming_Tool_Call.cs b/dotnet/sample/AutoGen.BasicSamples/GettingStart/Streaming_Tool_Call.cs new file mode 100644 index 000000000000..48ebd127b562 --- /dev/null +++ b/dotnet/sample/AutoGen.BasicSamples/GettingStart/Streaming_Tool_Call.cs @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Streaming_Tool_Call.cs + +using AutoGen.Core; +using AutoGen.OpenAI; +using AutoGen.OpenAI.Extension; +using Azure.AI.OpenAI; +using FluentAssertions; + +namespace AutoGen.BasicSample.GettingStart; + +internal class Streaming_Tool_Call +{ + public static async Task RunAsync() + { + #region Create_tools + var tools = new Tools(); + #endregion Create_tools + + #region Create_auto_invoke_middleware + var autoInvokeMiddleware = new FunctionCallMiddleware( + functions: [tools.GetWeatherFunctionContract], + functionMap: new Dictionary>>() + { + { tools.GetWeatherFunctionContract.Name, tools.GetWeatherWrapper }, + }); + #endregion Create_auto_invoke_middleware + + #region Create_Agent + var apiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY") ?? throw new Exception("Please set OPENAI_API_KEY environment variable."); + var model = "gpt-4o"; + var openaiClient = new OpenAIClient(apiKey); + var agent = new OpenAIChatAgent( + openAIClient: openaiClient, + name: "agent", + modelName: model, + systemMessage: "You are a helpful AI assistant") + .RegisterMessageConnector() + .RegisterStreamingMiddleware(autoInvokeMiddleware) + .RegisterPrintMessage(); + #endregion Create_Agent + + IMessage finalReply = null; + var question = new TextMessage(Role.User, "What's the weather in Seattle"); + + // In streaming function call + // function can only be invoked untill all the chunks are collected + // therefore, only one ToolCallAggregateMessage chunk will be return here. + await foreach (var message in agent.GenerateStreamingReplyAsync([question])) + { + finalReply = message; + } + + finalReply?.GetContent().Should().Be("The weather in Seattle is sunny."); + } +} diff --git a/dotnet/sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs b/dotnet/sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs index f1a230c123b1..b441fe389da2 100644 --- a/dotnet/sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs +++ b/dotnet/sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs @@ -11,6 +11,7 @@ namespace AutoGen.BasicSample; +#region Tools public partial class Tools { /// @@ -23,6 +24,8 @@ public async Task GetWeather(string city) return $"The weather in {city} is sunny."; } } +#endregion Tools + public class Use_Tools_With_Agent { public static async Task RunAsync() @@ -31,37 +34,53 @@ public static async Task RunAsync() var tools = new Tools(); #endregion Create_tools - #region Create_Agent - var apiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY") ?? throw new Exception("Please set OPENAI_API_KEY environment variable."); - var model = "gpt-3.5-turbo"; - var openaiClient = new OpenAIClient(apiKey); - var functionCallMiddleware = new FunctionCallMiddleware( + #region Create_auto_invoke_middleware + var autoInvokeMiddleware = new FunctionCallMiddleware( functions: [tools.GetWeatherFunctionContract], functionMap: new Dictionary>>() { { tools.GetWeatherFunctionContract.Name!, tools.GetWeatherWrapper }, }); + #endregion Create_auto_invoke_middleware + + #region Create_no_invoke_middleware + var noInvokeMiddleware = new FunctionCallMiddleware( + functions: [tools.GetWeatherFunctionContract]); + #endregion Create_no_invoke_middleware + + #region Create_Agent + var apiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY") ?? throw new Exception("Please set OPENAI_API_KEY environment variable."); + var model = "gpt-3.5-turbo"; + var openaiClient = new OpenAIClient(apiKey); var agent = new OpenAIChatAgent( openAIClient: openaiClient, name: "agent", modelName: model, systemMessage: "You are a helpful AI assistant") - .RegisterMessageConnector() // convert OpenAI message to AutoGen message - .RegisterMiddleware(functionCallMiddleware) // pass function definition to agent. - .RegisterPrintMessage(); // print the message content + .RegisterMessageConnector(); // convert OpenAI message to AutoGen message #endregion Create_Agent - #region Single_Turn_Tool_Call + #region Single_Turn_Auto_Invoke + var autoInvokeAgent = agent + .RegisterMiddleware(autoInvokeMiddleware) // pass function definition to agent. + .RegisterPrintMessage(); // print the message content var question = new TextMessage(Role.User, "What is the weather in Seattle?"); - var toolCallReply = await agent.SendAsync(question); - #endregion Single_Turn_Tool_Call + var reply = await autoInvokeAgent.SendAsync(question); + reply.Should().BeOfType(); + #endregion Single_Turn_Auto_Invoke + + #region Single_Turn_No_Invoke + var noInvokeAgent = agent + .RegisterMiddleware(noInvokeMiddleware) // pass function definition to agent. + .RegisterPrintMessage(); // print the message content - #region verify_too_call_reply - toolCallReply.Should().BeOfType(); - #endregion verify_too_call_reply + question = new TextMessage(Role.User, "What is the weather in Seattle?"); + reply = await noInvokeAgent.SendAsync(question); + reply.Should().BeOfType(); + #endregion Single_Turn_No_Invoke #region Multi_Turn_Tool_Call - var finalReply = await agent.SendAsync(chatHistory: [question, toolCallReply]); + var finalReply = await agent.SendAsync(chatHistory: [question, reply]); #endregion Multi_Turn_Tool_Call #region verify_reply @@ -70,16 +89,19 @@ public static async Task RunAsync() #region parallel_tool_call question = new TextMessage(Role.User, "What is the weather in Seattle, New York and Vancouver"); - toolCallReply = await agent.SendAsync(question); + reply = await agent.SendAsync(question); #endregion parallel_tool_call #region verify_parallel_tool_call_reply - toolCallReply.Should().BeOfType(); - (toolCallReply as ToolCallAggregateMessage)!.Message1.ToolCalls.Count().Should().Be(3); + reply.Should().BeOfType(); + (reply as ToolCallAggregateMessage)!.Message1.ToolCalls.Count().Should().Be(3); #endregion verify_parallel_tool_call_reply #region Multi_Turn_Parallel_Tool_Call - finalReply = await agent.SendAsync(chatHistory: [question, toolCallReply]); + finalReply = await agent.SendAsync(chatHistory: [question, reply]); + finalReply.Should().BeOfType(); + (finalReply as ToolCallAggregateMessage)!.Message1.ToolCalls.Count().Should().Be(3); #endregion Multi_Turn_Parallel_Tool_Call } + } diff --git a/dotnet/sample/AutoGen.OpenAI.Sample/AutoGen.OpenAI.Sample.csproj b/dotnet/sample/AutoGen.OpenAI.Sample/AutoGen.OpenAI.Sample.csproj new file mode 100644 index 000000000000..ffe18f8a616a --- /dev/null +++ b/dotnet/sample/AutoGen.OpenAI.Sample/AutoGen.OpenAI.Sample.csproj @@ -0,0 +1,21 @@ + + + + Exe + net8.0 + enable + enable + True + $(NoWarn);CS8981;CS8600;CS8602;CS8604;CS8618;CS0219;SKEXP0054;SKEXP0050;SKEXP0110 + true + + + + + + + + + + + diff --git a/dotnet/sample/AutoGen.OpenAI.Sample/Connect_To_Ollama.cs b/dotnet/sample/AutoGen.OpenAI.Sample/Connect_To_Ollama.cs new file mode 100644 index 000000000000..3823de2a5284 --- /dev/null +++ b/dotnet/sample/AutoGen.OpenAI.Sample/Connect_To_Ollama.cs @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Connect_To_Ollama.cs + +#region using_statement +using AutoGen.Core; +using AutoGen.OpenAI.Extension; +using Azure.AI.OpenAI; +using Azure.Core.Pipeline; +#endregion using_statement + +namespace AutoGen.OpenAI.Sample; + +#region CustomHttpClientHandler +public sealed class CustomHttpClientHandler : HttpClientHandler +{ + private string _modelServiceUrl; + + public CustomHttpClientHandler(string modelServiceUrl) + { + _modelServiceUrl = modelServiceUrl; + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + request.RequestUri = new Uri($"{_modelServiceUrl}{request.RequestUri.PathAndQuery}"); + + return base.SendAsync(request, cancellationToken); + } +} +#endregion CustomHttpClientHandler + +public class Connect_To_Ollama +{ + public static async Task RunAsync() + { + #region create_agent + using var client = new HttpClient(new CustomHttpClientHandler("http://localhost:11434")); + var option = new OpenAIClientOptions(OpenAIClientOptions.ServiceVersion.V2024_04_01_Preview) + { + Transport = new HttpClientTransport(client), + }; + + // api-key is not required for local server + // so you can use any string here + var openAIClient = new OpenAIClient("api-key", option); + var model = "llama3"; + + var agent = new OpenAIChatAgent( + openAIClient: openAIClient, + name: "assistant", + modelName: model, + systemMessage: "You are a helpful assistant designed to output JSON.", + seed: 0) + .RegisterMessageConnector() + .RegisterPrintMessage(); + #endregion create_agent + + #region send_message + await agent.SendAsync("Can you write a piece of C# code to calculate 100th of fibonacci?"); + #endregion send_message + } +} diff --git a/dotnet/sample/AutoGen.OpenAI.Sample/Program.cs b/dotnet/sample/AutoGen.OpenAI.Sample/Program.cs new file mode 100644 index 000000000000..5a38a3ff03b9 --- /dev/null +++ b/dotnet/sample/AutoGen.OpenAI.Sample/Program.cs @@ -0,0 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Program.cs + +using AutoGen.OpenAI.Sample; + +Tool_Call_With_Ollama_And_LiteLLM.RunAsync().Wait(); diff --git a/dotnet/sample/AutoGen.OpenAI.Sample/Tool_Call_With_Ollama_And_LiteLLM.cs b/dotnet/sample/AutoGen.OpenAI.Sample/Tool_Call_With_Ollama_And_LiteLLM.cs new file mode 100644 index 000000000000..b0b0adc0e6f5 --- /dev/null +++ b/dotnet/sample/AutoGen.OpenAI.Sample/Tool_Call_With_Ollama_And_LiteLLM.cs @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Tool_Call_With_Ollama_And_LiteLLM.cs + +using AutoGen.Core; +using AutoGen.OpenAI.Extension; +using Azure.AI.OpenAI; +using Azure.Core.Pipeline; + +namespace AutoGen.OpenAI.Sample; + +#region Function +public partial class Function +{ + [Function] + public async Task GetWeatherAsync(string city) + { + return await Task.FromResult("The weather in " + city + " is 72 degrees and sunny."); + } +} +#endregion Function + +public class Tool_Call_With_Ollama_And_LiteLLM +{ + public static async Task RunAsync() + { + // Before running this code, make sure you have + // - Ollama: + // - Install dolphincoder:latest in Ollama + // - Ollama running on http://localhost:11434 + // - LiteLLM + // - Install LiteLLM + // - Start LiteLLM with the following command: + // - litellm --model ollama_chat/dolphincoder --port 4000 + + # region Create_tools + var functions = new Function(); + var functionMiddleware = new FunctionCallMiddleware( + functions: [functions.GetWeatherAsyncFunctionContract], + functionMap: new Dictionary>> + { + { functions.GetWeatherAsyncFunctionContract.Name!, functions.GetWeatherAsyncWrapper }, + }); + #endregion Create_tools + #region Create_Agent + var liteLLMUrl = "http://localhost:4000"; + using var httpClient = new HttpClient(new CustomHttpClientHandler(liteLLMUrl)); + var option = new OpenAIClientOptions(OpenAIClientOptions.ServiceVersion.V2024_04_01_Preview) + { + Transport = new HttpClientTransport(httpClient), + }; + + // api-key is not required for local server + // so you can use any string here + var openAIClient = new OpenAIClient("api-key", option); + + var agent = new OpenAIChatAgent( + openAIClient: openAIClient, + name: "assistant", + modelName: "dolphincoder:latest", + systemMessage: "You are a helpful AI assistant") + .RegisterMessageConnector() + .RegisterMiddleware(functionMiddleware) + .RegisterPrintMessage(); + + var reply = await agent.SendAsync("what's the weather in new york"); + #endregion Create_Agent + } +} diff --git a/dotnet/sample/AutoGen.OpenAI.Sample/Use_Json_Mode.cs b/dotnet/sample/AutoGen.OpenAI.Sample/Use_Json_Mode.cs new file mode 100644 index 000000000000..d92983c5050f --- /dev/null +++ b/dotnet/sample/AutoGen.OpenAI.Sample/Use_Json_Mode.cs @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Use_Json_Mode.cs + +using System.Text.Json; +using System.Text.Json.Serialization; +using AutoGen.Core; +using AutoGen.OpenAI; +using AutoGen.OpenAI.Extension; +using Azure.AI.OpenAI; +using FluentAssertions; + +namespace AutoGen.BasicSample; + +public class Use_Json_Mode +{ + public static async Task RunAsync() + { + #region create_agent + var apiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY") ?? throw new Exception("Please set OPENAI_API_KEY environment variable."); + var model = "gpt-3.5-turbo"; + + var openAIClient = new OpenAIClient(apiKey); + var openAIClientAgent = new OpenAIChatAgent( + openAIClient: openAIClient, + name: "assistant", + modelName: model, + systemMessage: "You are a helpful assistant designed to output JSON.", + seed: 0, // explicitly set a seed to enable deterministic output + responseFormat: ChatCompletionsResponseFormat.JsonObject) // set response format to JSON object to enable JSON mode + .RegisterMessageConnector() + .RegisterPrintMessage(); + #endregion create_agent + + #region chat_with_agent + var reply = await openAIClientAgent.SendAsync("My name is John, I am 25 years old, and I live in Seattle."); + + var person = JsonSerializer.Deserialize(reply.GetContent()); + Console.WriteLine($"Name: {person.Name}"); + Console.WriteLine($"Age: {person.Age}"); + + if (!string.IsNullOrEmpty(person.Address)) + { + Console.WriteLine($"Address: {person.Address}"); + } + + Console.WriteLine("Done."); + #endregion chat_with_agent + + person.Name.Should().Be("John"); + person.Age.Should().Be(25); + person.Address.Should().BeNullOrEmpty(); + } +} + +#region person_class +public class Person +{ + [JsonPropertyName("name")] + public string Name { get; set; } + + [JsonPropertyName("age")] + public int Age { get; set; } + + [JsonPropertyName("address")] + public string Address { get; set; } +} +#endregion person_class diff --git a/dotnet/sample/AutoGen.WebAPI.Sample/AutoGen.WebAPI.Sample.csproj b/dotnet/sample/AutoGen.WebAPI.Sample/AutoGen.WebAPI.Sample.csproj new file mode 100644 index 000000000000..41f3b7d1d381 --- /dev/null +++ b/dotnet/sample/AutoGen.WebAPI.Sample/AutoGen.WebAPI.Sample.csproj @@ -0,0 +1,13 @@ + + + + net8.0 + enable + enable + + + + + + + diff --git a/dotnet/sample/AutoGen.WebAPI.Sample/Program.cs b/dotnet/sample/AutoGen.WebAPI.Sample/Program.cs new file mode 100644 index 000000000000..dbeb8494363d --- /dev/null +++ b/dotnet/sample/AutoGen.WebAPI.Sample/Program.cs @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Program.cs + +using System.Runtime.CompilerServices; +using AutoGen.Core; +using AutoGen.WebAPI; + +var alice = new DummyAgent("alice"); +var bob = new DummyAgent("bob"); + +var builder = WebApplication.CreateBuilder(args); +// Add services to the container. + +// run endpoint at port 5000 +builder.WebHost.UseUrls("http://localhost:5000"); +var app = builder.Build(); + +app.UseAgentAsOpenAIChatCompletionEndpoint(alice); +app.UseAgentAsOpenAIChatCompletionEndpoint(bob); + +app.Run(); + +public class DummyAgent : IStreamingAgent +{ + public DummyAgent(string name = "dummy") + { + Name = name; + } + + public string Name { get; } + + public async Task GenerateReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, CancellationToken cancellationToken = default) + { + return new TextMessage(Role.Assistant, $"I am dummy {this.Name}", this.Name); + } + + public async IAsyncEnumerable GenerateStreamingReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var reply = $"I am dummy {this.Name}"; + foreach (var c in reply) + { + yield return new TextMessageUpdate(Role.Assistant, c.ToString(), this.Name); + }; + } +} diff --git a/dotnet/src/AutoGen.Anthropic/Agent/AnthropicClientAgent.cs b/dotnet/src/AutoGen.Anthropic/Agent/AnthropicClientAgent.cs index e395bb4a225f..73510baeb71c 100644 --- a/dotnet/src/AutoGen.Anthropic/Agent/AnthropicClientAgent.cs +++ b/dotnet/src/AutoGen.Anthropic/Agent/AnthropicClientAgent.cs @@ -1,5 +1,9 @@ -using System; +// Copyright (c) Microsoft Corporation. All rights reserved. +// AnthropicClientAgent.cs + +using System; using System.Collections.Generic; +using System.Linq; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; @@ -16,6 +20,8 @@ public class AnthropicClientAgent : IStreamingAgent private readonly string _systemMessage; private readonly decimal _temperature; private readonly int _maxTokens; + private readonly Tool[]? _tools; + private readonly ToolChoice? _toolChoice; public AnthropicClientAgent( AnthropicClient anthropicClient, @@ -23,7 +29,9 @@ public AnthropicClientAgent( string modelName, string systemMessage = "You are a helpful AI assistant", decimal temperature = 0.7m, - int maxTokens = 1024) + int maxTokens = 1024, + Tool[]? tools = null, + ToolChoice? toolChoice = null) { Name = name; _anthropicClient = anthropicClient; @@ -31,6 +39,8 @@ public AnthropicClientAgent( _systemMessage = systemMessage; _temperature = temperature; _maxTokens = maxTokens; + _tools = tools; + _toolChoice = toolChoice; } public async Task GenerateReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, @@ -40,7 +50,7 @@ public async Task GenerateReplyAsync(IEnumerable messages, G return new MessageEnvelope(response, from: this.Name); } - public async IAsyncEnumerable GenerateStreamingReplyAsync(IEnumerable messages, + public async IAsyncEnumerable GenerateStreamingReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { await foreach (var message in _anthropicClient.StreamingChatCompletionsAsync( @@ -59,6 +69,9 @@ private ChatCompletionRequest CreateParameters(IEnumerable messages, G Model = _modelName, Stream = shouldStream, Temperature = (decimal?)options?.Temperature ?? _temperature, + Tools = _tools?.ToList(), + ToolChoice = _toolChoice ?? (_tools is { Length: > 0 } ? ToolChoice.Auto : null), + StopSequences = options?.StopSequence?.ToArray(), }; chatCompletionRequest.Messages = BuildMessages(messages); @@ -86,6 +99,22 @@ private List BuildMessages(IEnumerable messages) } } - return chatMessages; + // merge messages with the same role + // fixing #2884 + var mergedMessages = chatMessages.Aggregate(new List(), (acc, message) => + { + if (acc.Count > 0 && acc.Last().Role == message.Role) + { + acc.Last().Content.AddRange(message.Content); + } + else + { + acc.Add(message); + } + + return acc; + }); + + return mergedMessages; } } diff --git a/dotnet/src/AutoGen.Anthropic/AnthropicClient.cs b/dotnet/src/AutoGen.Anthropic/AnthropicClient.cs index 90bd33683f20..c58b2c1952ed 100644 --- a/dotnet/src/AutoGen.Anthropic/AnthropicClient.cs +++ b/dotnet/src/AutoGen.Anthropic/AnthropicClient.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // AnthropicClient.cs using System; @@ -24,12 +24,12 @@ public sealed class AnthropicClient : IDisposable private static readonly JsonSerializerOptions JsonSerializerOptions = new() { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - Converters = { new ContentBaseConverter() } + Converters = { new ContentBaseConverter(), new JsonPropertyNameEnumConverter() } }; private static readonly JsonSerializerOptions JsonDeserializerOptions = new() { - Converters = { new ContentBaseConverter() } + Converters = { new ContentBaseConverter(), new JsonPropertyNameEnumConverter() } }; public AnthropicClient(HttpClient httpClient, string baseUrl, string apiKey) @@ -48,7 +48,9 @@ public async Task CreateChatCompletionsAsync(ChatComplet var responseStream = await httpResponseMessage.Content.ReadAsStreamAsync(); if (httpResponseMessage.IsSuccessStatusCode) + { return await DeserializeResponseAsync(responseStream, cancellationToken); + } ErrorResponse res = await DeserializeResponseAsync(responseStream, cancellationToken); throw new Exception(res.Error?.Message); @@ -61,24 +63,58 @@ public async IAsyncEnumerable StreamingChatCompletionsAs using var reader = new StreamReader(await httpResponseMessage.Content.ReadAsStreamAsync()); var currentEvent = new SseEvent(); + while (await reader.ReadLineAsync() is { } line) { if (!string.IsNullOrEmpty(line)) { - currentEvent.Data = line.Substring("data:".Length).Trim(); + if (line.StartsWith("event:")) + { + currentEvent.EventType = line.Substring("event:".Length).Trim(); + } + else if (line.StartsWith("data:")) + { + currentEvent.Data = line.Substring("data:".Length).Trim(); + } } - else + else // an empty line indicates the end of an event { - if (currentEvent.Data == "[DONE]") - continue; + if (currentEvent.EventType == "content_block_start" && !string.IsNullOrEmpty(currentEvent.Data)) + { + var dataBlock = JsonSerializer.Deserialize(currentEvent.Data!); + if (dataBlock != null && dataBlock.ContentBlock?.Type == "tool_use") + { + currentEvent.ContentBlock = dataBlock.ContentBlock; + } + } - if (currentEvent.Data != null) + if (currentEvent.EventType is "message_start" or "content_block_delta" or "message_delta" && currentEvent.Data != null) { - yield return await JsonSerializer.DeserializeAsync( + var res = await JsonSerializer.DeserializeAsync( new MemoryStream(Encoding.UTF8.GetBytes(currentEvent.Data)), cancellationToken: cancellationToken) ?? throw new Exception("Failed to deserialize response"); + if (res.Delta?.Type == "input_json_delta" && !string.IsNullOrEmpty(res.Delta.PartialJson) && + currentEvent.ContentBlock != null) + { + currentEvent.ContentBlock.AppendDeltaParameters(res.Delta.PartialJson!); + } + else if (res.Delta is { StopReason: "tool_use" } && currentEvent.ContentBlock != null) + { + if (res.Content == null) + { + res.Content = [currentEvent.ContentBlock.CreateToolUseContent()]; + } + else + { + res.Content.Add(currentEvent.ContentBlock.CreateToolUseContent()); + } + + currentEvent = new SseEvent(); + } + + yield return res; } - else if (currentEvent.Data != null) + else if (currentEvent.EventType == "error" && currentEvent.Data != null) { var res = await JsonSerializer.DeserializeAsync( new MemoryStream(Encoding.UTF8.GetBytes(currentEvent.Data)), cancellationToken: cancellationToken); @@ -86,8 +122,10 @@ public async IAsyncEnumerable StreamingChatCompletionsAs throw new Exception(res?.Error?.Message); } - // Reset the current event for the next one - currentEvent = new SseEvent(); + if (currentEvent.ContentBlock == null) + { + currentEvent = new SseEvent(); + } } } } @@ -113,11 +151,50 @@ public void Dispose() private struct SseEvent { + public string EventType { get; set; } public string? Data { get; set; } + public ContentBlock? ContentBlock { get; set; } - public SseEvent(string? data = null) + public SseEvent(string eventType, string? data = null, ContentBlock? contentBlock = null) { + EventType = eventType; Data = data; + ContentBlock = contentBlock; } } + + private class ContentBlock + { + [JsonPropertyName("type")] + public string? Type { get; set; } + + [JsonPropertyName("id")] + public string? Id { get; set; } + + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("input")] + public object? Input { get; set; } + + public string? parameters { get; set; } + + public void AppendDeltaParameters(string deltaParams) + { + StringBuilder sb = new StringBuilder(parameters); + sb.Append(deltaParams); + parameters = sb.ToString(); + } + + public ToolUseContent CreateToolUseContent() + { + return new ToolUseContent { Id = Id, Name = Name, Input = parameters }; + } + } + + private class DataBlock + { + [JsonPropertyName("content_block")] + public ContentBlock? ContentBlock { get; set; } + } } diff --git a/dotnet/src/AutoGen.Anthropic/Converters/ContentBaseConverter.cs b/dotnet/src/AutoGen.Anthropic/Converters/ContentBaseConverter.cs index 4cb8fdbb34e0..3e620f934c28 100644 --- a/dotnet/src/AutoGen.Anthropic/Converters/ContentBaseConverter.cs +++ b/dotnet/src/AutoGen.Anthropic/Converters/ContentBaseConverter.cs @@ -1,12 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. -// ContentConverter.cs - -using AutoGen.Anthropic.DTO; - +// ContentBaseConverter.cs using System; using System.Text.Json; using System.Text.Json.Serialization; +using AutoGen.Anthropic.DTO; namespace AutoGen.Anthropic.Converters; public sealed class ContentBaseConverter : JsonConverter @@ -24,6 +22,10 @@ public override ContentBase Read(ref Utf8JsonReader reader, Type typeToConvert, return JsonSerializer.Deserialize(text, options) ?? throw new InvalidOperationException(); case "image": return JsonSerializer.Deserialize(text, options) ?? throw new InvalidOperationException(); + case "tool_use": + return JsonSerializer.Deserialize(text, options) ?? throw new InvalidOperationException(); + case "tool_result": + return JsonSerializer.Deserialize(text, options) ?? throw new InvalidOperationException(); } } diff --git a/dotnet/src/AutoGen.Anthropic/Converters/JsonPropertyNameEnumCoverter.cs b/dotnet/src/AutoGen.Anthropic/Converters/JsonPropertyNameEnumCoverter.cs new file mode 100644 index 000000000000..cd95d837cffd --- /dev/null +++ b/dotnet/src/AutoGen.Anthropic/Converters/JsonPropertyNameEnumCoverter.cs @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// JsonPropertyNameEnumCoverter.cs + +using System; +using System.Reflection; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace AutoGen.Anthropic.Converters; + +internal class JsonPropertyNameEnumConverter : JsonConverter where T : struct, Enum +{ + public override T Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + string value = reader.GetString() ?? throw new JsonException("Value was null."); + + foreach (var field in typeToConvert.GetFields()) + { + var attribute = field.GetCustomAttribute(); + if (attribute?.Name == value) + { + return (T)Enum.Parse(typeToConvert, field.Name); + } + } + + throw new JsonException($"Unable to convert \"{value}\" to enum {typeToConvert}."); + } + + public override void Write(Utf8JsonWriter writer, T value, JsonSerializerOptions options) + { + var field = value.GetType().GetField(value.ToString()); + var attribute = field.GetCustomAttribute(); + + if (attribute != null) + { + writer.WriteStringValue(attribute.Name); + } + else + { + writer.WriteStringValue(value.ToString()); + } + } +} + diff --git a/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionRequest.cs b/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionRequest.cs index 0c1749eaa989..463ee7fc2595 100644 --- a/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionRequest.cs +++ b/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionRequest.cs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // ChatCompletionRequest.cs -using System.Text.Json.Serialization; using System.Collections.Generic; +using System.Text.Json.Serialization; namespace AutoGen.Anthropic.DTO; @@ -37,6 +37,12 @@ public class ChatCompletionRequest [JsonPropertyName("top_p")] public decimal? TopP { get; set; } + [JsonPropertyName("tools")] + public List? Tools { get; set; } + + [JsonPropertyName("tool_choice")] + public ToolChoice? ToolChoice { get; set; } + public ChatCompletionRequest() { Messages = new List(); @@ -62,4 +68,6 @@ public ChatMessage(string role, List content) Role = role; Content = content; } + + public void AddContent(ContentBase content) => Content.Add(content); } diff --git a/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionResponse.cs b/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionResponse.cs index c6861f9c3150..fc33aa0e26b1 100644 --- a/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionResponse.cs +++ b/dotnet/src/AutoGen.Anthropic/DTO/ChatCompletionResponse.cs @@ -1,10 +1,11 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. +// ChatCompletionResponse.cs -namespace AutoGen.Anthropic.DTO; using System.Collections.Generic; using System.Text.Json.Serialization; +namespace AutoGen.Anthropic.DTO; public class ChatCompletionResponse { [JsonPropertyName("content")] @@ -49,9 +50,6 @@ public class StreamingMessage [JsonPropertyName("role")] public string? Role { get; set; } - [JsonPropertyName("content")] - public List? Content { get; set; } - [JsonPropertyName("model")] public string? Model { get; set; } @@ -85,6 +83,9 @@ public class Delta [JsonPropertyName("text")] public string? Text { get; set; } + [JsonPropertyName("partial_json")] + public string? PartialJson { get; set; } + [JsonPropertyName("usage")] public Usage? Usage { get; set; } } diff --git a/dotnet/src/AutoGen.Anthropic/DTO/Content.cs b/dotnet/src/AutoGen.Anthropic/DTO/Content.cs index dd2481bd58f3..353cf6ae824b 100644 --- a/dotnet/src/AutoGen.Anthropic/DTO/Content.cs +++ b/dotnet/src/AutoGen.Anthropic/DTO/Content.cs @@ -1,6 +1,7 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Content.cs +using System.Text.Json.Nodes; using System.Text.Json.Serialization; namespace AutoGen.Anthropic.DTO; @@ -40,3 +41,30 @@ public class ImageSource [JsonPropertyName("data")] public string? Data { get; set; } } + +public class ToolUseContent : ContentBase +{ + [JsonPropertyName("type")] + public override string Type => "tool_use"; + + [JsonPropertyName("id")] + public string? Id { get; set; } + + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("input")] + public JsonNode? Input { get; set; } +} + +public class ToolResultContent : ContentBase +{ + [JsonPropertyName("type")] + public override string Type => "tool_result"; + + [JsonPropertyName("tool_use_id")] + public string? Id { get; set; } + + [JsonPropertyName("content")] + public string? Content { get; set; } +} diff --git a/dotnet/src/AutoGen.Anthropic/DTO/ErrorResponse.cs b/dotnet/src/AutoGen.Anthropic/DTO/ErrorResponse.cs index d02a8f6d1cfc..1a94334c88ff 100644 --- a/dotnet/src/AutoGen.Anthropic/DTO/ErrorResponse.cs +++ b/dotnet/src/AutoGen.Anthropic/DTO/ErrorResponse.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // ErrorResponse.cs using System.Text.Json.Serialization; diff --git a/dotnet/src/AutoGen.Anthropic/DTO/Tool.cs b/dotnet/src/AutoGen.Anthropic/DTO/Tool.cs new file mode 100644 index 000000000000..2a46bc42a35b --- /dev/null +++ b/dotnet/src/AutoGen.Anthropic/DTO/Tool.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Tool.cs + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace AutoGen.Anthropic.DTO; + +public class Tool +{ + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("description")] + public string? Description { get; set; } + + [JsonPropertyName("input_schema")] + public InputSchema? InputSchema { get; set; } +} + +public class InputSchema +{ + [JsonPropertyName("type")] + public string? Type { get; set; } + + [JsonPropertyName("properties")] + public Dictionary? Properties { get; set; } + + [JsonPropertyName("required")] + public List? Required { get; set; } +} + +public class SchemaProperty +{ + [JsonPropertyName("type")] + public string? Type { get; set; } + + [JsonPropertyName("description")] + public string? Description { get; set; } +} diff --git a/dotnet/src/AutoGen.Anthropic/DTO/ToolChoice.cs b/dotnet/src/AutoGen.Anthropic/DTO/ToolChoice.cs new file mode 100644 index 000000000000..0a5c3790e1de --- /dev/null +++ b/dotnet/src/AutoGen.Anthropic/DTO/ToolChoice.cs @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ToolChoice.cs + +using System.Text.Json.Serialization; +using AutoGen.Anthropic.Converters; + +namespace AutoGen.Anthropic.DTO; + +[JsonConverter(typeof(JsonPropertyNameEnumConverter))] +public enum ToolChoiceType +{ + [JsonPropertyName("auto")] + Auto, // Default behavior + + [JsonPropertyName("any")] + Any, // Use any provided tool + + [JsonPropertyName("tool")] + Tool // Force a specific tool +} + +public class ToolChoice +{ + [JsonPropertyName("type")] + public ToolChoiceType Type { get; set; } + + [JsonPropertyName("name")] + public string? Name { get; set; } + + private ToolChoice(ToolChoiceType type, string? name = null) + { + Type = type; + Name = name; + } + + public static ToolChoice Auto => new(ToolChoiceType.Auto); + public static ToolChoice Any => new(ToolChoiceType.Any); + public static ToolChoice ToolUse(string name) => new(ToolChoiceType.Tool, name); +} diff --git a/dotnet/src/AutoGen.Anthropic/Middleware/AnthropicMessageConnector.cs b/dotnet/src/AutoGen.Anthropic/Middleware/AnthropicMessageConnector.cs index bb2f5820f74c..af06a0547849 100644 --- a/dotnet/src/AutoGen.Anthropic/Middleware/AnthropicMessageConnector.cs +++ b/dotnet/src/AutoGen.Anthropic/Middleware/AnthropicMessageConnector.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Net.Http; using System.Runtime.CompilerServices; +using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; using AutoGen.Anthropic.DTO; @@ -28,7 +29,7 @@ public async Task InvokeAsync(MiddlewareContext context, IAgent agent, : response; } - public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, + public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) { var messages = context.Messages; @@ -36,7 +37,7 @@ public async IAsyncEnumerable InvokeAsync(MiddlewareContext c await foreach (var reply in agent.GenerateStreamingReplyAsync(chatMessages, context.Options, cancellationToken)) { - if (reply is IStreamingMessage chatMessage) + if (reply is IMessage chatMessage) { var response = ProcessChatCompletionResponse(chatMessage, agent); if (response is not null) @@ -51,9 +52,20 @@ public async IAsyncEnumerable InvokeAsync(MiddlewareContext c } } - private IStreamingMessage? ProcessChatCompletionResponse(IStreamingMessage chatMessage, + private IMessage? ProcessChatCompletionResponse(IMessage chatMessage, IStreamingAgent agent) { + if (chatMessage.Content.Content is { Count: 1 } && + chatMessage.Content.Content[0] is ToolUseContent toolUseContent) + { + return new ToolCallMessage( + toolUseContent.Name ?? + throw new InvalidOperationException($"Expected {nameof(toolUseContent.Name)} to be specified"), + toolUseContent.Input?.ToString() ?? + throw new InvalidOperationException($"Expected {nameof(toolUseContent.Input)} to be specified"), + from: agent.Name); + } + var delta = chatMessage.Content.Delta; return delta != null && !string.IsNullOrEmpty(delta.Text) ? new TextMessageUpdate(role: Role.Assistant, delta.Text, from: agent.Name) @@ -71,16 +83,20 @@ private async Task> ProcessMessageAsync(IEnumerable ProcessTextMessage(textMessage, agent), ImageMessage imageMessage => - new MessageEnvelope(new ChatMessage("user", + (MessageEnvelope[])[new MessageEnvelope(new ChatMessage("user", new ContentBase[] { new ImageContent { Source = await ProcessImageSourceAsync(imageMessage) } } .ToList()), - from: agent.Name), + from: agent.Name)], MultiModalMessage multiModalMessage => await ProcessMultiModalMessageAsync(multiModalMessage, agent), - _ => message, + + ToolCallMessage toolCallMessage => ProcessToolCallMessage(toolCallMessage, agent), + ToolCallResultMessage toolCallResultMessage => ProcessToolCallResultMessage(toolCallResultMessage), + AggregateMessage toolCallAggregateMessage => ProcessToolCallAggregateMessage(toolCallAggregateMessage, agent), + _ => [message], }; - processedMessages.Add(processedMessage); + processedMessages.AddRange(processedMessage); } return processedMessages; @@ -93,15 +109,42 @@ private IMessage PostProcessMessage(ChatCompletionResponse response, IAgent from throw new ArgumentNullException(nameof(response.Content)); } - if (response.Content.Count != 1) + // When expecting a tool call, sometimes the response will contain two messages, one chat and one tool. + // The first message is typically a TextContent, of the LLM explaining what it is trying to do. + // The second message contains the tool call. + if (response.Content.Count > 1) { - throw new NotSupportedException($"{nameof(response.Content)} != 1"); + if (response.Content.Count == 2 && response.Content[0] is TextContent && + response.Content[1] is ToolUseContent toolUseContent) + { + return new ToolCallMessage(toolUseContent.Name ?? string.Empty, + toolUseContent.Input?.ToJsonString() ?? string.Empty, + from: from.Name); + } + + throw new NotSupportedException($"Expected {nameof(response.Content)} to have one output"); } - return new TextMessage(Role.Assistant, ((TextContent)response.Content[0]).Text ?? string.Empty, from: from.Name); + var content = response.Content[0]; + switch (content) + { + case TextContent textContent: + return new TextMessage(Role.Assistant, textContent.Text ?? string.Empty, from: from.Name); + + case ToolUseContent toolUseContent: + return new ToolCallMessage(toolUseContent.Name ?? string.Empty, + toolUseContent.Input?.ToJsonString() ?? string.Empty, + from: from.Name); + + case ImageContent: + throw new InvalidOperationException( + "Claude is an image understanding model only. It can interpret and analyze images, but it cannot generate, produce, edit, manipulate or create images"); + default: + throw new ArgumentOutOfRangeException(nameof(content)); + } } - private IMessage ProcessTextMessage(TextMessage textMessage, IAgent agent) + private IEnumerable> ProcessTextMessage(TextMessage textMessage, IAgent agent) { ChatMessage messages; @@ -139,10 +182,10 @@ private IMessage ProcessTextMessage(TextMessage textMessage, IAgent "user", textMessage.Content); } - return new MessageEnvelope(messages, from: textMessage.From); + return [new MessageEnvelope(messages, from: textMessage.From)]; } - private async Task ProcessMultiModalMessageAsync(MultiModalMessage multiModalMessage, IAgent agent) + private async Task> ProcessMultiModalMessageAsync(MultiModalMessage multiModalMessage, IAgent agent) { var content = new List(); foreach (var message in multiModalMessage.Content) @@ -158,8 +201,7 @@ private async Task ProcessMultiModalMessageAsync(MultiModalMessage mul } } - var chatMessage = new ChatMessage("user", content); - return MessageEnvelope.Create(chatMessage, agent.Name); + return [MessageEnvelope.Create(new ChatMessage("user", content), agent.Name)]; } private async Task ProcessImageSourceAsync(ImageMessage imageMessage) @@ -192,4 +234,52 @@ private async Task ProcessImageSourceAsync(ImageMessage imageMessag Data = Convert.ToBase64String(await response.Content.ReadAsByteArrayAsync()) }; } + + private IEnumerable ProcessToolCallMessage(ToolCallMessage toolCallMessage, IAgent agent) + { + var chatMessage = new ChatMessage("assistant", new List()); + foreach (var toolCall in toolCallMessage.ToolCalls) + { + chatMessage.AddContent(new ToolUseContent + { + Id = toolCall.ToolCallId, + Name = toolCall.FunctionName, + Input = JsonNode.Parse(toolCall.FunctionArguments) + }); + } + + return [MessageEnvelope.Create(chatMessage, toolCallMessage.From)]; + } + + private IEnumerable ProcessToolCallResultMessage(ToolCallResultMessage toolCallResultMessage) + { + var chatMessage = new ChatMessage("user", new List()); + foreach (var toolCall in toolCallResultMessage.ToolCalls) + { + chatMessage.AddContent(new ToolResultContent + { + Id = toolCall.ToolCallId ?? string.Empty, + Content = toolCall.Result, + }); + } + + return [MessageEnvelope.Create(chatMessage, toolCallResultMessage.From)]; + } + + private IEnumerable ProcessToolCallAggregateMessage(AggregateMessage aggregateMessage, IAgent agent) + { + if (aggregateMessage.From is { } from && from != agent.Name) + { + var contents = aggregateMessage.Message2.ToolCalls.Select(t => t.Result); + var messages = contents.Select(c => + new ChatMessage("assistant", c ?? throw new ArgumentNullException(nameof(c)))); + + return messages.Select(m => new MessageEnvelope(m, from: from)); + } + + var toolCallMessage = ProcessToolCallMessage(aggregateMessage.Message1, agent); + var toolCallResult = ProcessToolCallResultMessage(aggregateMessage.Message2); + + return toolCallMessage.Concat(toolCallResult); + } } diff --git a/dotnet/src/AutoGen.Anthropic/Utils/AnthropicConstants.cs b/dotnet/src/AutoGen.Anthropic/Utils/AnthropicConstants.cs index e70572cbddf2..6fd70cb4ee3e 100644 --- a/dotnet/src/AutoGen.Anthropic/Utils/AnthropicConstants.cs +++ b/dotnet/src/AutoGen.Anthropic/Utils/AnthropicConstants.cs @@ -1,5 +1,5 @@ // Copyright (c) Microsoft Corporation. All rights reserved. -// Constants.cs +// AnthropicConstants.cs namespace AutoGen.Anthropic.Utils; diff --git a/dotnet/src/AutoGen.Core/Agent/IAgent.cs b/dotnet/src/AutoGen.Core/Agent/IAgent.cs index b9149008480d..34a31055d1bf 100644 --- a/dotnet/src/AutoGen.Core/Agent/IAgent.cs +++ b/dotnet/src/AutoGen.Core/Agent/IAgent.cs @@ -7,10 +7,14 @@ using System.Threading.Tasks; namespace AutoGen.Core; -public interface IAgent + +public interface IAgentMetaInformation { public string Name { get; } +} +public interface IAgent : IAgentMetaInformation +{ /// /// Generate reply /// diff --git a/dotnet/src/AutoGen.Core/Agent/IStreamingAgent.cs b/dotnet/src/AutoGen.Core/Agent/IStreamingAgent.cs index 665f18bac12a..6b7794c921ad 100644 --- a/dotnet/src/AutoGen.Core/Agent/IStreamingAgent.cs +++ b/dotnet/src/AutoGen.Core/Agent/IStreamingAgent.cs @@ -11,7 +11,7 @@ namespace AutoGen.Core; /// public interface IStreamingAgent : IAgent { - public IAsyncEnumerable GenerateStreamingReplyAsync( + public IAsyncEnumerable GenerateStreamingReplyAsync( IEnumerable messages, GenerateReplyOptions? options = null, CancellationToken cancellationToken = default); diff --git a/dotnet/src/AutoGen.Core/Agent/MiddlewareStreamingAgent.cs b/dotnet/src/AutoGen.Core/Agent/MiddlewareStreamingAgent.cs index 52967d6ff1ce..c7643b1e4735 100644 --- a/dotnet/src/AutoGen.Core/Agent/MiddlewareStreamingAgent.cs +++ b/dotnet/src/AutoGen.Core/Agent/MiddlewareStreamingAgent.cs @@ -47,7 +47,7 @@ public Task GenerateReplyAsync(IEnumerable messages, Generat return _agent.GenerateReplyAsync(messages, options, cancellationToken); } - public IAsyncEnumerable GenerateStreamingReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, CancellationToken cancellationToken = default) + public IAsyncEnumerable GenerateStreamingReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, CancellationToken cancellationToken = default) { return _agent.GenerateStreamingReplyAsync(messages, options, cancellationToken); } @@ -83,7 +83,7 @@ public Task GenerateReplyAsync(IEnumerable messages, Generat return this.streamingMiddleware.InvokeAsync(context, (IAgent)innerAgent, cancellationToken); } - public IAsyncEnumerable GenerateStreamingReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, CancellationToken cancellationToken = default) + public IAsyncEnumerable GenerateStreamingReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, CancellationToken cancellationToken = default) { if (streamingMiddleware is null) { diff --git a/dotnet/src/AutoGen.Core/Extension/GroupChatExtension.cs b/dotnet/src/AutoGen.Core/Extension/GroupChatExtension.cs index e3e44622c817..45728023b96b 100644 --- a/dotnet/src/AutoGen.Core/Extension/GroupChatExtension.cs +++ b/dotnet/src/AutoGen.Core/Extension/GroupChatExtension.cs @@ -100,8 +100,7 @@ internal static IEnumerable ProcessConversationsForRolePlay( var msg = @$"From {x.From}: {x.GetContent()} -round # - {i}"; +round # {i}"; return new TextMessage(Role.User, content: msg); }); diff --git a/dotnet/src/AutoGen.Core/Function/FunctionAttribute.cs b/dotnet/src/AutoGen.Core/Function/FunctionAttribute.cs index 2c828c26d890..556c16436c63 100644 --- a/dotnet/src/AutoGen.Core/Function/FunctionAttribute.cs +++ b/dotnet/src/AutoGen.Core/Function/FunctionAttribute.cs @@ -35,7 +35,7 @@ public class FunctionContract /// /// The name of the function. /// - public string? Name { get; set; } + public string Name { get; set; } = null!; /// /// The description of the function. diff --git a/dotnet/src/AutoGen.Core/GroupChat/Graph.cs b/dotnet/src/AutoGen.Core/GroupChat/Graph.cs index 02f4da50bae0..acff955a292c 100644 --- a/dotnet/src/AutoGen.Core/GroupChat/Graph.cs +++ b/dotnet/src/AutoGen.Core/GroupChat/Graph.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading; using System.Threading.Tasks; namespace AutoGen.Core; @@ -12,9 +13,12 @@ public class Graph { private readonly List transitions = new List(); - public Graph(IEnumerable transitions) + public Graph(IEnumerable? transitions = null) { - this.transitions.AddRange(transitions); + if (transitions != null) + { + this.transitions.AddRange(transitions); + } } public void AddTransition(Transition transition) @@ -33,13 +37,13 @@ public void AddTransition(Transition transition) /// the from agent /// messages /// A list of agents that the messages can be transit to - public async Task> TransitToNextAvailableAgentsAsync(IAgent fromAgent, IEnumerable messages) + public async Task> TransitToNextAvailableAgentsAsync(IAgent fromAgent, IEnumerable messages, CancellationToken ct = default) { var nextAgents = new List(); var availableTransitions = transitions.FindAll(t => t.From == fromAgent) ?? Enumerable.Empty(); foreach (var transition in availableTransitions) { - if (await transition.CanTransitionAsync(messages)) + if (await transition.CanTransitionAsync(messages, ct)) { nextAgents.Add(transition.To); } @@ -56,7 +60,7 @@ public class Transition { private readonly IAgent _from; private readonly IAgent _to; - private readonly Func, Task>? _canTransition; + private readonly Func, CancellationToken, Task>? _canTransition; /// /// Create a new instance of . @@ -66,22 +70,44 @@ public class Transition /// from agent /// to agent /// detect if the transition is allowed, default to be always true - internal Transition(IAgent from, IAgent to, Func, Task>? canTransitionAsync = null) + internal Transition(IAgent from, IAgent to, Func, CancellationToken, Task>? canTransitionAsync = null) { _from = from; _to = to; _canTransition = canTransitionAsync; } + /// + /// Create a new instance of without transition condition check. + /// + /// " + public static Transition Create(TFromAgent from, TToAgent to) + where TFromAgent : IAgent + where TToAgent : IAgent + { + return new Transition(from, to, (fromAgent, toAgent, messages, _) => Task.FromResult(true)); + } + /// /// Create a new instance of . /// /// " - public static Transition Create(TFromAgent from, TToAgent to, Func, Task>? canTransitionAsync = null) + public static Transition Create(TFromAgent from, TToAgent to, Func, Task> canTransitionAsync) + where TFromAgent : IAgent + where TToAgent : IAgent + { + return new Transition(from, to, (fromAgent, toAgent, messages, _) => canTransitionAsync.Invoke((TFromAgent)fromAgent, (TToAgent)toAgent, messages)); + } + + /// + /// Create a new instance of with cancellation token. + /// + /// " + public static Transition Create(TFromAgent from, TToAgent to, Func, CancellationToken, Task> canTransitionAsync) where TFromAgent : IAgent where TToAgent : IAgent { - return new Transition(from, to, (fromAgent, toAgent, messages) => canTransitionAsync?.Invoke((TFromAgent)fromAgent, (TToAgent)toAgent, messages) ?? Task.FromResult(true)); + return new Transition(from, to, (fromAgent, toAgent, messages, ct) => canTransitionAsync.Invoke((TFromAgent)fromAgent, (TToAgent)toAgent, messages, ct)); } public IAgent From => _from; @@ -92,13 +118,13 @@ public static Transition Create(TFromAgent from, TToAgent /// Check if the transition is allowed. /// /// messages - public Task CanTransitionAsync(IEnumerable messages) + public Task CanTransitionAsync(IEnumerable messages, CancellationToken ct = default) { if (_canTransition == null) { return Task.FromResult(true); } - return _canTransition(this.From, this.To, messages); + return _canTransition(this.From, this.To, messages, ct); } } diff --git a/dotnet/src/AutoGen.Core/GroupChat/GroupChat.cs b/dotnet/src/AutoGen.Core/GroupChat/GroupChat.cs index 5e82931ab658..57e15c18ca62 100644 --- a/dotnet/src/AutoGen.Core/GroupChat/GroupChat.cs +++ b/dotnet/src/AutoGen.Core/GroupChat/GroupChat.cs @@ -15,6 +15,7 @@ public class GroupChat : IGroupChat private List agents = new List(); private IEnumerable initializeMessages = new List(); private Graph? workflow = null; + private readonly IOrchestrator orchestrator; public IEnumerable? Messages { get; private set; } @@ -36,6 +37,37 @@ public GroupChat( this.initializeMessages = initializeMessages ?? new List(); this.workflow = workflow; + if (admin is not null) + { + this.orchestrator = new RolePlayOrchestrator(admin, workflow); + } + else if (workflow is not null) + { + this.orchestrator = new WorkflowOrchestrator(workflow); + } + else + { + this.orchestrator = new RoundRobinOrchestrator(); + } + + this.Validation(); + } + + /// + /// Create a group chat which uses the to decide the next speaker(s). + /// + /// + /// + /// + public GroupChat( + IEnumerable members, + IOrchestrator orchestrator, + IEnumerable? initializeMessages = null) + { + this.agents = members.ToList(); + this.initializeMessages = initializeMessages ?? new List(); + this.orchestrator = orchestrator; + this.Validation(); } @@ -64,12 +96,6 @@ private void Validation() throw new Exception("All agents in the workflow must be in the group chat."); } } - - // must provide one of admin or workflow - if (this.admin == null && this.workflow == null) - { - throw new Exception("Must provide one of admin or workflow."); - } } /// @@ -81,6 +107,7 @@ private void Validation() /// current speaker /// conversation history /// next speaker. + [Obsolete("Please use RolePlayOrchestrator or WorkflowOrchestrator")] public async Task SelectNextSpeakerAsync(IAgent currentSpeaker, IEnumerable conversationHistory) { var agentNames = this.agents.Select(x => x.Name).ToList(); @@ -140,37 +167,40 @@ public void AddInitializeMessage(IMessage message) } public async Task> CallAsync( - IEnumerable? conversationWithName = null, + IEnumerable? chatHistory = null, int maxRound = 10, CancellationToken ct = default) { var conversationHistory = new List(); - if (conversationWithName != null) + conversationHistory.AddRange(this.initializeMessages); + if (chatHistory != null) { - conversationHistory.AddRange(conversationWithName); + conversationHistory.AddRange(chatHistory); } + var roundLeft = maxRound; - var lastSpeaker = conversationHistory.LastOrDefault()?.From switch + while (roundLeft > 0) { - null => this.agents.First(), - _ => this.agents.FirstOrDefault(x => x.Name == conversationHistory.Last().From) ?? throw new Exception("The agent is not in the group chat"), - }; - var round = 0; - while (round < maxRound) - { - var currentSpeaker = await this.SelectNextSpeakerAsync(lastSpeaker, conversationHistory); - var processedConversation = this.ProcessConversationForAgent(this.initializeMessages, conversationHistory); - var result = await currentSpeaker.GenerateReplyAsync(processedConversation) ?? throw new Exception("No result is returned."); + var orchestratorContext = new OrchestrationContext + { + Candidates = this.agents, + ChatHistory = conversationHistory, + }; + var nextSpeaker = await this.orchestrator.GetNextSpeakerAsync(orchestratorContext, ct); + if (nextSpeaker == null) + { + break; + } + + var result = await nextSpeaker.GenerateReplyAsync(conversationHistory, cancellationToken: ct); conversationHistory.Add(result); - // if message is terminate message, then terminate the conversation - if (result?.IsGroupChatTerminateMessage() ?? false) + if (result.IsGroupChatTerminateMessage()) { - break; + return conversationHistory; } - lastSpeaker = currentSpeaker; - round++; + roundLeft--; } return conversationHistory; diff --git a/dotnet/src/AutoGen.Core/IGroupChat.cs b/dotnet/src/AutoGen.Core/GroupChat/IGroupChat.cs similarity index 100% rename from dotnet/src/AutoGen.Core/IGroupChat.cs rename to dotnet/src/AutoGen.Core/GroupChat/IGroupChat.cs diff --git a/dotnet/src/AutoGen.Core/GroupChat/RoundRobinGroupChat.cs b/dotnet/src/AutoGen.Core/GroupChat/RoundRobinGroupChat.cs index b8de89b834fe..b95cd1958fc5 100644 --- a/dotnet/src/AutoGen.Core/GroupChat/RoundRobinGroupChat.cs +++ b/dotnet/src/AutoGen.Core/GroupChat/RoundRobinGroupChat.cs @@ -3,9 +3,6 @@ using System; using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; namespace AutoGen.Core; @@ -25,76 +22,12 @@ public SequentialGroupChat(IEnumerable agents, List? initializ /// /// A group chat that allows agents to talk in a round-robin manner. /// -public class RoundRobinGroupChat : IGroupChat +public class RoundRobinGroupChat : GroupChat { - private readonly List agents = new List(); - private readonly List initializeMessages = new List(); - public RoundRobinGroupChat( IEnumerable agents, List? initializeMessages = null) + : base(agents, initializeMessages: initializeMessages) { - this.agents.AddRange(agents); - this.initializeMessages = initializeMessages ?? new List(); - } - - /// - public void AddInitializeMessage(IMessage message) - { - this.SendIntroduction(message); - } - - public async Task> CallAsync( - IEnumerable? conversationWithName = null, - int maxRound = 10, - CancellationToken ct = default) - { - var conversationHistory = new List(); - if (conversationWithName != null) - { - conversationHistory.AddRange(conversationWithName); - } - - var lastSpeaker = conversationHistory.LastOrDefault()?.From switch - { - null => this.agents.First(), - _ => this.agents.FirstOrDefault(x => x.Name == conversationHistory.Last().From) ?? throw new Exception("The agent is not in the group chat"), - }; - var round = 0; - while (round < maxRound) - { - var currentSpeaker = this.SelectNextSpeaker(lastSpeaker); - var processedConversation = this.ProcessConversationForAgent(this.initializeMessages, conversationHistory); - var result = await currentSpeaker.GenerateReplyAsync(processedConversation) ?? throw new Exception("No result is returned."); - conversationHistory.Add(result); - - // if message is terminate message, then terminate the conversation - if (result?.IsGroupChatTerminateMessage() ?? false) - { - break; - } - - lastSpeaker = currentSpeaker; - round++; - } - - return conversationHistory; - } - - public void SendIntroduction(IMessage message) - { - this.initializeMessages.Add(message); - } - - private IAgent SelectNextSpeaker(IAgent currentSpeaker) - { - var index = this.agents.IndexOf(currentSpeaker); - if (index == -1) - { - throw new ArgumentException("The agent is not in the group chat", nameof(currentSpeaker)); - } - - var nextIndex = (index + 1) % this.agents.Count; - return this.agents[nextIndex]; } } diff --git a/dotnet/src/AutoGen.Core/Message/IMessage.cs b/dotnet/src/AutoGen.Core/Message/IMessage.cs index ad215d510e3b..9952cbf06792 100644 --- a/dotnet/src/AutoGen.Core/Message/IMessage.cs +++ b/dotnet/src/AutoGen.Core/Message/IMessage.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // IMessage.cs +using System; using System.Collections.Generic; namespace AutoGen.Core; @@ -35,19 +36,21 @@ namespace AutoGen.Core; /// /// /// -public interface IMessage : IStreamingMessage +public interface IMessage { + string? From { get; set; } } -public interface IMessage : IMessage, IStreamingMessage +public interface IMessage : IMessage { + T Content { get; } } /// /// The interface for messages that can get text content. /// This interface will be used by to get the content from the message. /// -public interface ICanGetTextContent : IMessage, IStreamingMessage +public interface ICanGetTextContent : IMessage { public string? GetContent(); } @@ -55,17 +58,18 @@ public interface ICanGetTextContent : IMessage, IStreamingMessage /// /// The interface for messages that can get a list of /// -public interface ICanGetToolCalls : IMessage, IStreamingMessage +public interface ICanGetToolCalls : IMessage { public IEnumerable GetToolCalls(); } - +[Obsolete("Use IMessage instead")] public interface IStreamingMessage { string? From { get; set; } } +[Obsolete("Use IMessage instead")] public interface IStreamingMessage : IStreamingMessage { T Content { get; } diff --git a/dotnet/src/AutoGen.Core/Message/MessageEnvelope.cs b/dotnet/src/AutoGen.Core/Message/MessageEnvelope.cs index f83bea279260..dc9709bbde5b 100644 --- a/dotnet/src/AutoGen.Core/Message/MessageEnvelope.cs +++ b/dotnet/src/AutoGen.Core/Message/MessageEnvelope.cs @@ -5,7 +5,7 @@ namespace AutoGen.Core; -public abstract class MessageEnvelope : IMessage, IStreamingMessage +public abstract class MessageEnvelope : IMessage { public MessageEnvelope(string? from = null, IDictionary? metadata = null) { @@ -23,7 +23,7 @@ public static MessageEnvelope Create(TContent content, strin public IDictionary Metadata { get; set; } } -public class MessageEnvelope : MessageEnvelope, IMessage, IStreamingMessage +public class MessageEnvelope : MessageEnvelope, IMessage { public MessageEnvelope(T content, string? from = null, IDictionary? metadata = null) : base(from, metadata) diff --git a/dotnet/src/AutoGen.Core/Message/TextMessage.cs b/dotnet/src/AutoGen.Core/Message/TextMessage.cs index addd8728a926..9419c2b3ba86 100644 --- a/dotnet/src/AutoGen.Core/Message/TextMessage.cs +++ b/dotnet/src/AutoGen.Core/Message/TextMessage.cs @@ -3,7 +3,7 @@ namespace AutoGen.Core; -public class TextMessage : IMessage, IStreamingMessage, ICanGetTextContent +public class TextMessage : IMessage, ICanGetTextContent { public TextMessage(Role role, string content, string? from = null) { @@ -51,7 +51,7 @@ public override string ToString() } } -public class TextMessageUpdate : IStreamingMessage, ICanGetTextContent +public class TextMessageUpdate : IMessage, ICanGetTextContent { public TextMessageUpdate(Role role, string? content, string? from = null) { diff --git a/dotnet/src/AutoGen.Core/Message/ToolCallAggregateMessage.cs b/dotnet/src/AutoGen.Core/Message/ToolCallAggregateMessage.cs index 7781b785ef8c..7d46d56135aa 100644 --- a/dotnet/src/AutoGen.Core/Message/ToolCallAggregateMessage.cs +++ b/dotnet/src/AutoGen.Core/Message/ToolCallAggregateMessage.cs @@ -1,5 +1,5 @@ // Copyright (c) Microsoft Corporation. All rights reserved. -// FunctionCallAggregateMessage.cs +// ToolCallAggregateMessage.cs using System.Collections.Generic; diff --git a/dotnet/src/AutoGen.Core/Message/ToolCallMessage.cs b/dotnet/src/AutoGen.Core/Message/ToolCallMessage.cs index 396dba3d3a17..8660b323044f 100644 --- a/dotnet/src/AutoGen.Core/Message/ToolCallMessage.cs +++ b/dotnet/src/AutoGen.Core/Message/ToolCallMessage.cs @@ -36,7 +36,7 @@ public override string ToString() } } -public class ToolCallMessage : IMessage, ICanGetToolCalls +public class ToolCallMessage : IMessage, ICanGetToolCalls, ICanGetTextContent { public ToolCallMessage(IEnumerable toolCalls, string? from = null) { @@ -80,6 +80,12 @@ public void Update(ToolCallMessageUpdate update) public string? From { get; set; } + /// + /// Some LLMs might also include text content in a tool call response, like GPT. + /// This field is used to store the text content in that case. + /// + public string? Content { get; set; } + public override string ToString() { var sb = new StringBuilder(); @@ -96,9 +102,14 @@ public IEnumerable GetToolCalls() { return this.ToolCalls; } + + public string? GetContent() + { + return this.Content; + } } -public class ToolCallMessageUpdate : IStreamingMessage +public class ToolCallMessageUpdate : IMessage { public ToolCallMessageUpdate(string functionName, string functionArgumentUpdate, string? from = null) { diff --git a/dotnet/src/AutoGen.Core/Middleware/FunctionCallMiddleware.cs b/dotnet/src/AutoGen.Core/Middleware/FunctionCallMiddleware.cs index d0788077b590..7d30f6d0928a 100644 --- a/dotnet/src/AutoGen.Core/Middleware/FunctionCallMiddleware.cs +++ b/dotnet/src/AutoGen.Core/Middleware/FunctionCallMiddleware.cs @@ -70,7 +70,7 @@ public async Task InvokeAsync(MiddlewareContext context, IAgent agent, return reply; } - public async IAsyncEnumerable InvokeAsync( + public async IAsyncEnumerable InvokeAsync( MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) @@ -86,16 +86,16 @@ public async IAsyncEnumerable InvokeAsync( var combinedFunctions = this.functions?.Concat(options.Functions ?? []) ?? options.Functions; options.Functions = combinedFunctions?.ToArray(); - IStreamingMessage? initMessage = default; + IMessage? mergedFunctionCallMessage = default; await foreach (var message in agent.GenerateStreamingReplyAsync(context.Messages, options, cancellationToken)) { if (message is ToolCallMessageUpdate toolCallMessageUpdate && this.functionMap != null) { - if (initMessage is null) + if (mergedFunctionCallMessage is null) { - initMessage = new ToolCallMessage(toolCallMessageUpdate); + mergedFunctionCallMessage = new ToolCallMessage(toolCallMessageUpdate); } - else if (initMessage is ToolCallMessage toolCall) + else if (mergedFunctionCallMessage is ToolCallMessage toolCall) { toolCall.Update(toolCallMessageUpdate); } @@ -104,13 +104,17 @@ public async IAsyncEnumerable InvokeAsync( throw new InvalidOperationException("The first message is ToolCallMessage, but the update message is not ToolCallMessageUpdate"); } } + else if (message is ToolCallMessage toolCallMessage1) + { + mergedFunctionCallMessage = toolCallMessage1; + } else { yield return message; } } - if (initMessage is ToolCallMessage toolCallMsg) + if (mergedFunctionCallMessage is ToolCallMessage toolCallMsg) { yield return await this.InvokeToolCallMessagesAfterInvokingAgentAsync(toolCallMsg, agent); } diff --git a/dotnet/src/AutoGen.Core/Middleware/IStreamingMiddleware.cs b/dotnet/src/AutoGen.Core/Middleware/IStreamingMiddleware.cs index bc7aec57f52b..d550bdb519ce 100644 --- a/dotnet/src/AutoGen.Core/Middleware/IStreamingMiddleware.cs +++ b/dotnet/src/AutoGen.Core/Middleware/IStreamingMiddleware.cs @@ -14,7 +14,7 @@ public interface IStreamingMiddleware : IMiddleware /// /// The streaming version of . /// - public IAsyncEnumerable InvokeAsync( + public IAsyncEnumerable InvokeAsync( MiddlewareContext context, IStreamingAgent agent, CancellationToken cancellationToken = default); diff --git a/dotnet/src/AutoGen.Core/Middleware/PrintMessageMiddleware.cs b/dotnet/src/AutoGen.Core/Middleware/PrintMessageMiddleware.cs index 099f78e5f176..a4e84de85a44 100644 --- a/dotnet/src/AutoGen.Core/Middleware/PrintMessageMiddleware.cs +++ b/dotnet/src/AutoGen.Core/Middleware/PrintMessageMiddleware.cs @@ -48,7 +48,7 @@ public async Task InvokeAsync(MiddlewareContext context, IAgent agent, } } - public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) { IMessage? recentUpdate = null; await foreach (var message in agent.GenerateStreamingReplyAsync(context.Messages, context.Options, cancellationToken)) diff --git a/dotnet/src/AutoGen.Core/Orchestrator/IOrchestrator.cs b/dotnet/src/AutoGen.Core/Orchestrator/IOrchestrator.cs new file mode 100644 index 000000000000..777834871f65 --- /dev/null +++ b/dotnet/src/AutoGen.Core/Orchestrator/IOrchestrator.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// IOrchestrator.cs + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace AutoGen.Core; + +public class OrchestrationContext +{ + public IEnumerable Candidates { get; set; } = Array.Empty(); + + public IEnumerable ChatHistory { get; set; } = Array.Empty(); +} + +public interface IOrchestrator +{ + /// + /// Return the next agent as the next speaker. return null if no agent is selected. + /// + /// orchestration context, such as candidate agents and chat history. + /// cancellation token + public Task GetNextSpeakerAsync( + OrchestrationContext context, + CancellationToken cancellationToken = default); +} diff --git a/dotnet/src/AutoGen.Core/Orchestrator/RolePlayOrchestrator.cs b/dotnet/src/AutoGen.Core/Orchestrator/RolePlayOrchestrator.cs new file mode 100644 index 000000000000..6798f23f2df8 --- /dev/null +++ b/dotnet/src/AutoGen.Core/Orchestrator/RolePlayOrchestrator.cs @@ -0,0 +1,116 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// RolePlayOrchestrator.cs + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace AutoGen.Core; + +public class RolePlayOrchestrator : IOrchestrator +{ + private readonly IAgent admin; + private readonly Graph? workflow = null; + public RolePlayOrchestrator(IAgent admin, Graph? workflow = null) + { + this.admin = admin; + this.workflow = workflow; + } + + public async Task GetNextSpeakerAsync( + OrchestrationContext context, + CancellationToken cancellationToken = default) + { + var candidates = context.Candidates.ToList(); + + if (candidates.Count == 0) + { + return null; + } + + if (candidates.Count == 1) + { + return candidates.First(); + } + + // if there's a workflow + // and the next available agent from the workflow is in the group chat + // then return the next agent from the workflow + if (this.workflow != null) + { + var lastMessage = context.ChatHistory.LastOrDefault(); + if (lastMessage == null) + { + return null; + } + var currentSpeaker = candidates.First(candidates => candidates.Name == lastMessage.From); + var nextAgents = await this.workflow.TransitToNextAvailableAgentsAsync(currentSpeaker, context.ChatHistory); + nextAgents = nextAgents.Where(nextAgent => candidates.Any(candidate => candidate.Name == nextAgent.Name)); + candidates = nextAgents.ToList(); + if (!candidates.Any()) + { + return null; + } + + if (candidates is { Count: 1 }) + { + return candidates.First(); + } + } + + // In this case, since there are more than one available agents from the workflow for the next speaker + // the admin will be invoked to decide the next speaker + var agentNames = candidates.Select(candidate => candidate.Name); + var rolePlayMessage = new TextMessage(Role.User, + content: $@"You are in a role play game. Carefully read the conversation history and carry on the conversation. +The available roles are: +{string.Join(",", agentNames)} + +Each message will start with 'From name:', e.g: +From {agentNames.First()}: +//your message//."); + + var chatHistoryWithName = this.ProcessConversationsForRolePlay(context.ChatHistory); + var messages = new IMessage[] { rolePlayMessage }.Concat(chatHistoryWithName); + + var response = await this.admin.GenerateReplyAsync( + messages: messages, + options: new GenerateReplyOptions + { + Temperature = 0, + MaxToken = 128, + StopSequence = [":"], + Functions = null, + }, + cancellationToken: cancellationToken); + + var name = response.GetContent() ?? throw new Exception("No name is returned."); + + // remove From + name = name!.Substring(5); + var candidate = candidates.FirstOrDefault(x => x.Name!.ToLower() == name.ToLower()); + + if (candidate != null) + { + return candidate; + } + + var errorMessage = $"The response from admin is {name}, which is either not in the candidates list or not in the correct format."; + throw new Exception(errorMessage); + } + + private IEnumerable ProcessConversationsForRolePlay(IEnumerable messages) + { + return messages.Select((x, i) => + { + var msg = @$"From {x.From}: +{x.GetContent()} + +round # {i}"; + + return new TextMessage(Role.User, content: msg); + }); + } +} diff --git a/dotnet/src/AutoGen.Core/Orchestrator/RoundRobinOrchestrator.cs b/dotnet/src/AutoGen.Core/Orchestrator/RoundRobinOrchestrator.cs new file mode 100644 index 000000000000..0f8b8e483c63 --- /dev/null +++ b/dotnet/src/AutoGen.Core/Orchestrator/RoundRobinOrchestrator.cs @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// RoundRobinOrchestrator.cs + +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace AutoGen.Core; + +/// +/// Return the next agent in a round-robin fashion. +/// +/// If the last message is from one of the candidates, the next agent will be the next candidate in the list. +/// +/// +/// Otherwise, no agent will be selected. In this case, the orchestrator will return an empty list. +/// +/// +/// This orchestrator always return a single agent. +/// +/// +public class RoundRobinOrchestrator : IOrchestrator +{ + public async Task GetNextSpeakerAsync( + OrchestrationContext context, + CancellationToken cancellationToken = default) + { + var lastMessage = context.ChatHistory.LastOrDefault(); + + if (lastMessage == null) + { + return null; + } + + var candidates = context.Candidates.ToList(); + var lastAgentIndex = candidates.FindIndex(a => a.Name == lastMessage.From); + if (lastAgentIndex == -1) + { + return null; + } + + var nextAgentIndex = (lastAgentIndex + 1) % candidates.Count; + return candidates[nextAgentIndex]; + } +} diff --git a/dotnet/src/AutoGen.Core/Orchestrator/WorkflowOrchestrator.cs b/dotnet/src/AutoGen.Core/Orchestrator/WorkflowOrchestrator.cs new file mode 100644 index 000000000000..b84850a07c75 --- /dev/null +++ b/dotnet/src/AutoGen.Core/Orchestrator/WorkflowOrchestrator.cs @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// WorkflowOrchestrator.cs + +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace AutoGen.Core; + +public class WorkflowOrchestrator : IOrchestrator +{ + private readonly Graph workflow; + + public WorkflowOrchestrator(Graph workflow) + { + this.workflow = workflow; + } + + public async Task GetNextSpeakerAsync( + OrchestrationContext context, + CancellationToken cancellationToken = default) + { + var lastMessage = context.ChatHistory.LastOrDefault(); + if (lastMessage == null) + { + return null; + } + + var candidates = context.Candidates.ToList(); + var currentSpeaker = candidates.FirstOrDefault(candidates => candidates.Name == lastMessage.From); + + if (currentSpeaker == null) + { + return null; + } + var nextAgents = await this.workflow.TransitToNextAvailableAgentsAsync(currentSpeaker, context.ChatHistory); + nextAgents = nextAgents.Where(nextAgent => candidates.Any(candidate => candidate.Name == nextAgent.Name)); + candidates = nextAgents.ToList(); + if (!candidates.Any()) + { + return null; + } + + if (candidates is { Count: 1 }) + { + return candidates.First(); + } + else + { + throw new System.Exception("There are more than one available agents from the workflow for the next speaker."); + } + } +} diff --git a/dotnet/src/AutoGen.DotnetInteractive/InteractiveService.cs b/dotnet/src/AutoGen.DotnetInteractive/InteractiveService.cs index 7490b64e1267..1ca19fcbcfff 100644 --- a/dotnet/src/AutoGen.DotnetInteractive/InteractiveService.cs +++ b/dotnet/src/AutoGen.DotnetInteractive/InteractiveService.cs @@ -19,7 +19,7 @@ public class InteractiveService : IDisposable private bool disposedValue; private const string DotnetInteractiveToolNotInstallMessage = "Cannot find a tool in the manifest file that has a command named 'dotnet-interactive'."; //private readonly ProcessJobTracker jobTracker = new ProcessJobTracker(); - private string installingDirectory; + private string? installingDirectory; public event EventHandler? DisplayEvent; @@ -30,7 +30,11 @@ public class InteractiveService : IDisposable public event EventHandler? HoverTextProduced; /// - /// Create an instance of InteractiveService + /// Install dotnet interactive tool to + /// and create an instance of . + /// + /// When using this constructor, you need to call to install dotnet interactive tool + /// and start the kernel. /// /// dotnet interactive installing directory public InteractiveService(string installingDirectory) @@ -38,8 +42,23 @@ public InteractiveService(string installingDirectory) this.installingDirectory = installingDirectory; } + /// + /// Create an instance of with a running kernel. + /// When using this constructor, you don't need to call to start the kernel. + /// + /// + public InteractiveService(Kernel kernel) + { + this.kernel = kernel; + } + public async Task StartAsync(string workingDirectory, CancellationToken ct = default) { + if (this.kernel != null) + { + return true; + } + this.kernel = await this.CreateKernelAsync(workingDirectory, true, ct); return true; } diff --git a/dotnet/src/AutoGen.Gemini/GeminiChatAgent.cs b/dotnet/src/AutoGen.Gemini/GeminiChatAgent.cs index b081faae8321..e759ba26d1e9 100644 --- a/dotnet/src/AutoGen.Gemini/GeminiChatAgent.cs +++ b/dotnet/src/AutoGen.Gemini/GeminiChatAgent.cs @@ -143,7 +143,7 @@ public async Task GenerateReplyAsync(IEnumerable messages, G return MessageEnvelope.Create(response, this.Name); } - public async IAsyncEnumerable GenerateStreamingReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public async IAsyncEnumerable GenerateStreamingReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { var request = BuildChatRequest(messages, options); var response = this.client.GenerateContentStreamAsync(request); diff --git a/dotnet/src/AutoGen.Gemini/IGeminiClient.cs b/dotnet/src/AutoGen.Gemini/IGeminiClient.cs index 2e209e02b030..d391a4508398 100644 --- a/dotnet/src/AutoGen.Gemini/IGeminiClient.cs +++ b/dotnet/src/AutoGen.Gemini/IGeminiClient.cs @@ -1,5 +1,5 @@ // Copyright (c) Microsoft Corporation. All rights reserved. -// IVertexGeminiClient.cs +// IGeminiClient.cs using System.Collections.Generic; using System.Threading; diff --git a/dotnet/src/AutoGen.Gemini/Middleware/GeminiMessageConnector.cs b/dotnet/src/AutoGen.Gemini/Middleware/GeminiMessageConnector.cs index cb18ba084d78..422fb4cd3458 100644 --- a/dotnet/src/AutoGen.Gemini/Middleware/GeminiMessageConnector.cs +++ b/dotnet/src/AutoGen.Gemini/Middleware/GeminiMessageConnector.cs @@ -39,7 +39,7 @@ public GeminiMessageConnector(bool strictMode = false) public string Name => nameof(GeminiMessageConnector); - public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) { var messages = ProcessMessage(context.Messages, agent); diff --git a/dotnet/src/AutoGen.Gemini/VertexGeminiClient.cs b/dotnet/src/AutoGen.Gemini/VertexGeminiClient.cs index c54f2280dfd3..12a11993cd69 100644 --- a/dotnet/src/AutoGen.Gemini/VertexGeminiClient.cs +++ b/dotnet/src/AutoGen.Gemini/VertexGeminiClient.cs @@ -1,5 +1,5 @@ // Copyright (c) Microsoft Corporation. All rights reserved. -// IGeminiClient.cs +// VertexGeminiClient.cs using System.Collections.Generic; using System.Threading; diff --git a/dotnet/src/AutoGen.Mistral/Agent/MistralClientAgent.cs b/dotnet/src/AutoGen.Mistral/Agent/MistralClientAgent.cs index cc2c74145504..db14d68a1217 100644 --- a/dotnet/src/AutoGen.Mistral/Agent/MistralClientAgent.cs +++ b/dotnet/src/AutoGen.Mistral/Agent/MistralClientAgent.cs @@ -78,7 +78,7 @@ public async Task GenerateReplyAsync( return new MessageEnvelope(response, from: this.Name); } - public async IAsyncEnumerable GenerateStreamingReplyAsync( + public async IAsyncEnumerable GenerateStreamingReplyAsync( IEnumerable messages, GenerateReplyOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) @@ -97,6 +97,7 @@ private ChatCompletionRequest BuildChatRequest(IEnumerable messages, G var chatHistory = BuildChatHistory(messages); var chatRequest = new ChatCompletionRequest(model: _model, messages: chatHistory.ToList(), temperature: options?.Temperature, randomSeed: _randomSeed) { + Stop = options?.StopSequence, MaxTokens = options?.MaxToken, ResponseFormat = _jsonOutput ? new ResponseFormat() { ResponseFormatType = "json_object" } : null, }; diff --git a/dotnet/src/AutoGen.Mistral/DTOs/ChatCompletionRequest.cs b/dotnet/src/AutoGen.Mistral/DTOs/ChatCompletionRequest.cs index 71a084673f13..affe2bb6dcc3 100644 --- a/dotnet/src/AutoGen.Mistral/DTOs/ChatCompletionRequest.cs +++ b/dotnet/src/AutoGen.Mistral/DTOs/ChatCompletionRequest.cs @@ -105,6 +105,9 @@ public class ChatCompletionRequest [JsonPropertyName("random_seed")] public int? RandomSeed { get; set; } + [JsonPropertyName("stop")] + public string[]? Stop { get; set; } + [JsonPropertyName("tools")] public List? Tools { get; set; } diff --git a/dotnet/src/AutoGen.Mistral/Middleware/MistralChatMessageConnector.cs b/dotnet/src/AutoGen.Mistral/Middleware/MistralChatMessageConnector.cs index 95592e97fcc5..78de12a5c01e 100644 --- a/dotnet/src/AutoGen.Mistral/Middleware/MistralChatMessageConnector.cs +++ b/dotnet/src/AutoGen.Mistral/Middleware/MistralChatMessageConnector.cs @@ -15,14 +15,14 @@ public class MistralChatMessageConnector : IStreamingMiddleware, IMiddleware { public string? Name => nameof(MistralChatMessageConnector); - public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) { var messages = context.Messages; var chatMessages = ProcessMessage(messages, agent); var chunks = new List(); await foreach (var reply in agent.GenerateStreamingReplyAsync(chatMessages, context.Options, cancellationToken)) { - if (reply is IStreamingMessage chatMessage) + if (reply is IMessage chatMessage) { chunks.Add(chatMessage.Content); var response = ProcessChatCompletionResponse(chatMessage, agent); @@ -167,7 +167,7 @@ private IMessage PostProcessMessage(ChatCompletionResponse response, IAgent from } } - private IStreamingMessage? ProcessChatCompletionResponse(IStreamingMessage message, IAgent agent) + private IMessage? ProcessChatCompletionResponse(IMessage message, IAgent agent) { var response = message.Content; if (response.VarObject != "chat.completion.chunk") diff --git a/dotnet/src/AutoGen.Ollama/Agent/OllamaAgent.cs b/dotnet/src/AutoGen.Ollama/Agent/OllamaAgent.cs index 9ef68388d605..87b176d8bcc5 100644 --- a/dotnet/src/AutoGen.Ollama/Agent/OllamaAgent.cs +++ b/dotnet/src/AutoGen.Ollama/Agent/OllamaAgent.cs @@ -53,7 +53,7 @@ public async Task GenerateReplyAsync( } } - public async IAsyncEnumerable GenerateStreamingReplyAsync( + public async IAsyncEnumerable GenerateStreamingReplyAsync( IEnumerable messages, GenerateReplyOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) diff --git a/dotnet/src/AutoGen.Ollama/DTOs/Message.cs b/dotnet/src/AutoGen.Ollama/DTOs/Message.cs index 2e0d891cc61e..75f622ff7f04 100644 --- a/dotnet/src/AutoGen.Ollama/DTOs/Message.cs +++ b/dotnet/src/AutoGen.Ollama/DTOs/Message.cs @@ -1,5 +1,5 @@ // Copyright (c) Microsoft Corporation. All rights reserved. -// ChatResponseUpdate.cs +// Message.cs using System.Collections.Generic; using System.Text.Json.Serialization; diff --git a/dotnet/src/AutoGen.Ollama/Embeddings/ITextEmbeddingService.cs b/dotnet/src/AutoGen.Ollama/Embeddings/ITextEmbeddingService.cs index 5ce0dc8cc40a..cce6dbb83076 100644 --- a/dotnet/src/AutoGen.Ollama/Embeddings/ITextEmbeddingService.cs +++ b/dotnet/src/AutoGen.Ollama/Embeddings/ITextEmbeddingService.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // ITextEmbeddingService.cs using System.Threading; diff --git a/dotnet/src/AutoGen.Ollama/Embeddings/OllamaTextEmbeddingService.cs b/dotnet/src/AutoGen.Ollama/Embeddings/OllamaTextEmbeddingService.cs index 2e431e7bcb81..ea4993eb813f 100644 --- a/dotnet/src/AutoGen.Ollama/Embeddings/OllamaTextEmbeddingService.cs +++ b/dotnet/src/AutoGen.Ollama/Embeddings/OllamaTextEmbeddingService.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // OllamaTextEmbeddingService.cs using System; diff --git a/dotnet/src/AutoGen.Ollama/Embeddings/TextEmbeddingsRequest.cs b/dotnet/src/AutoGen.Ollama/Embeddings/TextEmbeddingsRequest.cs index 7f2531c522ad..d776b183db0b 100644 --- a/dotnet/src/AutoGen.Ollama/Embeddings/TextEmbeddingsRequest.cs +++ b/dotnet/src/AutoGen.Ollama/Embeddings/TextEmbeddingsRequest.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // TextEmbeddingsRequest.cs using System.Text.Json.Serialization; diff --git a/dotnet/src/AutoGen.Ollama/Embeddings/TextEmbeddingsResponse.cs b/dotnet/src/AutoGen.Ollama/Embeddings/TextEmbeddingsResponse.cs index 580059c033b5..f3ce64b7032f 100644 --- a/dotnet/src/AutoGen.Ollama/Embeddings/TextEmbeddingsResponse.cs +++ b/dotnet/src/AutoGen.Ollama/Embeddings/TextEmbeddingsResponse.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // TextEmbeddingsResponse.cs using System.Text.Json.Serialization; diff --git a/dotnet/src/AutoGen.Ollama/Middlewares/OllamaMessageConnector.cs b/dotnet/src/AutoGen.Ollama/Middlewares/OllamaMessageConnector.cs index a21ec3a1c991..3919b238d659 100644 --- a/dotnet/src/AutoGen.Ollama/Middlewares/OllamaMessageConnector.cs +++ b/dotnet/src/AutoGen.Ollama/Middlewares/OllamaMessageConnector.cs @@ -30,14 +30,14 @@ public async Task InvokeAsync(MiddlewareContext context, IAgent agent, }; } - public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, + public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) { var messages = ProcessMessage(context.Messages, agent); var chunks = new List(); await foreach (var update in agent.GenerateStreamingReplyAsync(messages, context.Options, cancellationToken)) { - if (update is IStreamingMessage chatResponseUpdate) + if (update is IMessage chatResponseUpdate) { var response = chatResponseUpdate.Content switch { diff --git a/dotnet/src/AutoGen.OpenAI/Agent/GPTAgent.cs b/dotnet/src/AutoGen.OpenAI/Agent/GPTAgent.cs index cdc6cc464d17..5de481245b72 100644 --- a/dotnet/src/AutoGen.OpenAI/Agent/GPTAgent.cs +++ b/dotnet/src/AutoGen.OpenAI/Agent/GPTAgent.cs @@ -104,7 +104,7 @@ public async Task GenerateReplyAsync( return await _innerAgent.GenerateReplyAsync(messages, options, cancellationToken); } - public IAsyncEnumerable GenerateStreamingReplyAsync( + public IAsyncEnumerable GenerateStreamingReplyAsync( IEnumerable messages, GenerateReplyOptions? options = null, CancellationToken cancellationToken = default) diff --git a/dotnet/src/AutoGen.OpenAI/Agent/OpenAIChatAgent.cs b/dotnet/src/AutoGen.OpenAI/Agent/OpenAIChatAgent.cs index 37a4882f69e1..b192cde1024b 100644 --- a/dotnet/src/AutoGen.OpenAI/Agent/OpenAIChatAgent.cs +++ b/dotnet/src/AutoGen.OpenAI/Agent/OpenAIChatAgent.cs @@ -87,7 +87,7 @@ public async Task GenerateReplyAsync( return new MessageEnvelope(reply, from: this.Name); } - public async IAsyncEnumerable GenerateStreamingReplyAsync( + public async IAsyncEnumerable GenerateStreamingReplyAsync( IEnumerable messages, GenerateReplyOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) diff --git a/dotnet/src/AutoGen.OpenAI/Middleware/OpenAIChatRequestMessageConnector.cs b/dotnet/src/AutoGen.OpenAI/Middleware/OpenAIChatRequestMessageConnector.cs index 246e50cc6c59..e1dd0757fcf3 100644 --- a/dotnet/src/AutoGen.OpenAI/Middleware/OpenAIChatRequestMessageConnector.cs +++ b/dotnet/src/AutoGen.OpenAI/Middleware/OpenAIChatRequestMessageConnector.cs @@ -47,7 +47,7 @@ public async Task InvokeAsync(MiddlewareContext context, IAgent agent, return PostProcessMessage(reply); } - public async IAsyncEnumerable InvokeAsync( + public async IAsyncEnumerable InvokeAsync( MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) @@ -57,7 +57,7 @@ public async IAsyncEnumerable InvokeAsync( string? currentToolName = null; await foreach (var reply in streamingReply) { - if (reply is IStreamingMessage update) + if (reply is IMessage update) { if (update.Content.FunctionName is string functionName) { @@ -98,7 +98,7 @@ public IMessage PostProcessMessage(IMessage message) }; } - public IStreamingMessage? PostProcessStreamingMessage(IStreamingMessage update, string? currentToolName) + public IMessage? PostProcessStreamingMessage(IMessage update, string? currentToolName) { if (update.Content.ContentUpdate is string contentUpdate) { @@ -136,14 +136,13 @@ private IMessage PostProcessChatCompletions(IMessage message) private IMessage PostProcessChatResponseMessage(ChatResponseMessage chatResponseMessage, string? from) { - if (chatResponseMessage.Content is string content && !string.IsNullOrEmpty(content)) - { - return new TextMessage(Role.Assistant, content, from); - } - + var textContent = chatResponseMessage.Content; if (chatResponseMessage.FunctionCall is FunctionCall functionCall) { - return new ToolCallMessage(functionCall.Name, functionCall.Arguments, from); + return new ToolCallMessage(functionCall.Name, functionCall.Arguments, from) + { + Content = textContent, + }; } if (chatResponseMessage.ToolCalls.Where(tc => tc is ChatCompletionsFunctionToolCall).Any()) @@ -154,7 +153,15 @@ private IMessage PostProcessChatResponseMessage(ChatResponseMessage chatResponse var toolCalls = functionToolCalls.Select(tc => new ToolCall(tc.Name, tc.Arguments) { ToolCallId = tc.Id }); - return new ToolCallMessage(toolCalls, from); + return new ToolCallMessage(toolCalls, from) + { + Content = textContent, + }; + } + + if (textContent is string content && !string.IsNullOrEmpty(content)) + { + return new TextMessage(Role.Assistant, content, from); } throw new InvalidOperationException("Invalid ChatResponseMessage"); @@ -327,7 +334,8 @@ private IEnumerable ProcessToolCallMessage(IAgent agent, Too } var toolCall = message.ToolCalls.Select((tc, i) => new ChatCompletionsFunctionToolCall(tc.ToolCallId ?? $"{tc.FunctionName}_{i}", tc.FunctionName, tc.FunctionArguments)); - var chatRequestMessage = new ChatRequestAssistantMessage(string.Empty) { Name = message.From }; + var textContent = message.GetContent() ?? string.Empty; + var chatRequestMessage = new ChatRequestAssistantMessage(textContent) { Name = message.From }; foreach (var tc in toolCall) { chatRequestMessage.ToolCalls.Add(tc); diff --git a/dotnet/src/AutoGen.SemanticKernel/Middleware/SemanticKernelChatMessageContentConnector.cs b/dotnet/src/AutoGen.SemanticKernel/Middleware/SemanticKernelChatMessageContentConnector.cs index 6ce242eb1abe..a055c0afcb6a 100644 --- a/dotnet/src/AutoGen.SemanticKernel/Middleware/SemanticKernelChatMessageContentConnector.cs +++ b/dotnet/src/AutoGen.SemanticKernel/Middleware/SemanticKernelChatMessageContentConnector.cs @@ -47,7 +47,7 @@ public async Task InvokeAsync(MiddlewareContext context, IAgent agent, return PostProcessMessage(reply); } - public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public async IAsyncEnumerable InvokeAsync(MiddlewareContext context, IStreamingAgent agent, [EnumeratorCancellation] CancellationToken cancellationToken = default) { var chatMessageContents = ProcessMessage(context.Messages, agent) .Select(m => new MessageEnvelope(m)); @@ -67,11 +67,11 @@ private IMessage PostProcessMessage(IMessage input) }; } - private IStreamingMessage PostProcessStreamingMessage(IStreamingMessage input) + private IMessage PostProcessStreamingMessage(IMessage input) { return input switch { - IStreamingMessage streamingMessage => PostProcessMessage(streamingMessage), + IMessage streamingMessage => PostProcessMessage(streamingMessage), IMessage msg => PostProcessMessage(msg), _ => input, }; @@ -98,7 +98,7 @@ private IMessage PostProcessMessage(IMessage messageEnvelope } } - private IStreamingMessage PostProcessMessage(IStreamingMessage streamingMessage) + private IMessage PostProcessMessage(IMessage streamingMessage) { var chatMessageContent = streamingMessage.Content; if (chatMessageContent.ChoiceIndex > 0) diff --git a/dotnet/src/AutoGen.SemanticKernel/SemanticKernelAgent.cs b/dotnet/src/AutoGen.SemanticKernel/SemanticKernelAgent.cs index 21f652f56c4f..d12c54c1b3b2 100644 --- a/dotnet/src/AutoGen.SemanticKernel/SemanticKernelAgent.cs +++ b/dotnet/src/AutoGen.SemanticKernel/SemanticKernelAgent.cs @@ -65,7 +65,7 @@ public async Task GenerateReplyAsync(IEnumerable messages, G return new MessageEnvelope(reply.First(), from: this.Name); } - public async IAsyncEnumerable GenerateStreamingReplyAsync( + public async IAsyncEnumerable GenerateStreamingReplyAsync( IEnumerable messages, GenerateReplyOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) diff --git a/dotnet/src/AutoGen.SourceGenerator/SourceGeneratorFunctionContract.cs b/dotnet/src/AutoGen.SourceGenerator/SourceGeneratorFunctionContract.cs index 24e42affa3bd..aa4980379f4f 100644 --- a/dotnet/src/AutoGen.SourceGenerator/SourceGeneratorFunctionContract.cs +++ b/dotnet/src/AutoGen.SourceGenerator/SourceGeneratorFunctionContract.cs @@ -1,5 +1,5 @@ // Copyright (c) Microsoft Corporation. All rights reserved. -// FunctionContract.cs +// SourceGeneratorFunctionContract.cs namespace AutoGen.SourceGenerator { diff --git a/dotnet/src/AutoGen.SourceGenerator/Template/FunctionCallTemplate.cs b/dotnet/src/AutoGen.SourceGenerator/Template/FunctionCallTemplate.cs index 40adbdcde47c..8eeb117141d8 100644 --- a/dotnet/src/AutoGen.SourceGenerator/Template/FunctionCallTemplate.cs +++ b/dotnet/src/AutoGen.SourceGenerator/Template/FunctionCallTemplate.cs @@ -107,7 +107,7 @@ public virtual string TransformText() } if (functionContract.Description != null) { this.Write(" Description = @\""); - this.Write(this.ToStringHelper.ToStringWithCulture(functionContract.Description)); + this.Write(this.ToStringHelper.ToStringWithCulture(functionContract.Description.Replace("\"", "\"\""))); this.Write("\",\r\n"); } if (functionContract.ReturnType != null) { @@ -132,7 +132,7 @@ public virtual string TransformText() } if (parameter.Description != null) { this.Write(" Description = @\""); - this.Write(this.ToStringHelper.ToStringWithCulture(parameter.Description)); + this.Write(this.ToStringHelper.ToStringWithCulture(parameter.Description.Replace("\"", "\"\""))); this.Write("\",\r\n"); } if (parameter.Type != null) { @@ -152,12 +152,7 @@ public virtual string TransformText() } this.Write(" },\r\n"); } - this.Write(" };\r\n }\r\n\r\n public global::Azure.AI.OpenAI.FunctionDefin" + - "ition "); - this.Write(this.ToStringHelper.ToStringWithCulture(functionContract.GetFunctionDefinitionName())); - this.Write("\r\n {\r\n get => this."); - this.Write(this.ToStringHelper.ToStringWithCulture(functionContract.GetFunctionContractName())); - this.Write(".ToOpenAIFunctionDefinition();\r\n }\r\n"); + this.Write(" };\r\n }\r\n"); } this.Write(" }\r\n"); if (!String.IsNullOrEmpty(NameSpace)) { diff --git a/dotnet/src/AutoGen.SourceGenerator/Template/FunctionCallTemplate.tt b/dotnet/src/AutoGen.SourceGenerator/Template/FunctionCallTemplate.tt index 0d1b221c35c8..dc41f0af9d70 100644 --- a/dotnet/src/AutoGen.SourceGenerator/Template/FunctionCallTemplate.tt +++ b/dotnet/src/AutoGen.SourceGenerator/Template/FunctionCallTemplate.tt @@ -63,7 +63,7 @@ namespace <#=NameSpace#> Name = @"<#=functionContract.Name#>", <#}#> <#if (functionContract.Description != null) {#> - Description = @"<#=functionContract.Description#>", + Description = @"<#=functionContract.Description.Replace("\"", "\"\"")#>", <#}#> <#if (functionContract.ReturnType != null) {#> ReturnType = typeof(<#=functionContract.ReturnType#>), @@ -81,7 +81,7 @@ namespace <#=NameSpace#> Name = @"<#=parameter.Name#>", <#}#> <#if (parameter.Description != null) {#> - Description = @"<#=parameter.Description#>", + Description = @"<#= parameter.Description.Replace("\"", "\"\"") #>", <#}#> <#if (parameter.Type != null) {#> ParameterType = typeof(<#=parameter.Type#>), @@ -96,11 +96,6 @@ namespace <#=NameSpace#> <#}#> }; } - - public global::Azure.AI.OpenAI.FunctionDefinition <#=functionContract.GetFunctionDefinitionName()#> - { - get => this.<#=functionContract.GetFunctionContractName()#>.ToOpenAIFunctionDefinition(); - } <#}#> } <#if (!String.IsNullOrEmpty(NameSpace)) {#> diff --git a/dotnet/src/AutoGen.WebAPI/AutoGen.WebAPI.csproj b/dotnet/src/AutoGen.WebAPI/AutoGen.WebAPI.csproj new file mode 100644 index 000000000000..c5b720764761 --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/AutoGen.WebAPI.csproj @@ -0,0 +1,27 @@ + + + + net6.0;net8.0 + true + $(NoWarn);CS1591;CS1573 + + + + + + + + AutoGen.WebAPI + + Turn an `AutoGen.Core.IAgent` into a RESTful API. + + + + + + + + + + + diff --git a/dotnet/src/AutoGen.WebAPI/Extension.cs b/dotnet/src/AutoGen.WebAPI/Extension.cs new file mode 100644 index 000000000000..c8534e43e540 --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/Extension.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Extension.cs + +using AutoGen.Core; +using Microsoft.AspNetCore.Builder; + +namespace AutoGen.WebAPI; + +public static class Extension +{ + /// + /// Serve the agent as an OpenAI chat completion endpoint using . + /// If the request path is /v1/chat/completions and model name is the same as the agent name, + /// the request will be handled by the agent. + /// otherwise, the request will be passed to the next middleware. + /// + /// application builder + /// + public static IApplicationBuilder UseAgentAsOpenAIChatCompletionEndpoint(this IApplicationBuilder app, IAgent agent) + { + var middleware = new OpenAIChatCompletionMiddleware(agent); + return app.Use(middleware.InvokeAsync); + } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/Converter/OpenAIMessageConverter.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/Converter/OpenAIMessageConverter.cs new file mode 100644 index 000000000000..888a0f8dd8c8 --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/Converter/OpenAIMessageConverter.cs @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIMessageConverter.cs + +using System; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace AutoGen.WebAPI.OpenAI.DTO; + +internal class OpenAIMessageConverter : JsonConverter +{ + public override OpenAIMessage Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + using JsonDocument document = JsonDocument.ParseValue(ref reader); + var root = document.RootElement; + var role = root.GetProperty("role").GetString(); + var contentDocument = root.GetProperty("content"); + var isContentDocumentString = contentDocument.ValueKind == JsonValueKind.String; + switch (role) + { + case "system": + return JsonSerializer.Deserialize(root.GetRawText()) ?? throw new JsonException(); + case "user" when isContentDocumentString: + return JsonSerializer.Deserialize(root.GetRawText()) ?? throw new JsonException(); + case "user" when !isContentDocumentString: + return JsonSerializer.Deserialize(root.GetRawText()) ?? throw new JsonException(); + case "assistant": + return JsonSerializer.Deserialize(root.GetRawText()) ?? throw new JsonException(); + case "tool": + return JsonSerializer.Deserialize(root.GetRawText()) ?? throw new JsonException(); + default: + throw new JsonException(); + } + } + + public override void Write(Utf8JsonWriter writer, OpenAIMessage value, JsonSerializerOptions options) + { + switch (value) + { + case OpenAISystemMessage systemMessage: + JsonSerializer.Serialize(writer, systemMessage, options); + break; + case OpenAIUserMessage userMessage: + JsonSerializer.Serialize(writer, userMessage, options); + break; + case OpenAIAssistantMessage assistantMessage: + JsonSerializer.Serialize(writer, assistantMessage, options); + break; + case OpenAIToolMessage toolMessage: + JsonSerializer.Serialize(writer, toolMessage, options); + break; + default: + throw new JsonException(); + } + } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIAssistantMessage.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIAssistantMessage.cs new file mode 100644 index 000000000000..bfd090358453 --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIAssistantMessage.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIAssistantMessage.cs + +using System.Text.Json.Serialization; + +namespace AutoGen.WebAPI.OpenAI.DTO; + +internal class OpenAIAssistantMessage : OpenAIMessage +{ + [JsonPropertyName("role")] + public override string? Role { get; } = "assistant"; + + [JsonPropertyName("content")] + public string? Content { get; set; } + + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("tool_calls")] + public OpenAIToolCallObject[]? ToolCalls { get; set; } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletion.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletion.cs new file mode 100644 index 000000000000..041f4cfc848c --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletion.cs @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIChatCompletion.cs + +using System.Text.Json.Serialization; + +namespace AutoGen.WebAPI.OpenAI.DTO; + +internal class OpenAIChatCompletion +{ + [JsonPropertyName("id")] + public string? ID { get; set; } + + [JsonPropertyName("created")] + public long Created { get; set; } + + [JsonPropertyName("choices")] + public OpenAIChatCompletionChoice[]? Choices { get; set; } + + [JsonPropertyName("model")] + public string? Model { get; set; } + + [JsonPropertyName("system_fingerprint")] + public string? SystemFingerprint { get; set; } + + [JsonPropertyName("object")] + public string Object { get; set; } = "chat.completion"; + + [JsonPropertyName("usage")] + public OpenAIChatCompletionUsage? Usage { get; set; } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionChoice.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionChoice.cs new file mode 100644 index 000000000000..35b6fce59a8e --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionChoice.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIChatCompletionChoice.cs + +using System.Text.Json.Serialization; + +namespace AutoGen.WebAPI.OpenAI.DTO; + +internal class OpenAIChatCompletionChoice +{ + [JsonPropertyName("finish_reason")] + public string? FinishReason { get; set; } + + [JsonPropertyName("index")] + public int Index { get; set; } + + [JsonPropertyName("message")] + public OpenAIChatCompletionMessage? Message { get; set; } + + [JsonPropertyName("delta")] + public OpenAIChatCompletionMessage? Delta { get; set; } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionMessage.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionMessage.cs new file mode 100644 index 000000000000..de6be0dbf7a5 --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionMessage.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIChatCompletionMessage.cs + +using System.Text.Json.Serialization; + +namespace AutoGen.WebAPI.OpenAI.DTO; + +internal class OpenAIChatCompletionMessage +{ + [JsonPropertyName("role")] + public string Role { get; } = "assistant"; + + [JsonPropertyName("content")] + public string? Content { get; set; } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionOption.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionOption.cs new file mode 100644 index 000000000000..0b9137d43a39 --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionOption.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIChatCompletionOption.cs + +using System.Text.Json.Serialization; + +namespace AutoGen.WebAPI.OpenAI.DTO; + +internal class OpenAIChatCompletionOption +{ + [JsonPropertyName("messages")] + public OpenAIMessage[]? Messages { get; set; } + + [JsonPropertyName("model")] + public string? Model { get; set; } + + [JsonPropertyName("max_tokens")] + public int? MaxTokens { get; set; } + + [JsonPropertyName("temperature")] + public float Temperature { get; set; } = 1; + + /// + /// If set, partial message deltas will be sent, like in ChatGPT. Tokens will be sent as data-only server-sent events as they become available, with the stream terminated by a data: [DONE] message + /// + [JsonPropertyName("stream")] + public bool? Stream { get; set; } = false; + + [JsonPropertyName("stream_options")] + public OpenAIStreamOptions? StreamOptions { get; set; } + + [JsonPropertyName("stop")] + public string[]? Stop { get; set; } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionUsage.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionUsage.cs new file mode 100644 index 000000000000..f196ccb842ea --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIChatCompletionUsage.cs @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIChatCompletionUsage.cs + +using System.Text.Json.Serialization; + +namespace AutoGen.WebAPI.OpenAI.DTO; + +internal class OpenAIChatCompletionUsage +{ + [JsonPropertyName("completion_tokens")] + public int CompletionTokens { get; set; } + + [JsonPropertyName("prompt_tokens")] + public int PromptTokens { get; set; } + + [JsonPropertyName("total_tokens")] + public int TotalTokens { get; set; } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIImageUrlObject.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIImageUrlObject.cs new file mode 100644 index 000000000000..a50012c9fed1 --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIImageUrlObject.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIImageUrlObject.cs + +using System.Text.Json.Serialization; + +namespace AutoGen.WebAPI.OpenAI.DTO; + +internal class OpenAIImageUrlObject +{ + [JsonPropertyName("url")] + public string? Url { get; set; } + + [JsonPropertyName("detail")] + public string? Detail { get; set; } = "auto"; +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIMessage.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIMessage.cs new file mode 100644 index 000000000000..deb729b72003 --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIMessage.cs @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIMessage.cs + +using System.Text.Json.Serialization; + +namespace AutoGen.WebAPI.OpenAI.DTO; + +[JsonConverter(typeof(OpenAIMessageConverter))] +internal abstract class OpenAIMessage +{ + [JsonPropertyName("role")] + public abstract string? Role { get; } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIStreamOptions.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIStreamOptions.cs new file mode 100644 index 000000000000..e95991388b7f --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIStreamOptions.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIStreamOptions.cs + +using System.Text.Json.Serialization; + +namespace AutoGen.WebAPI.OpenAI.DTO; + +internal class OpenAIStreamOptions +{ + [JsonPropertyName("include_usage")] + public bool? IncludeUsage { get; set; } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAISystemMessage.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAISystemMessage.cs new file mode 100644 index 000000000000..f29b10826c4f --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAISystemMessage.cs @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAISystemMessage.cs + +using System.Text.Json.Serialization; + +namespace AutoGen.WebAPI.OpenAI.DTO; + +internal class OpenAISystemMessage : OpenAIMessage +{ + [JsonPropertyName("role")] + public override string? Role { get; } = "system"; + + [JsonPropertyName("content")] + public string? Content { get; set; } + + [JsonPropertyName("name")] + public string? Name { get; set; } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIToolCallObject.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIToolCallObject.cs new file mode 100644 index 000000000000..f3fc37f9c44f --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIToolCallObject.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIToolCallObject.cs + +using System.Text.Json.Serialization; + +namespace AutoGen.WebAPI.OpenAI.DTO; + +internal class OpenAIToolCallObject +{ + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("arguments")] + public string? Arguments { get; set; } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIToolMessage.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIToolMessage.cs new file mode 100644 index 000000000000..0c84c164cd96 --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIToolMessage.cs @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIToolMessage.cs + +using System.Text.Json.Serialization; + +namespace AutoGen.WebAPI.OpenAI.DTO; + +internal class OpenAIToolMessage : OpenAIMessage +{ + [JsonPropertyName("role")] + public override string? Role { get; } = "tool"; + + [JsonPropertyName("content")] + public string? Content { get; set; } + + [JsonPropertyName("tool_call_id")] + public string? ToolCallId { get; set; } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserImageContent.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserImageContent.cs new file mode 100644 index 000000000000..28b83ffb3058 --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserImageContent.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIUserImageContent.cs + +using System.Text.Json.Serialization; + +namespace AutoGen.WebAPI.OpenAI.DTO; + +internal class OpenAIUserImageContent : OpenAIUserMessageItem +{ + [JsonPropertyName("type")] + public override string MessageType { get; } = "image"; + + [JsonPropertyName("image_url")] + public string? Url { get; set; } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMessage.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMessage.cs new file mode 100644 index 000000000000..b5f1e7c50c12 --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMessage.cs @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIUserMessage.cs + +using System.Text.Json.Serialization; + +namespace AutoGen.WebAPI.OpenAI.DTO; + +internal class OpenAIUserMessage : OpenAIMessage +{ + [JsonPropertyName("role")] + public override string? Role { get; } = "user"; + + [JsonPropertyName("content")] + public string? Content { get; set; } + + [JsonPropertyName("name")] + public string? Name { get; set; } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMessageItem.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMessageItem.cs new file mode 100644 index 000000000000..94e7d91534a5 --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMessageItem.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIUserMessageItem.cs + +using System.Text.Json.Serialization; + +namespace AutoGen.WebAPI.OpenAI.DTO; + +internal abstract class OpenAIUserMessageItem +{ + [JsonPropertyName("type")] + public abstract string MessageType { get; } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMultiModalMessage.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMultiModalMessage.cs new file mode 100644 index 000000000000..789df5afaaae --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserMultiModalMessage.cs @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIUserMultiModalMessage.cs + +using System.Text.Json.Serialization; + +namespace AutoGen.WebAPI.OpenAI.DTO; + +internal class OpenAIUserMultiModalMessage : OpenAIMessage +{ + [JsonPropertyName("role")] + public override string? Role { get; } = "user"; + + [JsonPropertyName("content")] + public OpenAIUserMessageItem[]? Content { get; set; } + + [JsonPropertyName("name")] + public string? Name { get; set; } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserTextContent.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserTextContent.cs new file mode 100644 index 000000000000..d22d5aa4c7f3 --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/DTO/OpenAIUserTextContent.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIUserTextContent.cs + +using System.Text.Json.Serialization; + +namespace AutoGen.WebAPI.OpenAI.DTO; + +internal class OpenAIUserTextContent : OpenAIUserMessageItem +{ + [JsonPropertyName("type")] + public override string MessageType { get; } = "text"; + + [JsonPropertyName("text")] + public string? Content { get; set; } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAI/Service/OpenAIChatCompletionService.cs b/dotnet/src/AutoGen.WebAPI/OpenAI/Service/OpenAIChatCompletionService.cs new file mode 100644 index 000000000000..27481da006a2 --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAI/Service/OpenAIChatCompletionService.cs @@ -0,0 +1,157 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIChatCompletionService.cs + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using AutoGen.Core; +using AutoGen.WebAPI.OpenAI.DTO; + +namespace AutoGen.Server; + +internal class OpenAIChatCompletionService +{ + private readonly IAgent agent; + + public OpenAIChatCompletionService(IAgent agent) + { + this.agent = agent; + } + + public async Task GetChatCompletionAsync(OpenAIChatCompletionOption request) + { + var messages = this.ProcessMessages(request.Messages ?? Array.Empty()); + + var generateOption = this.ProcessReplyOptions(request); + + var reply = await this.agent.GenerateReplyAsync(messages, generateOption); + + var openAIChatCompletion = new OpenAIChatCompletion() + { + Created = DateTimeOffset.UtcNow.Ticks / TimeSpan.TicksPerMillisecond / 1000, + Model = this.agent.Name, + }; + + if (reply.GetContent() is string content) + { + var message = new OpenAIChatCompletionMessage() + { + Content = content, + }; + + var choice = new OpenAIChatCompletionChoice() + { + Message = message, + Index = 0, + FinishReason = "completed", + }; + + openAIChatCompletion.Choices = [choice]; + + return openAIChatCompletion; + } + + throw new NotImplementedException("Unsupported reply content type"); + } + + public async IAsyncEnumerable GetStreamingChatCompletionAsync(OpenAIChatCompletionOption request) + { + if (this.agent is IStreamingAgent streamingAgent) + { + var messages = this.ProcessMessages(request.Messages ?? Array.Empty()); + + var generateOption = this.ProcessReplyOptions(request); + + await foreach (var reply in streamingAgent.GenerateStreamingReplyAsync(messages, generateOption)) + { + var openAIChatCompletion = new OpenAIChatCompletion() + { + Created = DateTimeOffset.UtcNow.Ticks / TimeSpan.TicksPerMillisecond / 1000, + Model = this.agent.Name, + }; + + if (reply.GetContent() is string content) + { + var message = new OpenAIChatCompletionMessage() + { + Content = content, + }; + + var choice = new OpenAIChatCompletionChoice() + { + Delta = message, + Index = 0, + }; + + openAIChatCompletion.Choices = [choice]; + + yield return openAIChatCompletion; + } + else + { + throw new NotImplementedException("Unsupported reply content type"); + } + } + + var doneMessage = new OpenAIChatCompletion() + { + Created = DateTimeOffset.UtcNow.Ticks / TimeSpan.TicksPerMillisecond / 1000, + Model = this.agent.Name, + }; + + var doneChoice = new OpenAIChatCompletionChoice() + { + FinishReason = "stop", + Index = 0, + }; + + doneMessage.Choices = [doneChoice]; + + yield return doneMessage; + } + else + { + yield return await this.GetChatCompletionAsync(request); + } + } + + private IEnumerable ProcessMessages(IEnumerable messages) + { + return messages.Select(m => m switch + { + OpenAISystemMessage systemMessage when systemMessage.Content is string content => new TextMessage(Role.System, content, this.agent.Name), + OpenAIUserMessage userMessage when userMessage.Content is string content => new TextMessage(Role.User, content, this.agent.Name), + OpenAIAssistantMessage assistantMessage when assistantMessage.Content is string content => new TextMessage(Role.Assistant, content, this.agent.Name), + OpenAIUserMultiModalMessage userMultiModalMessage when userMultiModalMessage.Content is { Length: > 0 } => this.CreateMultiModaMessageFromOpenAIUserMultiModalMessage(userMultiModalMessage), + _ => throw new ArgumentException($"Unsupported message type {m.GetType()}") + }); + } + + private GenerateReplyOptions ProcessReplyOptions(OpenAIChatCompletionOption request) + { + return new GenerateReplyOptions() + { + Temperature = request.Temperature, + MaxToken = request.MaxTokens, + StopSequence = request.Stop, + }; + } + + private MultiModalMessage CreateMultiModaMessageFromOpenAIUserMultiModalMessage(OpenAIUserMultiModalMessage message) + { + if (message.Content is null) + { + throw new ArgumentNullException(nameof(message.Content)); + } + + IEnumerable items = message.Content.Select(item => item switch + { + OpenAIUserImageContent imageContent when imageContent.Url is string url => new ImageMessage(Role.User, url, this.agent.Name), + OpenAIUserTextContent textContent when textContent.Content is string content => new TextMessage(Role.User, content, this.agent.Name), + _ => throw new ArgumentException($"Unsupported content type {item.GetType()}") + }); + + return new MultiModalMessage(Role.User, items, this.agent.Name); + } +} diff --git a/dotnet/src/AutoGen.WebAPI/OpenAIChatCompletionMiddleware.cs b/dotnet/src/AutoGen.WebAPI/OpenAIChatCompletionMiddleware.cs new file mode 100644 index 000000000000..53b3699fd62e --- /dev/null +++ b/dotnet/src/AutoGen.WebAPI/OpenAIChatCompletionMiddleware.cs @@ -0,0 +1,92 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIChatCompletionMiddleware.cs + +using System.Text.Json; +using System.Threading.Tasks; +using AutoGen.Core; +using AutoGen.Server; +using AutoGen.WebAPI.OpenAI.DTO; +using Microsoft.AspNetCore.Http; + +namespace AutoGen.WebAPI; + +public class OpenAIChatCompletionMiddleware : Microsoft.AspNetCore.Http.IMiddleware +{ + private readonly IAgent _agent; + private readonly OpenAIChatCompletionService chatCompletionService; + + public OpenAIChatCompletionMiddleware(IAgent agent) + { + _agent = agent; + chatCompletionService = new OpenAIChatCompletionService(_agent); + } + + public async Task InvokeAsync(HttpContext context, RequestDelegate next) + { + // if HttpPost and path is /v1/chat/completions + // get the request body + // call chatCompletionService.GetChatCompletionAsync(request) + // return the response + + // else + // call next middleware + if (context.Request.Method == HttpMethods.Post && context.Request.Path == "/v1/chat/completions") + { + context.Request.EnableBuffering(); + var body = await context.Request.ReadFromJsonAsync(); + context.Request.Body.Position = 0; + if (body is null) + { + // return 400 Bad Request + context.Response.StatusCode = 400; + return; + } + + if (body.Model != _agent.Name) + { + await next(context); + return; + } + + if (body.Stream is true) + { + // Send as server side events + context.Response.Headers.Append("Content-Type", "text/event-stream"); + context.Response.Headers.Append("Cache-Control", "no-cache"); + context.Response.Headers.Append("Connection", "keep-alive"); + await foreach (var chatCompletion in chatCompletionService.GetStreamingChatCompletionAsync(body)) + { + if (chatCompletion?.Choices?[0].FinishReason is "stop") + { + // the stream is done + // send Data: [DONE]\n\n + await context.Response.WriteAsync("data: [DONE]\n\n"); + break; + } + else + { + // remove null + var option = new JsonSerializerOptions + { + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + }; + var data = JsonSerializer.Serialize(chatCompletion, option); + await context.Response.WriteAsync($"data: {data}\n\n"); + } + } + + return; + } + else + { + var chatCompletion = await chatCompletionService.GetChatCompletionAsync(body); + await context.Response.WriteAsJsonAsync(chatCompletion); + return; + } + } + else + { + await next(context); + } + } +} diff --git a/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientAgentTest.cs b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientAgentTest.cs index d29025b44aff..085917d419e9 100644 --- a/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientAgentTest.cs +++ b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientAgentTest.cs @@ -32,6 +32,30 @@ public async Task AnthropicAgentChatCompletionTestAsync() reply.From.Should().Be(agent.Name); } + [ApiKeyFact("ANTHROPIC_API_KEY")] + public async Task AnthropicAgentMergeMessageWithSameRoleTests() + { + // this test is added to fix issue #2884 + var client = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, AnthropicTestUtils.ApiKey); + + var agent = new AnthropicClientAgent( + client, + name: "AnthropicAgent", + AnthropicConstants.Claude3Haiku, + systemMessage: "You are a helpful AI assistant that convert user message to upper case") + .RegisterMessageConnector(); + + var uppCaseMessage = new TextMessage(Role.User, "abcdefg"); + var anotherUserMessage = new TextMessage(Role.User, "hijklmn"); + var assistantMessage = new TextMessage(Role.Assistant, "opqrst"); + var anotherAssistantMessage = new TextMessage(Role.Assistant, "uvwxyz"); + var yetAnotherUserMessage = new TextMessage(Role.User, "123456"); + + // just make sure it doesn't throw exception + var reply = await agent.SendAsync(chatHistory: [uppCaseMessage, anotherUserMessage, assistantMessage, anotherAssistantMessage, yetAnotherUserMessage]); + reply.GetContent().Should().NotBeNull(); + } + [ApiKeyFact("ANTHROPIC_API_KEY")] public async Task AnthropicAgentTestProcessImageAsync() { @@ -105,4 +129,101 @@ public async Task AnthropicAgentTestImageMessageAsync() reply.GetContent().Should().NotBeNullOrEmpty(); reply.From.Should().Be(agent.Name); } + + [ApiKeyFact("ANTHROPIC_API_KEY")] + public async Task AnthropicAgentTestToolAsync() + { + var client = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, AnthropicTestUtils.ApiKey); + + var function = new TypeSafeFunctionCall(); + var functionCallMiddleware = new FunctionCallMiddleware( + functions: new[] { function.WeatherReportFunctionContract }, + functionMap: new Dictionary>> + { + { function.WeatherReportFunctionContract.Name ?? string.Empty, function.WeatherReportWrapper }, + }); + + var agent = new AnthropicClientAgent( + client, + name: "AnthropicAgent", + AnthropicConstants.Claude3Haiku, + systemMessage: "You are an LLM that is specialized in finding the weather !", + tools: [AnthropicTestUtils.WeatherTool] + ) + .RegisterMessageConnector() + .RegisterStreamingMiddleware(functionCallMiddleware); + + var reply = await agent.SendAsync("What is the weather in Philadelphia?"); + reply.GetContent().Should().Be("Weather report for Philadelphia on today is sunny"); + } + + [ApiKeyFact("ANTHROPIC_API_KEY")] + public async Task AnthropicAgentFunctionCallMessageTest() + { + var client = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, AnthropicTestUtils.ApiKey); + var agent = new AnthropicClientAgent( + client, + name: "AnthropicAgent", + AnthropicConstants.Claude3Haiku, + systemMessage: "You are a helpful AI assistant.", + tools: [AnthropicTestUtils.WeatherTool] + ) + .RegisterMessageConnector(); + + var weatherFunctionArgumets = """ + { + "city": "Philadelphia", + "date": "6/14/2024" + } + """; + + var function = new AnthropicTestFunctionCalls(); + var functionCallResult = await function.GetWeatherReportWrapper(weatherFunctionArgumets); + var toolCall = new ToolCall(function.WeatherReportFunctionContract.Name!, weatherFunctionArgumets) + { + ToolCallId = "get_weather", + Result = functionCallResult, + }; + + IMessage[] chatHistory = [ + new TextMessage(Role.User, "what's the weather in Philadelphia?"), + new ToolCallMessage([toolCall], from: "assistant"), + new ToolCallResultMessage([toolCall], from: "user"), + ]; + + var reply = await agent.SendAsync(chatHistory: chatHistory); + + reply.Should().BeOfType(); + reply.GetContent().Should().Be("The weather report for Philadelphia on 6/14/2024 is sunny."); + } + + [ApiKeyFact("ANTHROPIC_API_KEY")] + public async Task AnthropicAgentFunctionCallMiddlewareMessageTest() + { + var client = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, AnthropicTestUtils.ApiKey); + var function = new AnthropicTestFunctionCalls(); + var functionCallMiddleware = new FunctionCallMiddleware( + functions: [function.WeatherReportFunctionContract], + functionMap: new Dictionary>> + { + { function.WeatherReportFunctionContract.Name!, function.GetWeatherReportWrapper } + }); + + var functionCallAgent = new AnthropicClientAgent( + client, + name: "AnthropicAgent", + AnthropicConstants.Claude3Haiku, + systemMessage: "You are a helpful AI assistant.", + tools: [AnthropicTestUtils.WeatherTool] + ) + .RegisterMessageConnector() + .RegisterStreamingMiddleware(functionCallMiddleware); + + var question = new TextMessage(Role.User, "what's the weather in Philadelphia?"); + var reply = await functionCallAgent.SendAsync(question); + + var finalReply = await functionCallAgent.SendAsync(chatHistory: [question, reply]); + finalReply.Should().BeOfType(); + finalReply.GetContent()!.ToLower().Should().Contain("sunny"); + } } diff --git a/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientTest.cs b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientTest.cs index a0b1f60cfb95..102e48b9b8ac 100644 --- a/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientTest.cs +++ b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicClientTest.cs @@ -1,5 +1,9 @@ -using System.Text; +// Copyright (c) Microsoft Corporation. All rights reserved. +// AnthropicClientTest.cs + +using System.Text; using System.Text.Json; +using System.Text.Json.Nodes; using System.Text.Json.Serialization; using AutoGen.Anthropic.DTO; using AutoGen.Anthropic.Utils; @@ -58,7 +62,9 @@ public async Task AnthropicClientStreamingChatCompletionTestAsync() foreach (ChatCompletionResponse result in results) { if (result.Delta is not null && !string.IsNullOrEmpty(result.Delta.Text)) + { sb.Append(result.Delta.Text); + } } string resultContent = sb.ToString(); @@ -108,6 +114,57 @@ public async Task AnthropicClientImageChatCompletionTestAsync() response.Usage.OutputTokens.Should().BeGreaterThan(0); } + [ApiKeyFact("ANTHROPIC_API_KEY")] + public async Task AnthropicClientTestToolsAsync() + { + var anthropicClient = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, AnthropicTestUtils.ApiKey); + + var request = new ChatCompletionRequest(); + request.Model = AnthropicConstants.Claude3Haiku; + request.Stream = false; + request.MaxTokens = 100; + request.Messages = new List() { new("user", "Use the stock price tool to look for MSFT. Your response should only be the tool.") }; + request.Tools = new List() { AnthropicTestUtils.StockTool }; + + ChatCompletionResponse response = + await anthropicClient.CreateChatCompletionsAsync(request, CancellationToken.None); + + Assert.NotNull(response.Content); + Assert.True(response.Content.First() is ToolUseContent); + ToolUseContent toolUseContent = ((ToolUseContent)response.Content.First()); + Assert.Equal("get_stock_price", toolUseContent.Name); + Assert.NotNull(toolUseContent.Input); + Assert.True(toolUseContent.Input is JsonNode); + JsonNode jsonNode = toolUseContent.Input; + Assert.Equal("{\"ticker\":\"MSFT\"}", jsonNode.ToJsonString()); + } + + [ApiKeyFact("ANTHROPIC_API_KEY")] + public async Task AnthropicClientTestToolChoiceAsync() + { + var anthropicClient = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, AnthropicTestUtils.ApiKey); + + var request = new ChatCompletionRequest(); + request.Model = AnthropicConstants.Claude3Haiku; + request.Stream = false; + request.MaxTokens = 100; + request.Messages = new List() { new("user", "What is the weather today? Your response should only be the tool.") }; + request.Tools = new List() { AnthropicTestUtils.StockTool, AnthropicTestUtils.WeatherTool }; + + // Force to use get_stock_price even though the prompt is about weather + request.ToolChoice = ToolChoice.ToolUse("get_stock_price"); + + ChatCompletionResponse response = + await anthropicClient.CreateChatCompletionsAsync(request, CancellationToken.None); + + Assert.NotNull(response.Content); + Assert.True(response.Content.First() is ToolUseContent); + ToolUseContent toolUseContent = ((ToolUseContent)response.Content.First()); + Assert.Equal("get_stock_price", toolUseContent.Name); + Assert.NotNull(toolUseContent.Input); + Assert.True(toolUseContent.Input is JsonNode); + } + private sealed class Person { [JsonPropertyName("name")] diff --git a/dotnet/test/AutoGen.Anthropic.Tests/AnthropicTestFunctionCalls.cs b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicTestFunctionCalls.cs new file mode 100644 index 000000000000..8b5466e3a519 --- /dev/null +++ b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicTestFunctionCalls.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// AnthropicTestFunctionCalls.cs + +using System.Text.Json; +using System.Text.Json.Serialization; +using AutoGen.Core; + +namespace AutoGen.Anthropic.Tests; + +public partial class AnthropicTestFunctionCalls +{ + private class GetWeatherSchema + { + [JsonPropertyName("city")] + public string? City { get; set; } + + [JsonPropertyName("date")] + public string? Date { get; set; } + } + + /// + /// Get weather report + /// + /// city + /// date + [Function] + public async Task WeatherReport(string city, string date) + { + return $"Weather report for {city} on {date} is sunny"; + } + + public Task GetWeatherReportWrapper(string arguments) + { + var schema = JsonSerializer.Deserialize( + arguments, + new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }); + + return WeatherReport(schema?.City ?? string.Empty, schema?.Date ?? string.Empty); + } +} diff --git a/dotnet/test/AutoGen.Anthropic.Tests/AnthropicTestUtils.cs b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicTestUtils.cs index de630da6d87c..a1faffec5344 100644 --- a/dotnet/test/AutoGen.Anthropic.Tests/AnthropicTestUtils.cs +++ b/dotnet/test/AutoGen.Anthropic.Tests/AnthropicTestUtils.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // AnthropicTestUtils.cs +using AutoGen.Anthropic.DTO; + namespace AutoGen.Anthropic.Tests; public static class AnthropicTestUtils @@ -13,4 +15,52 @@ public static async Task Base64FromImageAsync(string imageName) return Convert.ToBase64String( await File.ReadAllBytesAsync(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "images", imageName))); } + + public static Tool WeatherTool + { + get + { + return new Tool + { + Name = "WeatherReport", + Description = "Get the current weather", + InputSchema = new InputSchema + { + Type = "object", + Properties = new Dictionary + { + { "city", new SchemaProperty {Type = "string", Description = "The name of the city"} }, + { "date", new SchemaProperty {Type = "string", Description = "date of the day"} } + } + } + }; + } + } + + public static Tool StockTool + { + get + { + return new Tool + { + Name = "get_stock_price", + Description = "Get the current stock price for a given ticker symbol.", + InputSchema = new InputSchema + { + Type = "object", + Properties = new Dictionary + { + { + "ticker", new SchemaProperty + { + Type = "string", + Description = "The stock ticker symbol, e.g. AAPL for Apple Inc." + } + } + }, + Required = new List { "ticker" } + } + }; + } + } } diff --git a/dotnet/test/AutoGen.Anthropic.Tests/AutoGen.Anthropic.Tests.csproj b/dotnet/test/AutoGen.Anthropic.Tests/AutoGen.Anthropic.Tests.csproj index 0f22d9fe6764..ac479ed2e722 100644 --- a/dotnet/test/AutoGen.Anthropic.Tests/AutoGen.Anthropic.Tests.csproj +++ b/dotnet/test/AutoGen.Anthropic.Tests/AutoGen.Anthropic.Tests.csproj @@ -12,6 +12,7 @@ + diff --git a/dotnet/test/AutoGen.Gemini.Tests/GeminiAgentTests.cs b/dotnet/test/AutoGen.Gemini.Tests/GeminiAgentTests.cs index 872cce5e645b..c076aee18376 100644 --- a/dotnet/test/AutoGen.Gemini.Tests/GeminiAgentTests.cs +++ b/dotnet/test/AutoGen.Gemini.Tests/GeminiAgentTests.cs @@ -1,13 +1,13 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // GeminiAgentTests.cs -using AutoGen.Tests; -using Google.Cloud.AIPlatform.V1; using AutoGen.Core; -using FluentAssertions; using AutoGen.Gemini.Extension; -using static Google.Cloud.AIPlatform.V1.Part; +using AutoGen.Tests; +using FluentAssertions; +using Google.Cloud.AIPlatform.V1; using Xunit.Abstractions; +using static Google.Cloud.AIPlatform.V1.Part; namespace AutoGen.Gemini.Tests; public class GeminiAgentTests @@ -86,8 +86,8 @@ public async Task VertexGeminiAgentGenerateStreamingReplyForTextContentAsync() var message = MessageEnvelope.Create(textContent, from: agent.Name); var completion = agent.GenerateStreamingReplyAsync([message]); - var chunks = new List(); - IStreamingMessage finalReply = null!; + var chunks = new List(); + IMessage finalReply = null!; await foreach (var item in completion) { @@ -212,8 +212,8 @@ public async Task VertexGeminiAgentGenerateStreamingReplyWithToolsAsync() var message = MessageEnvelope.Create(textContent, from: agent.Name); - var chunks = new List(); - IStreamingMessage finalReply = null!; + var chunks = new List(); + IMessage finalReply = null!; var completion = agent.GenerateStreamingReplyAsync([message]); diff --git a/dotnet/test/AutoGen.Gemini.Tests/GeminiMessageTests.cs b/dotnet/test/AutoGen.Gemini.Tests/GeminiMessageTests.cs index 7ffb532ea9c1..12ba94734032 100644 --- a/dotnet/test/AutoGen.Gemini.Tests/GeminiMessageTests.cs +++ b/dotnet/test/AutoGen.Gemini.Tests/GeminiMessageTests.cs @@ -225,10 +225,10 @@ public async Task ItProcessStreamingTextMessageAsync() }) .Select(m => MessageEnvelope.Create(m)); - IStreamingMessage? finalReply = null; + IMessage? finalReply = null; await foreach (var reply in agent.GenerateStreamingReplyAsync(messageChunks)) { - reply.Should().BeAssignableTo(); + reply.Should().BeAssignableTo(); finalReply = reply; } diff --git a/dotnet/test/AutoGen.Gemini.Tests/VertexGeminiClientTests.cs b/dotnet/test/AutoGen.Gemini.Tests/VertexGeminiClientTests.cs index 2f06305ed59f..fba97aa522d5 100644 --- a/dotnet/test/AutoGen.Gemini.Tests/VertexGeminiClientTests.cs +++ b/dotnet/test/AutoGen.Gemini.Tests/VertexGeminiClientTests.cs @@ -1,5 +1,5 @@ // Copyright (c) Microsoft Corporation. All rights reserved. -// GeminiVertexClientTests.cs +// VertexGeminiClientTests.cs using AutoGen.Tests; using FluentAssertions; @@ -53,7 +53,7 @@ public async Task ItGenerateContentWithImageAsync() var model = "gemini-1.5-flash-001"; var text = "what's in the image"; - var imagePath = Path.Combine("testData", "images", "image.png"); + var imagePath = Path.Combine("testData", "images", "square.png"); var image = File.ReadAllBytes(imagePath); var request = new GenerateContentRequest { diff --git a/dotnet/test/AutoGen.Ollama.Tests/OllamaAgentTests.cs b/dotnet/test/AutoGen.Ollama.Tests/OllamaAgentTests.cs index c1fb466f0b09..8a416116ea92 100644 --- a/dotnet/test/AutoGen.Ollama.Tests/OllamaAgentTests.cs +++ b/dotnet/test/AutoGen.Ollama.Tests/OllamaAgentTests.cs @@ -65,8 +65,8 @@ public async Task GenerateStreamingReplyAsync_ReturnsValidMessages_WhenCalled() var msg = new Message("user", "hey how are you"); var messages = new IMessage[] { MessageEnvelope.Create(msg, from: modelName) }; - IStreamingMessage? finalReply = default; - await foreach (IStreamingMessage message in ollamaAgent.GenerateStreamingReplyAsync(messages)) + IMessage? finalReply = default; + await foreach (IMessage message in ollamaAgent.GenerateStreamingReplyAsync(messages)) { message.Should().NotBeNull(); message.From.Should().Be(ollamaAgent.Name); @@ -171,8 +171,8 @@ public async Task ItReturnValidStreamingMessageUsingLLavaAsync() var messages = new IMessage[] { MessageEnvelope.Create(imageMessage, from: modelName) }; - IStreamingMessage? finalReply = default; - await foreach (IStreamingMessage message in ollamaAgent.GenerateStreamingReplyAsync(messages)) + IMessage? finalReply = default; + await foreach (IMessage message in ollamaAgent.GenerateStreamingReplyAsync(messages)) { message.Should().NotBeNull(); message.From.Should().Be(ollamaAgent.Name); diff --git a/dotnet/test/AutoGen.Ollama.Tests/OllamaMessageTests.cs b/dotnet/test/AutoGen.Ollama.Tests/OllamaMessageTests.cs index b19291e97671..82cc462061da 100644 --- a/dotnet/test/AutoGen.Ollama.Tests/OllamaMessageTests.cs +++ b/dotnet/test/AutoGen.Ollama.Tests/OllamaMessageTests.cs @@ -57,10 +57,10 @@ public async Task ItProcessStreamingTextMessageAsync() }) .Select(m => MessageEnvelope.Create(m)); - IStreamingMessage? finalReply = null; + IMessage? finalReply = null; await foreach (var reply in agent.GenerateStreamingReplyAsync(messageChunks)) { - reply.Should().BeAssignableTo(); + reply.Should().BeAssignableTo(); finalReply = reply; } diff --git a/dotnet/test/AutoGen.Ollama.Tests/OllamaTextEmbeddingServiceTests.cs b/dotnet/test/AutoGen.Ollama.Tests/OllamaTextEmbeddingServiceTests.cs index 06522bdd8238..b7186a3c6ebc 100644 --- a/dotnet/test/AutoGen.Ollama.Tests/OllamaTextEmbeddingServiceTests.cs +++ b/dotnet/test/AutoGen.Ollama.Tests/OllamaTextEmbeddingServiceTests.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // OllamaTextEmbeddingServiceTests.cs using AutoGen.Tests; diff --git a/dotnet/test/AutoGen.OpenAI.Tests/AutoGen.OpenAI.Tests.csproj b/dotnet/test/AutoGen.OpenAI.Tests/AutoGen.OpenAI.Tests.csproj index ba499232beb9..04800a631ee6 100644 --- a/dotnet/test/AutoGen.OpenAI.Tests/AutoGen.OpenAI.Tests.csproj +++ b/dotnet/test/AutoGen.OpenAI.Tests/AutoGen.OpenAI.Tests.csproj @@ -8,6 +8,7 @@ + diff --git a/dotnet/test/AutoGen.OpenAI.Tests/MathClassTest.cs b/dotnet/test/AutoGen.OpenAI.Tests/MathClassTest.cs index aae314ff773e..01af3d4646c4 100644 --- a/dotnet/test/AutoGen.OpenAI.Tests/MathClassTest.cs +++ b/dotnet/test/AutoGen.OpenAI.Tests/MathClassTest.cs @@ -110,7 +110,7 @@ public async Task OpenAIAgentMathChatTestAsync() functions: [this.UpdateProgressFunctionContract], functionMap: new Dictionary>> { - { this.UpdateProgressFunction.Name!, this.UpdateProgressWrapper }, + { this.UpdateProgressFunctionContract.Name, this.UpdateProgressWrapper }, }); var admin = new OpenAIChatAgent( openAIClient: openaiClient, diff --git a/dotnet/test/AutoGen.OpenAI.Tests/OpenAIMessageTests.cs b/dotnet/test/AutoGen.OpenAI.Tests/OpenAIMessageTests.cs index 81581d068ee7..a9b852e0d8c1 100644 --- a/dotnet/test/AutoGen.OpenAI.Tests/OpenAIMessageTests.cs +++ b/dotnet/test/AutoGen.OpenAI.Tests/OpenAIMessageTests.cs @@ -278,9 +278,9 @@ public async Task ItProcessToolCallMessageAsync() var innerMessage = msgs.Last(); innerMessage!.Should().BeOfType>(); var chatRequestMessage = (ChatRequestAssistantMessage)((MessageEnvelope)innerMessage!).Content; - chatRequestMessage.Content.Should().BeNullOrEmpty(); chatRequestMessage.Name.Should().Be("assistant"); chatRequestMessage.ToolCalls.Count().Should().Be(1); + chatRequestMessage.Content.Should().Be("textContent"); chatRequestMessage.ToolCalls.First().Should().BeOfType(); var functionToolCall = (ChatCompletionsFunctionToolCall)chatRequestMessage.ToolCalls.First(); functionToolCall.Name.Should().Be("test"); @@ -291,7 +291,10 @@ public async Task ItProcessToolCallMessageAsync() .RegisterMiddleware(middleware); // user message - IMessage message = new ToolCallMessage("test", "test", "assistant"); + IMessage message = new ToolCallMessage("test", "test", "assistant") + { + Content = "textContent", + }; await agent.GenerateReplyAsync([message]); } @@ -526,13 +529,14 @@ public async Task ItConvertChatResponseMessageToToolCallMessageAsync() .RegisterMiddleware(middleware); // tool call message - var toolCallMessage = CreateInstance(ChatRole.Assistant, "", new[] { new ChatCompletionsFunctionToolCall("test", "test", "test") }, new FunctionCall("test", "test"), CreateInstance(), new Dictionary()); + var toolCallMessage = CreateInstance(ChatRole.Assistant, "textContent", new[] { new ChatCompletionsFunctionToolCall("test", "test", "test") }, new FunctionCall("test", "test"), CreateInstance(), new Dictionary()); var chatRequestMessage = MessageEnvelope.Create(toolCallMessage); var message = await agent.GenerateReplyAsync([chatRequestMessage]); message.Should().BeOfType(); message.GetToolCalls()!.Count().Should().Be(1); message.GetToolCalls()!.First().FunctionName.Should().Be("test"); message.GetToolCalls()!.First().FunctionArguments.Should().Be("test"); + message.GetContent().Should().Be("textContent"); } [Fact] diff --git a/dotnet/test/AutoGen.SemanticKernel.Tests/ApprovalTests/KernelFunctionExtensionTests.ItCreateFunctionContractsFromMethod.approved.txt b/dotnet/test/AutoGen.SemanticKernel.Tests/ApprovalTests/KernelFunctionExtensionTests.ItCreateFunctionContractsFromMethod.approved.txt index 677831d412b7..eb346da3b313 100644 --- a/dotnet/test/AutoGen.SemanticKernel.Tests/ApprovalTests/KernelFunctionExtensionTests.ItCreateFunctionContractsFromMethod.approved.txt +++ b/dotnet/test/AutoGen.SemanticKernel.Tests/ApprovalTests/KernelFunctionExtensionTests.ItCreateFunctionContractsFromMethod.approved.txt @@ -14,8 +14,7 @@ "Name": "message", "Description": "", "ParameterType": "System.String, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e", - "IsRequired": true, - "DefaultValue": "" + "IsRequired": true } ], "ReturnType": "System.String, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e", diff --git a/dotnet/test/AutoGen.SemanticKernel.Tests/ApprovalTests/KernelFunctionExtensionTests.ItCreateFunctionContractsFromTestPlugin.approved.txt b/dotnet/test/AutoGen.SemanticKernel.Tests/ApprovalTests/KernelFunctionExtensionTests.ItCreateFunctionContractsFromTestPlugin.approved.txt index ee835b1ba081..9ed3c675e4a0 100644 --- a/dotnet/test/AutoGen.SemanticKernel.Tests/ApprovalTests/KernelFunctionExtensionTests.ItCreateFunctionContractsFromTestPlugin.approved.txt +++ b/dotnet/test/AutoGen.SemanticKernel.Tests/ApprovalTests/KernelFunctionExtensionTests.ItCreateFunctionContractsFromTestPlugin.approved.txt @@ -16,8 +16,7 @@ "Name": "newState", "Description": "new state", "ParameterType": "System.Boolean, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e", - "IsRequired": true, - "DefaultValue": "" + "IsRequired": true } ], "ReturnType": "System.String, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e", diff --git a/dotnet/test/AutoGen.SourceGenerator.Tests/ApprovalTests/FunctionCallTemplateTests.TestFunctionCallTemplate.approved.txt b/dotnet/test/AutoGen.SourceGenerator.Tests/ApprovalTests/FunctionCallTemplateTests.TestFunctionCallTemplate.approved.txt index 0439febc52c7..f223d3124ddd 100644 --- a/dotnet/test/AutoGen.SourceGenerator.Tests/ApprovalTests/FunctionCallTemplateTests.TestFunctionCallTemplate.approved.txt +++ b/dotnet/test/AutoGen.SourceGenerator.Tests/ApprovalTests/FunctionCallTemplateTests.TestFunctionCallTemplate.approved.txt @@ -61,11 +61,6 @@ namespace AutoGen.SourceGenerator.Tests }, }; } - - public global::Azure.AI.OpenAI.FunctionDefinition AddAsyncFunction - { - get => this.AddAsyncFunctionContract.ToOpenAIFunctionDefinition(); - } } } diff --git a/dotnet/test/AutoGen.SourceGenerator.Tests/FunctionCallTemplateEncodingTests.cs b/dotnet/test/AutoGen.SourceGenerator.Tests/FunctionCallTemplateEncodingTests.cs new file mode 100644 index 000000000000..0b2e211c6386 --- /dev/null +++ b/dotnet/test/AutoGen.SourceGenerator.Tests/FunctionCallTemplateEncodingTests.cs @@ -0,0 +1,94 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// FunctionCallTemplateEncodingTests.cs + +using System.Text.Json; // Needed for JsonSerializer +using AutoGen.SourceGenerator.Template; // Needed for FunctionCallTemplate +using Xunit; // Needed for Fact and Assert + +namespace AutoGen.SourceGenerator.Tests +{ + public class FunctionCallTemplateEncodingTests + { + private readonly JsonSerializerOptions jsonSerializerOptions = new JsonSerializerOptions + { + WriteIndented = true, + }; + + [Fact] + public void FunctionDescription_Should_Encode_DoubleQuotes() + { + // Arrange + var functionContracts = new List + { + new SourceGeneratorFunctionContract + { + Name = "TestFunction", + Description = "This is a \"test\" function", + Parameters = new SourceGeneratorParameterContract[] + { + new SourceGeneratorParameterContract + { + Name = "param1", + Description = "This is a \"parameter\" description", + Type = "string", + IsOptional = false + } + }, + ReturnType = "void" + } + }; + + var template = new FunctionCallTemplate + { + NameSpace = "TestNamespace", + ClassName = "TestClass", + FunctionContracts = functionContracts + }; + + // Act + var result = template.TransformText(); + + // Assert + Assert.Contains("Description = @\"This is a \"\"test\"\" function\"", result); + Assert.Contains("Description = @\"This is a \"\"parameter\"\" description\"", result); + } + + [Fact] + public void ParameterDescription_Should_Encode_DoubleQuotes() + { + // Arrange + var functionContracts = new List + { + new SourceGeneratorFunctionContract + { + Name = "TestFunction", + Description = "This is a test function", + Parameters = new SourceGeneratorParameterContract[] + { + new SourceGeneratorParameterContract + { + Name = "param1", + Description = "This is a \"parameter\" description", + Type = "string", + IsOptional = false + } + }, + ReturnType = "void" + } + }; + + var template = new FunctionCallTemplate + { + NameSpace = "TestNamespace", + ClassName = "TestClass", + FunctionContracts = functionContracts + }; + + // Act + var result = template.TransformText(); + + // Assert + Assert.Contains("Description = @\"This is a \"\"parameter\"\" description\"", result); + } + } +} diff --git a/dotnet/test/AutoGen.SourceGenerator.Tests/FunctionExample.test.cs b/dotnet/test/AutoGen.SourceGenerator.Tests/FunctionExample.test.cs index f7b90e0b96ff..0096f2c157ce 100644 --- a/dotnet/test/AutoGen.SourceGenerator.Tests/FunctionExample.test.cs +++ b/dotnet/test/AutoGen.SourceGenerator.Tests/FunctionExample.test.cs @@ -5,6 +5,7 @@ using ApprovalTests; using ApprovalTests.Namers; using ApprovalTests.Reporters; +using AutoGen.OpenAI.Extension; using Azure.AI.OpenAI; using FluentAssertions; using Xunit; @@ -29,7 +30,7 @@ public void Add_Test() }; this.VerifyFunction(functionExamples.AddWrapper, args, 3); - this.VerifyFunctionDefinition(functionExamples.AddFunction); + this.VerifyFunctionDefinition(functionExamples.AddFunctionContract.ToOpenAIFunctionDefinition()); } [Fact] @@ -41,7 +42,7 @@ public void Sum_Test() }; this.VerifyFunction(functionExamples.SumWrapper, args, 6.0); - this.VerifyFunctionDefinition(functionExamples.SumFunction); + this.VerifyFunctionDefinition(functionExamples.SumFunctionContract.ToOpenAIFunctionDefinition()); } [Fact] @@ -57,7 +58,7 @@ public async Task DictionaryToString_Test() }; await this.VerifyAsyncFunction(functionExamples.DictionaryToStringAsyncWrapper, args, JsonSerializer.Serialize(args.xargs, jsonSerializerOptions)); - this.VerifyFunctionDefinition(functionExamples.DictionaryToStringAsyncFunction); + this.VerifyFunctionDefinition(functionExamples.DictionaryToStringAsyncFunctionContract.ToOpenAIFunctionDefinition()); } [Fact] @@ -96,7 +97,7 @@ public void Query_Test() }; this.VerifyFunction(functionExamples.QueryWrapper, args, new[] { "hello", "hello", "hello" }); - this.VerifyFunctionDefinition(functionExamples.QueryFunction); + this.VerifyFunctionDefinition(functionExamples.QueryFunctionContract.ToOpenAIFunctionDefinition()); } [UseReporter(typeof(DiffReporter))] diff --git a/dotnet/test/AutoGen.Tests/AutoGen.Tests.csproj b/dotnet/test/AutoGen.Tests/AutoGen.Tests.csproj index 4def281ed7b4..3dc669b5edd8 100644 --- a/dotnet/test/AutoGen.Tests/AutoGen.Tests.csproj +++ b/dotnet/test/AutoGen.Tests/AutoGen.Tests.csproj @@ -9,6 +9,7 @@ + diff --git a/dotnet/test/AutoGen.Tests/BasicSampleTest.cs b/dotnet/test/AutoGen.Tests/BasicSampleTest.cs index 8f2b9b2de51b..89925b7d3b39 100644 --- a/dotnet/test/AutoGen.Tests/BasicSampleTest.cs +++ b/dotnet/test/AutoGen.Tests/BasicSampleTest.cs @@ -37,11 +37,6 @@ public async Task AgentFunctionCallTestAsync() await Example03_Agent_FunctionCall.RunAsync(); } - [ApiKeyFact("AZURE_OPENAI_API_KEY", "AZURE_OPENAI_ENDPOINT", "AZURE_OPENAI_DEPLOY_NAME")] - public async Task OpenAIAgent_JsonMode() - { - await Example13_OpenAIAgent_JsonMode.RunAsync(); - } [ApiKeyFact("MISTRAL_API_KEY")] public async Task MistralClientAgent_TokenCount() @@ -49,12 +44,6 @@ public async Task MistralClientAgent_TokenCount() await Example14_MistralClientAgent_TokenCount.RunAsync(); } - [ApiKeyFact("OPENAI_API_KEY")] - public async Task DynamicGroupChatGetMLNetPRTestAsync() - { - await Example04_Dynamic_GroupChat_Coding_Task.RunAsync(); - } - [ApiKeyFact("AZURE_OPENAI_API_KEY", "AZURE_OPENAI_ENDPOINT", "AZURE_OPENAI_DEPLOY_NAME")] public async Task DynamicGroupChatCalculateFibonacciAsync() { diff --git a/dotnet/test/AutoGen.Tests/EchoAgent.cs b/dotnet/test/AutoGen.Tests/EchoAgent.cs index 9cead5ad2516..af5490218e8d 100644 --- a/dotnet/test/AutoGen.Tests/EchoAgent.cs +++ b/dotnet/test/AutoGen.Tests/EchoAgent.cs @@ -29,7 +29,7 @@ public Task GenerateReplyAsync( return Task.FromResult(lastMessage); } - public async IAsyncEnumerable GenerateStreamingReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public async IAsyncEnumerable GenerateStreamingReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { foreach (var message in messages) { diff --git a/dotnet/test/AutoGen.Tests/GroupChat/GraphTests.cs b/dotnet/test/AutoGen.Tests/GroupChat/GraphTests.cs new file mode 100644 index 000000000000..7eeea6743f04 --- /dev/null +++ b/dotnet/test/AutoGen.Tests/GroupChat/GraphTests.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// GraphTests.cs + +using Xunit; + +namespace AutoGen.Tests +{ + public class GraphTests + { + [Fact] + public void GraphTest() + { + var graph1 = new Graph(); + Assert.NotNull(graph1); + + var graph2 = new Graph(null); + Assert.NotNull(graph2); + } + } +} diff --git a/dotnet/test/AutoGen.Tests/Orchestrator/RolePlayOrchestratorTests.cs b/dotnet/test/AutoGen.Tests/Orchestrator/RolePlayOrchestratorTests.cs new file mode 100644 index 000000000000..5a2cebb66cff --- /dev/null +++ b/dotnet/test/AutoGen.Tests/Orchestrator/RolePlayOrchestratorTests.cs @@ -0,0 +1,362 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// RolePlayOrchestratorTests.cs + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using AutoGen.Anthropic; +using AutoGen.Anthropic.Extensions; +using AutoGen.Anthropic.Utils; +using AutoGen.Gemini; +using AutoGen.Mistral; +using AutoGen.Mistral.Extension; +using AutoGen.OpenAI; +using AutoGen.OpenAI.Extension; +using Azure.AI.OpenAI; +using FluentAssertions; +using Moq; +using Xunit; + +namespace AutoGen.Tests; + +public class RolePlayOrchestratorTests +{ + [Fact] + public async Task ItReturnNextSpeakerTestAsync() + { + var admin = Mock.Of(); + Mock.Get(admin).Setup(x => x.Name).Returns("Admin"); + Mock.Get(admin).Setup(x => x.GenerateReplyAsync( + It.IsAny>(), + It.IsAny(), + It.IsAny())) + .Callback, GenerateReplyOptions, CancellationToken>((messages, option, _) => + { + // verify prompt + var rolePlayPrompt = messages.First().GetContent(); + rolePlayPrompt.Should().Contain("You are in a role play game. Carefully read the conversation history and carry on the conversation"); + rolePlayPrompt.Should().Contain("The available roles are:"); + rolePlayPrompt.Should().Contain("Alice,Bob"); + rolePlayPrompt.Should().Contain("From Alice:"); + option.StopSequence.Should().BeEquivalentTo([":"]); + option.Temperature.Should().Be(0); + option.MaxToken.Should().Be(128); + option.Functions.Should().BeNull(); + }) + .ReturnsAsync(new TextMessage(Role.Assistant, "From Alice")); + + var alice = new EchoAgent("Alice"); + var bob = new EchoAgent("Bob"); + + var orchestrator = new RolePlayOrchestrator(admin); + var context = new OrchestrationContext + { + Candidates = [alice, bob], + ChatHistory = [], + }; + + var speaker = await orchestrator.GetNextSpeakerAsync(context); + speaker.Should().Be(alice); + } + + [Fact] + public async Task ItReturnNullWhenNoCandidateIsAvailableAsync() + { + var admin = Mock.Of(); + var orchestrator = new RolePlayOrchestrator(admin); + var context = new OrchestrationContext + { + Candidates = [], + ChatHistory = [], + }; + + var speaker = await orchestrator.GetNextSpeakerAsync(context); + speaker.Should().BeNull(); + } + + [Fact] + public async Task ItReturnCandidateWhenOnlyOneCandidateIsAvailableAsync() + { + var admin = Mock.Of(); + var alice = new EchoAgent("Alice"); + var orchestrator = new RolePlayOrchestrator(admin); + var context = new OrchestrationContext + { + Candidates = [alice], + ChatHistory = [], + }; + + var speaker = await orchestrator.GetNextSpeakerAsync(context); + speaker.Should().Be(alice); + } + + [Fact] + public async Task ItThrowExceptionWhenAdminFailsToFollowPromptAsync() + { + var admin = Mock.Of(); + Mock.Get(admin).Setup(x => x.Name).Returns("Admin"); + Mock.Get(admin).Setup(x => x.GenerateReplyAsync( + It.IsAny>(), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(new TextMessage(Role.Assistant, "I don't know")); // admin fails to follow the prompt and returns an invalid message + + var alice = new EchoAgent("Alice"); + var bob = new EchoAgent("Bob"); + + var orchestrator = new RolePlayOrchestrator(admin); + var context = new OrchestrationContext + { + Candidates = [alice, bob], + ChatHistory = [], + }; + + var action = async () => await orchestrator.GetNextSpeakerAsync(context); + + await action.Should().ThrowAsync() + .WithMessage("The response from admin is 't know, which is either not in the candidates list or not in the correct format."); + } + + [Fact] + public async Task ItSelectNextSpeakerFromWorkflowIfProvided() + { + var workflow = new Graph(); + var alice = new EchoAgent("Alice"); + var bob = new EchoAgent("Bob"); + var charlie = new EchoAgent("Charlie"); + workflow.AddTransition(Transition.Create(alice, bob)); + workflow.AddTransition(Transition.Create(bob, charlie)); + workflow.AddTransition(Transition.Create(charlie, alice)); + + var admin = Mock.Of(); + var orchestrator = new RolePlayOrchestrator(admin, workflow); + var context = new OrchestrationContext + { + Candidates = [alice, bob, charlie], + ChatHistory = + [ + new TextMessage(Role.User, "Hello, Bob", from: "Alice"), + ], + }; + + var speaker = await orchestrator.GetNextSpeakerAsync(context); + speaker.Should().Be(bob); + } + + [Fact] + public async Task ItReturnNullIfNoAvailableAgentFromWorkflowAsync() + { + var workflow = new Graph(); + var alice = new EchoAgent("Alice"); + var bob = new EchoAgent("Bob"); + workflow.AddTransition(Transition.Create(alice, bob)); + + var admin = Mock.Of(); + var orchestrator = new RolePlayOrchestrator(admin, workflow); + var context = new OrchestrationContext + { + Candidates = [alice, bob], + ChatHistory = + [ + new TextMessage(Role.User, "Hello, Alice", from: "Bob"), + ], + }; + + var speaker = await orchestrator.GetNextSpeakerAsync(context); + speaker.Should().BeNull(); + } + + [Fact] + public async Task ItUseCandidatesFromWorflowAsync() + { + var workflow = new Graph(); + var alice = new EchoAgent("Alice"); + var bob = new EchoAgent("Bob"); + var charlie = new EchoAgent("Charlie"); + workflow.AddTransition(Transition.Create(alice, bob)); + workflow.AddTransition(Transition.Create(alice, charlie)); + + var admin = Mock.Of(); + Mock.Get(admin).Setup(x => x.GenerateReplyAsync( + It.IsAny>(), + It.IsAny(), + It.IsAny())) + .Callback, GenerateReplyOptions, CancellationToken>((messages, option, _) => + { + messages.First().IsSystemMessage().Should().BeTrue(); + + // verify prompt + var rolePlayPrompt = messages.First().GetContent(); + rolePlayPrompt.Should().Contain("Bob,Charlie"); + rolePlayPrompt.Should().Contain("From Bob:"); + option.StopSequence.Should().BeEquivalentTo([":"]); + option.Temperature.Should().Be(0); + option.MaxToken.Should().Be(128); + option.Functions.Should().BeEmpty(); + }) + .ReturnsAsync(new TextMessage(Role.Assistant, "From Bob")); + var orchestrator = new RolePlayOrchestrator(admin, workflow); + var context = new OrchestrationContext + { + Candidates = [alice, bob], + ChatHistory = + [ + new TextMessage(Role.User, "Hello, Bob", from: "Alice"), + ], + }; + + var speaker = await orchestrator.GetNextSpeakerAsync(context); + speaker.Should().Be(bob); + } + + [ApiKeyFact("AZURE_OPENAI_API_KEY", "AZURE_OPENAI_ENDPOINT", "AZURE_OPENAI_DEPLOY_NAME")] + public async Task GPT_3_5_CoderReviewerRunnerTestAsync() + { + var endpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT") ?? throw new Exception("Please set AZURE_OPENAI_ENDPOINT environment variable."); + var key = Environment.GetEnvironmentVariable("AZURE_OPENAI_API_KEY") ?? throw new Exception("Please set AZURE_OPENAI_API_KEY environment variable."); + var deployName = Environment.GetEnvironmentVariable("AZURE_OPENAI_DEPLOY_NAME") ?? throw new Exception("Please set AZURE_OPENAI_DEPLOY_NAME environment variable."); + var openaiClient = new OpenAIClient(new Uri(endpoint), new Azure.AzureKeyCredential(key)); + var openAIChatAgent = new OpenAIChatAgent( + openAIClient: openaiClient, + name: "assistant", + modelName: deployName) + .RegisterMessageConnector(); + + await CoderReviewerRunnerTestAsync(openAIChatAgent); + } + + [ApiKeyFact("OPENAI_API_KEY")] + public async Task GPT_4o_CoderReviewerRunnerTestAsync() + { + var apiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY"); + var model = "gpt-4o"; + var openaiClient = new OpenAIClient(apiKey); + var openAIChatAgent = new OpenAIChatAgent( + openAIClient: openaiClient, + name: "assistant", + modelName: model) + .RegisterMessageConnector(); + + await CoderReviewerRunnerTestAsync(openAIChatAgent); + } + + [ApiKeyFact("OPENAI_API_KEY")] + public async Task GPT_4o_mini_CoderReviewerRunnerTestAsync() + { + var apiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY"); + var model = "gpt-4o-mini"; + var openaiClient = new OpenAIClient(apiKey); + var openAIChatAgent = new OpenAIChatAgent( + openAIClient: openaiClient, + name: "assistant", + modelName: model) + .RegisterMessageConnector(); + + await CoderReviewerRunnerTestAsync(openAIChatAgent); + } + + + [ApiKeyFact("GOOGLE_GEMINI_API_KEY")] + public async Task GoogleGemini_1_5_flash_001_CoderReviewerRunnerTestAsync() + { + var apiKey = Environment.GetEnvironmentVariable("GOOGLE_GEMINI_API_KEY") ?? throw new InvalidOperationException("GOOGLE_GEMINI_API_KEY is not set"); + var geminiAgent = new GeminiChatAgent( + name: "gemini", + model: "gemini-1.5-flash-001", + apiKey: apiKey) + .RegisterMessageConnector(); + + await CoderReviewerRunnerTestAsync(geminiAgent); + } + + + [ApiKeyFact("ANTHROPIC_API_KEY")] + public async Task Claude3_Haiku_CoderReviewerRunnerTestAsync() + { + var apiKey = Environment.GetEnvironmentVariable("ANTHROPIC_API_KEY") ?? throw new Exception("Please set ANTHROPIC_API_KEY environment variable."); + var client = new AnthropicClient(new HttpClient(), AnthropicConstants.Endpoint, apiKey); + + var agent = new AnthropicClientAgent( + client, + name: "AnthropicAgent", + AnthropicConstants.Claude3Haiku, + systemMessage: "You are a helpful AI assistant that convert user message to upper case") + .RegisterMessageConnector(); + + await CoderReviewerRunnerTestAsync(agent); + } + + [ApiKeyFact("MISTRAL_API_KEY")] + public async Task Mistra_7b_CoderReviewerRunnerTestAsync() + { + var apiKey = Environment.GetEnvironmentVariable("MISTRAL_API_KEY") ?? throw new InvalidOperationException("MISTRAL_API_KEY is not set."); + var client = new MistralClient(apiKey: apiKey); + + var agent = new MistralClientAgent( + client: client, + name: "MistralClientAgent", + model: "open-mistral-7b") + .RegisterMessageConnector(); + + await CoderReviewerRunnerTestAsync(agent); + } + + /// + /// This test is to mimic the conversation among coder, reviewer and runner. + /// The coder will write the code, the reviewer will review the code, and the runner will run the code. + /// + /// + /// + public async Task CoderReviewerRunnerTestAsync(IAgent admin) + { + var coder = new EchoAgent("Coder"); + var reviewer = new EchoAgent("Reviewer"); + var runner = new EchoAgent("Runner"); + var user = new EchoAgent("User"); + var initializeMessage = new List + { + new TextMessage(Role.User, "Hello, I am user, I will provide the coding task, please write the code first, then review and run it", from: "User"), + new TextMessage(Role.User, "Hello, I am coder, I will write the code", from: "Coder"), + new TextMessage(Role.User, "Hello, I am reviewer, I will review the code", from: "Reviewer"), + new TextMessage(Role.User, "Hello, I am runner, I will run the code", from: "Runner"), + new TextMessage(Role.User, "how to print 'hello world' using C#", from: user.Name), + }; + + var chatHistory = new List() + { + new TextMessage(Role.User, """ + ```csharp + Console.WriteLine("Hello World"); + ``` + """, from: coder.Name), + new TextMessage(Role.User, "The code looks good", from: reviewer.Name), + new TextMessage(Role.User, "The code runs successfully, the output is 'Hello World'", from: runner.Name), + }; + + var orchestrator = new RolePlayOrchestrator(admin); + foreach (var message in chatHistory) + { + var context = new OrchestrationContext + { + Candidates = [coder, reviewer, runner, user], + ChatHistory = initializeMessage, + }; + + var speaker = await orchestrator.GetNextSpeakerAsync(context); + speaker!.Name.Should().Be(message.From); + initializeMessage.Add(message); + } + + // the last next speaker should be the user + var lastSpeaker = await orchestrator.GetNextSpeakerAsync(new OrchestrationContext + { + Candidates = [coder, reviewer, runner, user], + ChatHistory = initializeMessage, + }); + + lastSpeaker!.Name.Should().Be(user.Name); + } +} diff --git a/dotnet/test/AutoGen.Tests/Orchestrator/RoundRobinOrchestratorTests.cs b/dotnet/test/AutoGen.Tests/Orchestrator/RoundRobinOrchestratorTests.cs new file mode 100644 index 000000000000..e14bf85cf215 --- /dev/null +++ b/dotnet/test/AutoGen.Tests/Orchestrator/RoundRobinOrchestratorTests.cs @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// RoundRobinOrchestratorTests.cs + +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using Xunit; + +namespace AutoGen.Tests; + +public class RoundRobinOrchestratorTests +{ + [Fact] + public async Task ItReturnNextAgentAsync() + { + var orchestrator = new RoundRobinOrchestrator(); + var context = new OrchestrationContext + { + Candidates = new List + { + new EchoAgent("Alice"), + new EchoAgent("Bob"), + new EchoAgent("Charlie"), + }, + }; + + var messages = new List + { + new TextMessage(Role.User, "Hello, Alice", from: "Alice"), + new TextMessage(Role.User, "Hello, Bob", from: "Bob"), + new TextMessage(Role.User, "Hello, Charlie", from: "Charlie"), + }; + + var expected = new List { "Bob", "Charlie", "Alice" }; + + var zip = messages.Zip(expected); + + foreach (var (msg, expect) in zip) + { + context.ChatHistory = [msg]; + var nextSpeaker = await orchestrator.GetNextSpeakerAsync(context); + Assert.Equal(expect, nextSpeaker!.Name); + } + } + + [Fact] + public async Task ItReturnNullIfNoCandidates() + { + var orchestrator = new RoundRobinOrchestrator(); + var context = new OrchestrationContext + { + Candidates = new List(), + ChatHistory = new List + { + new TextMessage(Role.User, "Hello, Alice", from: "Alice"), + }, + }; + + var result = await orchestrator.GetNextSpeakerAsync(context); + Assert.Null(result); + } + + [Fact] + public async Task ItReturnNullIfLastMessageIsNotFromCandidates() + { + var orchestrator = new RoundRobinOrchestrator(); + var context = new OrchestrationContext + { + Candidates = new List + { + new EchoAgent("Alice"), + new EchoAgent("Bob"), + new EchoAgent("Charlie"), + }, + ChatHistory = new List + { + new TextMessage(Role.User, "Hello, David", from: "David"), + }, + }; + + var result = await orchestrator.GetNextSpeakerAsync(context); + result.Should().BeNull(); + } + + [Fact] + public async Task ItReturnEmptyListIfNoChatHistory() + { + var orchestrator = new RoundRobinOrchestrator(); + var context = new OrchestrationContext + { + Candidates = new List + { + new EchoAgent("Alice"), + new EchoAgent("Bob"), + new EchoAgent("Charlie"), + }, + }; + + var result = await orchestrator.GetNextSpeakerAsync(context); + result.Should().BeNull(); + } +} diff --git a/dotnet/test/AutoGen.Tests/Orchestrator/WorkflowOrchestratorTests.cs b/dotnet/test/AutoGen.Tests/Orchestrator/WorkflowOrchestratorTests.cs new file mode 100644 index 000000000000..6599566a4466 --- /dev/null +++ b/dotnet/test/AutoGen.Tests/Orchestrator/WorkflowOrchestratorTests.cs @@ -0,0 +1,112 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// WorkflowOrchestratorTests.cs + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using Xunit; + +namespace AutoGen.Tests; + +public class WorkflowOrchestratorTests +{ + [Fact] + public async Task ItReturnNextAgentAsync() + { + var workflow = new Graph(); + var alice = new EchoAgent("Alice"); + var bob = new EchoAgent("Bob"); + var charlie = new EchoAgent("Charlie"); + workflow.AddTransition(Transition.Create(alice, bob)); + workflow.AddTransition(Transition.Create(bob, charlie)); + workflow.AddTransition(Transition.Create(charlie, alice)); + var orchestrator = new WorkflowOrchestrator(workflow); + var context = new OrchestrationContext + { + Candidates = [alice, bob, charlie] + }; + + var messages = new List + { + new TextMessage(Role.User, "Hello, Alice", from: "Alice"), + new TextMessage(Role.User, "Hello, Bob", from: "Bob"), + new TextMessage(Role.User, "Hello, Charlie", from: "Charlie"), + }; + + var expected = new List { "Bob", "Charlie", "Alice" }; + + var zip = messages.Zip(expected); + + foreach (var (msg, expect) in zip) + { + context.ChatHistory = [msg]; + var result = await orchestrator.GetNextSpeakerAsync(context); + Assert.Equal(expect, result!.Name); + } + } + + [Fact] + public async Task ItReturnNullIfNoCandidates() + { + var workflow = new Graph(); + var orchestrator = new WorkflowOrchestrator(workflow); + var context = new OrchestrationContext + { + Candidates = new List(), + ChatHistory = new List + { + new TextMessage(Role.User, "Hello, Alice", from: "Alice"), + }, + }; + + var nextAgent = await orchestrator.GetNextSpeakerAsync(context); + nextAgent.Should().BeNull(); + } + + [Fact] + public async Task ItReturnNullIfNoAgentIsAvailableFromWorkflowAsync() + { + var workflow = new Graph(); + var alice = new EchoAgent("Alice"); + var bob = new EchoAgent("Bob"); + workflow.AddTransition(Transition.Create(alice, bob)); + var orchestrator = new WorkflowOrchestrator(workflow); + var context = new OrchestrationContext + { + Candidates = [alice, bob], + ChatHistory = new List + { + new TextMessage(Role.User, "Hello, Bob", from: "Bob"), + }, + }; + + var nextSpeaker = await orchestrator.GetNextSpeakerAsync(context); + nextSpeaker.Should().BeNull(); + } + + [Fact] + public async Task ItThrowExceptionWhenMoreThanOneAvailableAgentsFromWorkflowAsync() + { + var workflow = new Graph(); + var alice = new EchoAgent("Alice"); + var bob = new EchoAgent("Bob"); + var charlie = new EchoAgent("Charlie"); + workflow.AddTransition(Transition.Create(alice, bob)); + workflow.AddTransition(Transition.Create(alice, charlie)); + var orchestrator = new WorkflowOrchestrator(workflow); + var context = new OrchestrationContext + { + Candidates = [alice, bob, charlie], + ChatHistory = new List + { + new TextMessage(Role.User, "Hello, Bob", from: "Alice"), + }, + }; + + var action = async () => await orchestrator.GetNextSpeakerAsync(context); + + await action.Should().ThrowExactlyAsync().WithMessage("There are more than one available agents from the workflow for the next speaker."); + } +} diff --git a/dotnet/test/AutoGen.Tests/SingleAgentTest.cs b/dotnet/test/AutoGen.Tests/SingleAgentTest.cs index 5a3a9734cd1a..b545bbdbe864 100644 --- a/dotnet/test/AutoGen.Tests/SingleAgentTest.cs +++ b/dotnet/test/AutoGen.Tests/SingleAgentTest.cs @@ -8,6 +8,7 @@ using System.Threading.Tasks; using AutoGen.LMStudio; using AutoGen.OpenAI; +using AutoGen.OpenAI.Extension; using Azure.AI.OpenAI; using FluentAssertions; using Xunit; @@ -64,7 +65,7 @@ public async Task GPTAgentVisionTestAsync() systemMessage: "You are a helpful AI assistant, return highest label from conversation", config: gpt3Config, temperature: 0, - functions: new[] { this.GetHighestLabelFunction }, + functions: new[] { this.GetHighestLabelFunctionContract.ToOpenAIFunctionDefinition() }, functionMap: new Dictionary>> { { nameof(GetHighestLabel), this.GetHighestLabelWrapper }, @@ -116,7 +117,7 @@ public async Task GPTAgentVisionTestAsync() public async Task GPTFunctionCallAgentTestAsync() { var config = this.CreateAzureOpenAIGPT35TurboConfig(); - var agentWithFunction = new GPTAgent("gpt", "You are a helpful AI assistant", config, 0, functions: new[] { this.EchoAsyncFunction }); + var agentWithFunction = new GPTAgent("gpt", "You are a helpful AI assistant", config, 0, functions: new[] { this.EchoAsyncFunctionContract.ToOpenAIFunctionDefinition() }); await EchoFunctionCallTestAsync(agentWithFunction); } @@ -233,7 +234,7 @@ public async Task GPTAgentFunctionCallSelfExecutionTestAsync() systemMessage: "You are a helpful AI assistant", config: config, temperature: 0, - functions: new[] { this.EchoAsyncFunction }, + functions: new[] { this.EchoAsyncFunctionContract.ToOpenAIFunctionDefinition() }, functionMap: new Dictionary>> { { nameof(EchoAsync), this.EchoAsyncWrapper }, @@ -297,7 +298,7 @@ public async Task EchoFunctionCallExecutionStreamingTestAsync(IStreamingAgent ag }; var replyStream = agent.GenerateStreamingReplyAsync(messages: new[] { helloWorld }, option); var answer = "[ECHO] Hello world"; - IStreamingMessage? finalReply = default; + IMessage? finalReply = default; await foreach (var reply in replyStream) { reply.From.Should().Be(agent.Name); diff --git a/dotnet/test/AutoGen.Tests/TwoAgentTest.cs b/dotnet/test/AutoGen.Tests/TwoAgentTest.cs index 90c1bfa9a148..100a22c04a71 100644 --- a/dotnet/test/AutoGen.Tests/TwoAgentTest.cs +++ b/dotnet/test/AutoGen.Tests/TwoAgentTest.cs @@ -56,7 +56,7 @@ public async Task TwoAgentWeatherChatTestAsync() name: "user", functionMap: new Dictionary>> { - { this.GetWeatherFunction.Name, this.GetWeatherWrapper }, + { this.GetWeatherFunctionContract.Name, this.GetWeatherWrapper }, }) .RegisterMiddleware(async (msgs, option, agent, ct) => { diff --git a/dotnet/test/AutoGen.Tests/WorkflowTest.cs b/dotnet/test/AutoGen.Tests/WorkflowTest.cs index d1d12010e39f..1079ec95515a 100644 --- a/dotnet/test/AutoGen.Tests/WorkflowTest.cs +++ b/dotnet/test/AutoGen.Tests/WorkflowTest.cs @@ -17,7 +17,7 @@ public async Task TransitionTestAsync() var alice = new EchoAgent("alice"); var bob = new EchoAgent("bob"); - var aliceToBob = Transition.Create(alice, bob, async (from, to, messages) => + var aliceToBob = Transition.Create(alice, bob, async (from, to, messages, _) => { if (messages.Any(m => m.GetContent() == "Hello")) { @@ -30,7 +30,7 @@ public async Task TransitionTestAsync() var canTransit = await aliceToBob.CanTransitionAsync([]); canTransit.Should().BeFalse(); - canTransit = await aliceToBob.CanTransitionAsync(new[] { new TextMessage(Role.Assistant, "Hello") }); + canTransit = await aliceToBob.CanTransitionAsync([new TextMessage(Role.Assistant, "Hello")]); canTransit.Should().BeTrue(); // if no function is provided, it should always return true diff --git a/dotnet/test/AutoGen.WebAPI.Tests/AutoGen.WebAPI.Tests.csproj b/dotnet/test/AutoGen.WebAPI.Tests/AutoGen.WebAPI.Tests.csproj new file mode 100644 index 000000000000..3a9caf38fc8e --- /dev/null +++ b/dotnet/test/AutoGen.WebAPI.Tests/AutoGen.WebAPI.Tests.csproj @@ -0,0 +1,28 @@ + + + + $(TestTargetFramework) + enable + enable + false + true + True + + + + + + + + + + + + + + + + + + + diff --git a/dotnet/test/AutoGen.WebAPI.Tests/EchoAgent.cs b/dotnet/test/AutoGen.WebAPI.Tests/EchoAgent.cs new file mode 100644 index 000000000000..957f8d1d799b --- /dev/null +++ b/dotnet/test/AutoGen.WebAPI.Tests/EchoAgent.cs @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// EchoAgent.cs + +using System.Runtime.CompilerServices; +using AutoGen.Core; + +namespace AutoGen.WebAPI.Tests; + +public class EchoAgent : IStreamingAgent +{ + public EchoAgent(string name) + { + Name = name; + } + public string Name { get; } + + public async Task GenerateReplyAsync( + IEnumerable messages, + GenerateReplyOptions? options = null, + CancellationToken cancellationToken = default) + { + return messages.Last(); + } + + public async IAsyncEnumerable GenerateStreamingReplyAsync( + IEnumerable messages, + GenerateReplyOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var lastMessage = messages.LastOrDefault(); + if (lastMessage == null) + { + yield break; + } + + // return each character of the last message as a separate message + if (lastMessage.GetContent() is string content) + { + foreach (var c in content) + { + yield return new TextMessageUpdate(Role.Assistant, c.ToString(), this.Name); + } + } + } +} diff --git a/dotnet/test/AutoGen.WebAPI.Tests/OpenAIChatCompletionMiddlewareTests.cs b/dotnet/test/AutoGen.WebAPI.Tests/OpenAIChatCompletionMiddlewareTests.cs new file mode 100644 index 000000000000..07bdc850936d --- /dev/null +++ b/dotnet/test/AutoGen.WebAPI.Tests/OpenAIChatCompletionMiddlewareTests.cs @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// OpenAIChatCompletionMiddlewareTests.cs + +using AutoGen.Core; +using AutoGen.OpenAI; +using AutoGen.OpenAI.Extension; +using Azure.AI.OpenAI; +using Azure.Core.Pipeline; +using FluentAssertions; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; + +namespace AutoGen.WebAPI.Tests; + +public class OpenAIChatCompletionMiddlewareTests +{ + [Fact] + public async Task ItReturnTextMessageWhenSendTextMessage() + { + var agent = new EchoAgent("test"); + var hostBuilder = CreateHostBuilder(agent); + using var host = await hostBuilder.StartAsync(); + var client = host.GetTestClient(); + var openaiClient = CreateOpenAIClient(client); + var openAIAgent = new OpenAIChatAgent(openaiClient, "test", "test") + .RegisterMessageConnector(); + + var response = await openAIAgent.SendAsync("Hey"); + + response.GetContent().Should().Be("Hey"); + response.Should().BeOfType(); + response.From.Should().Be("test"); + } + + [Fact] + public async Task ItReturnTextMessageWhenSendTextMessageUseStreaming() + { + var agent = new EchoAgent("test"); + var hostBuilder = CreateHostBuilder(agent); + using var host = await hostBuilder.StartAsync(); + var client = host.GetTestClient(); + var openaiClient = CreateOpenAIClient(client); + var openAIAgent = new OpenAIChatAgent(openaiClient, "test", "test") + .RegisterMessageConnector(); + + var message = new TextMessage(Role.User, "ABCDEFGHIJKLMN"); + var chunks = new List(); + await foreach (var chunk in openAIAgent.GenerateStreamingReplyAsync([message])) + { + chunk.Should().BeOfType(); + chunks.Add(chunk); + } + + var mergedChunks = string.Join("", chunks.Select(c => c.GetContent())); + mergedChunks.Should().Be("ABCDEFGHIJKLMN"); + chunks.Count.Should().Be(14); + } + + private IHostBuilder CreateHostBuilder(IAgent agent) + { + return new HostBuilder() + .ConfigureWebHost(webHost => + { + webHost.UseTestServer(); + webHost.Configure(app => + { + app.UseAgentAsOpenAIChatCompletionEndpoint(agent); + }); + }); + } + + private OpenAIClient CreateOpenAIClient(HttpClient client) + { + var clientOption = new OpenAIClientOptions(OpenAIClientOptions.ServiceVersion.V2024_02_15_Preview) + { + Transport = new HttpClientTransport(client), + }; + return new OpenAIClient("api-key", clientOption); + } +} diff --git a/dotnet/website/articles/Function-call-with-ollama-and-litellm.md b/dotnet/website/articles/Function-call-with-ollama-and-litellm.md new file mode 100644 index 000000000000..2dc595ba3adb --- /dev/null +++ b/dotnet/website/articles/Function-call-with-ollama-and-litellm.md @@ -0,0 +1,93 @@ +This example shows how to use function call with local LLM models where [Ollama](https://ollama.com/) as local model provider and [LiteLLM](https://docs.litellm.ai/docs/) proxy server which provides an openai-api compatible interface. + +[![](https://img.shields.io/badge/Open%20on%20Github-grey?logo=github)](https://github.com/microsoft/autogen/blob/main/dotnet/sample/AutoGen.OpenAI.Sample/Tool_Call_With_Ollama_And_LiteLLM.cs) + +To run this example, the following prerequisites are required: +- Install [Ollama](https://ollama.com/) and [LiteLLM](https://docs.litellm.ai/docs/) on your local machine. +- A local model that supports function call. In this example `dolphincoder:latest` is used. + +## Install Ollama and pull `dolphincoder:latest` model +First, install Ollama by following the instructions on the [Ollama website](https://ollama.com/). + +After installing Ollama, pull the `dolphincoder:latest` model by running the following command: +```bash +ollama pull dolphincoder:latest +``` + +## Install LiteLLM and start the proxy server + +You can install LiteLLM by following the instructions on the [LiteLLM website](https://docs.litellm.ai/docs/). +```bash +pip install 'litellm[proxy]' +``` + +Then, start the proxy server by running the following command: + +```bash +litellm --model ollama_chat/dolphincoder --port 4000 +``` + +This will start an openai-api compatible proxy server at `http://localhost:4000`. You can verify if the server is running by observing the following output in the terminal: + +```bash +#------------------------------------------------------------# +# # +# 'The worst thing about this product is...' # +# https://github.com/BerriAI/litellm/issues/new # +# # +#------------------------------------------------------------# + +INFO: Application startup complete. +INFO: Uvicorn running on http://0.0.0.0:4000 (Press CTRL+C to quit) +``` + +## Install AutoGen and AutoGen.SourceGenerator +In your project, install the AutoGen and AutoGen.SourceGenerator package using the following command: + +```bash +dotnet add package AutoGen +dotnet add package AutoGen.SourceGenerator +``` + +The `AutoGen.SourceGenerator` package is used to automatically generate type-safe `FunctionContract` instead of manually defining them. For more information, please check out [Create type-safe function](Create-type-safe-function-call.md). + +And in your project file, enable structural xml document support by setting the `GenerateDocumentationFile` property to `true`: + +```xml + + + true + +``` + +## Define `WeatherReport` function and create @AutoGen.Core.FunctionCallMiddleware + +Create a `public partial` class to host the methods you want to use in AutoGen agents. The method has to be a `public` instance method and its return type must be `Task`. After the methods are defined, mark them with `AutoGen.Core.FunctionAttribute` attribute. + +[!code-csharp[Define WeatherReport function](../../sample/AutoGen.OpenAI.Sample/Tool_Call_With_Ollama_And_LiteLLM.cs?name=Function)] + +Then create a @AutoGen.Core.FunctionCallMiddleware and add the `WeatherReport` function to the middleware. The middleware will pass the `FunctionContract` to the agent when generating a response, and process the tool call response when receiving a `ToolCallMessage`. +[!code-csharp[Define WeatherReport function](../../sample/AutoGen.OpenAI.Sample/Tool_Call_With_Ollama_And_LiteLLM.cs?name=Create_tools)] + +## Create @AutoGen.OpenAI.OpenAIChatAgent with `GetWeatherReport` tool and chat with it + +Because LiteLLM proxy server is openai-api compatible, we can use @AutoGen.OpenAI.OpenAIChatAgent to connect to it as a third-party openai-api provider. The agent is also registered with a @AutoGen.Core.FunctionCallMiddleware which contains the `WeatherReport` tool. Therefore, the agent can call the `WeatherReport` tool when generating a response. + +[!code-csharp[Create an agent with tools](../../sample/AutoGen.OpenAI.Sample/Tool_Call_With_Ollama_And_LiteLLM.cs?name=Create_Agent)] + +The reply from the agent will similar to the following: +```bash +AggregateMessage from assistant +-------------------- +ToolCallMessage: +ToolCallMessage from assistant +-------------------- +- GetWeatherAsync: {"city": "new york"} +-------------------- + +ToolCallResultMessage: +ToolCallResultMessage from assistant +-------------------- +- GetWeatherAsync: The weather in new york is 72 degrees and sunny. +-------------------- +``` \ No newline at end of file diff --git a/dotnet/website/articles/OpenAIChatAgent-connect-to-third-party-api.md b/dotnet/website/articles/OpenAIChatAgent-connect-to-third-party-api.md index 8321fc87a5c2..0873765b1a6c 100644 --- a/dotnet/website/articles/OpenAIChatAgent-connect-to-third-party-api.md +++ b/dotnet/website/articles/OpenAIChatAgent-connect-to-third-party-api.md @@ -1,7 +1,6 @@ The following example shows how to connect to third-party OpenAI API using @AutoGen.OpenAI.OpenAIChatAgent. -> [!NOTE] -> You can find the complete code of this example in [Example16_OpenAIChatAgent_ConnectToThirdPartyBackend](https://github.com/microsoft/autogen/tree/main/dotnet/sample/AutoGen.BasicSamples/Example16_OpenAIChatAgent_ConnectToThirdPartyBackend.cs). +[![](https://img.shields.io/badge/Open%20on%20Github-grey?logo=github)](https://github.com/microsoft/autogen/blob/main/dotnet/sample/AutoGen.OpenAI.Sample/Connect_To_Ollama.cs) ## Overview A lot of LLM applications/platforms support spinning up a chat server that is compatible with OpenAI API, such as LM Studio, Ollama, Mistral etc. This means that you can connect to these servers using the @AutoGen.OpenAI.OpenAIChatAgent. @@ -25,24 +24,24 @@ ollama serve ## Steps - Import the required namespaces: -[!code-csharp[](../../sample/AutoGen.BasicSamples/Example16_OpenAIChatAgent_ConnectToThirdPartyBackend.cs?name=using_statement)] +[!code-csharp[](../../sample/AutoGen.OpenAI.Sample/Connect_To_Ollama.cs?name=using_statement)] - Create a `CustomHttpClientHandler` class. The `CustomHttpClientHandler` class is used to customize the HttpClientHandler. In this example, we override the `SendAsync` method to redirect the request to local Ollama server, which is running on `http://localhost:11434`. -[!code-csharp[](../../sample/AutoGen.BasicSamples/Example16_OpenAIChatAgent_ConnectToThirdPartyBackend.cs?name=CustomHttpClientHandler)] +[!code-csharp[](../../sample/AutoGen.OpenAI.Sample/Connect_To_Ollama.cs?name=CustomHttpClientHandler)] - Create an `OpenAIChatAgent` instance and connect to the third-party API. Then create an @AutoGen.OpenAI.OpenAIChatAgent instance and connect to the OpenAI API from Ollama. You can customize the transport behavior of `OpenAIClient` by passing a customized `HttpClientTransport` instance. In the customized `HttpClientTransport` instance, we pass the `CustomHttpClientHandler` we just created which redirects all openai chat requests to the local Ollama server. -[!code-csharp[](../../sample/AutoGen.BasicSamples/Example16_OpenAIChatAgent_ConnectToThirdPartyBackend.cs?name=create_agent)] +[!code-csharp[](../../sample/AutoGen.OpenAI.Sample/Connect_To_Ollama.cs?name=create_agent)] - Chat with the `OpenAIChatAgent`. Finally, you can start chatting with the agent. In this example, we send a coding question to the agent and get the response. -[!code-csharp[](../../sample/AutoGen.BasicSamples/Example16_OpenAIChatAgent_ConnectToThirdPartyBackend.cs?name=send_message)] +[!code-csharp[](../../sample/AutoGen.OpenAI.Sample/Connect_To_Ollama.cs?name=send_message)] ## Sample Output The following is the sample output of the code snippet above: diff --git a/dotnet/website/articles/OpenAIChatAgent-use-json-mode.md b/dotnet/website/articles/OpenAIChatAgent-use-json-mode.md index a822cb046334..22f0ced00469 100644 --- a/dotnet/website/articles/OpenAIChatAgent-use-json-mode.md +++ b/dotnet/website/articles/OpenAIChatAgent-use-json-mode.md @@ -1,5 +1,7 @@ The following example shows how to enable JSON mode in @AutoGen.OpenAI.OpenAIChatAgent. +[![](https://img.shields.io/badge/Open%20on%20Github-grey?logo=github)](https://github.com/microsoft/autogen/blob/main/dotnet/sample/AutoGen.OpenAI.Sample/Use_Json_Mode.cs) + ## What is JSON mode? JSON mode is a new feature in OpenAI which allows you to instruct model to always respond with a valid JSON object. This is useful when you want to constrain the model output to JSON format only. @@ -8,20 +10,17 @@ JSON mode is a new feature in OpenAI which allows you to instruct model to alway ## How to enable JSON mode in OpenAIChatAgent. -> [!NOTE] -> You can find the complete example in the [Example13_OpenAIAgent_JsonMode](https://github.com/microsoft/autogen/tree/main/dotnet/sample/AutoGen.BasicSamples/Example13_OpenAIAgent_JsonMode.cs). - To enable JSON mode for @AutoGen.OpenAI.OpenAIChatAgent, set `responseFormat` to `ChatCompletionsResponseFormat.JsonObject` when creating the agent. Note that when enabling JSON mode, you also need to instruct the agent to output JSON format in its system message. -[!code-csharp[](../../sample/AutoGen.BasicSamples/Example13_OpenAIAgent_JsonMode.cs?name=create_agent)] +[!code-csharp[](../../sample/AutoGen.OpenAI.Sample/Use_Json_Mode.cs?name=create_agent)] After enabling JSON mode, the `openAIClientAgent` will always respond in JSON format when it receives a message. -[!code-csharp[](../../sample/AutoGen.BasicSamples/Example13_OpenAIAgent_JsonMode.cs?name=chat_with_agent)] +[!code-csharp[](../../sample/AutoGen.OpenAI.Sample/Use_Json_Mode.cs?name=chat_with_agent)] When running the example, the output from `openAIClientAgent` will be a valid JSON object which can be parsed as `Person` class defined below. Note that in the output, the `address` field is missing because the address information is not provided in user input. -[!code-csharp[](../../sample/AutoGen.BasicSamples/Example13_OpenAIAgent_JsonMode.cs?name=person_class)] +[!code-csharp[](../../sample/AutoGen.OpenAI.Sample/Use_Json_Mode.cs?name=person_class)] The output will be: ```bash diff --git a/dotnet/website/articles/getting-start.md b/dotnet/website/articles/getting-start.md index 53cc7c9758f3..9db8494ff154 100644 --- a/dotnet/website/articles/getting-start.md +++ b/dotnet/website/articles/getting-start.md @@ -17,6 +17,8 @@ Then you can start with the following code snippet to create a conversable agent [!code-csharp[](../../sample/AutoGen.BasicSamples/CodeSnippet/GetStartCodeSnippet.cs?name=snippet_GetStartCodeSnippet)] [!code-csharp[](../../sample/AutoGen.BasicSamples/CodeSnippet/GetStartCodeSnippet.cs?name=code_snippet_1)] +### Tutorial +Getting started with AutoGen.Net by following the [tutorial](../tutorial/Chat-with-an-agent.md) series. ### Examples You can find more examples under the [sample project](https://github.com/microsoft/autogen/tree/dotnet/dotnet/sample/AutoGen.BasicSamples). diff --git a/dotnet/website/articles/toc.yml b/dotnet/website/articles/toc.yml index 837ecd6f86e0..2335ebf092b5 100644 --- a/dotnet/website/articles/toc.yml +++ b/dotnet/website/articles/toc.yml @@ -1,5 +1,7 @@ - name: Getting start items: + - name: Overview + href: ../index.md - name: Installation href: Installation.md - name: agent @@ -24,6 +26,8 @@ href: Create-type-safe-function-call.md - name: Use function call in an agent href: Use-function-call.md + - name: Function call with local model + href: Function-call-with-ollama-and-litellm.md - name: middleware items: - name: middleware overview diff --git a/dotnet/website/docfx.json b/dotnet/website/docfx.json index e06f9797c1fb..221cd4721e3d 100644 --- a/dotnet/website/docfx.json +++ b/dotnet/website/docfx.json @@ -30,6 +30,10 @@ "files": [ "articles/**.md", "articles/**/toc.yml", + "tutorial/**.md", + "tutorial/**/toc.yml", + "release_note/**.md", + "release_note/**/toc.yml", "toc.yml", "*.md" ] diff --git a/dotnet/website/images/articles/CreateAgentWithTools/single-turn-tool-call-with-auto-invoke.png b/dotnet/website/images/articles/CreateAgentWithTools/single-turn-tool-call-with-auto-invoke.png new file mode 100644 index 000000000000..27914072b271 --- /dev/null +++ b/dotnet/website/images/articles/CreateAgentWithTools/single-turn-tool-call-with-auto-invoke.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0d8e2ab194e31dc70e39ba081a755c8e792d291bef4dc8b4c5cc372bed9ec50 +size 215389 diff --git a/dotnet/website/images/articles/CreateAgentWithTools/single-turn-tool-call-without-auto-invoke.png b/dotnet/website/images/articles/CreateAgentWithTools/single-turn-tool-call-without-auto-invoke.png new file mode 100644 index 000000000000..a0711e505e8c --- /dev/null +++ b/dotnet/website/images/articles/CreateAgentWithTools/single-turn-tool-call-without-auto-invoke.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f2e632fb24641eb2fac7fff995c9b3213023c45c3238531eec5a340072865f6 +size 202768 diff --git a/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/FinalStepsA.png b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/FinalStepsA.png new file mode 100644 index 000000000000..0403a8cf9742 --- /dev/null +++ b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/FinalStepsA.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:491f8f538c55ce8768179cabfd3789c71c4a07b7d809f85deba9b8f4b759c00e +size 42329 diff --git a/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/FinalStepsB.png b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/FinalStepsB.png new file mode 100644 index 000000000000..03a68735c082 --- /dev/null +++ b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/FinalStepsB.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e319fad11682c46c3dc511e2fc63e033f3f99efb06d4530e7f72d1f4af23848f +size 31528 diff --git a/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/FinalStepsC.png b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/FinalStepsC.png new file mode 100644 index 000000000000..7326ad14d040 --- /dev/null +++ b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/FinalStepsC.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8024b5336615e8c2c3497df7a5890a331bd5bdc7b15dd06abd7ec528ffe0932 +size 70169 diff --git a/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Step5.2OpenAIModel.png b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Step5.2OpenAIModel.png new file mode 100644 index 000000000000..b2b7481bbe78 --- /dev/null +++ b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Step5.2OpenAIModel.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:911f2f7c1ab4f9403386298d9769243c0aa8cc22c6f119342cc107a654d1463a +size 44041 diff --git a/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Step5.3ModelNameAndURL.png b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Step5.3ModelNameAndURL.png new file mode 100644 index 000000000000..d1c19f300806 --- /dev/null +++ b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Step5.3ModelNameAndURL.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec10a48ed3f0a6d8448e0ce425658f3857c2cf89e2badef8a8d3a8c3744fc3bf +size 51944 diff --git a/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Step6.png b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Step6.png new file mode 100644 index 000000000000..67c734454427 --- /dev/null +++ b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Step6.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f016faea51f64af3970fde41ac95249c4e0423b02573f058c36dc1e6ba15562d +size 50669 diff --git a/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Step6b.png b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Step6b.png new file mode 100644 index 000000000000..ebd19bff045a --- /dev/null +++ b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Step6b.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a23cbbf5d3d24eaf1da9370e0914f186815f2ecbf46131d2fd6eb5ff3264d96 +size 22569 diff --git a/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Terminal.png b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Terminal.png new file mode 100644 index 000000000000..9edefc3aebf3 --- /dev/null +++ b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/Terminal.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97328776c25fd0a61c76065db379406d8d3c96bd8773490c34c168cd7c69a855 +size 58527 diff --git a/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/TheModelTab.png b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/TheModelTab.png new file mode 100644 index 000000000000..55e7bd862613 --- /dev/null +++ b/dotnet/website/images/articles/UseAutoGenAsModelinAGStudio/TheModelTab.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d7f4f3a772278e6de320a3601a76f8a9862cab4a9c0da03fad3058b86fcfaf7 +size 45260 diff --git a/dotnet/website/index.md b/dotnet/website/index.md index 3bc691523e9b..164e5c1cf81c 100644 --- a/dotnet/website/index.md +++ b/dotnet/website/index.md @@ -1,4 +1 @@ ---- -_disableTocFilter: true ---- [!INCLUDE [](./articles/getting-start.md)] \ No newline at end of file diff --git a/dotnet/website/release_note/0.0.16.md b/dotnet/website/release_note/0.0.16.md new file mode 100644 index 000000000000..b9a190c5f793 --- /dev/null +++ b/dotnet/website/release_note/0.0.16.md @@ -0,0 +1,32 @@ +# AutoGen.Net 0.0.16 Release Notes + +We are excited to announce the release of **AutoGen.Net 0.0.16**. This release includes several new features, bug fixes, improvements, and important updates. Below are the detailed release notes: + +**[Milestone: AutoGen.Net 0.0.16](https://github.com/microsoft/autogen/milestone/4)** + +## 📦 New Features +1. **Deprecate `IStreamingMessage`** ([#3045](https://github.com/microsoft/autogen/issues/3045)) - Replaced `IStreamingMessage` and `IStreamingMessage` with `IMessage` and `IMessage`. +2. **Add example for using ollama + LiteLLM for function call** ([#3014](https://github.com/microsoft/autogen/issues/3014)) - Added a new tutorial to the website for integrating ollama with LiteLLM for function calls. +3. **Add ReAct sample** ([#2978](https://github.com/microsoft/autogen/issues/2978)) - Added a new sample demonstrating the ReAct pattern. +4. **Support tools Anthropic Models** ([#2771](https://github.com/microsoft/autogen/issues/2771)) - Introduced support for tools like `AnthropicClient`, `AnthropicClientAgent`, and `AnthropicMessageConnector`. +5. **Propose Orchestrator for managing group chat/agentic workflow** ([#2695](https://github.com/microsoft/autogen/issues/2695)) - Introduced a customizable orchestrator interface for managing group chats and agent workflows. +6. **Run Agent as Web API** ([#2519](https://github.com/microsoft/autogen/issues/2519)) - Introduced the ability to start an OpenAI-chat-compatible web API from an arbitrary agent. + +## 🐛 Bug Fixes +1. **SourceGenerator doesn't work when function's arguments are empty** ([#2976](https://github.com/microsoft/autogen/issues/2976)) - Fixed an issue where the SourceGenerator failed when function arguments were empty. +2. **Add content field in ToolCallMessage** ([#2975](https://github.com/microsoft/autogen/issues/2975)) - Added a content property in `ToolCallMessage` to handle text content returned by the OpenAI model during tool calls. +3. **AutoGen.SourceGenerator doesn’t encode `"` in structural comments** ([#2872](https://github.com/microsoft/autogen/issues/2872)) - Fixed an issue where structural comments containing `"` were not properly encoded, leading to compilation errors. + +## 🚀 Improvements +1. **Sample update - Add getting-start samples for BasicSample project** ([#2859](https://github.com/microsoft/autogen/issues/2859)) - Re-organized the `AutoGen.BasicSample` project to include only essential getting-started examples, simplifying complex examples. +2. **Graph constructor should consider null transitions** ([#2708](https://github.com/microsoft/autogen/issues/2708)) - Updated the Graph constructor to handle cases where transitions’ values are null. + +## ⚠️ API-Breakchange +1. **Deprecate `IStreamingMessage`** ([#3045](https://github.com/microsoft/autogen/issues/3045)) - **Migration guide:** Deprecating `IStreamingMessage` will introduce breaking changes, particularly for `IStreamingAgent` and `IStreamingMiddleware`. Replace all `IStreamingMessage` and `IStreamingMessage` with `IMessage` and `IMessage`. + +## 📚 Document Update +1. **Add example for using ollama + LiteLLM for function call** ([#3014](https://github.com/microsoft/autogen/issues/3014)) - Added a tutorial to the website for using ollama with LiteLLM. + +Thank you to all the contributors for making this release possible. We encourage everyone to upgrade to AutoGen.Net 0.0.16 to take advantage of these new features and improvements. If you encounter any issues or have any feedback, please let us know. + +Happy coding! 🚀 \ No newline at end of file diff --git a/dotnet/website/release_note/toc.yml b/dotnet/website/release_note/toc.yml new file mode 100644 index 000000000000..d3b8559a9a32 --- /dev/null +++ b/dotnet/website/release_note/toc.yml @@ -0,0 +1,5 @@ +- name: 0.0.16 + href: 0.0.16.md + +- name: 0.0.0 - 0.0.15 + href: update.md \ No newline at end of file diff --git a/dotnet/website/update.md b/dotnet/website/release_note/update.md similarity index 100% rename from dotnet/website/update.md rename to dotnet/website/release_note/update.md diff --git a/dotnet/website/toc.yml b/dotnet/website/toc.yml index 3931f5e79471..ad5d0e2b695d 100644 --- a/dotnet/website/toc.yml +++ b/dotnet/website/toc.yml @@ -1,11 +1,14 @@ - name: Docs href: articles/ + +- name: Tutorial + href: tutorial/ - name: API Reference href: api/ -- name: Update Log - href: update.md +- name: Release Notes + href: release_note/ - name: Other Languages dropdown: true diff --git a/dotnet/website/tutorial/Chat-with-an-agent.md b/dotnet/website/tutorial/Chat-with-an-agent.md new file mode 100644 index 000000000000..11a73de341d4 --- /dev/null +++ b/dotnet/website/tutorial/Chat-with-an-agent.md @@ -0,0 +1,53 @@ +This tutorial shows how to generate response using an @AutoGen.Core.IAgent by taking @AutoGen.OpenAI.OpenAIChatAgent as an example. + +> [!NOTE] +> AutoGen.Net provides the following agents to connect to different LLM platforms. Generating responses using these agents is similar to the example shown below. +> - @AutoGen.OpenAI.OpenAIChatAgent +> - @AutoGen.SemanticKernel.SemanticKernelAgent +> - @AutoGen.LMStudio.LMStudioAgent +> - @AutoGen.Mistral.MistralClientAgent +> - @AutoGen.Anthropic.AnthropicClientAgent +> - @AutoGen.Ollama.OllamaAgent +> - @AutoGen.Gemini.GeminiChatAgent + +> [!NOTE] +> The complete code example can be found in [Chat_With_Agent.cs](https://github.com/microsoft/autogen/blob/main/dotnet/sample/AutoGen.BasicSamples/GettingStart/Chat_With_Agent.cs) + +## Step 1: Install AutoGen + +First, install the AutoGen package using the following command: + +```bash +dotnet add package AutoGen +``` + +## Step 2: add Using Statements + +[!code-csharp[Using Statements](../../sample/AutoGen.BasicSamples/GettingStart/Chat_With_Agent.cs?name=Using)] + +## Step 3: Create an @AutoGen.OpenAI.OpenAIChatAgent + +> [!NOTE] +> The @AutoGen.OpenAI.Extension.OpenAIAgentExtension.RegisterMessageConnector* method registers an @AutoGen.OpenAI.OpenAIChatRequestMessageConnector middleware which converts OpenAI message types to AutoGen message types. This step is necessary when you want to use AutoGen built-in message types like @AutoGen.Core.TextMessage, @AutoGen.Core.ImageMessage, etc. +> For more information, see [Built-in-messages](../articles/Built-in-messages.md) + +[!code-csharp[Create an OpenAIChatAgent](../../sample/AutoGen.BasicSamples/GettingStart/Chat_With_Agent.cs?name=Create_Agent)] + +## Step 4: Generate Response +To generate response, you can use one of the overloaded method of @AutoGen.Core.AgentExtension.SendAsync* method. The following code shows how to generate response with text message: + +[!code-csharp[Generate Response](../../sample/AutoGen.BasicSamples/GettingStart/Chat_With_Agent.cs?name=Chat_With_Agent)] + +To generate response with chat history, you can pass the chat history to the @AutoGen.Core.AgentExtension.SendAsync* method: + +[!code-csharp[Generate Response with Chat History](../../sample/AutoGen.BasicSamples/GettingStart/Chat_With_Agent.cs?name=Chat_With_History)] + +To streamingly generate response, use @AutoGen.Core.IStreamingAgent.GenerateStreamingReplyAsync* + +[!code-csharp[Generate Streaming Response](../../sample/AutoGen.BasicSamples/GettingStart/Chat_With_Agent.cs?name=Streaming_Chat)] + +## Further Reading +- [Chat with google gemini](../articles/AutoGen.Gemini/Chat-with-google-gemini.md) +- [Chat with vertex gemini](../articles/AutoGen.Gemini/Chat-with-vertex-gemini.md) +- [Chat with Ollama](../articles/AutoGen.Ollama/Chat-with-llama.md) +- [Chat with Semantic Kernel Agent](../articles/AutoGen.SemanticKernel/SemanticKernelAgent-simple-chat.md) \ No newline at end of file diff --git a/dotnet/website/tutorial/Create-agent-with-tools.md b/dotnet/website/tutorial/Create-agent-with-tools.md new file mode 100644 index 000000000000..5d631890308a --- /dev/null +++ b/dotnet/website/tutorial/Create-agent-with-tools.md @@ -0,0 +1,105 @@ +This tutorial shows how to use tools in an agent. + +## What is tool +Tools are pre-defined functions in user's project that agent can invoke. Agent can use tools to perform actions like search web, perform calculations, etc. With tools, it can greatly extend the capabilities of an agent. + +> [!NOTE] +> To use tools with agent, the backend LLM model used by the agent needs to support tool calling. Here are some of the LLM models that support tool calling as of 06/21/2024 +> - GPT-3.5-turbo with version >= 0613 +> - GPT-4 series +> - Gemini series +> - OPEN_MISTRAL_7B +> - ... +> +> This tutorial uses the latest `GPT-3.5-turbo` as example. + +> [!NOTE] +> The complete code example can be found in [Use_Tools_With_Agent.cs](https://github.com/microsoft/autogen/blob/main/dotnet/sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs) + +## Key Concepts +- @AutoGen.Core.FunctionContract: The contract of a function that agent can invoke. It contains the function name, description, parameters schema, and return type. +- @AutoGen.Core.ToolCallMessage: A message type that represents a tool call request in AutoGen.Net. +- @AutoGen.Core.ToolCallResultMessage: A message type that represents a tool call result in AutoGen.Net. +- @AutoGen.Core.ToolCallAggregateMessage: An aggregate message type that represents a tool call request and its result in a single message in AutoGen.Net. +- @AutoGen.Core.FunctionCallMiddleware: A middleware that pass the @AutoGen.Core.FunctionContract to the agent when generating response, and process the tool call response when receiving a @AutoGen.Core.ToolCallMessage. + +> [!Tip] +> You can Use AutoGen.SourceGenerator to automatically generate type-safe @AutoGen.Core.FunctionContract instead of manually defining them. For more information, please check out [Create type-safe function](../articles/Create-type-safe-function-call.md). + +## Install AutoGen and AutoGen.SourceGenerator +First, install the AutoGen and AutoGen.SourceGenerator package using the following command: + +```bash +dotnet add package AutoGen +dotnet add package AutoGen.SourceGenerator +``` + +Also, you might need to enable structural xml document support by setting `GenerateDocumentationFile` property to true in your project file. This allows source generator to leverage the documentation of the function when generating the function definition. + +```xml + + + true + +``` + +## Add Using Statements + +[!code-csharp[Using Statements](../../sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs?name=Using)] + +## Create agent + +Create an @AutoGen.OpenAI.OpenAIChatAgent with `GPT-3.5-turbo` as the backend LLM model. + +[!code-csharp[Create an agent with tools](../../sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs?name=Create_Agent)] + +## Define `Tool` class and create tools +Create a `public partial` class to host the tools you want to use in AutoGen agents. The method has to be a `public` instance method and its return type must be `Task`. After the methods is defined, mark them with @AutoGen.Core.FunctionAttribute attribute. + +In the following example, we define a `GetWeather` tool that returns the weather information of a city. + +[!code-csharp[Define Tool class](../../sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs?name=Tools)] +[!code-csharp[Create tools](../../sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs?name=Create_tools)] + +## Tool call without auto-invoke +In this case, when receiving a @AutoGen.Core.ToolCallMessage, the agent will not automatically invoke the tool. Instead, the agent will return the original message back to the user. The user can then decide whether to invoke the tool or not. + +![single-turn tool call without auto-invoke](../images/articles/CreateAgentWithTools/single-turn-tool-call-without-auto-invoke.png) + +To implement this, you can create the @AutoGen.Core.FunctionCallMiddleware without passing the `functionMap` parameter to the constructor so that the middleware will not automatically invoke the tool once it receives a @AutoGen.Core.ToolCallMessage from its inner agent. + +[!code-csharp[Single-turn tool call without auto-invoke](../../sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs?name=Create_no_invoke_middleware)] + +After creating the function call middleware, you can register it to the agent using `RegisterMiddleware` method, which will return a new agent which can use the methods defined in the `Tool` class. + +[!code-csharp[Generate Response](../../sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs?name=Single_Turn_No_Invoke)] + +## Tool call with auto-invoke +In this case, the agent will automatically invoke the tool when receiving a @AutoGen.Core.ToolCallMessage and return the @AutoGen.Core.ToolCallAggregateMessage which contains both the tool call request and the tool call result. + +![single-turn tool call with auto-invoke](../images/articles/CreateAgentWithTools/single-turn-tool-call-with-auto-invoke.png) + +To implement this, you can create the @AutoGen.Core.FunctionCallMiddleware with the `functionMap` parameter so that the middleware will automatically invoke the tool once it receives a @AutoGen.Core.ToolCallMessage from its inner agent. + +[!code-csharp[Single-turn tool call with auto-invoke](../../sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs?name=Create_auto_invoke_middleware)] + +After creating the function call middleware, you can register it to the agent using `RegisterMiddleware` method, which will return a new agent which can use the methods defined in the `Tool` class. + +[!code-csharp[Generate Response](../../sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs?name=Single_Turn_Auto_Invoke)] + +## Send the tool call result back to LLM to generate further response +In some cases, you may want to send the tool call result back to the LLM to generate further response. To do this, you can send the tool call response from agent back to the LLM by calling the `SendAsync` method of the agent. + +[!code-csharp[Generate Response](../../sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs?name=Multi_Turn_Tool_Call)] + +## Parallel tool call +Some LLM models support parallel tool call, which returns multiple tool calls in one single message. Note that @AutoGen.Core.FunctionCallMiddleware has already handled the parallel tool call for you. When it receives a @AutoGen.Core.ToolCallMessage that contains multiple tool calls, it will automatically invoke all the tools in the sequantial order and return the @AutoGen.Core.ToolCallAggregateMessage which contains all the tool call requests and results. + +[!code-csharp[Generate Response](../../sample/AutoGen.BasicSamples/GettingStart/Use_Tools_With_Agent.cs?name=parallel_tool_call)] + +## Further Reading +- [Function call with openai](../articles/OpenAIChatAgent-use-function-call.md) +- [Function call with gemini](../articles/AutoGen.Gemini/Function-call-with-gemini.md) +- [Function call with local model](../articles/Function-call-with-ollama-and-litellm.md) +- [Use kernel plugin in other agents](../articles/AutoGen.SemanticKernel/Use-kernel-plugin-in-other-agents.md) +- [function call in mistral](../articles/MistralChatAgent-use-function-call.md) \ No newline at end of file diff --git a/dotnet/website/tutorial/Image-chat-with-agent.md b/dotnet/website/tutorial/Image-chat-with-agent.md new file mode 100644 index 000000000000..1e6d4b0ae2b4 --- /dev/null +++ b/dotnet/website/tutorial/Image-chat-with-agent.md @@ -0,0 +1,50 @@ +This tutorial shows how to perform image chat with an agent using the @AutoGen.OpenAI.OpenAIChatAgent as an example. + +> [!NOTE] +> To chat image with an agent, the model behind the agent needs to support image input. Here is a partial list of models that support image input: +> - gpt-4o +> - gemini-1.5 +> - llava +> - claude-3 +> - ... +> +> In this example, we are using the gpt-4o model as the backend model for the agent. + +> [!NOTE] +> The complete code example can be found in [Image_Chat_With_Agent.cs](https://github.com/microsoft/autogen/blob/main/dotnet/sample/AutoGen.BasicSamples/GettingStart/Image_Chat_With_Agent.cs) + +## Step 1: Install AutoGen + +First, install the AutoGen package using the following command: + +```bash +dotnet add package AutoGen +``` + +## Step 2: Add Using Statements + +[!code-csharp[Using Statements](../../sample/AutoGen.BasicSamples/GettingStart/Image_Chat_With_Agent.cs?name=Using)] + +## Step 3: Create an @AutoGen.OpenAI.OpenAIChatAgent + +[!code-csharp[Create an OpenAIChatAgent](../../sample/AutoGen.BasicSamples/GettingStart/Image_Chat_With_Agent.cs?name=Create_Agent)] + +## Step 4: Prepare Image Message + +In AutoGen, you can create an image message using either @AutoGen.Core.ImageMessage or @AutoGen.Core.MultiModalMessage. The @AutoGen.Core.ImageMessage takes a single image as input, whereas the @AutoGen.Core.MultiModalMessage allows you to pass multiple modalities like text or image. + +Here is how to create an image message using @AutoGen.Core.ImageMessage: +[!code-csharp[Create Image Message](../../sample/AutoGen.BasicSamples/GettingStart/Image_Chat_With_Agent.cs?name=Prepare_Image_Input)] + +Here is how to create a multimodal message using @AutoGen.Core.MultiModalMessage: +[!code-csharp[Create MultiModal Message](../../sample/AutoGen.BasicSamples/GettingStart/Image_Chat_With_Agent.cs?name=Prepare_Multimodal_Input)] + +## Step 5: Generate Response + +To generate response, you can use one of the overloaded methods of @AutoGen.Core.AgentExtension.SendAsync* method. The following code shows how to generate response with an image message: + +[!code-csharp[Generate Response](../../sample/AutoGen.BasicSamples/GettingStart/Image_Chat_With_Agent.cs?name=Chat_With_Agent)] + +## Further Reading +- [Image chat with gemini](../articles/AutoGen.Gemini/Image-chat-with-gemini.md) +- [Image chat with llava](../articles/AutoGen.Ollama/Chat-with-llava.md) \ No newline at end of file diff --git a/dotnet/website/tutorial/Use-AutoGen.Net-agent-as-model-in-AG-Studio.md b/dotnet/website/tutorial/Use-AutoGen.Net-agent-as-model-in-AG-Studio.md new file mode 100644 index 000000000000..a47cb01f649e --- /dev/null +++ b/dotnet/website/tutorial/Use-AutoGen.Net-agent-as-model-in-AG-Studio.md @@ -0,0 +1,84 @@ +This tutorial shows how to use AutoGen.Net agent as model in AG Studio + +## Step 1. Create Dotnet empty web app and install AutoGen and AutoGen.WebAPI package + +```bash +dotnet new web +dotnet add package AutoGen +dotnet add package AutoGen.WebAPI +``` + +## Step 2. Replace the Program.cs with following code + +```bash +using AutoGen.Core; +using AutoGen.Service; + +var builder = WebApplication.CreateBuilder(args); +var app = builder.Build(); + +var helloWorldAgent = new HelloWorldAgent(); +app.UseAgentAsOpenAIChatCompletionEndpoint(helloWorldAgent); + +app.Run(); + +class HelloWorldAgent : IAgent +{ + public string Name => "HelloWorld"; + + public Task GenerateReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, CancellationToken cancellationToken = default) + { + return Task.FromResult(new TextMessage(Role.Assistant, "Hello World!", from: this.Name)); + } +} +``` + +## Step 3: Start the web app + +Run the following command to start web api + +```bash +dotnet RUN +``` + +The web api will listen at `http://localhost:5264/v1/chat/completion + +![terminal](../images/articles/UseAutoGenAsModelinAGStudio/Terminal.png) + +## Step 4: In another terminal, start autogen-studio + +```bash +autogenstudio ui +``` + +## Step 5: Navigate to AutoGen Studio UI and add hello world agent as openai Model + +### Step 5.1: Go to model tab + +![The Model Tab](../images/articles/UseAutoGenAsModelinAGStudio/TheModelTab.png) + +### Step 5.2: Select "OpenAI model" card + +![Open AI model Card](../images/articles/UseAutoGenAsModelinAGStudio/Step5.2OpenAIModel.png) + +### Step 5.3: Fill the model name and url + +The model name needs to be same with agent name + +![Fill the model name and url](../images/articles/UseAutoGenAsModelinAGStudio/Step5.3ModelNameAndURL.png) + +## Step 6: Create a hello world agent that uses the hello world model + +![Create a hello world agent that uses the hello world model](../images/articles/UseAutoGenAsModelinAGStudio/Step6.png) + +![Agent Configuration](../images/articles/UseAutoGenAsModelinAGStudio/Step6b.png) + +## Final Step: Use the hello world agent in workflow + +![Use the hello world agent in workflow](../images/articles/UseAutoGenAsModelinAGStudio/FinalStepsA.png) + +![Use the hello world agent in workflow](../images/articles/UseAutoGenAsModelinAGStudio/FinalStepsA.png) + +![Use the hello world agent in workflow](../images/articles/UseAutoGenAsModelinAGStudio/FinalStepsB.png) + +![Use the hello world agent in workflow](../images/articles/UseAutoGenAsModelinAGStudio/FinalStepsC.png) diff --git a/dotnet/website/tutorial/toc.yml b/dotnet/website/tutorial/toc.yml new file mode 100644 index 000000000000..167baa70e4fd --- /dev/null +++ b/dotnet/website/tutorial/toc.yml @@ -0,0 +1,11 @@ +- name: Chat with an agent + href: Chat-with-an-agent.md + +- name: Image chat with agent + href: Image-chat-with-agent.md + +- name: Create agent with tools + href: Create-agent-with-tools.md + +- name: Use AutoGen.Net agent as model in AG Studio + href: Use-AutoGen.Net-agent-as-model-in-AG-Studio.md \ No newline at end of file diff --git a/notebook/JSON_mode_example.ipynb b/notebook/JSON_mode_example.ipynb index f3adce9dec35..c4b65c4d9f4d 100644 --- a/notebook/JSON_mode_example.ipynb +++ b/notebook/JSON_mode_example.ipynb @@ -26,7 +26,7 @@ "\n", "\n", "## Requirements\n", - "JSON mode is a feature of OpenAI API, however strong models (such as claude 3 Opus), can generate appropriate json as well.\n", + "JSON mode is a feature of OpenAI API, however strong models (such as Claude 3 Opus), can generate appropriate json as well.\n", "AutoGen requires `Python>=3.8`. To run this notebook example, please install:\n", "```bash\n", "pip install pyautogen\n", @@ -71,7 +71,7 @@ "source": [ "## Model Configuration\n", "\n", - "we Need to set two different Configs for this to work. \n", + "We need to set two different Configs for this to work. \n", "One for JSON mode\n", "One for Text mode. \n", "This is because the group chat manager requires text mode. " diff --git a/notebook/agentchat_pgvector_RetrieveChat.ipynb b/notebook/agentchat_RetrieveChat_pgvector.ipynb similarity index 100% rename from notebook/agentchat_pgvector_RetrieveChat.ipynb rename to notebook/agentchat_RetrieveChat_pgvector.ipynb diff --git a/notebook/agentchat_qdrant_RetrieveChat.ipynb b/notebook/agentchat_RetrieveChat_qdrant.ipynb similarity index 50% rename from notebook/agentchat_qdrant_RetrieveChat.ipynb rename to notebook/agentchat_RetrieveChat_qdrant.ipynb index 10d6b55e11c6..b5bc2f681d22 100644 --- a/notebook/agentchat_qdrant_RetrieveChat.ipynb +++ b/notebook/agentchat_RetrieveChat_qdrant.ipynb @@ -31,172 +31,19 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "Requirement already satisfied: pyautogen>=0.2.3 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pyautogen[retrievechat]>=0.2.3) (0.2.21)\n", - "Requirement already satisfied: flaml[automl] in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (2.1.2)\n", - "Requirement already satisfied: qdrant_client[fastembed] in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (1.6.4)\n", - "Requirement already satisfied: openai>=1.3 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pyautogen>=0.2.3->pyautogen[retrievechat]>=0.2.3) (1.12.0)\n", - "Requirement already satisfied: diskcache in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pyautogen>=0.2.3->pyautogen[retrievechat]>=0.2.3) (5.6.3)\n", - "Requirement already satisfied: termcolor in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pyautogen>=0.2.3->pyautogen[retrievechat]>=0.2.3) (2.3.0)\n", - "Requirement already satisfied: numpy<2,>=1.17.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pyautogen>=0.2.3->pyautogen[retrievechat]>=0.2.3) (1.26.4)\n", - "Requirement already satisfied: python-dotenv in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pyautogen>=0.2.3->pyautogen[retrievechat]>=0.2.3) (1.0.0)\n", - "Requirement already satisfied: tiktoken in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pyautogen>=0.2.3->pyautogen[retrievechat]>=0.2.3) (0.5.1)\n", - "Requirement already satisfied: pydantic!=2.6.0,<3,>=1.10 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pyautogen>=0.2.3->pyautogen[retrievechat]>=0.2.3) (2.6.4)\n", - "Requirement already satisfied: docker in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pyautogen>=0.2.3->pyautogen[retrievechat]>=0.2.3) (7.0.0)\n", - "Requirement already satisfied: lightgbm>=2.3.1 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from flaml[automl]) (4.1.0)\n", - "Requirement already satisfied: xgboost>=0.90 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from flaml[automl]) (2.0.1)\n", - "Requirement already satisfied: scipy>=1.4.1 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from flaml[automl]) (1.10.1)\n", - "Requirement already satisfied: pandas>=1.1.4 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from flaml[automl]) (2.2.0)\n", - "Requirement already satisfied: scikit-learn>=0.24 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from flaml[automl]) (1.3.2)\n", - "Requirement already satisfied: grpcio>=1.41.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from qdrant_client[fastembed]) (1.60.0)\n", - "Requirement already satisfied: grpcio-tools>=1.41.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from qdrant_client[fastembed]) (1.59.2)\n", - "Requirement already satisfied: httpx>=0.14.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from httpx[http2]>=0.14.0->qdrant_client[fastembed]) (0.25.1)\n", - "Requirement already satisfied: portalocker<3.0.0,>=2.7.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from qdrant_client[fastembed]) (2.8.2)\n", - "Requirement already satisfied: urllib3<2.0.0,>=1.26.14 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from qdrant_client[fastembed]) (1.26.18)\n", - "Requirement already satisfied: fastembed==0.1.1 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from qdrant_client[fastembed]) (0.1.1)\n", - "Requirement already satisfied: onnx<2.0,>=1.11 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from fastembed==0.1.1->qdrant_client[fastembed]) (1.15.0)\n", - "Requirement already satisfied: onnxruntime<2.0,>=1.15 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from fastembed==0.1.1->qdrant_client[fastembed]) (1.15.1)\n", - "Requirement already satisfied: requests<3.0,>=2.31 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from fastembed==0.1.1->qdrant_client[fastembed]) (2.31.0)\n", - "Requirement already satisfied: tokenizers<0.14,>=0.13 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from fastembed==0.1.1->qdrant_client[fastembed]) (0.13.3)\n", - "Requirement already satisfied: tqdm<5.0,>=4.65 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from fastembed==0.1.1->qdrant_client[fastembed]) (4.66.2)\n", - "Requirement already satisfied: chromadb in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pyautogen[retrievechat]>=0.2.3) (0.4.22)\n", - "Requirement already satisfied: sentence-transformers in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pyautogen[retrievechat]>=0.2.3) (2.3.1)\n", - "Requirement already satisfied: pypdf in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pyautogen[retrievechat]>=0.2.3) (4.0.1)\n", - "Requirement already satisfied: ipython in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pyautogen[retrievechat]>=0.2.3) (8.17.2)\n", - "Requirement already satisfied: protobuf<5.0dev,>=4.21.6 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from grpcio-tools>=1.41.0->qdrant_client[fastembed]) (4.23.4)\n", - "Requirement already satisfied: setuptools in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from grpcio-tools>=1.41.0->qdrant_client[fastembed]) (68.2.2)\n", - "Requirement already satisfied: anyio in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from httpx>=0.14.0->httpx[http2]>=0.14.0->qdrant_client[fastembed]) (3.7.1)\n", - "Requirement already satisfied: certifi in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from httpx>=0.14.0->httpx[http2]>=0.14.0->qdrant_client[fastembed]) (2024.2.2)\n", - "Requirement already satisfied: httpcore in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from httpx>=0.14.0->httpx[http2]>=0.14.0->qdrant_client[fastembed]) (1.0.1)\n", - "Requirement already satisfied: idna in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from httpx>=0.14.0->httpx[http2]>=0.14.0->qdrant_client[fastembed]) (3.6)\n", - "Requirement already satisfied: sniffio in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from httpx>=0.14.0->httpx[http2]>=0.14.0->qdrant_client[fastembed]) (1.3.0)\n", - "Requirement already satisfied: h2<5,>=3 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from httpx[http2]>=0.14.0->qdrant_client[fastembed]) (4.1.0)\n", - "Requirement already satisfied: distro<2,>=1.7.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from openai>=1.3->pyautogen>=0.2.3->pyautogen[retrievechat]>=0.2.3) (1.8.0)\n", - "Requirement already satisfied: typing-extensions<5,>=4.7 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from openai>=1.3->pyautogen>=0.2.3->pyautogen[retrievechat]>=0.2.3) (4.9.0)\n", - "Requirement already satisfied: python-dateutil>=2.8.2 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pandas>=1.1.4->flaml[automl]) (2.8.2)\n", - "Requirement already satisfied: pytz>=2020.1 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pandas>=1.1.4->flaml[automl]) (2024.1)\n", - "Requirement already satisfied: tzdata>=2022.7 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pandas>=1.1.4->flaml[automl]) (2024.1)\n", - "Requirement already satisfied: annotated-types>=0.4.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pydantic!=2.6.0,<3,>=1.10->pyautogen>=0.2.3->pyautogen[retrievechat]>=0.2.3) (0.6.0)\n", - "Requirement already satisfied: pydantic-core==2.16.3 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pydantic!=2.6.0,<3,>=1.10->pyautogen>=0.2.3->pyautogen[retrievechat]>=0.2.3) (2.16.3)\n", - "Requirement already satisfied: joblib>=1.1.1 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from scikit-learn>=0.24->flaml[automl]) (1.3.2)\n", - "Requirement already satisfied: threadpoolctl>=2.0.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from scikit-learn>=0.24->flaml[automl]) (3.2.0)\n", - "Requirement already satisfied: build>=1.0.3 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (1.0.3)\n", - "Requirement already satisfied: chroma-hnswlib==0.7.3 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (0.7.3)\n", - "Requirement already satisfied: fastapi>=0.95.2 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (0.104.1)\n", - "Requirement already satisfied: uvicorn>=0.18.3 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb->pyautogen[retrievechat]>=0.2.3) (0.24.0)\n", - "Requirement already satisfied: posthog>=2.4.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (3.0.2)\n", - "Requirement already satisfied: pulsar-client>=3.1.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (3.3.0)\n", - "Requirement already satisfied: opentelemetry-api>=1.2.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (1.20.0)\n", - "Requirement already satisfied: opentelemetry-exporter-otlp-proto-grpc>=1.2.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (1.20.0)\n", - "Requirement already satisfied: opentelemetry-instrumentation-fastapi>=0.41b0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (0.41b0)\n", - "Requirement already satisfied: opentelemetry-sdk>=1.2.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (1.20.0)\n", - "Requirement already satisfied: pypika>=0.48.9 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (0.48.9)\n", - "Requirement already satisfied: overrides>=7.3.1 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (7.4.0)\n", - "Requirement already satisfied: importlib-resources in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (6.1.1)\n", - "Requirement already satisfied: bcrypt>=4.0.1 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (4.0.1)\n", - "Requirement already satisfied: typer>=0.9.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (0.9.0)\n", - "Requirement already satisfied: kubernetes>=28.1.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (28.1.0)\n", - "Requirement already satisfied: tenacity>=8.2.3 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (8.2.3)\n", - "Requirement already satisfied: PyYAML>=6.0.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (6.0.1)\n", - "Requirement already satisfied: mmh3>=4.0.1 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from chromadb->pyautogen[retrievechat]>=0.2.3) (4.0.1)\n", - "Requirement already satisfied: packaging>=14.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from docker->pyautogen>=0.2.3->pyautogen[retrievechat]>=0.2.3) (23.2)\n", - "Requirement already satisfied: decorator in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from ipython->pyautogen[retrievechat]>=0.2.3) (5.1.1)\n", - "Requirement already satisfied: jedi>=0.16 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from ipython->pyautogen[retrievechat]>=0.2.3) (0.19.1)\n", - "Requirement already satisfied: matplotlib-inline in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from ipython->pyautogen[retrievechat]>=0.2.3) (0.1.6)\n", - "Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from ipython->pyautogen[retrievechat]>=0.2.3) (3.0.39)\n", - "Requirement already satisfied: pygments>=2.4.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from ipython->pyautogen[retrievechat]>=0.2.3) (2.16.1)\n", - "Requirement already satisfied: stack-data in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from ipython->pyautogen[retrievechat]>=0.2.3) (0.6.3)\n", - "Requirement already satisfied: traitlets>=5 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from ipython->pyautogen[retrievechat]>=0.2.3) (5.14.2)\n", - "Requirement already satisfied: exceptiongroup in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from ipython->pyautogen[retrievechat]>=0.2.3) (1.1.3)\n", - "Requirement already satisfied: pexpect>4.3 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from ipython->pyautogen[retrievechat]>=0.2.3) (4.8.0)\n", - "Requirement already satisfied: transformers<5.0.0,>=4.32.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from sentence-transformers->pyautogen[retrievechat]>=0.2.3) (4.33.3)\n", - "Requirement already satisfied: torch>=1.11.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from sentence-transformers->pyautogen[retrievechat]>=0.2.3) (2.2.0)\n", - "Requirement already satisfied: nltk in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from sentence-transformers->pyautogen[retrievechat]>=0.2.3) (3.8.1)\n", - "Requirement already satisfied: sentencepiece in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from sentence-transformers->pyautogen[retrievechat]>=0.2.3) (0.1.99)\n", - "Requirement already satisfied: huggingface-hub>=0.15.1 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from sentence-transformers->pyautogen[retrievechat]>=0.2.3) (0.20.3)\n", - "Requirement already satisfied: Pillow in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from sentence-transformers->pyautogen[retrievechat]>=0.2.3) (10.2.0)\n", - "Requirement already satisfied: regex>=2022.1.18 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from tiktoken->pyautogen>=0.2.3->pyautogen[retrievechat]>=0.2.3) (2023.12.25)\n", - "Requirement already satisfied: pyproject_hooks in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from build>=1.0.3->chromadb->pyautogen[retrievechat]>=0.2.3) (1.0.0)\n", - "Requirement already satisfied: tomli>=1.1.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from build>=1.0.3->chromadb->pyautogen[retrievechat]>=0.2.3) (2.0.1)\n", - "Requirement already satisfied: starlette<0.28.0,>=0.27.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from fastapi>=0.95.2->chromadb->pyautogen[retrievechat]>=0.2.3) (0.27.0)\n", - "Requirement already satisfied: hyperframe<7,>=6.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from h2<5,>=3->httpx[http2]>=0.14.0->qdrant_client[fastembed]) (6.0.1)\n", - "Requirement already satisfied: hpack<5,>=4.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from h2<5,>=3->httpx[http2]>=0.14.0->qdrant_client[fastembed]) (4.0.0)\n", - "Requirement already satisfied: filelock in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from huggingface-hub>=0.15.1->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (3.13.1)\n", - "Requirement already satisfied: fsspec>=2023.5.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from huggingface-hub>=0.15.1->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (2024.2.0)\n", - "Requirement already satisfied: parso<0.9.0,>=0.8.3 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from jedi>=0.16->ipython->pyautogen[retrievechat]>=0.2.3) (0.8.3)\n", - "Requirement already satisfied: six>=1.9.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from kubernetes>=28.1.0->chromadb->pyautogen[retrievechat]>=0.2.3) (1.16.0)\n", - "Requirement already satisfied: google-auth>=1.0.1 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from kubernetes>=28.1.0->chromadb->pyautogen[retrievechat]>=0.2.3) (2.23.4)\n", - "Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from kubernetes>=28.1.0->chromadb->pyautogen[retrievechat]>=0.2.3) (1.6.4)\n", - "Requirement already satisfied: requests-oauthlib in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from kubernetes>=28.1.0->chromadb->pyautogen[retrievechat]>=0.2.3) (1.3.1)\n", - "Requirement already satisfied: oauthlib>=3.2.2 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from kubernetes>=28.1.0->chromadb->pyautogen[retrievechat]>=0.2.3) (3.2.2)\n", - "Requirement already satisfied: coloredlogs in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from onnxruntime<2.0,>=1.15->fastembed==0.1.1->qdrant_client[fastembed]) (15.0.1)\n", - "Requirement already satisfied: flatbuffers in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from onnxruntime<2.0,>=1.15->fastembed==0.1.1->qdrant_client[fastembed]) (23.5.26)\n", - "Requirement already satisfied: sympy in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from onnxruntime<2.0,>=1.15->fastembed==0.1.1->qdrant_client[fastembed]) (1.12)\n", - "Requirement already satisfied: deprecated>=1.2.6 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from opentelemetry-api>=1.2.0->chromadb->pyautogen[retrievechat]>=0.2.3) (1.2.14)\n", - "Requirement already satisfied: importlib-metadata<7.0,>=6.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from opentelemetry-api>=1.2.0->chromadb->pyautogen[retrievechat]>=0.2.3) (6.11.0)\n", - "Requirement already satisfied: backoff<3.0.0,>=1.10.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb->pyautogen[retrievechat]>=0.2.3) (2.2.1)\n", - "Requirement already satisfied: googleapis-common-protos~=1.52 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb->pyautogen[retrievechat]>=0.2.3) (1.61.0)\n", - "Requirement already satisfied: opentelemetry-exporter-otlp-proto-common==1.20.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb->pyautogen[retrievechat]>=0.2.3) (1.20.0)\n", - "Requirement already satisfied: opentelemetry-proto==1.20.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb->pyautogen[retrievechat]>=0.2.3) (1.20.0)\n", - "Requirement already satisfied: opentelemetry-instrumentation-asgi==0.41b0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb->pyautogen[retrievechat]>=0.2.3) (0.41b0)\n", - "Requirement already satisfied: opentelemetry-instrumentation==0.41b0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb->pyautogen[retrievechat]>=0.2.3) (0.41b0)\n", - "Requirement already satisfied: opentelemetry-semantic-conventions==0.41b0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb->pyautogen[retrievechat]>=0.2.3) (0.41b0)\n", - "Requirement already satisfied: opentelemetry-util-http==0.41b0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb->pyautogen[retrievechat]>=0.2.3) (0.41b0)\n", - "Requirement already satisfied: wrapt<2.0.0,>=1.0.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from opentelemetry-instrumentation==0.41b0->opentelemetry-instrumentation-fastapi>=0.41b0->chromadb->pyautogen[retrievechat]>=0.2.3) (1.16.0)\n", - "Requirement already satisfied: asgiref~=3.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from opentelemetry-instrumentation-asgi==0.41b0->opentelemetry-instrumentation-fastapi>=0.41b0->chromadb->pyautogen[retrievechat]>=0.2.3) (3.7.2)\n", - "Requirement already satisfied: ptyprocess>=0.5 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pexpect>4.3->ipython->pyautogen[retrievechat]>=0.2.3) (0.7.0)\n", - "Requirement already satisfied: monotonic>=1.5 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from posthog>=2.4.0->chromadb->pyautogen[retrievechat]>=0.2.3) (1.6)\n", - "Requirement already satisfied: wcwidth in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython->pyautogen[retrievechat]>=0.2.3) (0.2.9)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from requests<3.0,>=2.31->fastembed==0.1.1->qdrant_client[fastembed]) (3.3.2)\n", - "Requirement already satisfied: networkx in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from torch>=1.11.0->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (3.2.1)\n", - "Requirement already satisfied: jinja2 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from torch>=1.11.0->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (3.1.3)\n", - "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from torch>=1.11.0->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (12.1.105)\n", - "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from torch>=1.11.0->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (12.1.105)\n", - "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from torch>=1.11.0->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (12.1.105)\n", - "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from torch>=1.11.0->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (8.9.2.26)\n", - "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from torch>=1.11.0->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (12.1.3.1)\n", - "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from torch>=1.11.0->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (11.0.2.54)\n", - "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from torch>=1.11.0->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (10.3.2.106)\n", - "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from torch>=1.11.0->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (11.4.5.107)\n", - "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from torch>=1.11.0->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (12.1.0.106)\n", - "Requirement already satisfied: nvidia-nccl-cu12==2.19.3 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from torch>=1.11.0->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (2.19.3)\n", - "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from torch>=1.11.0->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (12.1.105)\n", - "Requirement already satisfied: triton==2.2.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from torch>=1.11.0->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (2.2.0)\n", - "Requirement already satisfied: nvidia-nvjitlink-cu12 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.11.0->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (12.3.52)\n", - "Requirement already satisfied: safetensors>=0.3.1 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from transformers<5.0.0,>=4.32.0->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (0.3.2)\n", - "Requirement already satisfied: click<9.0.0,>=7.1.1 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from typer>=0.9.0->chromadb->pyautogen[retrievechat]>=0.2.3) (8.1.7)\n", - "Requirement already satisfied: h11>=0.8 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from uvicorn>=0.18.3->uvicorn[standard]>=0.18.3->chromadb->pyautogen[retrievechat]>=0.2.3) (0.14.0)\n", - "Requirement already satisfied: httptools>=0.5.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb->pyautogen[retrievechat]>=0.2.3) (0.6.1)\n", - "Requirement already satisfied: uvloop!=0.15.0,!=0.15.1,>=0.14.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb->pyautogen[retrievechat]>=0.2.3) (0.19.0)\n", - "Requirement already satisfied: watchfiles>=0.13 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb->pyautogen[retrievechat]>=0.2.3) (0.21.0)\n", - "Requirement already satisfied: websockets>=10.4 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb->pyautogen[retrievechat]>=0.2.3) (11.0.3)\n", - "Requirement already satisfied: executing>=1.2.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from stack-data->ipython->pyautogen[retrievechat]>=0.2.3) (2.0.1)\n", - "Requirement already satisfied: asttokens>=2.1.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from stack-data->ipython->pyautogen[retrievechat]>=0.2.3) (2.4.1)\n", - "Requirement already satisfied: pure-eval in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from stack-data->ipython->pyautogen[retrievechat]>=0.2.3) (0.2.2)\n", - "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb->pyautogen[retrievechat]>=0.2.3) (5.3.2)\n", - "Requirement already satisfied: pyasn1-modules>=0.2.1 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb->pyautogen[retrievechat]>=0.2.3) (0.3.0)\n", - "Requirement already satisfied: rsa<5,>=3.1.4 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb->pyautogen[retrievechat]>=0.2.3) (4.9)\n", - "Requirement already satisfied: zipp>=0.5 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from importlib-metadata<7.0,>=6.0->opentelemetry-api>=1.2.0->chromadb->pyautogen[retrievechat]>=0.2.3) (3.17.0)\n", - "Requirement already satisfied: humanfriendly>=9.1 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from coloredlogs->onnxruntime<2.0,>=1.15->fastembed==0.1.1->qdrant_client[fastembed]) (10.0)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from jinja2->torch>=1.11.0->sentence-transformers->pyautogen[retrievechat]>=0.2.3) (2.1.5)\n", - "Requirement already satisfied: mpmath>=0.19 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from sympy->onnxruntime<2.0,>=1.15->fastembed==0.1.1->qdrant_client[fastembed]) (1.3.0)\n", - "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /home/lijiang1/anaconda3/envs/autogen/lib/python3.10/site-packages (from pyasn1-modules>=0.2.1->google-auth>=1.0.1->kubernetes>=28.1.0->chromadb->pyautogen[retrievechat]>=0.2.3) (0.5.0)\n", "Note: you may need to restart the kernel to use updated packages.\n" ] } ], "source": [ - "%pip install \"pyautogen[retrievechat-qdrant]\" \"flaml[automl]\"" + "%pip install \"pyautogen[retrievechat-qdrant]\" \"flaml[automl]\" -q" ] }, { @@ -211,23 +58,24 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "models to use: ['gpt4-1106-preview', 'gpt-35-turbo', 'gpt-35-turbo-0613']\n" + "models to use: ['gpt4-1106-preview', 'gpt-4o', 'gpt-35-turbo', 'gpt-35-turbo-0613']\n" ] } ], "source": [ "from qdrant_client import QdrantClient\n", + "from sentence_transformers import SentenceTransformer\n", "\n", "import autogen\n", - "from autogen.agentchat.contrib.qdrant_retrieve_user_proxy_agent import QdrantRetrieveUserProxyAgent\n", "from autogen.agentchat.contrib.retrieve_assistant_agent import RetrieveAssistantAgent\n", + "from autogen.agentchat.contrib.retrieve_user_proxy_agent import RetrieveUserProxyAgent\n", "\n", "# Accepted file formats for that can be stored in\n", "# a vector database instance\n", @@ -253,7 +101,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -261,7 +109,7 @@ "output_type": "stream", "text": [ "Accepted file formats for `docs_path`:\n", - "['yml', 'ppt', 'org', 'doc', 'epub', 'rst', 'log', 'docx', 'htm', 'html', 'tsv', 'csv', 'json', 'yaml', 'xlsx', 'pptx', 'rtf', 'msg', 'odt', 'pdf', 'jsonl', 'md', 'xml', 'txt']\n" + "['rtf', 'jsonl', 'xml', 'json', 'md', 'rst', 'docx', 'msg', 'pdf', 'log', 'xlsx', 'org', 'txt', 'csv', 'pptx', 'tsv', 'yml', 'epub', 'yaml', 'ppt', 'htm', 'doc', 'odt', 'html']\n" ] } ], @@ -284,9 +132,24 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "67171b10626248ba8b5bff0f5a4d6895", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Fetching 5 files: 0%| | 0/5 [00:00>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "> Running step 4f4f291b-5e13-495f-9871-4207e4c4bcb9. Step input: \n", + "What can i find in Tokyo related to Hayao Miyazaki and its moveis like Spirited Away?.\n", + "\n", + "\u001b[1;3;38;5;200mThought: The current language of the user is: English. I need to use a tool to help me answer the question.\n", + "Action: search_data\n", + "Action Input: {'query': 'Hayao Miyazaki Tokyo'}\n", + "\u001b[0m\u001b[1;3;34mObservation: Hayao Miyazaki (宮崎 駿 or 宮﨑 駿, Miyazaki Hayao, Japanese: [mijaꜜzaki hajao]; born January 5, 1941) is a Japanese animator, filmmaker, and manga artist. A founder of Studio Ghibli, he has attained international acclaim as a masterful storyteller and creator of Japanese animated feature films, and is widely regarded as one of the most accomplished filmmakers in the history of animation.\n", + "Born in Tokyo City in the Empire of Japan, Miyazaki expressed interest in manga and animation from an early age, and he joined Toei Animation in 1963. During his early years at Toei Animation he worked as an in-between artist and later collaborated with director Isao Takahata. Notable films to which Miyazaki contributed at Toei include Doggie March and Gulliver's Travels Beyond the Moon. He provided key animation to other films at Toei, such as Puss in Boots and Animal Treasure Island, before moving to A-Pro in 1971, where he co-directed Lupin the Third Part I alongside Takahata. After moving to Zuiyō Eizō (later known as Nippon Animation) in 1973, Miyazaki worked as an animator on World Masterpiece Theater, and directed the television series Future Boy Conan (1978). He joined Tokyo Movie Shinsha in 1979 to direct his first feature film The Castle of Cagliostro as well as the television series Sherlock Hound. In the same period, he began writing and illustrating the manga Nausicaä of the Valley of the Wind (1982–1994) and directed the 1984 film adaptation produced by Topcraft.\n", + "Miyazaki co-founded Studio Ghibli in 1985. He directed numerous films with Ghibli, including Laputa: Castle in the Sky (1986), My Neighbor Totoro (1988), Kiki's Delivery Service (1989), and Porco Rosso (1992). The films were met with critical and commercial success in Japan. Miyazaki's film Princess Mononoke was the first animated film ever to win the Japan Academy Film Prize for Picture of the Year, and briefly became the highest-grossing film in Japan following its release in 1997; its distribution to the Western world greatly increased Ghibli's popularity and influence outside Japan. His 2001 film Spirited Away became the highest-grossing film in Japanese history, winning the Academy Award for Best Animated Feature, and is frequently ranked among the greatest films of the 21st century. Miyazaki's later films—Howl's Moving Castle (2004), Ponyo (2008), and The Wind Rises (2013)—also enjoyed critical and commercial success. Following the release of The Wind Rises, Miyazaki announced his retirement from feature films, though he later returned to write and direct his twelfth feature film The Boy and the Heron (2023), for which he won his second Academy Award for Best Animated Feature.\n", + "Miyazaki's works are characterized by the recurrence of themes such as humanity's relationship with nature and technology, the wholesomeness of natural and traditional patterns of living, the importance of art and craftsmanship, and the difficulty of maintaining a pacifist ethic in a violent world. The protagonists of his films are often strong girls or young women, and several of his films present morally ambiguous antagonists with redeeming qualities. Miyazaki's works have been highly praised and awarded; he was named a Person of Cultural Merit for outstanding cultural contributions in November 2012, and received the Academy Honorary Award for his impact on animation and cinema in November 2014. Miyazaki has frequently been cited as an inspiration for numerous animators, directors, and writers.\n", + "\n", + "\n", + "== Early life ==\n", + "Hayao Miyazaki was born on January 5, 1941, in Tokyo City, Empire of Japan, the second of four sons. His father, Katsuji Miyazaki (born 1915), was the director of Miyazaki Airplane, his brother's company, which manufactured rudders for fighter planes during World War II. The business allowed his family to remain affluent during Miyazaki's early life. Miyazaki's father enjoyed purchasing paintings and demonstrating them to guests, but otherwise had little known artistic understanding. He said that he was in the Imperial Japanese Army around 1940; after declaring to his commanding officer that he wished not to fight because of his wife and young child, he was discharged after a lecture about disloyalty. According to Miyazaki, his father often told him about his exploits, claiming that he continued to attend nightclubs after turning 70. Katsuji Miyazaki died on March 18, 1993. After his death, Miyazaki felt that he had often looked at his father negatively and that he had never said anything \"lofty or inspiring\". He regretted not having a serious discussion with his father, and felt that he had inherited his \"anarchistic feelings and his lack of concern about embracing contradictions\".\n", + "\n", + "Miyazaki has noted that some of his earliest memories are of \"bombed-out cities\". In 1944, when he was three years old, Miyazaki's family evacuated to Utsunomiya. After the bombing of Utsunomiya in July 1945, he and his family evacuated to Kanuma. The bombing left a lasting impression on Miyazaki, then aged four. As a child, Miyazaki suffered from digestive problems, and was told that he would not live beyond 20, making him feel like an outcast. From 1947 to 1955, Miyazaki's mother Yoshiko suffered from spinal tuberculosis; she spent the first few years in hospital before being nursed from home. Yoshiko was frugal, and described as a strict, intellectual woman who regularly questioned \"socially accepted norms\". She was closest with Miyazaki, and had a strong influence on him and his later work. Yoshiko Miyazaki died in July 1983 at the age of 72.\n", + "Miyazaki began school in 1947, at an elementary school in Utsunomiya, completing the first through third grades. After his family moved back to Suginami-ku, Miyazaki completed the fourth grade at Ōmiya Elementary School, and fifth grade at Eifuku Elementary School, which was newly established after splitting off from Ōmiya Elementary. After graduating from Eifuku as part of the first graduating class, he attended Ōmiya Junior High School. He aspired to become a manga artist, but discovered he could not draw people; instead, he only drew planes, tanks, and battleships for several years. Miyazaki was influenced by several manga artists, such as Tetsuji Fukushima, Soji Yamakawa and Osamu Tezuka. Miyazaki destroyed much of his early work, believing it was \"bad form\" to copy Tezuka's style as it was hindering his own development as an artist. Around this time, Miyazaki would often see movies with his father, who was an avid moviegoer; memorable films for Miyazaki include Meshi (1951) and Tasogare Sakaba (1955).\n", + "After graduating from Ōmiya Junior High, Miyazaki attended Toyotama High School. During his third and final year, Miyazaki's interest in animation was sparked by Panda and the Magic Serpent (1958), Japan's first feature-length animated film in color; he had sneaked out to watch the film instead of studying for his entrance exams. Miyazaki later recounted that he fell in love with the film's heroine, Bai-Niang, and that the film moved him to tears and left a profound impression; he wrote that he was \"moved to the depths of [his] soul\" and that the \"pure, earnest world of the film\" affirmed a side of him that \"yearned desperately to affirm the world rather than negate it\". After graduating from Toyotama, Miyazaki attended Gakushuin University in the department of political economy, majoring in Japanese Industrial Theory. He joined the \"Children's Literature Research Club\", the \"closest thing back then to a comics club\"; he was sometimes the sole member of the club. In his free time, Miyazaki would visit his art teacher from middle school and sketch in his studio, where the two would drink and \"talk about politics, life, all sorts of things\". Around this time, he also drew manga; he never completed any stories, but accumulated thousands of pages of the beginnings of stories. He also frequently approached manga publishers to rent their stories. In 1960, Miyazaki was a bystander during the Anpo protests, having developed an interest after seeing photographs in Asahi Graph; by that point, he was too late to participate in the demonstrations. Miyazaki graduated from Gakushuin in 1963 with degrees in political science and economics.\n", + "\n", + "\n", + "== Career ==\n", + "\n", + "\n", + "=== Early career ===\n", + "\n", + "In 1963, Miyazaki was employed at Toei Animation; this was the last year the company hired regularly. After gaining employment, he began renting a four-and-a-half tatami (7.4 m2; 80 sq ft) apartment in Nerima, Tokyo; the rent was ¥6,000. His salary at Toei was ¥19,500. Miyazaki worked as an in-between artist on the theatrical feature anime Doggie March and the television anime Wolf Boy Ken (both 1963). He also worked on Gulliver's Travels Beyond the Moon (1965). He was a leader in a labor dispute soon after his arrival, and became chief secretary of Toei's labor union in 1964. Miyazaki later worked as chief animator, concept artist, and scene designer on The Great Adventure of Horus, Prince of the Sun (1968). Throughout the film's production, Miyazaki worked closely with his mentor, Yasuo Ōtsuka, whose approach to animation profoundly influenced Miyazaki's work. Directed by Isao Takahata, with whom Miyazaki would continue to collaborate for the remainder of his career, the film was highly praised, and deemed a pivotal work in the evolution of animation. Miyazaki moved to a residence in Ōizumigakuenchō in April 1969, after the birth of his second son.\n", + "Miyazaki provided key animation for The Wonderful World of Puss 'n Boots (1969), directed by Kimio Yabuki. He created a 12-chapter manga series as a promotional tie-in for the film; the series ran in the Sunday edition of Tokyo Shimbun from January to March 1969. Miyazaki later proposed scenes in the screenplay for Flying Phantom Ship (1969), in which military tanks would cause mass hysteria in downtown Tokyo, and was hired to storyboard and animate the scenes. Under the pseudonym Akitsu Saburō (秋津 三朗), Miyazaki wrote and illustrated the manga People of the Desert, published in 26 installments between September 1969 and March 1970 in Boys and Girls Newspaper (少年少女新聞, Shōnen shōjo shinbun). He was influenced by illustrated stories such as Fukushima's Evil Lord of the Desert (沙漠の魔王, Sabaku no maō). In 1970, Miyazaki moved residence to Tokorozawa. In 1971, he developed structure, characters and designs for Hiroshi Ikeda's adaptation of Animal Treasure Island; he created the 13-part manga adaptation, printed in Tokyo Shimbun from January to March 1971. Miyazaki also provided key animation for Ali Baba and the Forty Thieves.\n", + "Miyazaki left Toei Animation in August 1971, and was hired at A-Pro, where he directed, or co-directed with Takahata, 23 episodes of Lupin the Third Part I, often using the pseudonym Teruki Tsutomu (照樹 務). The two also began pre-production on a series based on Astrid Lindgren's Pippi Longstocking books, designing extensive storyboards; the series was canceled after Miyazaki and Takahata were unable to meet with Lindgren, and permission was refused to complete the project. In 1972 and 1973, Miyazaki wrote, designed and animated two Panda! Go, Panda! shorts, directed by Takahata. After moving from A-Pro to Zuiyō Eizō in June 1973, Miyazaki and Takahata worked on World Masterpiece Theater, which featured their animation series Heidi, Girl of the Alps, an adaptation of Johanna Spyri's Heidi. Zuiyō Eizō continued as Nippon Animation in July 1975. Miyazaki also directed the television series Future Boy Conan (1978), an adaptation of Alexander Key's The Incredible Tide.\n", + "\n", + "\n", + "=== Breakthrough films ===\n", + "Miyazaki left Nippon Animation in 1979, during the production of Anne of Green Gables; he provided scene design and organization on the first fifteen episodes. He moved to Telecom Animation Film, a subsidiary of TMS Entertainment, to direct his first feature anime film, The Castle of Cagliostro (1979), a Lupin III film. In his role at Telecom, Miyazaki helped train the second wave of employees. Miyazaki directed six episodes of Sherlock Hound in 1981, until issues with Sir Arthur Conan Doyle's estate led to a suspension in production; Miyazaki was busy with other projects by the time the issues were resolved, and the remaining episodes were directed by Kyosuke Mikuriya. They were broadcast from November 1984 to May 1985. Miyazaki also wrote the graphic novel The Journey of Shuna, inspired by the Tibetan folk tale \"Prince who became a dog\". The novel was published by Tokuma Shoten in June 1983, dramatized for radio broadcast in 1987, and published in English as Shuna's Journey in 2022. Hayao Miyazaki's Daydream Data Notes was also irregularly published from November 1984 to October 1994 in Model Graphix; selections of the stories received radio broadcast in 1995.\n", + "After the release of The Castle of Cagliostro, Miyazaki began working on his ideas for an animated film adaptation of Richard Corben's comic book Rowlf and pitched the idea to Yutaka Fujioka at TMS. In November 1980, a proposal was drawn up to acquire the film rights. Around that time, Miyazaki was also approached for a series of magazine articles by the editorial staff of Animage. During subsequent conversations, he showed his sketchbooks and discussed basic outlines for envisioned animation projects with editors Toshio Suzuki and Osamu Kameyama, who saw the potential for collaboration on their development into animation. Two projects were proposed: Warring States Demon Castle (戦国魔城, Sengoku ma-jō), to be set in the Sengoku period; and the adaptation of Corben's Rowlf. Both were rejected, as the company was unwilling to fund anime projects not based on existing manga, and the rights for the adaptation of Rowlf could not be secured. An agreement was reached that Miyazaki could start developing his sketches and ideas into a manga for the magazine with the proviso that it would never be made into a film. The manga—titled Nausicaä of the Valley of the Wind—ran from February 1982 to March 1994. The story, as re-printed in the tankōbon volumes, spans seven volumes for a combined total of 1060 pages. Miyazaki drew the episodes primarily in pencil, and it was printed monochrome in sepia-toned ink. Miyazaki resigned from Telecom Animation Film in November 1982.\n", + "\n", + "Following the completion of Nausicaä of the Valley of the Wind's first two volumes, Suzuki and the other editors of Animage encouraged Miyazaki to work on a film adaptation; some documentaries claim he began writing the manga after his film pitch was rejected, but Miyazaki said the manga came first. Miyazaki's imagination was sparked by the mercury poisoning of Minamata Bay and how nature responded and thrived in a poisoned environment, using it to create the film's polluted world. By this time, Miyazaki had moved to the animation studio Topcraft and was finding some of the staff to be unreliable. He eventually decided to bring on several of his previous collaborators for the film's production, including Takahata who would serve as producer. Pre-production began on May 31, 1983; Miyazaki encountered difficulties in creating the screenplay, with only sixteen chapters of the manga to work with. Takahata enlisted experimental and minimalist musician Joe Hisaishi to compose the film's score. Nausicaä of the Valley of the Wind was released on March 11, 1984. It grossed ¥1.48 billion at the box office, and made an additional ¥742 million in distribution income. It is often seen as Miyazaki's pivotal work, cementing his reputation as an animator. It was lauded for its positive portrayal of women, particularly that of main character Nausicaä. Several critics have labeled Nausicaä of the Valley of the Wind as possessing anti-war and feminist themes; Miyazaki argues otherwise, stating that he only wishes to entertain. The successful cooperation on the creation of the manga and the film laid the foundation for other collaborative projects. In April 1984, Miyazaki opened his own office in Suginami Ward, naming it Nibariki.\n", + "\n", + "\n", + "=== Studio Ghibli ===\n", + "\n", + "\n", + "==== Early films (1985–1996) ====\n", + "On June 15, 1985, Miyazaki and Takahata founded the animation production company Studio Ghibli as a subsidiary of Tokuma Shoten. Studio Ghibli's first film was Laputa: Castle in the Sky (1986), directed by Miyazaki. Some of the architecture in the film was also inspired by a Welsh mining town; Miyazaki witnessed the mining strike upon his first visit to Wales in 1984 and admired the miners' dedication to their work and community. Laputa was released on August 2, 1986, by the Toei Company. It sold around 775,000 tickets; Miyazaki and Suzuki expressed their disappointment with the film's box office figures. Miyazaki's following film, My Neighbor Totoro, was released alongside Takahata's Grave of the Fireflies in April 1988 to ensure Studio Ghibli's financial status. My Neighbor Totoro features the theme of the relationship between the environment and humanity, showing that harmony is the result of respecting the environment. While the film received critical acclaim, it was commercially unsuccessful at the box office. However, merchandising was successful, and the film was labeled as a cult classic.\n", + "In 1987, Studio Ghibli acquired the rights to create a film adaptation of Eiko Kadono's novel Kiki's Delivery Service. Miyazaki's work on My Neighbor Totoro prevented him from directing the adaptation; Sunao Katabuchi was chosen as director, and Nobuyuki Isshiki was hired as script writer. Miyazaki's dissatisfaction of Isshiki's first draft led him to make changes to the project, ultimately taking the role of director. Kadono was unhappy with the differences between the book and the screenplay. Miyazaki and Suzuki visited Kadono and invited her to the studio; she allowed the project to continue. The film was originally intended to be a 60-minute special, but expanded into a feature film after Miyazaki completed the storyboards and screenplay. Kiki's Delivery Service premiered on July 29, 1989. It earned ¥2.15 billion at the box office, and was the highest-grossing film in Japan in 1989.\n", + "From March to May 1989, Miyazaki's manga Hikōtei Jidai was published in the magazine Model Graphix. Miyazaki began production on a 45-minute in-flight film for Japan Airlines based on the manga; Suzuki ultimately extended the film into the feature-length film, titled Porco Rosso, as expectations grew. The outbreak of the Yugoslav Wars in 1991 affected Miyazaki, prompting a more sombre tone for the film; Miyazaki would later refer to the film as \"foolish\", as its mature tones were unsuitable for children. The film featured anti-war themes, which Miyazaki would later revisit. The airline remained a major investor in the film, resulting in its initial premiere as an in-flight film, prior to its theatrical release on July 18, 1992. The film was commercially successful and remained one of the highest-grossing films in Japan for several years.\n", + "Studio Ghibli set up its headquarters in Koganei, Tokyo in August 1992. In November 1992, two television spots directed by Miyazaki were broadcast by Nippon Television Network (NTV): Sora Iro no Tane, a 90-second spot adapted from the illustrated story Sora Iro no Tane by Rieko Nakagawa and Yuriko Omura; and Nandarou, a series of five advertisements featuring an undefinable creature. Miyazaki designed the storyboards and wrote the screenplay for Whisper of the Heart (1995), directed by Yoshifumi Kondō.\n", + "\n", + "\n", + "==== Global emergence (1997–2008) ====\n", + "Miyazaki began work on the initial storyboards for Princess Mononoke in August 1994, based on preliminary thoughts and sketches from the late 1970s. While experiencing writer's block during production, Miyazaki accepted a request for the creation of On Your Mark, a music video for the song of the same name by Chage and Aska. In the production of the video, Miyazaki experimented with computer animation to supplement traditional animation. On Your Mark premiered as a short before Whisper of the Heart. Despite the video's popularity, Suzuki said that it was not given \"100 percent\" focus.\n", + "\n", + "In May 1995, Miyazaki took a group of artists and animators to the ancient forests of Yakushima and the mountains of Shirakami-Sanchi, taking photographs and making sketches. The landscapes in the film were inspired by Yakushima. In Princess Mononoke, Miyazaki revisited the ecological and political themes of Nausicaä of the Valley of the Wind. Miyazaki supervised the 144,000 cels in the film, about 80,000 of which were key animation. Princess Mononoke was produced with an estimated budget of ¥2.35 billion (approximately US$23.5 million), making it the most expensive Japanese animated film at the time. Approximately fifteen minutes of the film uses computer animation: about five minutes uses techniques such as 3D rendering, digital composition, and texture mapping; the remaining ten minutes uses digital ink and paint. While the original intention was to digitally paint 5,000 of the film's frames, time constraints doubled this, though it remained below ten percent of the final film.\n", + "Upon its premiere on July 12, 1997, Princess Mononoke was critically acclaimed, becoming the first animated film to win the Japan Academy Film Prize for Picture of the Year. The film was also commercially successful, becoming the highest-grossing film in Japan for several months. Miramax Films purchased the film's distributions rights for North America; while it was largely unsuccessful at the box office, grossing about US$2.3 million, it was seen as the introduction of Studio Ghibli to global markets. Miyazaki claimed Princess Mononoke would be his final film. Tokuma Shoten merged with Studio Ghibli in June 1997. Miyazaki left Studio Ghibli on January 14, 1998, to create a new studio called Butaya, to be succeeded by Kondō; however, Kondō's death impacted Miyazaki, and he returned to Studio Ghibli on January 16, 1999.\n", + "Miyazaki's next film was conceived while on vacation at a mountain cabin with his family and five young girls who were family friends. Miyazaki realized that he had not created a film for 10-year-old girls, and set out to do so. He read shōjō manga magazines like Nakayoshi and Ribon for inspiration, but felt they only offered subjects on \"crushes and romance\", which is not what the girls \"held dear in their hearts\". He decided to produce the film about a female heroine whom they could look up to. Production of the film, titled Spirited Away, commenced in 2000 on a budget of ¥1.9 billion (US$15 million). As with Princess Mononoke, the staff experimented with computer animation, but kept the technology at a level to enhance the story, not to \"steal the show\". Spirited Away deals with symbols of human greed, symbolizing the 1980s Japanese asset price bubble, and a liminal journey through the realm of spirits. The film was released on July 20, 2001; it received critical acclaim, and is considered among the greatest films of the 2000s. It won the Japan Academy Film Prize for Picture of the Year, and the Academy Award for Best Animated Feature. The film was also commercially successful, earning ¥30.4 billion (US$289.1 million) at the box office. It became the highest-grossing film in Japan, a record it maintained for almost 20 years. Following the death of Tokuma in September 2000, Miyazaki served as the head of his funeral committee.\n", + "In September 2001, Studio Ghibli announced the production of Howl's Moving Castle, based on the novel by Diana Wynne Jones. Mamoru Hosoda of Toei Animation was originally selected to direct the film, but disagreements between Hosoda and Studio Ghibli executives led to the project's abandonment. After six months, Studio Ghibli resurrected the project. Miyazaki was inspired to direct the film upon reading Jones' novel, and was struck by the image of a castle moving around the countryside; the novel does not explain how the castle moved, which led to Miyazaki's designs. He traveled to Colmar and Riquewihr in Alsace, France, to study the architecture and the surroundings for the film's setting. Additional inspiration came from the concepts of future technology in Albert Robida's work. It was released on November 20, 2004, and received widespread critical acclaim. The film received the Osella Award for Technical Excellence at the 61st Venice International Film Festival, and was nominated for the Academy Award for Best Animated Feature. In Japan, the film grossed a record $14.5 million in its first week of release. It remains among the highest-grossing films in Japan, with a worldwide gross of over ¥19.3 billion. Miyazaki received the honorary Golden Lion for Lifetime Achievement award at the 62nd Venice International Film Festival in 2005.\n", + "In March 2005, Studio Ghibli split from Tokuma Shoten. In the 1980s, Miyazaki had contacted Ursula K. Le Guin expressing interest in producing an adaptation of her Earthsea novels; unaware of Miyazaki's work, Le Guin declined. Upon watching My Neighbor Totoro several years later, Le Guin expressed approval to the concept of the adaptation. She met with Suzuki in August 2005, who wanted Miyazaki's son Goro to direct the film, as Miyazaki had wished to retire. Disappointed that Miyazaki was not directing, but under the impression that he would supervise his son's work, Le Guin approved of the film's production. Miyazaki later publicly opposed and criticized Gorō's appointment as director. Upon Miyazaki's viewing of the film, he wrote a message for his son: \"It was made honestly, so it was good\".\n", + "Miyazaki designed the covers for several manga novels in 2006, including A Trip to Tynemouth; he also worked as editor, and created a short manga for the book. Miyazaki's next film, Ponyo, began production in May 2006. It was initially inspired by \"The Little Mermaid\" by Hans Christian Andersen, though began to take its own form as production continued. Miyazaki aimed for the film to celebrate the innocence and cheerfulness of a child's universe. He intended for it to only use traditional animation, and was intimately involved with the artwork. He preferred to draw the sea and waves himself, as he enjoyed experimenting. Ponyo features 170,000 frames—a record for Miyazaki. The film's seaside village was inspired by Tomonoura, a town in Setonaikai National Park, where Miyazaki stayed in 2005. The main character, Sōsuke, is based on Gorō. Following its release on July 19, 2008, Ponyo was critically acclaimed, receiving Animation of the Year at the 32nd Japan Academy Film Prize. The film was also a commercial success, earning ¥10 billion (US$93.2 million) in its first month and ¥15.5 billion by the end of 2008, placing it among the highest-grossing films in Japan.\n", + "\n", + "\n", + "==== Later films (2009–present) ====\n", + "\n", + "In early 2009, Miyazaki began writing a manga called Kaze Tachinu (風立ちぬ, The Wind Rises), telling the story of Mitsubishi A6M Zero fighter designer Jiro Horikoshi. The manga was first published in two issues of the Model Graphix magazine, published on February 25 and March 25, 2009. Miyazaki later co-wrote the screenplay for Arrietty (2010) and From Up on Poppy Hill (2011), directed by Hiromasa Yonebayashi and Gorō Miyazaki respectively. Miyazaki wanted his next film to be a sequel to Ponyo, but Suzuki convinced him to instead adapt Kaze Tachinu to film. In November 2012, Studio Ghibli announced the production of The Wind Rises, based on Kaze Tachinu, to be released alongside Takahata's The Tale of the Princess Kaguya.\n", + "Miyazaki was inspired to create The Wind Rises after reading a quote from Horikoshi: \"All I wanted to do was to make something beautiful\". Several scenes in The Wind Rises were inspired by Tatsuo Hori's novel The Wind Has Risen (風立ちぬ), in which Hori wrote about his life experiences with his fiancée before she died from tuberculosis. The female lead character's name, Naoko Satomi, was borrowed from Hori's novel Naoko (菜穂子). The Wind Rises continues to reflect Miyazaki's pacifist stance, continuing the themes of his earlier works, despite stating that condemning war was not the intention of the film. The film premiered on July 20, 2013, and received critical acclaim; it was named Animation of the Year at the 37th Japan Academy Film Prize, and was nominated for Best Animated Feature at the 86th Academy Awards. It was also commercially successful, grossing ¥11.6 billion (US$110 million) at the Japanese box office, becoming the highest-grossing film in Japan in 2013.\n", + "In September 2013, Miyazaki announced that he was retiring from the production of feature films due to his age, but wished to continue working on the displays at the Studio Ghibli Museum. Miyazaki was awarded the Academy Honorary Award at the Governors Awards in November 2014. He developed Boro the Caterpillar, an animated short film which was first discussed during pre-production for Princess Mononoke. It was screened exclusively at the Studio Ghibli Museum in July 2017. Around this time, Miyazaki was working on a manga titled Teppo Samurai. In February 2019, a four-part documentary was broadcast on the NHK network titled 10 Years with Hayao Miyazaki, documenting production of his films in his private studio. In 2019, Miyazaki approved a musical adaptation of Nausicaä of the Valley of the Wind, as it was performed by a kabuki troupe.\n", + "In August 2016, Miyazaki proposed a new feature-length film, Kimi-tachi wa Dō Ikiru ka (titled The Boy and the Heron in English), on which he began animation work without receiving official approval. The film opened in Japanese theaters on July 14, 2023. It was preceded by a minimal marketing campaign, forgoing trailers, commercials, and advertisements, a response from Suzuki to his perceived oversaturation of marketing materials in mainstream films. Despite claims that The Boy and the Heron would be Miyazaki's final film, Studio Ghibli vice president Junichi Nishioka said in September 2023 that Miyazaki continued to attend the office daily to plan his next film. Suzuki said he could no longer convince Miyazaki to retire. The Boy and the Heron won Miyazaki his second Academy Award for Best Animated Feature at the 96th Academy Awards, becoming the oldest director to do so; Miyazaki did not attend the show due to his advanced age.\n", + "\n", + "\n", + "== Views ==\n", + "\n", + "Miyazaki has often criticized the state of the animation industry, stating that some animators lack a foundational understanding of their subjects and do not prioritize realism. He is particularly critical of Japanese animation, saying that anime is \"produced by humans who can't stand looking at other humans ... that's why the industry is full of otaku !\". He has frequently criticized otaku, including \"fanatics\" of guns and fighter aircraft, declaring it a \"fetish\" and refusing to identify himself as such. He bemoaned the state of Disney animated films in 1988, saying \"they show nothing but contempt for the audience\".\n", + "In 2013, Miyazaki criticized Japanese Prime Minister Shinzo Abe's policies and the proposed Constitutional amendment that would allow Abe to revise the clause outlawing war as a means to settle international disputes. Miyazaki felt Abe wished to \"leave his name in history as a great man who revised the Constitution and its interpretation\", describing it as \"despicable\" and stating \"People who don't think enough shouldn't meddle with the constitution\". In 2015, Miyazaki disapproved Abe's denial of Japan's military aggression, stating Japan \"should clearly say that [they] inflicted enormous damage on China and express deep remorse over it\". He felt the government should give a \"proper apology\" to Korean comfort women who were forced to service the Japanese army during World War II and suggested the Senkaku Islands be \"split in half\" or controlled by both Japan and China. After the release of The Wind Rises in 2013, some online critics labeled Miyazaki a \"traitor\" and \"anti-Japanese\", describing the film as overly \"left-wing\"; Miyazaki recognized leftist values in his movies, citing his influence by and appreciation of communism as defined by Karl Marx, but criticized the Soviet Union's political system.\n", + "In 2003, Miyazaki refused to attend the 75th Academy Awards in Hollywood in protest of the United States's involvement in the Iraq War, and later said he \"didn't want to visit a country that was bombing Iraq\". He did not publicly express this opinion at the request of his producer until 2009 when he lifted his boycott and attended San Diego Comic Con International as a favor to his friend John Lasseter. Miyazaki also expressed his opinion about the terrorist attack at the offices of the French satirical magazine Charlie Hebdo, criticizing the magazine's decision to publish the content cited as the catalyst for the incident; he felt caricatures should be made of politicians, not cultures. In November 2016, Miyazaki stated that he believed \"many of the people who voted for Brexit and Trump\" were affected by the increase in unemployment due to companies \"building cars in Mexico because of low wages and [selling] them in the US\". He did not think that Donald Trump would be elected president, calling it \"a terrible thing\", and said that Trump's political opponent Hillary Clinton was \"terrible as well\".\n", + "\n", + "\n", + "== Themes ==\n", + "Miyazaki's works are characterized by the recurrence of themes such as feminism, environmentalism, pacifism, love, and family. His narratives are also notable for not pitting a hero against an unsympathetic antagonist; Miyazaki felt Spirited Away's Chihiro \"manages not because she has destroyed the 'evil', but because she has acquired the ability to survive\".\n", + "Miyazaki's films often emphasize environmentalism and the Earth's fragility. Margaret Talbot stated that Miyazaki dislikes modern technology, and believes much of modern culture is \"thin and shallow and fake\"; he anticipates a time with \"no more high-rises\". Miyazaki felt frustrated growing up in the Shōwa period from 1955 to 1965 because \"nature—the mountains and rivers—was being destroyed in the name of economic progress\". Peter Schellhase of The Imaginative Conservative identified that several antagonists of Miyazaki's films \"attempt to dominate nature in pursuit of political domination, and are ultimately destructive to both nature and human civilization\". Miyazaki is critical of exploitation under both communism and capitalism, as well as globalization and its effects on modern life, believing that \"a company is common property of the people that work there\". Ram Prakash Dwivedi identified values of Mahatma Gandhi in the films of Miyazaki.\n", + "Several of Miyazaki's films feature anti-war themes. Daisuke Akimoto of Animation Studies categorized Porco Rosso as \"anti-war propaganda\" and felt the protagonist, Porco, transforms into a pig partly due to his extreme distaste of militarism. Akimoto also argues that The Wind Rises reflects Miyazaki's \"antiwar pacifism\", despite the latter stating that the film does not attempt to \"denounce\" war. Schellhase also identifies Princess Mononoke as a pacifist film due to the protagonist, Ashitaka; instead of joining the campaign of revenge against humankind, as his ethnic history would lead him to do, Ashitaka strives for peace. David Loy and Linda Goodhew argue that both Nausicaä of the Valley of the Wind and Princess Mononoke do not depict traditional evil, but the Buddhist roots of evil: greed, ill will, and delusion; according to Buddhism, the roots of evil must transform into \"generosity, loving-kindness and wisdom\" in order to overcome suffering, and both Nausicaä and Ashitaka accomplish this. When characters in Miyazaki's films are forced to engage in violence, it is shown as being a difficult task; in Howl's Moving Castle, Howl is forced to fight an inescapable battle in defense of those he loves, and it almost destroys him, though he is ultimately saved by Sophie's love and bravery.\n", + "Suzuki described Miyazaki as a feminist in reference to his attitude to female workers. Miyazaki has described his female characters as \"brave, self-sufficient girls that don't think twice about fighting for what they believe in with all their heart\", stating that they may \"need a friend, or a supporter, but never a saviour\" and that \"any woman is just as capable of being a hero as any man\". Nausicaä of the Valley of the Wind was lauded for its positive portrayal of women, particularly protagonist Nausicaä. Schellhase noted that the female characters in Miyazaki's films are not objectified or sexualized, and possess complex and individual characteristics absent from Hollywood productions. Schellhase also identified a \"coming of age\" element for the heroines in Miyazaki's films, as they each discover \"individual personality and strengths\". Gabrielle Bellot of The Atlantic wrote that, in his films, Miyazaki \"shows a keen understanding of the complexities of what it might mean to be a woman\". In particular, Bellot cites Nausicaä of the Valley of the Wind, praising the film's challenging of gender expectations, and the strong and independent nature of Nausicaä. Bellot also noted that Princess Mononoke's San represents the \"conflict between selfhood and expression\".\n", + "Miyazaki is concerned with the sense of wonder in young people, seeking to maintain themes of love and family in his films. Michael Toscano of Curator found that Miyazaki \"fears Japanese children are dimmed by a culture of overconsumption, overprotection, utilitarian education, careerism, techno-industrialism, and a secularism that is swallowing Japan's native animism\". Schellhase wrote that several of Miyazaki's works feature themes of love and romance, but felt emphasis is placed on \"the way lonely and vulnerable individuals are integrated into relationships of mutual reliance and responsibility, which generally benefit everyone around them\". He also found that many of the protagonists in Miyazaki's films present an idealized image of families, whereas others are dysfunctional.\n", + "\n", + "\n", + "== Creation process and influences ==\n", + "Miyazaki forgoes traditional screenplays in his productions, instead developing the film's narrative as he designs the storyboards. \"We never know where the story will go but we just keep working on the film as it develops,\" he said. In each of his films, Miyazaki has employed traditional animation methods, drawing each frame by hand; computer-generated imagery has been employed in several of his later films, beginning with Princess Mononoke, to \"enrich the visual look\", though he ensures that each film can \"retain the right ratio between working by hand and computer ... and still be able to call my films 2D\". He oversees every frame of his films. For character designs, Miyazaki draws original drafts used by animation directors to create reference sheets, which are then corrected by Miyazaki in his style.\n", + "Miyazaki has cited several Japanese artists as his influences, including Sanpei Shirato, Osamu Tezuka, Soji Yamakawa, and Isao Takahata. A number of Western authors have also influenced his works, including Frédéric Back, Lewis Carroll, Roald Dahl, Jean Giraud, Paul Grimault, Ursula K. Le Guin, and Yuri Norstein, as well as animation studio Aardman Animations (specifically the works of Nick Park). Specific works that have influenced Miyazaki include Animal Farm (1945), The Snow Queen (1957), and The King and the Mockingbird (1980); The Snow Queen is said to be the true catalyst for Miyazaki's filmography, influencing his training and work. When animating young children, Miyazaki often takes inspiration from his friends' children, as well as memories of his own childhood.\n", + "\n", + "\n", + "== Personal life ==\n", + "\n", + "Miyazaki married fellow animator Akemi Ōta in October 1965; the two had met while colleagues at Toei Animation. The couple have two sons: Goro, born in January 1967, and Keisuke, born in April 1969. Miyazaki felt that becoming a father changed him, as he tried to produce work that would please his children. Miyazaki initially fulfilled a promise to his wife that they would both continue to work after Goro's birth, dropping him off at preschool for the day; however, upon seeing Goro's exhaustion walking home one day, Miyazaki decided that they could not continue, and his wife stayed at home to raise their children. Miyazaki's dedication to his work harmed his relationship with his children, as he was often absent. Goro watched his father's works in an attempt to \"understand\" him, since the two rarely talked. Miyazaki said that he \"tried to be a good father, but in the end [he] wasn't a very good parent\". During the production of Tales from Earthsea in 2006, Goro said that his father \"gets zero marks as a father but full marks as a director of animated films\".\n", + "Goro worked at a landscape design firm before beginning to work at the Ghibli Museum; he designed the garden on its rooftop and eventually became its curator. Keisuke studied forestry at Shinshu University and works as a wood artist; he designed a woodcut print that appears in Whisper of the Heart. Miyazaki's niece, Mei Okuyama, who was the inspiration behind the character Mei in My Neighbor Totoro, is married to animation artist Daisuke Tsutsumi.\n", + "\n", + "\n", + "== Legacy ==\n", + "Miyazaki was described as the \"godfather of animation in Japan\" by BBC's Tessa Wong in 2016, citing his craftsmanship and humanity, the themes of his films, and his inspiration to younger artists. Courtney Lanning of Arkansas Democrat-Gazette named him one of the world's greatest animators, comparing him to Osamu Tezuka and Walt Disney. Swapnil Dhruv Bose of Far Out Magazine wrote that Miyazaki's work \"has shaped not only the future of animation but also filmmaking in general\", and that it helped \"generation after generation of young viewers to observe the magic that exists in the mundane\". Richard James Havis of South China Morning Post called him a \"genius ... who sets exacting standards for himself, his peers and studio staff\". Paste's Toussaint Egan described Miyazaki as \"one of anime's great auteurs\", whose \"stories of such singular thematic vision and unmistakable aesthetic\" captured viewers otherwise unfamiliar with anime. Miyazaki became the subject of an exhibit at the Academy Museum of Motion Pictures in Los Angeles in 2021, featuring over 400 objects from his films.\n", + "Miyazaki has frequently been cited as an inspiration to numerous animators, directors and writers around the world, including Wes Anderson, James Cameron, Dean DeBlois, Guillermo del Toro, Pete Docter, Mamoru Hosoda, Bong Joon-ho, Travis Knight, John Lasseter, Nick Park, Henry Selick, Makoto Shinkai, and Steven Spielberg. Glen Keane said Miyazaki is a \"huge influence\" on Walt Disney Animation Studios and has been \"part of our heritage\" ever since The Rescuers Down Under (1990). The Disney Renaissance era was also prompted by competition with the development of Miyazaki's films. Artists from Pixar and Aardman Studios signed a tribute stating, \"You're our inspiration, Miyazaki-san!\" He has also been cited as inspiration for video game designers including Shigeru Miyamoto on The Legend of Zelda and Hironobu Sakaguchi on Final Fantasy, as well as the television series Avatar: The Last Airbender, and the video game Ori and the Blind Forest (2015).\n", + "Studio Ghibli has searched for some time for Miyazaki and Suzuki's successor to lead the studio; Kondō, the director of Whisper of the Heart, was initially considered, but died from a sudden heart attack in 1998. Some candidates were considered by 2023—including Miyazaki's son Goro, who declined—but the studio was not able to find a successor.\n", + "\n", + "\n", + "== Selected filmography ==\n", + "\n", + "The Castle of Cagliostro (1979)\n", + "Nausicaä of the Valley of the Wind (1984)\n", + "Laputa: Castle in the Sky (1986)\n", + "My Neighbor Totoro (1988)\n", + "Kiki's Delivery Service (1989)\n", + "Porco Rosso (1992)\n", + "Princess Mononoke (1997)\n", + "Spirited Away (2001)\n", + "Howl's Moving Castle (2004)\n", + "Ponyo (2008)\n", + "The Wind Rises (2013)\n", + "The Boy and the Heron (2023)\n", + "\n", + "\n", + "== Awards and nominations ==\n", + "\n", + "Miyazaki won the Ōfuji Noburō Award at the Mainichi Film Awards for The Castle of Cagliostro (1979), Nausicaä of the Valley of the Wind (1984), Laputa: Castle in the Sky (1986), and My Neighbor Totoro (1988), and the Mainichi Film Award for Best Animation Film for Kiki's Delivery Service (1989), Porco Rosso (1992), Princess Mononoke (1997), Spirited Away (2001), and Whale Hunt (2001). Spirited Away and The Boy and the Heron were awarded the Academy Award for Best Animated Feature, while Howl's Moving Castle (2004) and The Wind Rises (2013) received nominations. He was named a Person of Cultural Merit by the Japanese government in November 2012, for outstanding cultural contributions. In 2024, Time named him one of the 100 most influential people in the world, and Gold House honored him on its Most Impactful Asians A100 list. His other accolades include several Annie Awards, Japan Academy Film Prizes, Kinema Junpo Awards, and Tokyo Anime Awards.\n", + "\n", + "\n", + "== Notes ==\n", + "\n", + "\n", + "== References ==\n", + "\n", + "\n", + "== Sources ==\n", + "\n", + "\n", + "== External links ==\n", + "\n", + "Studio Ghibli (in Japanese)\n", + "Hayao Miyazaki at Anime News Network's encyclopedia\n", + "Hayao Miyazaki at IMDb\n", + "Hayao Miyazaki at Library of Congress, with 14 library catalogue records\n", + "\u001b[0m> Running step 015c82f1-c7ff-4de0-9930-ab2467424c1c. Step input: None\n", + "\u001b[1;3;38;5;200mThought: I have found detailed information about Hayao Miyazaki, his works, and his life in Tokyo. Now, I can provide a summary of the relevant details related to Tokyo and his movies like \"Spirited Away.\"\n", + "Answer: Hayao Miyazaki, a renowned Japanese animator, filmmaker, and manga artist, was born in Tokyo City, Japan. He expressed interest in manga and animation from a young age and joined Toei Animation in 1963. Miyazaki has contributed to various notable films and series, including \"The Castle of Cagliostro,\" \"Princess Mononoke,\" \"Spirited Away,\" and more. He co-founded Studio Ghibli in 1985, where he directed several successful films like \"My Neighbor Totoro,\" \"Kiki's Delivery Service,\" and \"Porco Rosso.\" His films often explore themes such as humanity's relationship with nature, the importance of art and craftsmanship, and the challenges of maintaining pacifism in a violent world.\n", + "\n", + "One of Miyazaki's most famous works, \"Spirited Away,\" became the highest-grossing film in Japanese history and won the Academy Award for Best Animated Feature. The film is known for its rich storytelling and captivating animation. Miyazaki's works are characterized by strong female protagonists, morally ambiguous antagonists, and themes of environmentalism, love, and family. His films have received critical acclaim and commercial success, influencing a generation of animators and filmmakers worldwide.\n", + "\n", + "Miyazaki's legacy extends beyond his films, as he has been recognized for his craftsmanship, thematic depth, and influence on the animation industry. His dedication to hand-drawn animation and storytelling has inspired numerous artists and filmmakers globally.\n", + "\u001b[0m\u001b[33mtrip_specialist\u001b[0m (to chat_manager):\n", + "\n", + "Hayao Miyazaki, a renowned Japanese animator, filmmaker, and manga artist, was born in Tokyo City, Japan. He expressed interest in manga and animation from a young age and joined Toei Animation in 1963. Miyazaki has contributed to various notable films and series, including \"The Castle of Cagliostro,\" \"Princess Mononoke,\" \"Spirited Away,\" and more. He co-founded Studio Ghibli in 1985, where he directed several successful films like \"My Neighbor Totoro,\" \"Kiki's Delivery Service,\" and \"Porco Rosso.\" His films often explore themes such as humanity's relationship with nature, the importance of art and craftsmanship, and the challenges of maintaining pacifism in a violent world.\n", + "\n", + "One of Miyazaki's most famous works, \"Spirited Away,\" became the highest-grossing film in Japanese history and won the Academy Award for Best Animated Feature. The film is known for its rich storytelling and captivating animation. Miyazaki's works are characterized by strong female protagonists, morally ambiguous antagonists, and themes of environmentalism, love, and family. His films have received critical acclaim and commercial success, influencing a generation of animators and filmmakers worldwide.\n", + "\n", + "Miyazaki's legacy extends beyond his films, as he has been recognized for his craftsmanship, thematic depth, and influence on the animation industry. His dedication to hand-drawn animation and storytelling has inspired numerous artists and filmmakers globally.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Admin\n", + "\u001b[0m\n" + ] + } + ], "source": [ "chat_result = user_proxy.initiate_chat(\n", " manager,\n", @@ -217,7 +390,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.9" + "version": "3.10.14" } }, "nbformat": 4, diff --git a/notebook/agentchat_nested_chats_chess_altmodels.ipynb b/notebook/agentchat_nested_chats_chess_altmodels.ipynb new file mode 100644 index 000000000000..69d3edbcfb50 --- /dev/null +++ b/notebook/agentchat_nested_chats_chess_altmodels.ipynb @@ -0,0 +1,584 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Conversational Chess using non-OpenAI clients\n", + "\n", + "This notebook provides tips for using non-OpenAI models when using functions/tools.\n", + "\n", + "The code is based on [this notebook](/docs/notebooks/agentchat_nested_chats_chess),\n", + "which provides a detailed look at nested chats for tool use. Please refer to that\n", + "notebook for more on nested chats as this will be concentrated on tweaks to\n", + "improve performance with non-OpenAI models.\n", + "\n", + "The notebook represents a chess game between two players with a nested chat to\n", + "determine the available moves and select a move to make.\n", + "\n", + "This game contains a couple of functions/tools that the LLMs must use correctly by the\n", + "LLMs:\n", + "- `get_legal_moves` to get a list of current legal moves.\n", + "- `make_move` to make a move.\n", + "\n", + "Two agents will be used to represent the white and black players, each associated with\n", + "a different LLM cloud provider and model:\n", + "- Anthropic's Sonnet 3.5 will be Player_White\n", + "- Mistral's Mixtral 8x7B (using Together.AI) will be Player_Black\n", + "\n", + "As this involves function calling, we use larger, more capable, models from these providers.\n", + "\n", + "The nested chat will be supported be a board proxy agent who is set up to execute\n", + "the tools and manage the game.\n", + "\n", + "Tips to improve performance with these non-OpenAI models will be noted throughout **in bold**." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Installation\n", + "\n", + "First, you need to install the `pyautogen` and `chess` packages to use AutoGen. We'll include Anthropic and Together.AI libraries." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "! pip install -qqq pyautogen[anthropic,together] chess" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setting up LLMs\n", + "\n", + "We'll use the Anthropic (`api_type` is `anthropic`) and Together.AI (`api_type` is `together`) client classes, with their respective models, which both support function calling." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "import chess\n", + "import chess.svg\n", + "from IPython.display import display\n", + "from typing_extensions import Annotated\n", + "\n", + "from autogen import ConversableAgent, register_function\n", + "\n", + "# Let's set our two player configs, specifying clients and models\n", + "\n", + "# Anthropic's Sonnet for player white\n", + "player_white_config_list = [\n", + " {\n", + " \"api_type\": \"anthropic\",\n", + " \"model\": \"claude-3-5-sonnet-20240620\",\n", + " \"api_key\": os.getenv(\"ANTHROPIC_API_KEY\"),\n", + " \"cache_seed\": None,\n", + " },\n", + "]\n", + "\n", + "# Mistral's Mixtral 8x7B for player black (through Together.AI)\n", + "player_black_config_list = [\n", + " {\n", + " \"api_type\": \"together\",\n", + " \"model\": \"mistralai/Mixtral-8x7B-Instruct-v0.1\",\n", + " \"api_key\": os.environ.get(\"TOGETHER_API_KEY\"),\n", + " \"cache_seed\": None,\n", + " },\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We'll setup game variables and the two functions for getting the available moves and then making a move." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize the board.\n", + "board = chess.Board()\n", + "\n", + "# Keep track of whether a move has been made.\n", + "made_move = False\n", + "\n", + "\n", + "def get_legal_moves() -> Annotated[\n", + " str,\n", + " \"Call this tool to list of all legal chess moves on the board, output is a list in UCI format, e.g. e2e4,e7e5,e7e8q.\",\n", + "]:\n", + " return \"Possible moves are: \" + \",\".join([str(move) for move in board.legal_moves])\n", + "\n", + "\n", + "def make_move(\n", + " move: Annotated[\n", + " str,\n", + " \"Call this tool to make a move after you have the list of legal moves and want to make a move. Takes UCI format, e.g. e2e4 or e7e5 or e7e8q.\",\n", + " ]\n", + ") -> Annotated[str, \"Result of the move.\"]:\n", + " move = chess.Move.from_uci(move)\n", + " board.push_uci(str(move))\n", + " global made_move\n", + " made_move = True\n", + " # Display the board.\n", + " display(\n", + " chess.svg.board(board, arrows=[(move.from_square, move.to_square)], fill={move.from_square: \"gray\"}, size=200)\n", + " )\n", + " # Get the piece name.\n", + " piece = board.piece_at(move.to_square)\n", + " piece_symbol = piece.unicode_symbol()\n", + " piece_name = (\n", + " chess.piece_name(piece.piece_type).capitalize()\n", + " if piece_symbol.isupper()\n", + " else chess.piece_name(piece.piece_type)\n", + " )\n", + " return f\"Moved {piece_name} ({piece_symbol}) from {chess.SQUARE_NAMES[move.from_square]} to {chess.SQUARE_NAMES[move.to_square]}.\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Creating agents\n", + "\n", + "Our main player agents are created next, with a few tweaks to help our models play:\n", + "\n", + "- Explicitly **telling agents their names** (as the name field isn't sent to the LLM).\n", + "- Providing simple instructions on the **order of functions** (not all models will need it).\n", + "- Asking the LLM to **include their name in the response** so the message content will include their names, helping the LLM understand who has made which moves.\n", + "- Ensure **no spaces are in the agent names** so that their name is distinguishable in the conversation.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "player_white = ConversableAgent(\n", + " name=\"Player_White\",\n", + " system_message=\"You are a chess player and you play as white, your name is 'Player_White'. \"\n", + " \"First call the function get_legal_moves() to get list of legal moves. \"\n", + " \"Then call the function make_move(move) to make a move. \"\n", + " \"Then tell Player_Black you have made your move and it is their turn. \"\n", + " \"Make sure you tell Player_Black you are Player_White.\",\n", + " llm_config={\"config_list\": player_white_config_list, \"cache_seed\": None},\n", + ")\n", + "\n", + "player_black = ConversableAgent(\n", + " name=\"Player_Black\",\n", + " system_message=\"You are a chess player and you play as black, your name is 'Player_Black'. \"\n", + " \"First call the function get_legal_moves() to get list of legal moves. \"\n", + " \"Then call the function make_move(move) to make a move. \"\n", + " \"Then tell Player_White you have made your move and it is their turn. \"\n", + " \"Make sure you tell Player_White you are Player_Black.\",\n", + " llm_config={\"config_list\": player_black_config_list, \"cache_seed\": None},\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we create a proxy agent that will be used to move the pieces on the board." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "# Check if the player has made a move, and reset the flag if move is made.\n", + "def check_made_move(msg):\n", + " global made_move\n", + " if made_move:\n", + " made_move = False\n", + " return True\n", + " else:\n", + " return False\n", + "\n", + "\n", + "board_proxy = ConversableAgent(\n", + " name=\"Board_Proxy\",\n", + " llm_config=False,\n", + " # The board proxy will only terminate the conversation if the player has made a move.\n", + " is_termination_msg=check_made_move,\n", + " # The auto reply message is set to keep the player agent retrying until a move is made.\n", + " default_auto_reply=\"Please make a move.\",\n", + " human_input_mode=\"NEVER\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Our functions are then assigned to the agents so they can be passed to the LLM to choose from.\n", + "\n", + "We have tweaked the descriptions to provide **more guidance on when** to use it." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "register_function(\n", + " make_move,\n", + " caller=player_white,\n", + " executor=board_proxy,\n", + " name=\"make_move\",\n", + " description=\"Call this tool to make a move after you have the list of legal moves.\",\n", + ")\n", + "\n", + "register_function(\n", + " get_legal_moves,\n", + " caller=player_white,\n", + " executor=board_proxy,\n", + " name=\"get_legal_moves\",\n", + " description=\"Call this to get a legal moves before making a move.\",\n", + ")\n", + "\n", + "register_function(\n", + " make_move,\n", + " caller=player_black,\n", + " executor=board_proxy,\n", + " name=\"make_move\",\n", + " description=\"Call this tool to make a move after you have the list of legal moves.\",\n", + ")\n", + "\n", + "register_function(\n", + " get_legal_moves,\n", + " caller=player_black,\n", + " executor=board_proxy,\n", + " name=\"get_legal_moves\",\n", + " description=\"Call this to get a legal moves before making a move.\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Almost there, we now create nested chats between players and the board proxy agent to work out the available moves and make the move." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "player_white.register_nested_chats(\n", + " trigger=player_black,\n", + " chat_queue=[\n", + " {\n", + " # The initial message is the one received by the player agent from\n", + " # the other player agent.\n", + " \"sender\": board_proxy,\n", + " \"recipient\": player_white,\n", + " # The final message is sent to the player agent.\n", + " \"summary_method\": \"last_msg\",\n", + " }\n", + " ],\n", + ")\n", + "\n", + "player_black.register_nested_chats(\n", + " trigger=player_white,\n", + " chat_queue=[\n", + " {\n", + " # The initial message is the one received by the player agent from\n", + " # the other player agent.\n", + " \"sender\": board_proxy,\n", + " \"recipient\": player_black,\n", + " # The final message is sent to the player agent.\n", + " \"summary_method\": \"last_msg\",\n", + " }\n", + " ],\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Playing the game\n", + "\n", + "Now the game can begin!" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mPlayer_Black\u001b[0m (to Player_White):\n", + "\n", + "Let's play chess! Your move.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[34m\n", + "********************************************************************************\u001b[0m\n", + "\u001b[34mStarting a new chat....\u001b[0m\n", + "\u001b[34m\n", + "********************************************************************************\u001b[0m\n", + "\u001b[33mBoard_Proxy\u001b[0m (to Player_White):\n", + "\n", + "Let's play chess! Your move.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mPlayer_White\u001b[0m (to Board_Proxy):\n", + "\n", + "Certainly! I'd be happy to play chess with you. As White, I'll make the first move. Let me start by checking the legal moves available to me.\n", + "\u001b[32m***** Suggested tool call (toolu_015sLMucefMVqS5ZNyWVGjgu): get_legal_moves *****\u001b[0m\n", + "Arguments: \n", + "{}\n", + "\u001b[32m*********************************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION get_legal_moves...\u001b[0m\n", + "\u001b[33mBoard_Proxy\u001b[0m (to Player_White):\n", + "\n", + "\u001b[33mBoard_Proxy\u001b[0m (to Player_White):\n", + "\n", + "\u001b[32m***** Response from calling tool (toolu_015sLMucefMVqS5ZNyWVGjgu) *****\u001b[0m\n", + "Possible moves are: g1h3,g1f3,b1c3,b1a3,h2h3,g2g3,f2f3,e2e3,d2d3,c2c3,b2b3,a2a3,h2h4,g2g4,f2f4,e2e4,d2d4,c2c4,b2b4,a2a4\n", + "\u001b[32m***********************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mPlayer_White\u001b[0m (to Board_Proxy):\n", + "\n", + "Thank you for initiating a game of chess! As Player_White, I'll make the first move. After analyzing the legal moves, I've decided to make a classic opening move.\n", + "\u001b[32m***** Suggested tool call (toolu_01VjmBhHcGw5RTRKYC4Y5MeV): make_move *****\u001b[0m\n", + "Arguments: \n", + "{\"move\": \"e2e4\"}\n", + "\u001b[32m***************************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION make_move...\u001b[0m\n" + ] + }, + { + "data": { + "image/svg+xml": [ + "
r n b q k b n r\n",
+       "p p p p p p p p\n",
+       ". . . . . . . .\n",
+       ". . . . . . . .\n",
+       ". . . . P . . .\n",
+       ". . . . . . . .\n",
+       "P P P P . P P P\n",
+       "R N B Q K B N R
" + ], + "text/plain": [ + "'
r n b q k b n r\\np p p p p p p p\\n. . . . . . . .\\n. . . . . . . .\\n. . . . P . . .\\n. . . . . . . .\\nP P P P . P P P\\nR N B Q K B N R
'" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mBoard_Proxy\u001b[0m (to Player_White):\n", + "\n", + "\u001b[33mBoard_Proxy\u001b[0m (to Player_White):\n", + "\n", + "\u001b[32m***** Response from calling tool (toolu_01VjmBhHcGw5RTRKYC4Y5MeV) *****\u001b[0m\n", + "Moved pawn (♙) from e2 to e4.\n", + "\u001b[32m***********************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mPlayer_White\u001b[0m (to Board_Proxy):\n", + "\n", + "Hello, Player_Black! I'm Player_White, and I've just made my move. I've chosen to play the classic opening move e2e4, moving my king's pawn forward two squares. This opens up lines for both my queen and king's bishop, and stakes a claim to the center of the board. It's now your turn to make a move. Good luck!\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mPlayer_White\u001b[0m (to Player_Black):\n", + "\n", + "Hello, Player_Black! I'm Player_White, and I've just made my move. I've chosen to play the classic opening move e2e4, moving my king's pawn forward two squares. This opens up lines for both my queen and king's bishop, and stakes a claim to the center of the board. It's now your turn to make a move. Good luck!\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[34m\n", + "********************************************************************************\u001b[0m\n", + "\u001b[34mStarting a new chat....\u001b[0m\n", + "\u001b[34m\n", + "********************************************************************************\u001b[0m\n", + "\u001b[33mBoard_Proxy\u001b[0m (to Player_Black):\n", + "\n", + "Hello, Player_Black! I'm Player_White, and I've just made my move. I've chosen to play the classic opening move e2e4, moving my king's pawn forward two squares. This opens up lines for both my queen and king's bishop, and stakes a claim to the center of the board. It's now your turn to make a move. Good luck!\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mPlayer_Black\u001b[0m (to Board_Proxy):\n", + "\n", + "\u001b[32m***** Suggested tool call (call_z6jagiqn59m784w1n0zhmiop): get_legal_moves *****\u001b[0m\n", + "Arguments: \n", + "{}\n", + "\u001b[32m********************************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION get_legal_moves...\u001b[0m\n", + "\u001b[33mBoard_Proxy\u001b[0m (to Player_Black):\n", + "\n", + "\u001b[33mBoard_Proxy\u001b[0m (to Player_Black):\n", + "\n", + "\u001b[32m***** Response from calling tool (call_z6jagiqn59m784w1n0zhmiop) *****\u001b[0m\n", + "Possible moves are: g8h6,g8f6,b8c6,b8a6,h7h6,g7g6,f7f6,e7e6,d7d6,c7c6,b7b6,a7a6,h7h5,g7g5,f7f5,e7e5,d7d5,c7c5,b7b5,a7a5\n", + "\u001b[32m**********************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mPlayer_Black\u001b[0m (to Board_Proxy):\n", + "\n", + "\u001b[32m***** Suggested tool call (call_59t20pl0ab68z4xx2workgbc): make_move *****\u001b[0m\n", + "Arguments: \n", + "{\"move\":\"g8h6\"}\n", + "\u001b[32m**************************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION make_move...\u001b[0m\n" + ] + }, + { + "data": { + "image/svg+xml": [ + "
r n b q k b . r\n",
+       "p p p p p p p p\n",
+       ". . . . . . . n\n",
+       ". . . . . . . .\n",
+       ". . . . P . . .\n",
+       ". . . . . . . .\n",
+       "P P P P . P P P\n",
+       "R N B Q K B N R
" + ], + "text/plain": [ + "'
r n b q k b . r\\np p p p p p p p\\n. . . . . . . n\\n. . . . . . . .\\n. . . . P . . .\\n. . . . . . . .\\nP P P P . P P P\\nR N B Q K B N R
'" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mBoard_Proxy\u001b[0m (to Player_Black):\n", + "\n", + "\u001b[33mBoard_Proxy\u001b[0m (to Player_Black):\n", + "\n", + "\u001b[32m***** Response from calling tool (call_59t20pl0ab68z4xx2workgbc) *****\u001b[0m\n", + "Moved knight (♞) from g8 to h6.\n", + "\u001b[32m**********************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mPlayer_Black\u001b[0m (to Board_Proxy):\n", + "\n", + "\u001b[32m***** Suggested tool call (call_jwv1d86srs1fnvu33cky9tgv): make_move *****\u001b[0m\n", + "Arguments: \n", + "{\"move\":\"g8h6\"}\n", + "\u001b[32m**************************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mPlayer_Black\u001b[0m (to Player_White):\n", + "\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n" + ] + } + ], + "source": [ + "# Clear the board.\n", + "board = chess.Board()\n", + "\n", + "chat_result = player_black.initiate_chat(\n", + " player_white,\n", + " message=\"Let's play chess! Your move.\",\n", + " max_turns=10,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "At this stage, it's hard to tell who's going to win, but they're playing well and using the functions correctly." + ] + } + ], + "metadata": { + "front_matter": { + "description": "LLM-backed agents playing chess with each other using nested chats.", + "tags": [ + "nested chat", + "tool use", + "orchestration" + ] + }, + "kernelspec": { + "display_name": "autogen", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebook/agentchat_nestedchat.ipynb b/notebook/agentchat_nestedchat.ipynb index 3cd4d0a99ed7..f81f20398591 100644 --- a/notebook/agentchat_nestedchat.ipynb +++ b/notebook/agentchat_nestedchat.ipynb @@ -100,7 +100,7 @@ " system_message=\"\"\"\n", " You are a professional writer, known for your insightful and engaging articles.\n", " You transform complex concepts into compelling narratives.\n", - " You should imporve the quality of the content based on the feedback from the user.\n", + " You should improve the quality of the content based on the feedback from the user.\n", " \"\"\",\n", ")\n", "\n", diff --git a/pyproject.toml b/pyproject.toml index 7981ef4b43d5..107c438a7f41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,6 @@ exclude = [ "math_utils\\.py$", "**/cap/py/autogencap/proto/*", ] -ignore-init-module-imports = true unfixable = ["F401"] [tool.ruff.lint.mccabe] diff --git a/setup.py b/setup.py index 738e09d9061c..9117ed45ceac 100644 --- a/setup.py +++ b/setup.py @@ -72,10 +72,7 @@ "mathchat": ["sympy", "pydantic==1.10.9", "wolframalpha"], "retrievechat": retrieve_chat, "retrievechat-pgvector": retrieve_chat_pgvector, - "retrievechat-qdrant": [ - *retrieve_chat, - "qdrant_client[fastembed]<1.9.2", - ], + "retrievechat-qdrant": [*retrieve_chat, "qdrant_client", "fastembed>=0.3.1"], "autobuild": ["chromadb", "sentence-transformers", "huggingface-hub", "pysqlite3"], "teachable": ["chromadb"], "lmm": ["replicate", "pillow"], @@ -91,6 +88,8 @@ "long-context": ["llmlingua<0.3"], "anthropic": ["anthropic>=0.23.1"], "mistral": ["mistralai>=0.2.0"], + "groq": ["groq>=0.9.0"], + "cohere": ["cohere>=5.5.8"], } setuptools.setup( diff --git a/test/agentchat/contrib/vectordb/test_qdrant.py b/test/agentchat/contrib/vectordb/test_qdrant.py new file mode 100644 index 000000000000..c2f4270f2953 --- /dev/null +++ b/test/agentchat/contrib/vectordb/test_qdrant.py @@ -0,0 +1,68 @@ +import os +import sys + +import pytest + +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) + +try: + import uuid + + from qdrant_client import QdrantClient + + from autogen.agentchat.contrib.vectordb.qdrant import QdrantVectorDB +except ImportError: + skip = True +else: + skip = False + + +@pytest.mark.skipif(skip, reason="dependency is not installed") +def test_qdrant(): + # test create collection + client = QdrantClient(location=":memory:") + db = QdrantVectorDB(client=client) + collection_name = uuid.uuid4().hex + db.create_collection(collection_name, overwrite=True, get_or_create=True) + assert client.collection_exists(collection_name) + + # test_delete_collection + db.delete_collection(collection_name) + assert not client.collection_exists(collection_name) + + # test_get_collection + db.create_collection(collection_name, overwrite=True, get_or_create=True) + collection_info = db.get_collection(collection_name) + # Assert default FastEmbed model dimensions + assert collection_info.config.params.vectors.size == 384 + + # test_insert_docs + docs = [{"content": "doc1", "id": 1}, {"content": "doc2", "id": 2}] + db.insert_docs(docs, collection_name, upsert=False) + res = db.get_docs_by_ids([1, 2], collection_name) + assert res[0]["id"] == 1 + assert res[0]["content"] == "doc1" + assert res[1]["id"] == 2 + assert res[1]["content"] == "doc2" + + # test_update_docs and get_docs_by_ids + docs = [{"content": "doc11", "id": 1}, {"content": "doc22", "id": 2}] + db.update_docs(docs, collection_name) + res = db.get_docs_by_ids([1, 2], collection_name) + assert res[0]["id"] == 1 + assert res[0]["content"] == "doc11" + assert res[1]["id"] == 2 + assert res[1]["content"] == "doc22" + + # test_retrieve_docs + queries = ["doc22", "doc11"] + res = db.retrieve_docs(queries, collection_name) + assert [[r[0]["id"] for r in rr] for rr in res] == [[2, 1], [1, 2]] + + # test_delete_docs + db.delete_docs([1], collection_name) + assert db.client.count(collection_name).count == 1 + + +if __name__ == "__main__": + test_qdrant() diff --git a/test/oai/test_anthropic.py b/test/oai/test_anthropic.py index 379ab47f6756..53926dbd18d6 100644 --- a/test/oai/test_anthropic.py +++ b/test/oai/test_anthropic.py @@ -50,17 +50,47 @@ def anthropic_client(): @pytest.mark.skipif(skip, reason=reason) def test_initialization_missing_api_key(): os.environ.pop("ANTHROPIC_API_KEY", None) - with pytest.raises(ValueError, match="API key is required to use the Anthropic API."): + os.environ.pop("AWS_ACCESS_KEY", None) + os.environ.pop("AWS_SECRET_KEY", None) + os.environ.pop("AWS_SESSION_TOKEN", None) + os.environ.pop("AWS_REGION", None) + with pytest.raises(ValueError, match="API key or AWS credentials are required to use the Anthropic API."): AnthropicClient() AnthropicClient(api_key="dummy_api_key") +@pytest.fixture() +def anthropic_client_with_aws_credentials(): + return AnthropicClient( + aws_access_key="dummy_access_key", + aws_secret_key="dummy_secret_key", + aws_session_token="dummy_session_token", + aws_region="us-west-2", + ) + + @pytest.mark.skipif(skip, reason=reason) def test_intialization(anthropic_client): assert anthropic_client.api_key == "dummy_api_key", "`api_key` should be correctly set in the config" +@pytest.mark.skipif(skip, reason=reason) +def test_intialization_with_aws_credentials(anthropic_client_with_aws_credentials): + assert ( + anthropic_client_with_aws_credentials.aws_access_key == "dummy_access_key" + ), "`aws_access_key` should be correctly set in the config" + assert ( + anthropic_client_with_aws_credentials.aws_secret_key == "dummy_secret_key" + ), "`aws_secret_key` should be correctly set in the config" + assert ( + anthropic_client_with_aws_credentials.aws_session_token == "dummy_session_token" + ), "`aws_session_token` should be correctly set in the config" + assert ( + anthropic_client_with_aws_credentials.aws_region == "us-west-2" + ), "`aws_region` should be correctly set in the config" + + # Test cost calculation @pytest.mark.skipif(skip, reason=reason) def test_cost_calculation(mock_completion): diff --git a/test/oai/test_client.py b/test/oai/test_client.py index debad5fae3b0..ea6f7ba7c5f7 100755 --- a/test/oai/test_client.py +++ b/test/oai/test_client.py @@ -140,10 +140,10 @@ def test_customized_cost(): env_or_file=OAI_CONFIG_LIST, file_location=KEY_LOC, filter_dict={"tags": ["gpt-3.5-turbo-instruct"]} ) for config in config_list: - config.update({"price": [1, 1]}) + config.update({"price": [1000, 1000]}) client = OpenAIWrapper(config_list=config_list, cache_seed=None) response = client.create(prompt="1+3=") - assert response.cost >= 4, "Due to customized pricing, cost should be greater than 4" + assert response.cost >= 4 and response.cost < 10, "Due to customized pricing, cost should be > 4 and < 10" @pytest.mark.skipif(skip, reason="openai>=1 not installed") diff --git a/test/oai/test_cohere.py b/test/oai/test_cohere.py new file mode 100644 index 000000000000..83ef56b17087 --- /dev/null +++ b/test/oai/test_cohere.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 -m pytest + +import os + +import pytest + +try: + from autogen.oai.cohere import CohereClient, calculate_cohere_cost + + skip = False +except ImportError: + CohereClient = object + skip = True + + +reason = "Cohere dependency not installed!" + + +@pytest.fixture() +def cohere_client(): + return CohereClient(api_key="dummy_api_key") + + +@pytest.mark.skipif(skip, reason=reason) +def test_initialization_missing_api_key(): + os.environ.pop("COHERE_API_KEY", None) + with pytest.raises( + AssertionError, + match="Please include the api_key in your config list entry for Cohere or set the COHERE_API_KEY env variable.", + ): + CohereClient() + + CohereClient(api_key="dummy_api_key") + + +@pytest.mark.skipif(skip, reason=reason) +def test_intialization(cohere_client): + assert cohere_client.api_key == "dummy_api_key", "`api_key` should be correctly set in the config" + + +@pytest.mark.skipif(skip, reason=reason) +def test_calculate_cohere_cost(): + assert ( + calculate_cohere_cost(0, 0, model="command-r") == 0.0 + ), "Cost should be 0 for 0 input_tokens and 0 output_tokens" + assert calculate_cohere_cost(100, 200, model="command-r-plus") == 0.0033 + + +@pytest.mark.skipif(skip, reason=reason) +def test_load_config(cohere_client): + params = { + "model": "command-r-plus", + "stream": False, + "temperature": 1, + "p": 0.8, + "max_tokens": 100, + } + expected_params = { + "model": "command-r-plus", + "temperature": 1, + "p": 0.8, + "seed": None, + "max_tokens": 100, + "frequency_penalty": 0, + "presence_penalty": 0, + "k": 0, + } + result = cohere_client.parse_params(params) + assert result == expected_params, "Config should be correctly loaded" diff --git a/test/oai/test_groq.py b/test/oai/test_groq.py new file mode 100644 index 000000000000..f55edbd8c7a6 --- /dev/null +++ b/test/oai/test_groq.py @@ -0,0 +1,249 @@ +from unittest.mock import MagicMock, patch + +import pytest + +try: + from autogen.oai.groq import GroqClient, calculate_groq_cost + + skip = False +except ImportError: + GroqClient = object + InternalServerError = object + skip = True + + +# Fixtures for mock data +@pytest.fixture +def mock_response(): + class MockResponse: + def __init__(self, text, choices, usage, cost, model): + self.text = text + self.choices = choices + self.usage = usage + self.cost = cost + self.model = model + + return MockResponse + + +@pytest.fixture +def groq_client(): + return GroqClient(api_key="fake_api_key") + + +skip_reason = "Groq dependency is not installed" + + +# Test initialization and configuration +@pytest.mark.skipif(skip, reason=skip_reason) +def test_initialization(): + + # Missing any api_key + with pytest.raises(AssertionError) as assertinfo: + GroqClient() # Should raise an AssertionError due to missing api_key + + assert "Please include the api_key in your config list entry for Groq or set the GROQ_API_KEY env variable." in str( + assertinfo.value + ) + + # Creation works + GroqClient(api_key="fake_api_key") # Should create okay now. + + +# Test standard initialization +@pytest.mark.skipif(skip, reason=skip_reason) +def test_valid_initialization(groq_client): + assert groq_client.api_key == "fake_api_key", "Config api_key should be correctly set" + + +# Test parameters +@pytest.mark.skipif(skip, reason=skip_reason) +def test_parsing_params(groq_client): + # All parameters + params = { + "model": "llama3-8b-8192", + "frequency_penalty": 1.5, + "presence_penalty": 1.5, + "max_tokens": 1000, + "seed": 42, + "stream": False, + "temperature": 1, + "top_p": 0.8, + } + expected_params = { + "model": "llama3-8b-8192", + "frequency_penalty": 1.5, + "presence_penalty": 1.5, + "max_tokens": 1000, + "seed": 42, + "stream": False, + "temperature": 1, + "top_p": 0.8, + } + result = groq_client.parse_params(params) + assert result == expected_params + + # Only model, others set as defaults + params = { + "model": "llama3-8b-8192", + } + expected_params = { + "model": "llama3-8b-8192", + "frequency_penalty": None, + "presence_penalty": None, + "max_tokens": None, + "seed": None, + "stream": False, + "temperature": 1, + "top_p": None, + } + result = groq_client.parse_params(params) + assert result == expected_params + + # Incorrect types, defaults should be set, will show warnings but not trigger assertions + params = { + "model": "llama3-8b-8192", + "frequency_penalty": "1.5", + "presence_penalty": "1.5", + "max_tokens": "1000", + "seed": "42", + "stream": "False", + "temperature": "1", + "top_p": "0.8", + } + result = groq_client.parse_params(params) + assert result == expected_params + + # Values outside bounds, should warn and set to defaults + params = { + "model": "llama3-8b-8192", + "frequency_penalty": 5000, + "presence_penalty": -500, + "temperature": 3, + } + result = groq_client.parse_params(params) + assert result == expected_params + + # No model + params = { + "frequency_penalty": 1, + } + + with pytest.raises(AssertionError) as assertinfo: + result = groq_client.parse_params(params) + + assert "Please specify the 'model' in your config list entry to nominate the Groq model to use." in str( + assertinfo.value + ) + + +# Test cost calculation +@pytest.mark.skipif(skip, reason=skip_reason) +def test_cost_calculation(mock_response): + response = mock_response( + text="Example response", + choices=[{"message": "Test message 1"}], + usage={"prompt_tokens": 500, "completion_tokens": 300, "total_tokens": 800}, + cost=None, + model="llama3-70b-8192", + ) + assert ( + calculate_groq_cost(response.usage["prompt_tokens"], response.usage["completion_tokens"], response.model) + == 0.000532 + ), "Cost for this should be $0.000532" + + +# Test text generation +@pytest.mark.skipif(skip, reason=skip_reason) +@patch("autogen.oai.groq.GroqClient.create") +def test_create_response(mock_chat, groq_client): + # Mock GroqClient.chat response + mock_groq_response = MagicMock() + mock_groq_response.choices = [ + MagicMock(finish_reason="stop", message=MagicMock(content="Example Groq response", tool_calls=None)) + ] + mock_groq_response.id = "mock_groq_response_id" + mock_groq_response.model = "llama3-70b-8192" + mock_groq_response.usage = MagicMock(prompt_tokens=10, completion_tokens=20) # Example token usage + + mock_chat.return_value = mock_groq_response + + # Test parameters + params = { + "messages": [{"role": "user", "content": "Hello"}, {"role": "assistant", "content": "World"}], + "model": "llama3-70b-8192", + } + + # Call the create method + response = groq_client.create(params) + + # Assertions to check if response is structured as expected + assert ( + response.choices[0].message.content == "Example Groq response" + ), "Response content should match expected output" + assert response.id == "mock_groq_response_id", "Response ID should match the mocked response ID" + assert response.model == "llama3-70b-8192", "Response model should match the mocked response model" + assert response.usage.prompt_tokens == 10, "Response prompt tokens should match the mocked response usage" + assert response.usage.completion_tokens == 20, "Response completion tokens should match the mocked response usage" + + +# Test functions/tools +@pytest.mark.skipif(skip, reason=skip_reason) +@patch("autogen.oai.groq.GroqClient.create") +def test_create_response_with_tool_call(mock_chat, groq_client): + # Mock `groq_response = client.chat(**groq_params)` + mock_function = MagicMock(name="currency_calculator") + mock_function.name = "currency_calculator" + mock_function.arguments = '{"base_currency": "EUR", "quote_currency": "USD", "base_amount": 123.45}' + + mock_function_2 = MagicMock(name="get_weather") + mock_function_2.name = "get_weather" + mock_function_2.arguments = '{"location": "Chicago"}' + + mock_chat.return_value = MagicMock( + choices=[ + MagicMock( + finish_reason="tool_calls", + message=MagicMock( + content="Sample text about the functions", + tool_calls=[ + MagicMock(id="gdRdrvnHh", function=mock_function), + MagicMock(id="abRdrvnHh", function=mock_function_2), + ], + ), + ) + ], + id="mock_groq_response_id", + model="llama3-70b-8192", + usage=MagicMock(prompt_tokens=10, completion_tokens=20), + ) + + # Construct parameters + converted_functions = [ + { + "type": "function", + "function": { + "description": "Currency exchange calculator.", + "name": "currency_calculator", + "parameters": { + "type": "object", + "properties": { + "base_amount": {"type": "number", "description": "Amount of currency in base_currency"}, + }, + "required": ["base_amount"], + }, + }, + } + ] + groq_messages = [ + {"role": "user", "content": "How much is 123.45 EUR in USD?"}, + {"role": "assistant", "content": "World"}, + ] + + # Call the create method + response = groq_client.create({"messages": groq_messages, "tools": converted_functions, "model": "llama3-70b-8192"}) + + # Assertions to check if the functions and content are included in the response + assert response.choices[0].message.content == "Sample text about the functions" + assert response.choices[0].message.tool_calls[0].function.name == "currency_calculator" + assert response.choices[0].message.tool_calls[1].function.name == "get_weather" diff --git a/test/oai/test_utils.py b/test/oai/test_utils.py index 99f8d8d24e8b..96956d07d90d 100755 --- a/test/oai/test_utils.py +++ b/test/oai/test_utils.py @@ -424,6 +424,12 @@ def test_is_valid_api_key(): assert is_valid_api_key("sk-asajsdjsd22372X23kjdfdfdf2329ffUUDSDS") assert is_valid_api_key("sk-asajsdjsd22372X23kjdfdfdf2329ffUUDSDS1212121221212sssXX") assert is_valid_api_key("sk-proj-asajsdjsd22372X23kjdfdfdf2329ffUUDSDS12121212212") + assert is_valid_api_key("sk-0-asajsdjsd22372X23kjdfdfdf2329ffUUDSDS12121212212") + assert is_valid_api_key("sk-aut0gen-asajsdjsd22372X23kjdfdfdf2329ffUUDSDS12121212212") + assert is_valid_api_key("sk-aut0-gen-asajsdjsd22372X23kjdfdfdf2329ffUUDSDS12121212212") + assert is_valid_api_key("sk-aut0--gen-asajsdjsd22372X23kjdfdfdf2329ffUUDSDS12121212212") + assert not is_valid_api_key("sk-aut0-gen--asajsdjsd22372X23kjdfdfdf2329ffUUDSDS12121212212") + assert not is_valid_api_key("sk--aut0-gen-asajsdjsd22372X23kjdfdfdf2329ffUUDSDS12121212212") assert is_valid_api_key(MOCK_OPEN_AI_API_KEY) diff --git a/test/test_browser_utils.py b/test/test_browser_utils.py index 5551130a6054..659dcce84ae7 100755 --- a/test/test_browser_utils.py +++ b/test/test_browser_utils.py @@ -78,7 +78,7 @@ def test_simple_text_browser(): top_viewport = browser.visit_page(BLOG_POST_URL) assert browser.viewport == top_viewport assert browser.page_title.strip() == BLOG_POST_TITLE.strip() - assert BLOG_POST_STRING in browser.page_content + assert BLOG_POST_STRING in browser.page_content.replace("\n\n", " ").replace("\\", "") # Check if page splitting works approx_pages = math.ceil(len(browser.page_content) / viewport_size) # May be fewer, since it aligns to word breaks diff --git a/website/blog/2023-06-28-MathChat/index.mdx b/website/blog/2023-06-28-MathChat/index.mdx index 4c1007c611b8..be2423de9eed 100644 --- a/website/blog/2023-06-28-MathChat/index.mdx +++ b/website/blog/2023-06-28-MathChat/index.mdx @@ -75,7 +75,7 @@ We found that compared to basic prompting, which demonstrates the innate capabil For categories like Algebra and Prealgebra, PoT and PS showed little improvement, and in some instances, even led to a decrease in accuracy. However, MathChat was able to enhance total accuracy by around 6% compared to PoT and PS, showing competitive performance across all categories. Remarkably, MathChat improved accuracy in the Algebra category by about 15% over other methods. Note that categories like Intermediate Algebra and Precalculus remained challenging for all methods, with only about 20% of problems solved accurately. -The code for experiments can be found at this [repository](https://github.com/kevin666aa/FLAML/tree/gpt_math_solver/flaml/autogen/math). +The code for experiments can be found at this [repository](https://github.com/yiranwu0/FLAML/tree/gpt_math_solver/flaml/autogen/math). We now provide an implementation of MathChat using the interactive agents in AutoGen. See this [notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_MathChat.ipynb) for example usage. ## Future Directions diff --git a/website/blog/2023-10-18-RetrieveChat/index.mdx b/website/blog/2023-10-18-RetrieveChat/index.mdx index 1685b22f5d8e..12ee03051321 100644 --- a/website/blog/2023-10-18-RetrieveChat/index.mdx +++ b/website/blog/2023-10-18-RetrieveChat/index.mdx @@ -483,4 +483,4 @@ The online app and the source code are hosted in [HuggingFace](https://huggingfa You can check out more example notebooks for RAG use cases: - [Automated Code Generation and Question Answering with Retrieval Augmented Agents](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_RetrieveChat.ipynb) - [Group Chat with Retrieval Augmented Generation (with 5 group member agents and 1 manager agent)](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_groupchat_RAG.ipynb) -- [Automated Code Generation and Question Answering with Qdrant based Retrieval Augmented Agents](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_qdrant_RetrieveChat.ipynb) +- [Automated Code Generation and Question Answering with Qdrant based Retrieval Augmented Agents](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_RetrieveChat_qdrant.ipynb) diff --git a/website/blog/2023-11-13-OAI-assistants/index.mdx b/website/blog/2023-11-13-OAI-assistants/index.mdx index e73e31ad591f..07216a25969c 100644 --- a/website/blog/2023-11-13-OAI-assistants/index.mdx +++ b/website/blog/2023-11-13-OAI-assistants/index.mdx @@ -112,6 +112,6 @@ Checkout more examples [here](https://github.com/microsoft/autogen/tree/main/not `GPTAssistantAgent` was made possible through collaboration with [@IANTHEREAL](https://github.com/IANTHEREAL), [Jiale Liu](https://leoljl.github.io), -[Yiran Wu](https://github.com/kevin666aa), +[Yiran Wu](https://github.com/yiranwu0), [Qingyun Wu](https://qingyun-wu.github.io/), [Chi Wang](https://www.microsoft.com/en-us/research/people/chiw/), and many other AutoGen maintainers. diff --git a/website/blog/2024-06-21-AgentEval/img/agenteval_ov_v3.png b/website/blog/2024-06-21-AgentEval/img/agenteval_ov_v3.png new file mode 100644 index 000000000000..fe31283d72d0 --- /dev/null +++ b/website/blog/2024-06-21-AgentEval/img/agenteval_ov_v3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e17aebbc38a8ba55e4b18b9352a680edcca4b3d6625b16f6d1ab3131da799b63 +size 236624 diff --git a/website/blog/2024-06-21-AgentEval/index.mdx b/website/blog/2024-06-21-AgentEval/index.mdx new file mode 100644 index 000000000000..87ffc857e839 --- /dev/null +++ b/website/blog/2024-06-21-AgentEval/index.mdx @@ -0,0 +1,202 @@ +--- +title: "AgentEval: A Developer Tool to Assess Utility of LLM-powered Applications" +authors: + - jluey + - julianakiseleva +tags: [LLM, GPT, evaluation, task utility] +--- + + +![Fig.1: An AgentEval framework with verification step](img/agenteval_ov_v3.png) + +

Fig.1 illustrates the general flow of AgentEval with verification step

+ + + +TL;DR: +* As a developer, how can you assess the utility and effectiveness of an LLM-powered application in helping end users with their tasks? +* To shed light on the question above, we previously introduced [`AgentEval`](https://microsoft.github.io/autogen/blog/2023/11/20/AgentEval/) — a framework to assess the multi-dimensional utility of any LLM-powered application crafted to assist users in specific tasks. We have now embedded it as part of the AutoGen library to ease developer adoption. +* Here, we introduce an updated version of AgentEval that includes a verification process to estimate the robustness of the QuantifierAgent. More details can be found in [this paper](https://arxiv.org/abs/2405.02178). + + +## Introduction + +Previously introduced [`AgentEval`](https://microsoft.github.io/autogen/blog/2023/11/20/AgentEval/) is a comprehensive framework designed to bridge the gap in assessing the utility of LLM-powered applications. It leverages recent advancements in LLMs to offer a scalable and cost-effective alternative to traditional human evaluations. The framework comprises three main agents: `CriticAgent`, `QuantifierAgent`, and `VerifierAgent`, each playing a crucial role in assessing the task utility of an application. + +**CriticAgent: Defining the Criteria** + +The CriticAgent's primary function is to suggest a set of criteria for evaluating an application based on the task description and examples of successful and failed executions. For instance, in the context of a math tutoring application, the CriticAgent might propose criteria such as efficiency, clarity, and correctness. These criteria are essential for understanding the various dimensions of the application's performance. It’s highly recommended that application developers validate the suggested criteria leveraging their domain expertise. + +**QuantifierAgent: Quantifying the Performance** + +Once the criteria are established, the QuantifierAgent takes over to quantify how well the application performs against each criterion. This quantification process results in a multi-dimensional assessment of the application's utility, providing a detailed view of its strengths and weaknesses. + +**VerifierAgent: Ensuring Robustness and Relevance** + +VerifierAgent ensures the criteria used to evaluate a utility are effective for the end-user, maintaining both robustness and high discriminative power. It does this through two main actions: + +1. Criteria Stability: + * Ensures criteria are essential, non-redundant, and consistently measurable. + * Iterates over generating and quantifying criteria, eliminating redundancies, and evaluating their stability. + * Retains only the most robust criteria. + +2. Discriminative Power: + + * Tests the system's reliability by introducing adversarial examples (noisy or compromised data). + * Assesses the system's ability to distinguish these from standard cases. + * If the system fails, it indicates the need for better criteria to handle varied conditions effectively. + +## A Flexible and Scalable Framework + +One of AgentEval's key strengths is its flexibility. It can be applied to a wide range of tasks where success may or may not be clearly defined. For tasks with well-defined success criteria, such as household chores, the framework can evaluate whether multiple successful solutions exist and how they compare. For more open-ended tasks, such as generating an email template, AgentEval can assess the utility of the system's suggestions. + +Furthermore, AgentEval allows for the incorporation of human expertise. Domain experts can participate in the evaluation process by suggesting relevant criteria or verifying the usefulness of the criteria identified by the agents. This human-in-the-loop approach ensures that the evaluation remains grounded in practical, real-world considerations. + +## Empirical Validation + +To validate AgentEval, the framework was tested on two applications: math problem solving and ALFWorld, a household task simulation. The math dataset comprised 12,500 challenging problems, each with step-by-step solutions, while the ALFWorld dataset involved multi-turn interactions in a simulated environment. In both cases, AgentEval successfully identified relevant criteria, quantified performance, and verified the robustness of the evaluations, demonstrating its effectiveness and versatility. + +## How to use `AgentEval` + +AgentEval currently has two main stages; criteria generation and criteria quantification (criteria verification is still under development). Both stages make use of sequential LLM-powered agents to make their determinations. + +**Criteria Generation:** + +During criteria generation, AgentEval uses example execution message chains to create a set of criteria for quantifying how well an application performed for a given task. + +``` +def generate_criteria( + llm_config: Optional[Union[Dict, Literal[False]]] = None, + task: Task = None, + additional_instructions: str = "", + max_round=2, + use_subcritic: bool = False, +) +``` + +Parameters: +* llm_config (dict or bool): llm inference configuration. +* task ([Task](https://github.com/microsoft/autogen/tree/main/autogen/agentchat/contrib/agent_eval/task.py)): The task to evaluate. +* additional_instructions (str, optional): Additional instructions for the criteria agent. +* max_round (int, optional): The maximum number of rounds to run the conversation. +* use_subcritic (bool, optional): Whether to use the Subcritic agent to generate subcriteria. The Subcritic agent will break down a generated criteria into smaller criteria to be assessed. + +Example code: +``` +config_list = autogen.config_list_from_json("OAI_CONFIG_LIST") +task = Task( + **{ + "name": "Math problem solving", + "description": "Given any question, the system needs to solve the problem as consisely and accurately as possible", + "successful_response": response_successful, + "failed_response": response_failed, + } +) + +criteria = generate_criteria(task=task, llm_config={"config_list": config_list}) +``` + +Note: Only one sample execution chain (success/failure) is required for the task object but AgentEval will perform better with an example for each case. + + +Example Output: +``` +[ + { + "name": "Accuracy", + "description": "The solution must be correct and adhere strictly to mathematical principles and techniques appropriate for the problem.", + "accepted_values": ["Correct", "Minor errors", "Major errors", "Incorrect"] + }, + { + "name": "Conciseness", + "description": "The explanation and method provided should be direct and to the point, avoiding unnecessary steps or complexity.", + "accepted_values": ["Very concise", "Concise", "Somewhat verbose", "Verbose"] + }, + { + "name": "Relevance", + "description": "The content of the response must be relevant to the question posed and should address the specific problem requirements.", + "accepted_values": ["Highly relevant", "Relevant", "Somewhat relevant", "Not relevant"] + } +] +``` + + + +**Criteria Quantification:** + +During the quantification stage, AgentEval will use the generated criteria (or user defined criteria) to assess a given execution chain to determine how well the application performed. + +``` +def quantify_criteria( + llm_config: Optional[Union[Dict, Literal[False]]], + criteria: List[Criterion], + task: Task, + test_case: str, + ground_truth: str, +) +``` + +Parameters: +* llm_config (dict or bool): llm inference configuration. +* criteria ([Criterion](https://github.com/microsoft/autogen/tree/main/autogen/agentchat/contrib/agent_eval/criterion.py)): A list of criteria for evaluating the utility of a given task. This can either be generated by the `generate_criteria` function or manually created. +* task ([Task](https://github.com/microsoft/autogen/tree/main/autogen/agentchat/contrib/agent_eval/task.py)): The task to evaluate. It should match the one used during the `generate_criteria` step. +* test_case (str): The execution chain to assess. Typically this is a json list of messages but could be any string representation of a conversation chain. +* ground_truth (str): The ground truth for the test case. + +Example Code: +``` +test_case="""[ + { + "content": "Find $24^{-1} \\pmod{11^2}$. That is, find the residue $b$ for which $24b \\equiv 1\\pmod{11^2}$.\n\nExpress your answer as an integer from $0$ to $11^2-1$, inclusive.", + "role": "user" + }, + { + "content": "To find the modular inverse of 24 modulo 11^2, we can use the Extended Euclidean Algorithm. Here is a Python function to compute the modular inverse using this algorithm:\n\n```python\ndef mod_inverse(a, m):\n..." + "role": "assistant" + } + ]""" + +quantifier_output = quantify_criteria( + llm_config={"config_list": config_list}, + criteria=criteria, + task=task, + test_case=test_case, + ground_truth="true", +) +``` + +The output will be a json object consisting of the ground truth and a dictionary mapping each criteria to it's score. + +``` +{ + "actual_success": true, + "estimated_performance": { + "Accuracy": "Correct", + "Conciseness": "Concise", + "Relevance": "Highly relevant" + } +} +``` + +## What is next? +* Enabling AgentEval in AutoGen Studio for a nocode solution. +* Fully implementing VerifierAgent in the AgentEval framework. + +## Conclusion + +AgentEval represents a significant advancement in the evaluation of LLM-powered applications. By combining the strengths of CriticAgent, QuantifierAgent, and VerifierAgent, the framework offers a robust, scalable, and flexible solution for assessing task utility. This innovative approach not only helps developers understand the current performance of their applications but also provides valuable insights that can drive future improvements. As the field of intelligent agents continues to evolve, frameworks like AgentEval will play a crucial role in ensuring that these applications meet the diverse and dynamic needs of their users. + + +## Further reading + +Please refer to our [paper](https://arxiv.org/abs/2405.02178) and [codebase](https://github.com/microsoft/autogen/tree/main/autogen/agentchat/contrib/agent_eval) for more details about AgentEval. + +If you find this blog useful, please consider citing: +```bobtex +@article{arabzadeh2024assessing, + title={Assessing and Verifying Task Utility in LLM-Powered Applications}, + author={Arabzadeh, Negar and Huo, Siging and Mehta, Nikhil and Wu, Qinqyun and Wang, Chi and Awadallah, Ahmed and Clarke, Charles LA and Kiseleva, Julia}, + journal={arXiv preprint arXiv:2405.02178}, + year={2024} +} +``` diff --git a/website/blog/2024-06-24-AltModels-Classes/img/agentstogether.jpeg b/website/blog/2024-06-24-AltModels-Classes/img/agentstogether.jpeg new file mode 100644 index 000000000000..fd859fc62076 --- /dev/null +++ b/website/blog/2024-06-24-AltModels-Classes/img/agentstogether.jpeg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:964628601b60ddbab8940fea45014dcd841b89783bb0b7e9ac9d3690f1c41798 +size 659594 diff --git a/website/blog/2024-06-24-AltModels-Classes/index.mdx b/website/blog/2024-06-24-AltModels-Classes/index.mdx new file mode 100644 index 000000000000..9c94094e7e4c --- /dev/null +++ b/website/blog/2024-06-24-AltModels-Classes/index.mdx @@ -0,0 +1,393 @@ +--- +title: Enhanced Support for Non-OpenAI Models +authors: + - marklysze + - Hk669 +tags: [mistral ai,anthropic,together.ai,gemini] +--- + +![agents](img/agentstogether.jpeg) + +## TL;DR + +- **AutoGen has expanded integrations with a variety of cloud-based model providers beyond OpenAI.** +- **Leverage models and platforms from Gemini, Anthropic, Mistral AI, Together.AI, and Groq for your AutoGen agents.** +- **Utilise models specifically for chat, language, image, and coding.** +- **LLM provider diversification can provide cost and resilience benefits.** + +In addition to the recently released AutoGen [Google Gemini](https://ai.google.dev/) client, new client classes for [Mistral AI](https://mistral.ai/), [Anthropic](https://www.anthropic.com/), [Together.AI](https://www.together.ai/), and [Groq](https://groq.com/) enable you to utilize over 75 different large language models in your AutoGen agent workflow. + +These new client classes tailor AutoGen's underlying messages to each provider's unique requirements and remove that complexity from the developer, who can then focus on building their AutoGen workflow. + +Using them is as simple as installing the client-specific library and updating your LLM config with the relevant `api_type` and `model`. We'll demonstrate how to use them below. + +The community is continuing to enhance and build new client classes as cloud-based inference providers arrive. So, watch this space, and feel free to [discuss](https://discord.gg/pAbnFJrkgZ) or [develop](https://github.com/microsoft/autogen/pulls) another one. + +## Benefits of choice + +The need to use only the best models to overcome workflow-breaking LLM inconsistency has diminished considerably over the last 12 months. + +These new classes provide access to the very largest trillion-parameter models from OpenAI, Google, and Anthropic, continuing to provide the most consistent +and competent agent experiences. However, it's worth trying smaller models from the likes of Meta, Mistral AI, Microsoft, Qwen, and many others. Perhaps they +are capable enough for a task, or sub-task, or even better suited (such as a coding model)! + +Using smaller models will have cost benefits, but they also allow you to test models that you could run locally, allowing you to determine if you can remove cloud inference costs +altogether or even run an AutoGen workflow offline. + +On the topic of cost, these client classes also include provider-specific token cost calculations so you can monitor the cost impact of your workflows. With costs per million +tokens as low as 10 cents (and some are even free!), cost savings can be noticeable. + +## Mix and match + +How does Google's Gemini 1.5 Pro model stack up against Anthropic's Opus or Meta's Llama 3? + +Now you have the ability to quickly change your agent configs and find out. If you want to run all three in the one workflow, +AutoGen's ability to associate specific configurations to each agent means you can select the best LLM for each agent. + +## Capabilities + +The common requirements of text generation and function/tool calling are supported by these client classes. + +Multi-modal support, such as for image/audio/video, is an area of active development. The [Google Gemini](https://microsoft.github.io/autogen/docs/topics/non-openai-models/cloud-gemini) client class can be +used to create a multimodal agent. + +## Tips + +Here are some tips when working with these client classes: + +- **Most to least capable** - start with larger models and get your workflow working, then iteratively try smaller models. +- **Right model** - choose one that's suited to your task, whether it's coding, function calling, knowledge, or creative writing. +- **Agent names** - these cloud providers do not use the `name` field on a message, so be sure to use your agent's name in their `system_message` and `description` fields, as well as instructing the LLM to 'act as' them. This is particularly important for "auto" speaker selection in group chats as we need to guide the LLM to choose the next agent based on a name, so tweak `select_speaker_message_template`, `select_speaker_prompt_template`, and `select_speaker_auto_multiple_template` with more guidance. +- **Context length** - as your conversation gets longer, models need to support larger context lengths, be mindful of what the model supports and consider using [Transform Messages](https://microsoft.github.io/autogen/docs/topics/handling_long_contexts/intro_to_transform_messages) to manage context size. +- **Provider parameters** - providers have parameters you can set such as temperature, maximum tokens, top-k, top-p, and safety. See each client class in AutoGen's [API Reference](https://microsoft.github.io/autogen/docs/reference/oai/gemini) or [documentation](https://microsoft.github.io/autogen/docs/topics/non-openai-models/cloud-gemini) for details. +- **Prompts** - prompt engineering is critical in guiding smaller LLMs to do what you need. [ConversableAgent](https://microsoft.github.io/autogen/docs/reference/agentchat/conversable_agent), [GroupChat](https://microsoft.github.io/autogen/docs/reference/agentchat/groupchat), [UserProxyAgent](https://microsoft.github.io/autogen/docs/reference/agentchat/user_proxy_agent), and [AssistantAgent](https://microsoft.github.io/autogen/docs/reference/agentchat/assistant_agent) all have customizable prompt attributes that you can tailor. Here are some prompting tips from [Anthropic](https://docs.anthropic.com/en/docs/build-with-claude/prompt-engineering/overview)([+Library](https://docs.anthropic.com/en/prompt-library/library)), [Mistral AI](https://docs.mistral.ai/guides/prompting_capabilities/), [Together.AI](https://docs.together.ai/docs/examples), and [Meta](https://llama.meta.com/docs/how-to-guides/prompting/). +- **Help!** - reach out on the AutoGen [Discord](https://discord.gg/pAbnFJrkgZ) or [log an issue](https://github.com/microsoft/autogen/issues) if you need help with or can help improve these client classes. + +Now it's time to try them out. + +## Quickstart + +### Installation + +Install the appropriate client based on the model you wish to use. + +```sh +pip install pyautogen["mistral"] # for Mistral AI client +pip install pyautogen["anthropic"] # for Anthropic client +pip install pyautogen["together"] # for Together.AI client +pip install pyautogen["groq"] # for Groq client +``` + +### Configuration Setup + +Add your model configurations to the `OAI_CONFIG_LIST`. Ensure you specify the `api_type` to initialize the respective client (Anthropic, Mistral, or Together). + +```yaml +[ + { + "model": "your anthropic model name", + "api_key": "your Anthropic api_key", + "api_type": "anthropic" + }, + { + "model": "your mistral model name", + "api_key": "your Mistral AI api_key", + "api_type": "mistral" + }, + { + "model": "your together.ai model name", + "api_key": "your Together.AI api_key", + "api_type": "together" + }, + { + "model": "your groq model name", + "api_key": "your Groq api_key", + "api_type": "groq" + } +] +``` + +### Usage + +The `[config_list_from_json](https://microsoft.github.io/autogen/docs/reference/oai/openai_utils/#config_list_from_json)` function loads a list of configurations from an environment variable or a json file. + +```py +import autogen +from autogen import AssistantAgent, UserProxyAgent + +config_list = autogen.config_list_from_json( + "OAI_CONFIG_LIST" +) +``` + +### Construct Agents + +Construct a simple conversation between a User proxy and an Assistant agent + +```py +user_proxy = UserProxyAgent( + name="User_proxy", + code_execution_config={ + "last_n_messages": 2, + "work_dir": "groupchat", + "use_docker": False, # Please set use_docker = True if docker is available to run the generated code. Using docker is safer than running the generated code directly. + }, + human_input_mode="ALWAYS", + is_termination_msg=lambda msg: not msg["content"] +) + +assistant = AssistantAgent( + name="assistant", + llm_config = {"config_list": config_list} +) +``` + +### Start chat + +```py + +user_proxy.intiate_chat(assistant, message="Write python code to print Hello World!") + +``` + +**NOTE: To integrate this setup into GroupChat, follow the [tutorial](https://microsoft.github.io/autogen/docs/notebooks/agentchat_groupchat) with the same config as above.** + + +## Function Calls + +Now, let's look at how Anthropic's Sonnet 3.5 is able to suggest multiple function calls in a single response. + +This example is a simple travel agent setup with an agent for function calling and a user proxy agent for executing the functions. + +One thing you'll note here is Anthropic's models are more verbose than OpenAI's and will typically provide chain-of-thought or general verbiage when replying. Therefore we provide more explicit instructions to `functionbot` to not reply with more than necessary. Even so, it can't always help itself! + +Let's start with setting up our configuration and agents. + +```py +import os +import autogen +import json +from typing import Literal +from typing_extensions import Annotated + +# Anthropic configuration, using api_type='anthropic' +anthropic_llm_config = { + "config_list": + [ + { + "api_type": "anthropic", + "model": "claude-3-5-sonnet-20240620", + "api_key": os.getenv("ANTHROPIC_API_KEY"), + "cache_seed": None + } + ] +} + +# Our functionbot, who will be assigned two functions and +# given directions to use them. +functionbot = autogen.AssistantAgent( + name="functionbot", + system_message="For currency exchange tasks, only use " + "the functions you have been provided with. Do not " + "reply with helpful tips. Once you've recommended functions " + "reply with 'TERMINATE'.", + is_termination_msg=lambda x: x.get("content", "") and (x.get("content", "").rstrip().endswith("TERMINATE") or x.get("content", "") == ""), + llm_config=anthropic_llm_config, +) + +# Our user proxy agent, who will be used to manage the customer +# request and conversation with the functionbot, terminating +# when we have the information we need. +user_proxy = autogen.UserProxyAgent( + name="user_proxy", + system_message="You are a travel agent that provides " + "specific information to your customers. Get the " + "information you need and provide a great summary " + "so your customer can have a great trip. If you " + "have the information you need, simply reply with " + "'TERMINATE'.", + is_termination_msg=lambda x: x.get("content", "") and (x.get("content", "").rstrip().endswith("TERMINATE") or x.get("content", "") == ""), + human_input_mode="NEVER", + max_consecutive_auto_reply=10, +) +``` + +We define the two functions. +```py +CurrencySymbol = Literal["USD", "EUR"] + +def exchange_rate(base_currency: CurrencySymbol, quote_currency: CurrencySymbol) -> float: + if base_currency == quote_currency: + return 1.0 + elif base_currency == "USD" and quote_currency == "EUR": + return 1 / 1.1 + elif base_currency == "EUR" and quote_currency == "USD": + return 1.1 + else: + raise ValueError(f"Unknown currencies {base_currency}, {quote_currency}") + +def get_current_weather(location, unit="fahrenheit"): + """Get the weather for some location""" + if "chicago" in location.lower(): + return json.dumps({"location": "Chicago", "temperature": "13", "unit": unit}) + elif "san francisco" in location.lower(): + return json.dumps({"location": "San Francisco", "temperature": "55", "unit": unit}) + elif "new york" in location.lower(): + return json.dumps({"location": "New York", "temperature": "11", "unit": unit}) + else: + return json.dumps({"location": location, "temperature": "unknown"}) +``` + +And then associate them with the `user_proxy` for execution and `functionbot` for the LLM to consider using them. + +```py +@user_proxy.register_for_execution() +@functionbot.register_for_llm(description="Currency exchange calculator.") +def currency_calculator( + base_amount: Annotated[float, "Amount of currency in base_currency"], + base_currency: Annotated[CurrencySymbol, "Base currency"] = "USD", + quote_currency: Annotated[CurrencySymbol, "Quote currency"] = "EUR", +) -> str: + quote_amount = exchange_rate(base_currency, quote_currency) * base_amount + return f"{quote_amount} {quote_currency}" + +@user_proxy.register_for_execution() +@functionbot.register_for_llm(description="Weather forecast for US cities.") +def weather_forecast( + location: Annotated[str, "City name"], +) -> str: + weather_details = get_current_weather(location=location) + weather = json.loads(weather_details) + return f"{weather['location']} will be {weather['temperature']} degrees {weather['unit']}" +``` + +Finally, we start the conversation with a request for help from our customer on their upcoming trip to New York and the Euro they would like exchanged to USD. + +Importantly, we're also using Anthropic's Sonnet to provide a summary through the `summary_method`. Using `summary_prompt`, we guide Sonnet to give us an email output. + +```py +# start the conversation +res = user_proxy.initiate_chat( + functionbot, + message="My customer wants to travel to New York and " + "they need to exchange 830 EUR to USD. Can you please " + "provide them with a summary of the weather and " + "exchanged currently in USD?", + summary_method="reflection_with_llm", + summary_args={ + "summary_prompt": """Summarize the conversation by + providing an email response with the travel information + for the customer addressed as 'Dear Customer'. Do not + provide any additional conversation or apologise, + just provide the relevant information and the email.""" + }, +) +``` + +After the conversation has finished, we'll print out the summary. + +```py +print(f"Here's the LLM summary of the conversation:\n\n{res.summary['content']}") +``` + +Here's the resulting output. + +```text +user_proxy (to functionbot): + +My customer wants to travel to New York and they need to exchange 830 EUR +to USD. Can you please provide them with a summary of the weather and +exchanged currently in USD? + +-------------------------------------------------------------------------------- +functionbot (to user_proxy): + +Certainly! I'd be happy to help your customer with information about the +weather in New York and the currency exchange from EUR to USD. Let's use +the available tools to get this information. + +***** Suggested tool call (toolu_016wBUKVX2TToBaMqmiGvhav): weather_forecast ***** +Arguments: +{"location": "New York"} +********************************************************************************** +***** Suggested tool call (toolu_01Nxjeew2BspfKdZ85on3XkP): currency_calculator ***** +Arguments: +{"base_amount": 830, "base_currency": "EUR", "quote_currency": "USD"} +************************************************************************************* + +-------------------------------------------------------------------------------- + +>>>>>>>> EXECUTING FUNCTION weather_forecast... + +>>>>>>>> EXECUTING FUNCTION currency_calculator... +user_proxy (to functionbot): + +user_proxy (to functionbot): + +***** Response from calling tool (toolu_016wBUKVX2TToBaMqmiGvhav) ***** +New York will be 11 degrees fahrenheit +*********************************************************************** + +-------------------------------------------------------------------------------- +user_proxy (to functionbot): + +***** Response from calling tool (toolu_01Nxjeew2BspfKdZ85on3XkP) ***** +913.0000000000001 USD +*********************************************************************** + +-------------------------------------------------------------------------------- +functionbot (to user_proxy): + +Thank you for providing the information. I can now give your customer a +summary of the weather in New York and the currency exchange from EUR to USD. + +Weather in New York: +The current forecast for New York indicates that it will be 11 degrees +Fahrenheit. This is quite cold, so your customer should be prepared with +warm clothing for their trip. + +Currency Exchange: +For 830 EUR, your customer will receive approximately 913 USD. This is based +on the current exchange rate provided by our currency calculator. + +To summarize: +1. Weather in New York: 11°F (very cold) +2. Currency exchange: 830 EUR = 913 USD + +Your customer should pack warm clothes for the cold weather in New York and +can expect to have about 913 USD for their trip after exchanging 830 EUR. + +TERMINATE + +-------------------------------------------------------------------------------- +Here's the LLM summary of the conversation: + +Certainly. I'll provide an email response to the customer with the travel +information as requested. + +Dear Customer, + +We are pleased to provide you with the following information for your +upcoming trip to New York: + +Weather Forecast: +The current forecast for New York indicates a temperature of 11 degrees +Fahrenheit. Please be prepared for very cold weather and pack appropriate +warm clothing. + +Currency Exchange: +We have calculated the currency exchange for you. Your 830 EUR will be +equivalent to approximately 913 USD at the current exchange rate. + +We hope this information helps you prepare for your trip to New York. Have +a safe and enjoyable journey! + +Best regards, +Travel Assistance Team +``` + +So we can see how Anthropic's Sonnet is able to suggest multiple tools in a single response, with AutoGen executing them both and providing the results back to Sonnet. Sonnet then finishes with a nice email summary that can be the basis for continued real-life conversation with the customer. + +## More tips and tricks + +For an interesting chess game between Anthropic's Sonnet and Mistral's Mixtral, we've put together a sample notebook that highlights some of the tips and tricks for working with non-OpenAI LLMs. [See the notebook here](https://microsoft.github.io/autogen/docs/notebooks/agentchat_nested_chats_chess_altmodels). diff --git a/website/blog/authors.yml b/website/blog/authors.yml index 302bb8fceaaf..0e023514465d 100644 --- a/website/blog/authors.yml +++ b/website/blog/authors.yml @@ -13,8 +13,8 @@ qingyunwu: yiranwu: name: Yiran Wu title: PhD student at Pennsylvania State University - url: https://github.com/kevin666aa - image_url: https://github.com/kevin666aa.png + url: https://github.com/yiranwu0 + image_url: https://github.com/yiranwu0.png jialeliu: name: Jiale Liu @@ -123,3 +123,20 @@ yifanzeng: title: PhD student at Oregon State University url: https://xhmy.github.io/ image_url: https://xhmy.github.io/assets/img/photo.JPG + +jluey: + name: James Woffinden-Luey + title: Senior Research Engineer at Microsoft Research + url: https://github.com/jluey1 + +Hk669: + name: Hrushikesh Dokala + title: CS Undergraduate Based in India + url: https://github.com/Hk669 + image_url: https://github.com/Hk669.png + +marklysze: + name: Mark Sze + title: AI Freelancer + url: https://github.com/marklysze + image_url: https://github.com/marklysze.png diff --git a/website/docs/Examples.md b/website/docs/Examples.md index 2ec83d1e0f21..5efd71748f9d 100644 --- a/website/docs/Examples.md +++ b/website/docs/Examples.md @@ -11,7 +11,7 @@ Links to notebook examples: - Automated Task Solving with Code Generation, Execution & Debugging - [View Notebook](/docs/notebooks/agentchat_auto_feedback_from_code_execution) - Automated Code Generation and Question Answering with Retrieval Augmented Agents - [View Notebook](/docs/notebooks/agentchat_RetrieveChat) -- Automated Code Generation and Question Answering with [Qdrant](https://qdrant.tech/) based Retrieval Augmented Agents - [View Notebook](/docs/notebooks/agentchat_qdrant_RetrieveChat) +- Automated Code Generation and Question Answering with [Qdrant](https://qdrant.tech/) based Retrieval Augmented Agents - [View Notebook](/docs/notebooks/agentchat_RetrieveChat_qdrant) ### Multi-Agent Collaboration (>3 Agents) @@ -80,6 +80,9 @@ Links to notebook examples: - OpenAI Assistant in a Group Chat - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_oai_assistant_groupchat.ipynb) - GPTAssistantAgent based Multi-Agent Tool Use - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/gpt_assistant_agent_function_call.ipynb) +### Non-OpenAI Models +- Conversational Chess using non-OpenAI Models - [View Notebook](/docs/notebooks/agentchat_nested_chats_chess_altmodels) + ### Multimodal Agent - Multimodal Agent Chat with DALLE and GPT-4V - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_dalle_and_gpt4v.ipynb) diff --git a/website/docs/Getting-Started.mdx b/website/docs/Getting-Started.mdx index 0f8c7322411d..3e162a098327 100644 --- a/website/docs/Getting-Started.mdx +++ b/website/docs/Getting-Started.mdx @@ -3,11 +3,12 @@ import TabItem from "@theme/TabItem"; # Getting Started -AutoGen is a framework that enables development of LLM applications using -multiple agents that can converse with each other to solve tasks. AutoGen agents -are customizable, conversable, and seamlessly allow human participation. They -can operate in various modes that employ combinations of LLMs, human inputs, and -tools. +AutoGen is an open-source programming framework for building AI agents and facilitating +cooperation among multiple agents to solve tasks. AutoGen aims to provide an easy-to-use +and flexible framework for accelerating development and research on agentic AI, +like PyTorch for Deep Learning. It offers features such as agents that can converse +with other agents, LLM and tool use support, autonomous and human-in-the-loop workflows, +and multi-agent conversation patterns. ![AutoGen Overview](/img/autogen_agentchat.png) diff --git a/website/docs/contributor-guide/contributing.md b/website/docs/contributor-guide/contributing.md index b90d81f227c8..b1b6b848f667 100644 --- a/website/docs/contributor-guide/contributing.md +++ b/website/docs/contributor-guide/contributing.md @@ -1,6 +1,6 @@ # Contributing to AutoGen -This project welcomes and encourages all forms of contributions, including but not limited to: +The project welcomes contributions from developers and organizations worldwide. Our goal is to foster a collaborative and inclusive community where diverse perspectives and expertise can drive innovation and enhance the project's capabilities. Whether you are an individual contributor or represent an organization, we invite you to join us in shaping the future of this project. Together, we can build something truly remarkable. Possible contributions include but not limited to: - Pushing patches. - Code review of pull requests. @@ -32,3 +32,7 @@ To see what we are working on and what we plan to work on, please check our ## Becoming a Reviewer There is currently no formal reviewer solicitation process. Current reviewers identify reviewers from active contributors. If you are willing to become a reviewer, you are welcome to let us know on discord. + +## Contact Maintainers + +The project is currently maintained by a [dynamic group of volunteers](https://butternut-swordtail-8a5.notion.site/410675be605442d3ada9a42eb4dfef30?v=fa5d0a79fd3d4c0f9c112951b2831cbb&pvs=4) from several different organizations. Contact project administrators Chi Wang and Qingyun Wu via auto-gen@outlook.com if you are interested in becoming a maintainer. diff --git a/website/docs/ecosystem/azure_cosmos_db.md b/website/docs/ecosystem/azure_cosmos_db.md new file mode 100644 index 000000000000..0d1313bc14b7 --- /dev/null +++ b/website/docs/ecosystem/azure_cosmos_db.md @@ -0,0 +1,13 @@ +# Azure Cosmos DB + +> "OpenAI relies on Cosmos DB to dynamically scale their ChatGPT service – one of the fastest-growing consumer apps ever – enabling high reliability and low maintenance." +> – Satya Nadella, Microsoft chairman and chief executive officer + +Azure Cosmos DB is a fully managed [NoSQL](https://learn.microsoft.com/en-us/azure/cosmos-db/distributed-nosql), [relational](https://learn.microsoft.com/en-us/azure/cosmos-db/distributed-relational), and [vector database](https://learn.microsoft.com/azure/cosmos-db/vector-database). It offers single-digit millisecond response times, automatic and instant scalability, along with guaranteed speed at any scale. Your business continuity is assured with up to 99.999% availability backed by SLA. + +Your can simplify your application development by using this single database service for all your AI agent memory system needs, from [geo-replicated distributed cache](https://medium.com/@marcodesanctis2/using-azure-cosmos-db-as-your-persistent-geo-replicated-distributed-cache-b381ad80f8a0) to tracing/logging to [vector database](https://learn.microsoft.com/en-us/azure/cosmos-db/vector-database). + +Learn more about how Azure Cosmos DB enhances the performance of your [AI agent](https://learn.microsoft.com/en-us/azure/cosmos-db/ai-agents). + +- [Try Azure Cosmos DB free](https://learn.microsoft.com/en-us/azure/cosmos-db/try-free) +- [Use Azure Cosmos DB lifetime free tier](https://learn.microsoft.com/en-us/azure/cosmos-db/free-tier) diff --git a/website/docs/ecosystem/pgvector.md b/website/docs/ecosystem/pgvector.md index c455595eda4e..99afa676e7e4 100644 --- a/website/docs/ecosystem/pgvector.md +++ b/website/docs/ecosystem/pgvector.md @@ -2,4 +2,4 @@ [PGVector](https://github.com/pgvector/pgvector) is an open-source vector similarity search for Postgres. -- [PGVector + AutoGen Code Examples](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_pgvector_RetrieveChat.ipynb) +- [PGVector + AutoGen Code Examples](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_RetrieveChat_pgvector.ipynb) diff --git a/website/docs/installation/Optional-Dependencies.md b/website/docs/installation/Optional-Dependencies.md index 13991023f81b..2d0067c9950e 100644 --- a/website/docs/installation/Optional-Dependencies.md +++ b/website/docs/installation/Optional-Dependencies.md @@ -75,7 +75,7 @@ Example notebooks: [Group Chat with Retrieval Augmented Generation (with 5 group member agents and 1 manager agent)](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_groupchat_RAG.ipynb) -[Automated Code Generation and Question Answering with Qdrant based Retrieval Augmented Agents](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_qdrant_RetrieveChat.ipynb) +[Automated Code Generation and Question Answering with Qdrant based Retrieval Augmented Agents](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_RetrieveChat_qdrant.ipynb) ## Teachability diff --git a/website/docs/topics/groupchat/customized_speaker_selection.ipynb b/website/docs/topics/groupchat/customized_speaker_selection.ipynb index 830215a5e90b..2b800f3c8678 100644 --- a/website/docs/topics/groupchat/customized_speaker_selection.ipynb +++ b/website/docs/topics/groupchat/customized_speaker_selection.ipynb @@ -6,7 +6,34 @@ "source": [ "# Customize Speaker Selection\n", "\n", - "In GroupChat, we can also customize the speaker selection by passing in a function to `speaker_selection_method`:\n", + "```{=mdx}\n", + "![group_chat](../../../blog/2024-02-29-StateFlow/img/sf_example_1.png)\n", + "```\n", + "\n", + "In GroupChat, we can customize the speaker selection by passing a function to the `GroupChat` object. With this function, you can build a more **deterministic** agent workflow. We recommend following a **StateFlow** pattern when crafting this function. Please refer to the [StateFlow blog](/blog/2024/02/29/StateFlow) for more details.\n", + "\n", + "\n", + "## An example research workflow\n", + "We provide a simple example to build a StateFlow model for research with customized speaker selection.\n", + "\n", + "We first define the following agents:\n", + "\n", + "- Initializer: Start the workflow by sending a task.\n", + "- Coder: Retrieve papers from the internet by writing code.\n", + "- Executor: Execute the code.\n", + "- Scientist: Read the papers and write a summary.\n", + "\n", + "In the figure above, we define a simple workflow for research with 4 states: *Init*, *Retrieve*, *Research*, and *End*. Within each state, we will call different agents to perform the tasks.\n", + "\n", + "- *Init*: We use the initializer to start the workflow.\n", + "- *Retrieve*: We will first call the coder to write code and then call the executor to execute the code.\n", + "- *Research*: We will call the scientist to read the papers and write a summary.\n", + "- *End*: We will end the workflow.\n", + "\n", + "## Create your speaker selection function\n", + "\n", + "Below is a skeleton of the speaker selection function. Fill in the function to define the speaker selection logic.\n", + "\n", "```python\n", "def custom_speaker_selection_func(\n", " last_speaker: Agent, \n", @@ -35,28 +62,7 @@ ")\n", "```\n", "The last speaker and the groupchat object are passed to the function. \n", - "Commonly used variables from groupchat are `groupchat.messages` and `groupchat.agents`, which is the message history and the agents in the group chat respectively. You can access other attributes of the groupchat, such as `groupchat.allowed_speaker_transitions_dict` for pre-defined `allowed_speaker_transitions_dict`.\n", - "\n", - "Heres is a simple example to build workflow for research with customized speaker selection.\n", - "\n", - "\n", - "```{=mdx}\n", - "![group_chat](../../../blog/2024-02-29-StateFlow/img/sf_example_1.png)\n", - "```\n", - "\n", - "We define the following agents:\n", - "\n", - "- Initializer: Start the workflow by sending a task.\n", - "- Coder: Retrieve papers from the internet by writing code.\n", - "- Executor: Execute the code.\n", - "- Scientist: Read the papers and write a summary.\n", - "\n", - "In the Figure, we define a simple workflow for research with 4 states: Init, Retrieve, Research and End. Within each state, we will call different agents to perform the tasks.\n", - "\n", - "Init: We use the initializer to start the workflow.\n", - "Retrieve: We will first call the coder to write code and then call the executor to execute the code.\n", - "Research: We will call the scientist to read the papers and write a summary.\n", - "End: We will end the workflow." + "Commonly used variables from groupchat are `groupchat.messages` and `groupchat.agents`, which is the message history and the agents in the group chat respectively. You can access other attributes of the groupchat, such as `groupchat.allowed_speaker_transitions_dict` for pre-defined `allowed_speaker_transitions_dict`." ] }, { diff --git a/website/docs/topics/non-openai-models/cloud-cohere.ipynb b/website/docs/topics/non-openai-models/cloud-cohere.ipynb new file mode 100644 index 000000000000..fed5911475f4 --- /dev/null +++ b/website/docs/topics/non-openai-models/cloud-cohere.ipynb @@ -0,0 +1,534 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Cohere\n", + "\n", + "[Cohere](https://cohere.com/) is a cloud based platform serving their own LLMs, in particular the Command family of models.\n", + "\n", + "Cohere's API differs from OpenAI's, which is the native API used by AutoGen, so to use Cohere's LLMs you need to use this library.\n", + "\n", + "You will need a Cohere account and create an API key. [See their website for further details](https://cohere.com/)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Features\n", + "\n", + "When using this client class, AutoGen's messages are automatically tailored to accommodate the specific requirements of Cohere's API.\n", + "\n", + "Additionally, this client class provides support for function/tool calling and will track token usage and cost correctly as per Cohere's API costs (as of July 2024).\n", + "\n", + "## Getting started\n", + "\n", + "First you need to install the `pyautogen` package to use AutoGen with the Cohere API library.\n", + "\n", + "``` bash\n", + "pip install pyautogen[cohere]\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Cohere provides a number of models to use, included below. See the list of [models here](https://docs.cohere.com/docs/models).\n", + "\n", + "See the sample `OAI_CONFIG_LIST` below showing how the Cohere client class is used by specifying the `api_type` as `cohere`.\n", + "\n", + "```python\n", + "[\n", + " {\n", + " \"model\": \"gpt-35-turbo\",\n", + " \"api_key\": \"your OpenAI Key goes here\",\n", + " },\n", + " {\n", + " \"model\": \"gpt-4-vision-preview\",\n", + " \"api_key\": \"your OpenAI Key goes here\",\n", + " },\n", + " {\n", + " \"model\": \"dalle\",\n", + " \"api_key\": \"your OpenAI Key goes here\",\n", + " },\n", + " {\n", + " \"model\": \"command-r-plus\",\n", + " \"api_key\": \"your Cohere API Key goes here\",\n", + " \"api_type\": \"cohere\"\n", + " },\n", + " {\n", + " \"model\": \"command-r\",\n", + " \"api_key\": \"your Cohere API Key goes here\",\n", + " \"api_type\": \"cohere\"\n", + " },\n", + " {\n", + " \"model\": \"command\",\n", + " \"api_key\": \"your Cohere API Key goes here\",\n", + " \"api_type\": \"cohere\"\n", + " }\n", + "]\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As an alternative to the `api_key` key and value in the config, you can set the environment variable `COHERE_API_KEY` to your Cohere key.\n", + "\n", + "Linux/Mac:\n", + "``` bash\n", + "export COHERE_API_KEY=\"your_cohere_api_key_here\"\n", + "```\n", + "\n", + "Windows:\n", + "``` bash\n", + "set COHERE_API_KEY=your_cohere_api_key_here\n", + "```\n", + "\n", + "## API parameters\n", + "\n", + "The following parameters can be added to your config for the Cohere API. See [this link](https://docs.cohere.com/reference/chat) for further information on them and their default values.\n", + "\n", + "- temperature (number > 0)\n", + "- p (number 0.01..0.99)\n", + "- k (number 0..500)\n", + "- max_tokens (null, integer >= 0)\n", + "- seed (null, integer)\n", + "- frequency_penalty (number 0..1)\n", + "- presence_penalty (number 0..1)\n", + "\n", + "Example:\n", + "```python\n", + "[\n", + " {\n", + " \"model\": \"command-r\",\n", + " \"api_key\": \"your Cohere API Key goes here\",\n", + " \"api_type\": \"cohere\",\n", + " \"temperature\": 0.5,\n", + " \"p\": 0.2,\n", + " \"k\": 100,\n", + " \"max_tokens\": 2048,\n", + " \"seed\": 42,\n", + " \"frequency_penalty\": 0.5,\n", + " \"presence_penalty\": 0.2\n", + " }\n", + "]\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Two-Agent Coding Example\n", + "\n", + "In this example, we run a two-agent chat with an AssistantAgent (primarily a coding agent) to generate code to count the number of prime numbers between 1 and 10,000 and then it will be executed.\n", + "\n", + "We'll use Cohere's Command R model which is suitable for coding." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "config_list = [\n", + " {\n", + " # Let's choose the Command-R model\n", + " \"model\": \"command-r\",\n", + " # Provide your Cohere's API key here or put it into the COHERE_API_KEY environment variable.\n", + " \"api_key\": os.environ.get(\"COHERE_API_KEY\"),\n", + " # We specify the API Type as 'cohere' so it uses the Cohere client class\n", + " \"api_type\": \"cohere\",\n", + " }\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Importantly, we have tweaked the system message so that the model doesn't return the termination keyword, which we've changed to FINISH, with the code block." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "\n", + "from autogen import AssistantAgent, UserProxyAgent\n", + "from autogen.coding import LocalCommandLineCodeExecutor\n", + "\n", + "# Setting up the code executor\n", + "workdir = Path(\"coding\")\n", + "workdir.mkdir(exist_ok=True)\n", + "code_executor = LocalCommandLineCodeExecutor(work_dir=workdir)\n", + "\n", + "# Setting up the agents\n", + "\n", + "# The UserProxyAgent will execute the code that the AssistantAgent provides\n", + "user_proxy_agent = UserProxyAgent(\n", + " name=\"User\",\n", + " code_execution_config={\"executor\": code_executor},\n", + " is_termination_msg=lambda msg: \"FINISH\" in msg.get(\"content\"),\n", + ")\n", + "\n", + "system_message = \"\"\"You are a helpful AI assistant who writes code and the user executes it.\n", + "Solve tasks using your coding and language skills.\n", + "In the following cases, suggest python code (in a python coding block) for the user to execute.\n", + "Solve the task step by step if you need to. If a plan is not provided, explain your plan first. Be clear which step uses code, and which step uses your language skill.\n", + "When using code, you must indicate the script type in the code block. The user cannot provide any other feedback or perform any other action beyond executing the code you suggest. The user can't modify your code. So do not suggest incomplete code which requires users to modify. Don't use a code block if it's not intended to be executed by the user.\n", + "Don't include multiple code blocks in one response. Do not ask users to copy and paste the result. Instead, use 'print' function for the output when relevant. Check the execution result returned by the user.\n", + "If the result indicates there is an error, fix the error and output the code again. Suggest the full code instead of partial code or code changes. If the error can't be fixed or if the task is not solved even after the code is executed successfully, analyze the problem, revisit your assumption, collect additional info you need, and think of a different approach to try.\n", + "When you find an answer, verify the answer carefully. Include verifiable evidence in your response if possible.\n", + "IMPORTANT: Wait for the user to execute your code and then you can reply with the word \"FINISH\". DO NOT OUTPUT \"FINISH\" after your code block.\"\"\"\n", + "\n", + "# The AssistantAgent, using Cohere's model, will take the coding request and return code\n", + "assistant_agent = AssistantAgent(\n", + " name=\"Cohere Assistant\",\n", + " system_message=system_message,\n", + " llm_config={\"config_list\": config_list},\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mUser\u001b[0m (to Cohere Assistant):\n", + "\n", + "Provide code to count the number of prime numbers from 1 to 10000.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mCohere Assistant\u001b[0m (to User):\n", + "\n", + "Here's the code to count the number of prime numbers from 1 to 10,000:\n", + "```python\n", + "# Prime Number Counter\n", + "count = 0\n", + "for num in range(2, 10001):\n", + " if num > 1:\n", + " for div in range(2, num):\n", + " if (num % div) == 0:\n", + " break\n", + " else:\n", + " count += 1\n", + "print(count)\n", + "```\n", + "\n", + "My plan is to use two nested loops. The outer loop iterates through numbers from 2 to 10,000. The inner loop checks if there's any divisor for the current number in the range from 2 to the number itself. If there's no such divisor, the number is prime and the counter is incremented.\n", + "\n", + "Please execute the code and let me know the output.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> NO HUMAN INPUT RECEIVED.\u001b[0m\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK (inferred language is python)...\u001b[0m\n", + "\u001b[33mUser\u001b[0m (to Cohere Assistant):\n", + "\n", + "exitcode: 0 (execution succeeded)\n", + "Code output: 1229\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mCohere Assistant\u001b[0m (to User):\n", + "\n", + "That's correct! The code you executed successfully found 1229 prime numbers within the specified range.\n", + "\n", + "FINISH.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> NO HUMAN INPUT RECEIVED.\u001b[0m\n" + ] + } + ], + "source": [ + "# Start the chat, with the UserProxyAgent asking the AssistantAgent the message\n", + "chat_result = user_proxy_agent.initiate_chat(\n", + " assistant_agent,\n", + " message=\"Provide code to count the number of prime numbers from 1 to 10000.\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Tool Call Example\n", + "\n", + "In this example, instead of writing code, we will show how Cohere's Command R+ model can perform parallel tool calling, where it recommends calling more than one tool at a time.\n", + "\n", + "We'll use a simple travel agent assistant program where we have a couple of tools for weather and currency conversion.\n", + "\n", + "We start by importing libraries and setting up our configuration to use Command R+ and the `cohere` client class." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import os\n", + "from typing import Literal\n", + "\n", + "from typing_extensions import Annotated\n", + "\n", + "import autogen\n", + "\n", + "config_list = [\n", + " {\"api_type\": \"cohere\", \"model\": \"command-r-plus\", \"api_key\": os.getenv(\"COHERE_API_KEY\"), \"cache_seed\": None}\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Create our two agents." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "# Create the agent for tool calling\n", + "chatbot = autogen.AssistantAgent(\n", + " name=\"chatbot\",\n", + " system_message=\"\"\"For currency exchange and weather forecasting tasks,\n", + " only use the functions you have been provided with.\n", + " Output 'HAVE FUN!' when an answer has been provided.\"\"\",\n", + " llm_config={\"config_list\": config_list},\n", + ")\n", + "\n", + "# Note that we have changed the termination string to be \"HAVE FUN!\"\n", + "user_proxy = autogen.UserProxyAgent(\n", + " name=\"user_proxy\",\n", + " is_termination_msg=lambda x: x.get(\"content\", \"\") and \"HAVE FUN!\" in x.get(\"content\", \"\"),\n", + " human_input_mode=\"NEVER\",\n", + " max_consecutive_auto_reply=1,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Create the two functions, annotating them so that those descriptions can be passed through to the LLM.\n", + "\n", + "We associate them with the agents using `register_for_execution` for the user_proxy so it can execute the function and `register_for_llm` for the chatbot (powered by the LLM) so it can pass the function definitions to the LLM." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "# Currency Exchange function\n", + "\n", + "CurrencySymbol = Literal[\"USD\", \"EUR\"]\n", + "\n", + "# Define our function that we expect to call\n", + "\n", + "\n", + "def exchange_rate(base_currency: CurrencySymbol, quote_currency: CurrencySymbol) -> float:\n", + " if base_currency == quote_currency:\n", + " return 1.0\n", + " elif base_currency == \"USD\" and quote_currency == \"EUR\":\n", + " return 1 / 1.1\n", + " elif base_currency == \"EUR\" and quote_currency == \"USD\":\n", + " return 1.1\n", + " else:\n", + " raise ValueError(f\"Unknown currencies {base_currency}, {quote_currency}\")\n", + "\n", + "\n", + "# Register the function with the agent\n", + "\n", + "\n", + "@user_proxy.register_for_execution()\n", + "@chatbot.register_for_llm(description=\"Currency exchange calculator.\")\n", + "def currency_calculator(\n", + " base_amount: Annotated[float, \"Amount of currency in base_currency\"],\n", + " base_currency: Annotated[CurrencySymbol, \"Base currency\"] = \"USD\",\n", + " quote_currency: Annotated[CurrencySymbol, \"Quote currency\"] = \"EUR\",\n", + ") -> str:\n", + " quote_amount = exchange_rate(base_currency, quote_currency) * base_amount\n", + " return f\"{format(quote_amount, '.2f')} {quote_currency}\"\n", + "\n", + "\n", + "# Weather function\n", + "\n", + "\n", + "# Example function to make available to model\n", + "def get_current_weather(location, unit=\"fahrenheit\"):\n", + " \"\"\"Get the weather for some location\"\"\"\n", + " if \"chicago\" in location.lower():\n", + " return json.dumps({\"location\": \"Chicago\", \"temperature\": \"13\", \"unit\": unit})\n", + " elif \"san francisco\" in location.lower():\n", + " return json.dumps({\"location\": \"San Francisco\", \"temperature\": \"55\", \"unit\": unit})\n", + " elif \"new york\" in location.lower():\n", + " return json.dumps({\"location\": \"New York\", \"temperature\": \"11\", \"unit\": unit})\n", + " else:\n", + " return json.dumps({\"location\": location, \"temperature\": \"unknown\"})\n", + "\n", + "\n", + "# Register the function with the agent\n", + "\n", + "\n", + "@user_proxy.register_for_execution()\n", + "@chatbot.register_for_llm(description=\"Weather forecast for US cities.\")\n", + "def weather_forecast(\n", + " location: Annotated[str, \"City name\"],\n", + ") -> str:\n", + " weather_details = get_current_weather(location=location)\n", + " weather = json.loads(weather_details)\n", + " return f\"{weather['location']} will be {weather['temperature']} degrees {weather['unit']}\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We pass through our customers message and run the chat.\n", + "\n", + "Finally, we ask the LLM to summarise the chat and print that out." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33muser_proxy\u001b[0m (to chatbot):\n", + "\n", + "What's the weather in New York and can you tell me how much is 123.45 EUR in USD so I can spend it on my holiday? Throw a few holiday tips in as well.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mchatbot\u001b[0m (to user_proxy):\n", + "\n", + "I will use the weather_forecast function to find out the weather in New York, and the currency_calculator function to convert 123.45 EUR to USD. I will then search for 'holiday tips' to find some extra information to include in my answer.\n", + "\u001b[32m***** Suggested tool call (45212): weather_forecast *****\u001b[0m\n", + "Arguments: \n", + "{\"location\": \"New York\"}\n", + "\u001b[32m*********************************************************\u001b[0m\n", + "\u001b[32m***** Suggested tool call (16564): currency_calculator *****\u001b[0m\n", + "Arguments: \n", + "{\"base_amount\": 123.45, \"base_currency\": \"EUR\", \"quote_currency\": \"USD\"}\n", + "\u001b[32m************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION weather_forecast...\u001b[0m\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION currency_calculator...\u001b[0m\n", + "\u001b[33muser_proxy\u001b[0m (to chatbot):\n", + "\n", + "\u001b[33muser_proxy\u001b[0m (to chatbot):\n", + "\n", + "\u001b[32m***** Response from calling tool (45212) *****\u001b[0m\n", + "New York will be 11 degrees fahrenheit\n", + "\u001b[32m**********************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33muser_proxy\u001b[0m (to chatbot):\n", + "\n", + "\u001b[32m***** Response from calling tool (16564) *****\u001b[0m\n", + "135.80 USD\n", + "\u001b[32m**********************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mchatbot\u001b[0m (to user_proxy):\n", + "\n", + "The weather in New York is 11 degrees Fahrenheit. \n", + "\n", + "€123.45 is worth $135.80. \n", + "\n", + "Here are some holiday tips:\n", + "- Make sure to pack layers for the cold weather\n", + "- Try the local cuisine, New York is famous for its pizza\n", + "- Visit Central Park and take in the views from the top of the Rockefeller Centre\n", + "\n", + "HAVE FUN!\n", + "\n", + "--------------------------------------------------------------------------------\n", + "LLM SUMMARY: The weather in New York is 11 degrees Fahrenheit. 123.45 EUR is worth 135.80 USD. Holiday tips: make sure to pack warm clothes and have a great time!\n" + ] + } + ], + "source": [ + "# start the conversation\n", + "res = user_proxy.initiate_chat(\n", + " chatbot,\n", + " message=\"What's the weather in New York and can you tell me how much is 123.45 EUR in USD so I can spend it on my holiday? Throw a few holiday tips in as well.\",\n", + " summary_method=\"reflection_with_llm\",\n", + ")\n", + "\n", + "print(f\"LLM SUMMARY: {res.summary['content']}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can see that Command R+ recommended we call both tools and passed through the right parameters. The `user_proxy` executed them and this was passed back to Command R+ to interpret them and respond. Finally, Command R+ was asked to summarise the whole conversation." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "autogen", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb b/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb new file mode 100644 index 000000000000..eaec2b72b268 --- /dev/null +++ b/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb @@ -0,0 +1,796 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "# Use AutoGen with Gemini via VertexAI\n", + "\n", + "This notebook demonstrates how to use Autogen with Gemini via Vertex AI, which enables enhanced authentication method that also supports enterprise requirements using service accounts or even a personal Google cloud account.\n", + "\n", + "## Requirements\n", + "\n", + "AutoGen requires `Python>=3.8`. To run this notebook example, please install with the [gemini] option:\n", + "```bash\n", + "pip install \"pyautogen[gemini]\"\n", + "```\n", + "\n", + "### Google Cloud Account\n", + "To use VertexAI a Google Cloud account is needed. If you do not have one yet, just sign up for a free trial [here](https://cloud.google.com).\n", + "\n", + "Login to your account at [console.cloud.google.com](https://console.cloud.google.com)\n", + "\n", + "In the next step we create a Google Cloud project, which is needed for VertexAI. The official guide for creating a project is available is [here](https://developers.google.com/workspace/guides/create-project). \n", + "\n", + "We will name our project Autogen-with-Gemini.\n", + "\n", + "### Enable Google Cloud APIs\n", + "\n", + "If you wish to use Gemini with your personal account, then creating a Google Cloud account is enough. However, if a service account is needed, then a few extra steps are needed.\n", + "\n", + "#### Enable API for Gemini\n", + " * For enabling Gemini for Google Cloud search for \"api\" and select Enabled APIs & services. \n", + " * Then click ENABLE APIS AND SERVICES. \n", + " * Search for Gemini, and select Gemini for Google Cloud.
A direct link will look like this for our autogen-with-gemini project:\n", + "https://console.cloud.google.com/apis/library/cloudaicompanion.googleapis.com?project=autogen-with-gemini&supportedpurview=project\n", + "* Click ENABLE for Gemini for Google Cloud.\n", + "\n", + "### Enable API for Vertex AI\n", + "* For enabling Vertex AI for Google Cloud search for \"api\" and select Enabled APIs & services. \n", + "* Then click ENABLE APIS AND SERVICES. \n", + "* Search for Vertex AI, and select Vertex AI API.
A direct link for our autogen-with-gemini will be: https://console.cloud.google.com/apis/library/aiplatform.googleapis.com?project=autogen-with-gemini\n", + "* Click ENABLE Vertex AI API for Google Cloud.\n", + "\n", + "### Create a Service Account\n", + "\n", + "You can find an overview of service accounts [can be found in the cloud console](https://console.cloud.google.com/iam-admin/serviceaccounts)\n", + "\n", + "Detailed guide: https://cloud.google.com/iam/docs/service-accounts-create\n", + "\n", + "A service account can be created within the scope of a project, so a project needs to be selected.\n", + "\n", + "
\n", + "\n", + "
\n", + "\n", + "For the sake of simplicity we will assign the Editor role to our service account for autogen on our Autogen-with-Gemini Google Cloud project.\n", + "\n", + "* Under IAM & Admin > Service Account select the newly created service accounts, and click the option \"Manage keys\" among the items. \n", + "* From the \"ADD KEY\" dropdown select \"Create new key\" and select the JSON format and click CREATE.\n", + " * The new key will be downloaded automatically. \n", + "* You can then upload the service account key file to the from where you will be running autogen. \n", + " * Please consider restricting the permissions on the key file. For example, you could run `chmod 600 autogen-with-gemini-service-account-key.json` if your keyfile is called autogen-with-gemini-service-account-key.json." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "execution": { + "iopub.execute_input": "2023-02-13T23:40:52.317406Z", + "iopub.status.busy": "2023-02-13T23:40:52.316561Z", + "iopub.status.idle": "2023-02-13T23:40:52.321193Z", + "shell.execute_reply": "2023-02-13T23:40:52.320628Z" + } + }, + "outputs": [], + "source": [ + "# %pip install \"pyautogen[gemini]\"" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "execution": { + "iopub.execute_input": "2023-02-13T23:40:54.634335Z", + "iopub.status.busy": "2023-02-13T23:40:54.633929Z", + "iopub.status.idle": "2023-02-13T23:40:56.105700Z", + "shell.execute_reply": "2023-02-13T23:40:56.105085Z" + }, + "slideshow": { + "slide_type": "slide" + } + }, + "outputs": [], + "source": [ + "import autogen" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Configure Authentication\n", + "\n", + "Authentication happens using standard [Google Cloud authentication methods](https://cloud.google.com/docs/authentication),
which means\n", + "that either an already active session can be reused, or by specifying the Google application credentials of a service account.\n", + "\n", + "#### Use Service Account Keyfile\n", + "\n", + "The Google Cloud service account can be specified by setting the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path to the JSON key file of the service account.
\n", + "\n", + "We could even just directly set the environment variable, or we can add the `\"google_application_credentials\"` key with the respective value for our model in the OAI_CONFIG_LIST.\n", + "\n", + "#### Use the Google Default Credentials\n", + "\n", + "If you are using [Cloud Shell](https://shell.cloud.google.com/cloudshell) or [Cloud Shell editor](https://shell.cloud.google.com/cloudshell/editor) in Google Cloud,
then you are already authenticated. If you have the Google Cloud SDK installed locally,
then you can login by running `gcloud auth login` in the command line. \n", + "\n", + "Detailed instructions for installing the Google Cloud SDK can be found [here](https://cloud.google.com/sdk/docs/install)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Example Config List\n", + "The config could look like the following (change `project_id` and `google_application_credentials`):\n", + "```python\n", + "config_list = [\n", + " {\n", + " \"model\": \"gemini-pro\",\n", + " \"api_type\": \"google\",\n", + " \"project_id\": \"autogen-with-gemini\",\n", + " \"location\": \"us-west1\"\n", + " },\n", + " {\n", + " \"model\": \"gemini-1.5-pro-001\",\n", + " \"api_type\": \"google\",\n", + " \"project_id\": \"autogen-with-gemini\",\n", + " \"location\": \"us-west1\"\n", + " },\n", + " {\n", + " \"model\": \"gemini-1.5-pro\",\n", + " \"api_type\": \"google\",\n", + " \"project\": \"autogen-with-gemini\",\n", + " \"location\": \"us-west1\",\n", + " \"google_application_credentials\": \"autogen-with-gemini-service-account-key.json\"\n", + " },\n", + " {\n", + " \"model\": \"gemini-pro-vision\",\n", + " \"api_type\": \"google\",\n", + " \"project_id\": \"autogen-with-gemini\",\n", + " \"location\": \"us-west1\"\n", + " }\n", + "]\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Configure Safety Settings for VertexAI\n", + "Configuring safety settings for VertexAI is slightly different, as we have to use the speicialized safety setting object types instead of plain strings" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "from vertexai.generative_models import (\n", + " GenerationConfig,\n", + " GenerativeModel,\n", + " HarmBlockThreshold,\n", + " HarmCategory,\n", + " Part,\n", + ")\n", + "\n", + "safety_settings = {\n", + " HarmCategory.HARM_CATEGORY_HARASSMENT: HarmBlockThreshold.BLOCK_ONLY_HIGH,\n", + " HarmCategory.HARM_CATEGORY_HATE_SPEECH: HarmBlockThreshold.BLOCK_ONLY_HIGH,\n", + " HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: HarmBlockThreshold.BLOCK_ONLY_HIGH,\n", + " HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_ONLY_HIGH,\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union\n", + "\n", + "import chromadb\n", + "from PIL import Image\n", + "from termcolor import colored\n", + "\n", + "from autogen import Agent, AssistantAgent, ConversableAgent, UserProxyAgent\n", + "from autogen.agentchat.contrib.img_utils import _to_pil, get_image_data\n", + "from autogen.agentchat.contrib.multimodal_conversable_agent import MultimodalConversableAgent\n", + "from autogen.agentchat.contrib.retrieve_assistant_agent import RetrieveAssistantAgent\n", + "from autogen.agentchat.contrib.retrieve_user_proxy_agent import RetrieveUserProxyAgent\n", + "from autogen.code_utils import DEFAULT_MODEL, UNKNOWN, content_str, execute_code, extract_code, infer_lang" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "config_list_gemini = autogen.config_list_from_json(\n", + " \"OAI_CONFIG_LIST\",\n", + " filter_dict={\n", + " \"model\": [\"gemini-1.5-pro\"],\n", + " },\n", + ")\n", + "\n", + "config_list_gemini_vision = autogen.config_list_from_json(\n", + " \"OAI_CONFIG_LIST\",\n", + " filter_dict={\n", + " \"model\": [\"gemini-pro-vision\"],\n", + " },\n", + ")\n", + "\n", + "for config_list in [config_list_gemini, config_list_gemini_vision]:\n", + " for config_list_item in config_list:\n", + " config_list_item[\"safety_settings\"] = safety_settings\n", + "\n", + "seed = 25 # for caching" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33muser_proxy\u001b[0m (to assistant):\n", + "\n", + "\n", + " Compute the integral of the function f(x)=x^2 on the interval 0 to 1 using a Python script, \n", + " which returns the value of the definite integral.\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33massistant\u001b[0m (to user_proxy):\n", + "\n", + "Plan:\n", + "1. (Code) Use Python's numerical integration library to compute the integral.\n", + "2. (Language) Output the result.\n", + "\n", + "```python\n", + "# filename: integral.py\n", + "import scipy.integrate\n", + "\n", + "f = lambda x: x**2\n", + "result, error = scipy.integrate.quad(f, 0, 1)\n", + "\n", + "print(f\"The definite integral of x^2 from 0 to 1 is: {result}\")\n", + "```\n", + "\n", + "Let me know when you have executed the code. \n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001b[0m\n", + "\u001b[33muser_proxy\u001b[0m (to assistant):\n", + "\n", + "exitcode: 0 (execution succeeded)\n", + "Code output: \n", + "The definite integral of x^2 from 0 to 1 is: 0.33333333333333337\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33massistant\u001b[0m (to user_proxy):\n", + "\n", + "The code executed successfully and returned the value of the definite integral as approximately 0.33333333333333337. \n", + "\n", + "This aligns with the analytical solution:\n", + "\n", + "The integral of x^2 is (x^3)/3. Evaluating this from 0 to 1 gives us (1^3)/3 - (0^3)/3 = 1/3 = 0.33333...\n", + "\n", + "Therefore, the answer is verified to be correct.\n", + "\n", + "TERMINATE\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + } + ], + "source": [ + "assistant = AssistantAgent(\n", + " \"assistant\", llm_config={\"config_list\": config_list_gemini, \"seed\": seed}, max_consecutive_auto_reply=3\n", + ")\n", + "\n", + "user_proxy = UserProxyAgent(\n", + " \"user_proxy\",\n", + " code_execution_config={\"work_dir\": \"coding\", \"use_docker\": False},\n", + " human_input_mode=\"NEVER\",\n", + " is_termination_msg=lambda x: content_str(x.get(\"content\")).find(\"TERMINATE\") >= 0,\n", + ")\n", + "\n", + "result = user_proxy.initiate_chat(\n", + " assistant,\n", + " message=\"\"\"\n", + " Compute the integral of the function f(x)=x^2 on the interval 0 to 1 using a Python script,\n", + " which returns the value of the definite integral.\"\"\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Example with Gemini Multimodal\n", + "Authentication is the same for vision models as for the text based Gemini models" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33muser_proxy\u001b[0m (to Gemini Vision):\n", + "\n", + "Describe what is in this image?\n", + ".\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mGemini Vision\u001b[0m (to user_proxy):\n", + "\n", + " The image shows a taxonomy of different types of conversational agents. The taxonomy is based on two dimensions: agent customization and flexible conversation patterns. Agent customization refers to the ability of the agent to be tailored to the individual user. Flexible conversation patterns refer to the ability of the agent to engage in different types of conversations, such as joint chat and hierarchical chat.\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + }, + { + "data": { + "text/plain": [ + "ChatResult(chat_id=None, chat_history=[{'content': 'Describe what is in this image?\\n.', 'role': 'assistant'}, {'content': ' The image shows a taxonomy of different types of conversational agents. The taxonomy is based on two dimensions: agent customization and flexible conversation patterns. Agent customization refers to the ability of the agent to be tailored to the individual user. Flexible conversation patterns refer to the ability of the agent to engage in different types of conversations, such as joint chat and hierarchical chat.', 'role': 'user'}], summary=' The image shows a taxonomy of different types of conversational agents. The taxonomy is based on two dimensions: agent customization and flexible conversation patterns. Agent customization refers to the ability of the agent to be tailored to the individual user. Flexible conversation patterns refer to the ability of the agent to engage in different types of conversations, such as joint chat and hierarchical chat.', cost={'usage_including_cached_inference': {'total_cost': 0.0002385, 'gemini-pro-vision': {'cost': 0.0002385, 'prompt_tokens': 267, 'completion_tokens': 70, 'total_tokens': 337}}, 'usage_excluding_cached_inference': {'total_cost': 0.0002385, 'gemini-pro-vision': {'cost': 0.0002385, 'prompt_tokens': 267, 'completion_tokens': 70, 'total_tokens': 337}}}, human_input=[])" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "image_agent = MultimodalConversableAgent(\n", + " \"Gemini Vision\", llm_config={\"config_list\": config_list_gemini_vision, \"seed\": seed}, max_consecutive_auto_reply=1\n", + ")\n", + "\n", + "user_proxy = UserProxyAgent(\"user_proxy\", human_input_mode=\"NEVER\", max_consecutive_auto_reply=0)\n", + "\n", + "user_proxy.initiate_chat(\n", + " image_agent,\n", + " message=\"\"\"Describe what is in this image?\n", + ".\"\"\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "front_matter": { + "description": "Using Gemini with AutoGen via VertexAI", + "tags": [ + "gemini", + "vertexai" + ] + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + }, + "vscode": { + "interpreter": { + "hash": "949777d72b0d2535278d3dc13498b2535136f6dfe0678499012e853ee9abcab1" + } + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "state": { + "2d910cfd2d2a4fc49fc30fbbdc5576a7": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "454146d0f7224f038689031002906e6f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_e4ae2b6f5a974fd4bafb6abb9d12ff26", + "IPY_MODEL_577e1e3cc4db4942b0883577b3b52755", + "IPY_MODEL_b40bdfb1ac1d4cffb7cefcb870c64d45" + ], + "layout": "IPY_MODEL_dc83c7bff2f241309537a8119dfc7555", + "tabbable": null, + "tooltip": null + } + }, + "577e1e3cc4db4942b0883577b3b52755": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_2d910cfd2d2a4fc49fc30fbbdc5576a7", + "max": 1, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_74a6ba0c3cbc4051be0a83e152fe1e62", + "tabbable": null, + "tooltip": null, + "value": 1 + } + }, + "6086462a12d54bafa59d3c4566f06cb2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "74a6ba0c3cbc4051be0a83e152fe1e62": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "7d3f3d9e15894d05a4d188ff4f466554": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "b40bdfb1ac1d4cffb7cefcb870c64d45": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_f1355871cc6f4dd4b50d9df5af20e5c8", + "placeholder": "​", + "style": "IPY_MODEL_ca245376fd9f4354af6b2befe4af4466", + "tabbable": null, + "tooltip": null, + "value": " 1/1 [00:00<00:00, 44.69it/s]" + } + }, + "ca245376fd9f4354af6b2befe4af4466": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "dc83c7bff2f241309537a8119dfc7555": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e4ae2b6f5a974fd4bafb6abb9d12ff26": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_6086462a12d54bafa59d3c4566f06cb2", + "placeholder": "​", + "style": "IPY_MODEL_7d3f3d9e15894d05a4d188ff4f466554", + "tabbable": null, + "tooltip": null, + "value": "100%" + } + }, + "f1355871cc6f4dd4b50d9df5af20e5c8": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + } + }, + "version_major": 2, + "version_minor": 0 + } + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/website/docs/topics/non-openai-models/cloud-groq.ipynb b/website/docs/topics/non-openai-models/cloud-groq.ipynb new file mode 100644 index 000000000000..d2289cbdcd45 --- /dev/null +++ b/website/docs/topics/non-openai-models/cloud-groq.ipynb @@ -0,0 +1,524 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Groq\n", + "\n", + "[Groq](https://groq.com/) is a cloud based platform serving a number of popular open weight models at high inference speeds. Models include Meta's Llama 3, Mistral AI's Mixtral, and Google's Gemma.\n", + "\n", + "Although Groq's API is aligned well with OpenAI's, which is the native API used by AutoGen, this library provides the ability to set specific parameters as well as track API costs.\n", + "\n", + "You will need a Groq account and create an API key. [See their website for further details](https://groq.com/)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Groq provides a number of models to use, included below. See the list of [models here (requires login)](https://console.groq.com/docs/models).\n", + "\n", + "See the sample `OAI_CONFIG_LIST` below showing how the Groq client class is used by specifying the `api_type` as `groq`.\n", + "\n", + "```python\n", + "[\n", + " {\n", + " \"model\": \"gpt-35-turbo\",\n", + " \"api_key\": \"your OpenAI Key goes here\",\n", + " },\n", + " {\n", + " \"model\": \"gpt-4-vision-preview\",\n", + " \"api_key\": \"your OpenAI Key goes here\",\n", + " },\n", + " {\n", + " \"model\": \"dalle\",\n", + " \"api_key\": \"your OpenAI Key goes here\",\n", + " },\n", + " {\n", + " \"model\": \"llama3-8b-8192\",\n", + " \"api_key\": \"your Groq API Key goes here\",\n", + " \"api_type\": \"groq\"\n", + " },\n", + " {\n", + " \"model\": \"llama3-70b-8192\",\n", + " \"api_key\": \"your Groq API Key goes here\",\n", + " \"api_type\": \"groq\"\n", + " },\n", + " {\n", + " \"model\": \"Mixtral 8x7b\",\n", + " \"api_key\": \"your Groq API Key goes here\",\n", + " \"api_type\": \"groq\"\n", + " },\n", + " {\n", + " \"model\": \"gemma-7b-it\",\n", + " \"api_key\": \"your Groq API Key goes here\",\n", + " \"api_type\": \"groq\"\n", + " }\n", + "]\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As an alternative to the `api_key` key and value in the config, you can set the environment variable `GROQ_API_KEY` to your Groq key.\n", + "\n", + "Linux/Mac:\n", + "``` bash\n", + "export GROQ_API_KEY=\"your_groq_api_key_here\"\n", + "```\n", + "\n", + "Windows:\n", + "``` bash\n", + "set GROQ_API_KEY=your_groq_api_key_here\n", + "```\n", + "\n", + "## API parameters\n", + "\n", + "The following parameters can be added to your config for the Groq API. See [this link](https://console.groq.com/docs/text-chat) for further information on them.\n", + "\n", + "- frequency_penalty (number 0..1)\n", + "- max_tokens (integer >= 0)\n", + "- presence_penalty (number -2..2)\n", + "- seed (integer)\n", + "- temperature (number 0..2)\n", + "- top_p (number)\n", + "\n", + "Example:\n", + "```python\n", + "[\n", + " {\n", + " \"model\": \"llama3-8b-8192\",\n", + " \"api_key\": \"your Groq API Key goes here\",\n", + " \"api_type\": \"groq\",\n", + " \"frequency_penalty\": 0.5,\n", + " \"max_tokens\": 2048,\n", + " \"presence_penalty\": 0.2,\n", + " \"seed\": 42,\n", + " \"temperature\": 0.5,\n", + " \"top_p\": 0.2\n", + " }\n", + "]\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Two-Agent Coding Example\n", + "\n", + "In this example, we run a two-agent chat with an AssistantAgent (primarily a coding agent) to generate code to count the number of prime numbers between 1 and 10,000 and then it will be executed.\n", + "\n", + "We'll use Meta's Llama 3 model which is suitable for coding." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "config_list = [\n", + " {\n", + " # Let's choose the Llama 3 model\n", + " \"model\": \"llama3-8b-8192\",\n", + " # Put your Groq API key here or put it into the GROQ_API_KEY environment variable.\n", + " \"api_key\": os.environ.get(\"GROQ_API_KEY\"),\n", + " # We specify the API Type as 'groq' so it uses the Groq client class\n", + " \"api_type\": \"groq\",\n", + " }\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Importantly, we have tweaked the system message so that the model doesn't return the termination keyword, which we've changed to FINISH, with the code block." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "\n", + "from autogen import AssistantAgent, UserProxyAgent\n", + "from autogen.coding import LocalCommandLineCodeExecutor\n", + "\n", + "# Setting up the code executor\n", + "workdir = Path(\"coding\")\n", + "workdir.mkdir(exist_ok=True)\n", + "code_executor = LocalCommandLineCodeExecutor(work_dir=workdir)\n", + "\n", + "# Setting up the agents\n", + "\n", + "# The UserProxyAgent will execute the code that the AssistantAgent provides\n", + "user_proxy_agent = UserProxyAgent(\n", + " name=\"User\",\n", + " code_execution_config={\"executor\": code_executor},\n", + " is_termination_msg=lambda msg: \"FINISH\" in msg.get(\"content\"),\n", + ")\n", + "\n", + "system_message = \"\"\"You are a helpful AI assistant who writes code and the user executes it.\n", + "Solve tasks using your coding and language skills.\n", + "In the following cases, suggest python code (in a python coding block) for the user to execute.\n", + "Solve the task step by step if you need to. If a plan is not provided, explain your plan first. Be clear which step uses code, and which step uses your language skill.\n", + "When using code, you must indicate the script type in the code block. The user cannot provide any other feedback or perform any other action beyond executing the code you suggest. The user can't modify your code. So do not suggest incomplete code which requires users to modify. Don't use a code block if it's not intended to be executed by the user.\n", + "Don't include multiple code blocks in one response. Do not ask users to copy and paste the result. Instead, use 'print' function for the output when relevant. Check the execution result returned by the user.\n", + "If the result indicates there is an error, fix the error and output the code again. Suggest the full code instead of partial code or code changes. If the error can't be fixed or if the task is not solved even after the code is executed successfully, analyze the problem, revisit your assumption, collect additional info you need, and think of a different approach to try.\n", + "When you find an answer, verify the answer carefully. Include verifiable evidence in your response if possible.\n", + "IMPORTANT: Wait for the user to execute your code and then you can reply with the word \"FINISH\". DO NOT OUTPUT \"FINISH\" after your code block.\"\"\"\n", + "\n", + "# The AssistantAgent, using Groq's model, will take the coding request and return code\n", + "assistant_agent = AssistantAgent(\n", + " name=\"Groq Assistant\",\n", + " system_message=system_message,\n", + " llm_config={\"config_list\": config_list},\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mUser\u001b[0m (to Groq Assistant):\n", + "\n", + "Provide code to count the number of prime numbers from 1 to 10000.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mGroq Assistant\u001b[0m (to User):\n", + "\n", + "Here's the plan to count the number of prime numbers from 1 to 10000:\n", + "\n", + "First, we need to write a helper function to check if a number is prime. A prime number is a number that is divisible only by 1 and itself.\n", + "\n", + "Then, we can use a loop to iterate through all numbers from 1 to 10000, check if each number is prime using our helper function, and count the number of prime numbers found.\n", + "\n", + "Here's the Python code to implement this plan:\n", + "```python\n", + "def is_prime(n):\n", + " if n <= 1:\n", + " return False\n", + " for i in range(2, int(n ** 0.5) + 1):\n", + " if n % i == 0:\n", + " return False\n", + " return True\n", + "\n", + "count = 0\n", + "for i in range(2, 10001):\n", + " if is_prime(i):\n", + " count += 1\n", + "\n", + "print(count)\n", + "```\n", + "Please execute this code, and I'll wait for the result.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> NO HUMAN INPUT RECEIVED.\u001b[0m\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK (inferred language is python)...\u001b[0m\n", + "\u001b[33mUser\u001b[0m (to Groq Assistant):\n", + "\n", + "exitcode: 0 (execution succeeded)\n", + "Code output: 1229\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mGroq Assistant\u001b[0m (to User):\n", + "\n", + "FINISH\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> NO HUMAN INPUT RECEIVED.\u001b[0m\n" + ] + } + ], + "source": [ + "# Start the chat, with the UserProxyAgent asking the AssistantAgent the message\n", + "chat_result = user_proxy_agent.initiate_chat(\n", + " assistant_agent,\n", + " message=\"Provide code to count the number of prime numbers from 1 to 10000.\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Tool Call Example\n", + "\n", + "In this example, instead of writing code, we will show how we can use Meta's Llama 3 model to perform parallel tool calling, where it recommends calling more than one tool at a time, using Groq's cloud inference.\n", + "\n", + "We'll use a simple travel agent assistant program where we have a couple of tools for weather and currency conversion.\n", + "\n", + "We start by importing libraries and setting up our configuration to use Meta's Llama 3 model and the `groq` client class." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import os\n", + "from typing import Literal\n", + "\n", + "from typing_extensions import Annotated\n", + "\n", + "import autogen\n", + "\n", + "config_list = [\n", + " {\"api_type\": \"groq\", \"model\": \"llama3-8b-8192\", \"api_key\": os.getenv(\"GROQ_API_KEY\"), \"cache_seed\": None}\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Create our two agents." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "# Create the agent for tool calling\n", + "chatbot = autogen.AssistantAgent(\n", + " name=\"chatbot\",\n", + " system_message=\"\"\"For currency exchange and weather forecasting tasks,\n", + " only use the functions you have been provided with.\n", + " Output 'HAVE FUN!' when an answer has been provided.\"\"\",\n", + " llm_config={\"config_list\": config_list},\n", + ")\n", + "\n", + "# Note that we have changed the termination string to be \"HAVE FUN!\"\n", + "user_proxy = autogen.UserProxyAgent(\n", + " name=\"user_proxy\",\n", + " is_termination_msg=lambda x: x.get(\"content\", \"\") and \"HAVE FUN!\" in x.get(\"content\", \"\"),\n", + " human_input_mode=\"NEVER\",\n", + " max_consecutive_auto_reply=1,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Create the two functions, annotating them so that those descriptions can be passed through to the LLM.\n", + "\n", + "We associate them with the agents using `register_for_execution` for the user_proxy so it can execute the function and `register_for_llm` for the chatbot (powered by the LLM) so it can pass the function definitions to the LLM." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "# Currency Exchange function\n", + "\n", + "CurrencySymbol = Literal[\"USD\", \"EUR\"]\n", + "\n", + "# Define our function that we expect to call\n", + "\n", + "\n", + "def exchange_rate(base_currency: CurrencySymbol, quote_currency: CurrencySymbol) -> float:\n", + " if base_currency == quote_currency:\n", + " return 1.0\n", + " elif base_currency == \"USD\" and quote_currency == \"EUR\":\n", + " return 1 / 1.1\n", + " elif base_currency == \"EUR\" and quote_currency == \"USD\":\n", + " return 1.1\n", + " else:\n", + " raise ValueError(f\"Unknown currencies {base_currency}, {quote_currency}\")\n", + "\n", + "\n", + "# Register the function with the agent\n", + "\n", + "\n", + "@user_proxy.register_for_execution()\n", + "@chatbot.register_for_llm(description=\"Currency exchange calculator.\")\n", + "def currency_calculator(\n", + " base_amount: Annotated[float, \"Amount of currency in base_currency\"],\n", + " base_currency: Annotated[CurrencySymbol, \"Base currency\"] = \"USD\",\n", + " quote_currency: Annotated[CurrencySymbol, \"Quote currency\"] = \"EUR\",\n", + ") -> str:\n", + " quote_amount = exchange_rate(base_currency, quote_currency) * base_amount\n", + " return f\"{format(quote_amount, '.2f')} {quote_currency}\"\n", + "\n", + "\n", + "# Weather function\n", + "\n", + "\n", + "# Example function to make available to model\n", + "def get_current_weather(location, unit=\"fahrenheit\"):\n", + " \"\"\"Get the weather for some location\"\"\"\n", + " if \"chicago\" in location.lower():\n", + " return json.dumps({\"location\": \"Chicago\", \"temperature\": \"13\", \"unit\": unit})\n", + " elif \"san francisco\" in location.lower():\n", + " return json.dumps({\"location\": \"San Francisco\", \"temperature\": \"55\", \"unit\": unit})\n", + " elif \"new york\" in location.lower():\n", + " return json.dumps({\"location\": \"New York\", \"temperature\": \"11\", \"unit\": unit})\n", + " else:\n", + " return json.dumps({\"location\": location, \"temperature\": \"unknown\"})\n", + "\n", + "\n", + "# Register the function with the agent\n", + "\n", + "\n", + "@user_proxy.register_for_execution()\n", + "@chatbot.register_for_llm(description=\"Weather forecast for US cities.\")\n", + "def weather_forecast(\n", + " location: Annotated[str, \"City name\"],\n", + ") -> str:\n", + " weather_details = get_current_weather(location=location)\n", + " weather = json.loads(weather_details)\n", + " return f\"{weather['location']} will be {weather['temperature']} degrees {weather['unit']}\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We pass through our customers message and run the chat.\n", + "\n", + "Finally, we ask the LLM to summarise the chat and print that out." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33muser_proxy\u001b[0m (to chatbot):\n", + "\n", + "What's the weather in New York and can you tell me how much is 123.45 EUR in USD so I can spend it on my holiday? Throw a few holiday tips in as well.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mchatbot\u001b[0m (to user_proxy):\n", + "\n", + "\u001b[32m***** Suggested tool call (call_hg7g): weather_forecast *****\u001b[0m\n", + "Arguments: \n", + "{\"location\":\"New York\"}\n", + "\u001b[32m*************************************************************\u001b[0m\n", + "\u001b[32m***** Suggested tool call (call_hrsf): currency_calculator *****\u001b[0m\n", + "Arguments: \n", + "{\"base_amount\":123.45,\"base_currency\":\"EUR\",\"quote_currency\":\"USD\"}\n", + "\u001b[32m****************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION weather_forecast...\u001b[0m\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION currency_calculator...\u001b[0m\n", + "\u001b[33muser_proxy\u001b[0m (to chatbot):\n", + "\n", + "\u001b[33muser_proxy\u001b[0m (to chatbot):\n", + "\n", + "\u001b[32m***** Response from calling tool (call_hg7g) *****\u001b[0m\n", + "New York will be 11 degrees fahrenheit\n", + "\u001b[32m**************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33muser_proxy\u001b[0m (to chatbot):\n", + "\n", + "\u001b[32m***** Response from calling tool (call_hrsf) *****\u001b[0m\n", + "135.80 USD\n", + "\u001b[32m**************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mchatbot\u001b[0m (to user_proxy):\n", + "\n", + "\u001b[32m***** Suggested tool call (call_ahwk): weather_forecast *****\u001b[0m\n", + "Arguments: \n", + "{\"location\":\"New York\"}\n", + "\u001b[32m*************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "LLM SUMMARY: Based on the conversation, it's predicted that New York will be 11 degrees Fahrenheit. You also found out that 123.45 EUR is equal to 135.80 USD. Here are a few holiday tips:\n", + "\n", + "* Pack warm clothing for your visit to New York, as the temperature is expected to be quite chilly.\n", + "* Consider exchanging your money at a local currency exchange or an ATM since the exchange rate might not be as favorable in tourist areas.\n", + "* Make sure to check the estimated expenses for your holiday and adjust your budget accordingly.\n", + "\n", + "I hope you have a great trip!\n" + ] + } + ], + "source": [ + "# start the conversation\n", + "res = user_proxy.initiate_chat(\n", + " chatbot,\n", + " message=\"What's the weather in New York and can you tell me how much is 123.45 EUR in USD so I can spend it on my holiday? Throw a few holiday tips in as well.\",\n", + " summary_method=\"reflection_with_llm\",\n", + ")\n", + "\n", + "print(f\"LLM SUMMARY: {res.summary['content']}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Using its fast inference, Groq required less than 2 seconds for the whole chat!\n", + "\n", + "Additionally, Llama 3 was able to call both tools and pass through the right parameters. The `user_proxy` then executed them and this was passed back for Llama 3 to summarise the whole conversation." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "autogen", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/website/docs/topics/retrieval_augmentation.md b/website/docs/topics/retrieval_augmentation.md index ca7222a404f3..366893cb9825 100644 --- a/website/docs/topics/retrieval_augmentation.md +++ b/website/docs/topics/retrieval_augmentation.md @@ -119,8 +119,8 @@ ragproxyagent.initiate_chat( ## More Examples and Notebooks For more detailed examples and notebooks showcasing the usage of retrieval augmented agents in AutoGen, refer to the following: - Automated Code Generation and Question Answering with Retrieval Augmented Agents - [View Notebook](/docs/notebooks/agentchat_RetrieveChat) -- Automated Code Generation and Question Answering with [PGVector](https://github.com/pgvector/pgvector) based Retrieval Augmented Agents - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_pgvector_RetrieveChat.ipynb) -- Automated Code Generation and Question Answering with [Qdrant](https://qdrant.tech/) based Retrieval Augmented Agents - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_qdrant_RetrieveChat.ipynb) +- Automated Code Generation and Question Answering with [PGVector](https://github.com/pgvector/pgvector) based Retrieval Augmented Agents - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_RetrieveChat_pgvector.ipynb) +- Automated Code Generation and Question Answering with [Qdrant](https://qdrant.tech/) based Retrieval Augmented Agents - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_RetrieveChat_qdrant.ipynb) - Chat with OpenAI Assistant with Retrieval Augmentation - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_oai_assistant_retrieval.ipynb) - **RAG**: Group Chat with Retrieval Augmented Generation (with 5 group member agents and 1 manager agent) - [View Notebook](/docs/notebooks/agentchat_groupchat_RAG) diff --git a/website/docs/tutorial/chat-termination.ipynb b/website/docs/tutorial/chat-termination.ipynb index dcd5bdacecab..8da0aec6b92f 100644 --- a/website/docs/tutorial/chat-termination.ipynb +++ b/website/docs/tutorial/chat-termination.ipynb @@ -324,7 +324,7 @@ "You can configure both parameters in `initiate_chat` and also configuration of agents.\n", "\n", "That said, it is important to note that when a termination condition is triggered,\n", - "the conversation may not always terminated immediately. The actual termination\n", + "the conversation may not always terminate immediately. The actual termination\n", "depends on the `human_input_mode` argument of the `ConversableAgent` class.\n", "For example, when mode is `NEVER` the termination conditions above will end the conversations.\n", "But when mode is `ALWAYS` or `TERMINATE`, it will not terminate immediately.\n", diff --git a/website/docs/tutorial/conversation-patterns.ipynb b/website/docs/tutorial/conversation-patterns.ipynb index eeaaa409b784..7ea8f0bfa517 100644 --- a/website/docs/tutorial/conversation-patterns.ipynb +++ b/website/docs/tutorial/conversation-patterns.ipynb @@ -12,7 +12,18 @@ "In this chapter, we will first dig a little bit more into the two-agent \n", "chat pattern and chat result, \n", "then we will show you several conversation patterns that involve \n", - "more than two agents.\n" + "more than two agents.\n", + "\n", + "### An Overview\n", + "\n", + "1. **Two-agent chat**: the simplest form of conversation pattern where two agents chat with each other.\n", + "2. **Sequential chat**: a sequence of chats between two agents, chained together by a carryover mechanism, which brings the summary of the previous chat to the context of the next chat.\n", + "3. **Group Chat**: a single chat involving more than two agents. An important question in group chat is: What agent should be next to speak? To support different scenarios, we provide different ways to organize agents in a group chat:\n", + " - We support several strategies to select the next agent: `round_robin`, `random`, `manual` (human selection), and `auto` (Default, using an LLM to decide).\n", + " - We provide a way to constrain the selection of the next speaker (See examples below).\n", + " - We allow you to pass in a function to customize the selection of the next speaker. With this feature, you can build a **StateFlow** model which allows a deterministic workflow among your agents.\n", + " Please refer to this [guide](/docs/topics/groupchat/customized_speaker_selection) and this [blog post](/blog/2024/02/29/StateFlow) on StateFlow for more details.\n", + "4. **Nested Chat**: package a workflow into a single agent for reuse in a larger workflow." ] }, { diff --git a/website/docs/tutorial/human-in-the-loop.ipynb b/website/docs/tutorial/human-in-the-loop.ipynb index 04fbdd038b51..afcdeeaf42bf 100644 --- a/website/docs/tutorial/human-in-the-loop.ipynb +++ b/website/docs/tutorial/human-in-the-loop.ipynb @@ -10,7 +10,7 @@ "\n", "But many applications may require putting humans in-the-loop with agents. For example, to allow human feedback to steer agents in the right direction, specify goals, etc. In this chapter, we will show how AutoGen supports human intervention.\n", "\n", - "In AutoGen's `ConversableAgent`, the human-the-loop component sits in front\n", + "In AutoGen's `ConversableAgent`, the human-in-the-loop component sits in front\n", "of the auto-reply components. It can intercept the incoming messages and\n", "decide whether to pass them to the auto-reply components or to provide\n", "human feedback. The figure below illustrates the design.\n", @@ -285,9 +285,9 @@ "## Human Input Mode = `TERMINATE`\n", "\n", "In this mode, human input is only requested when a termination condition is\n", - "met. **If the human choose to intercept and reply, the counter will be reset**; if \n", - "the human choose to skip, automatic reply mechanism will be used; if the human\n", - "choose to terminate, the conversation will be terminated.\n", + "met. **If the human chooses to intercept and reply, the counter will be reset**; if \n", + "the human chooses to skip, the automatic reply mechanism will be used; if the human\n", + "chooses to terminate, the conversation will be terminated.\n", "\n", "Let us see this mode in action by playing the same game again, but this time\n", "the guessing agent will only have two chances to guess the number, and if it \n", diff --git a/website/docs/tutorial/tool-use.ipynb b/website/docs/tutorial/tool-use.ipynb index b5e21559225c..792bfefde878 100644 --- a/website/docs/tutorial/tool-use.ipynb +++ b/website/docs/tutorial/tool-use.ipynb @@ -21,7 +21,7 @@ "````mdx-code-block\n", ":::note\n", "Tool use is currently only available for LLMs\n", - "that support OpenAI-comaptible tool call API.\n", + "that support OpenAI-compatible tool call API.\n", ":::\n", "````" ] diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index efc13096b0f7..f0c0f84a3945 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -14,7 +14,7 @@ customPostCssPlugin = () => { module.exports = { title: "AutoGen", - tagline: "Enable Next-Gen Large Language Model Applications", + tagline: "An Open-Source Programming Framework for Agentic AI", url: "https://microsoft.github.io", baseUrl: "/autogen/", onBrokenLinks: "throw", @@ -281,6 +281,10 @@ module.exports = { to: "/docs/notebooks/agentchat_nested_chats_chess", from: ["/docs/notebooks/agentchat_chess"], }, + { + to: "/docs/notebooks/agentchat_nested_chats_chess_altmodels", + from: ["/docs/notebooks/agentchat_chess_altmodels"], + }, { to: "/docs/contributor-guide/contributing", from: ["/docs/Contribute"], diff --git a/website/src/data/gallery.json b/website/src/data/gallery.json index 5ad6932fd92e..10ed9f6866db 100644 --- a/website/src/data/gallery.json +++ b/website/src/data/gallery.json @@ -1,7 +1,7 @@ [ { "title": "AutoTx - Crypto Transactions Agent", - "link": "https://blog.polywrap.io/p/autotx-your-ai-powered-transaction", + "link": "https://www.agentcoin.org/blog/autotx", "description": "Generates on-chain transactions, which are submitted to a smart account so users can easily approve & execute them.", "image": "autotx.png", "tags": ["tools", "groupchat", "app", "blockchain"] @@ -219,5 +219,14 @@ "description": "Automatically creating linear tasks and assigning them to the right person, project, and team from GitHub commits using AutoGen Agents.", "image": "composio-autogen.png", "tags": ["tools"] + }, + { + "title": "Agentok - Autogen Visualized", + "link": "https://github.com/hughlv/agentok", + "description": "Offering intuitive visual tools that streamline the creation and management of complex AutoGen workflows.", + "image": "agentok.png", + "tags": [ + "tools", "ui", "app" + ] } ] diff --git a/website/static/img/create_gcp_svc.png b/website/static/img/create_gcp_svc.png new file mode 100644 index 000000000000..aefd4af762fe --- /dev/null +++ b/website/static/img/create_gcp_svc.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff7aa39b25ffcaba97cfd1a87cb1b44d45d3814241c83122f84408e03ad575b0 +size 101583