diff --git a/.circleci/config.yml b/.circleci/config.yml index 5f4628d26dc5..d070c55dca72 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -65,6 +65,7 @@ jobs: pip install "pydantic==2.7.1" pip install "diskcache==5.6.1" pip install "Pillow==10.3.0" + pip install "ijson==3.2.3" - save_cache: paths: - ./venv @@ -126,6 +127,7 @@ jobs: pip install jinja2 pip install tokenizers pip install openai + pip install ijson - run: name: Run tests command: | @@ -180,6 +182,7 @@ jobs: pip install numpydoc pip install prisma pip install fastapi + pip install ijson pip install "httpx==0.24.1" pip install "gunicorn==21.2.0" pip install "anyio==3.7.1" @@ -202,6 +205,7 @@ jobs: -e REDIS_PORT=$REDIS_PORT \ -e AZURE_FRANCE_API_KEY=$AZURE_FRANCE_API_KEY \ -e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \ + -e MISTRAL_API_KEY=$MISTRAL_API_KEY \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml new file mode 100644 index 000000000000..58e7cfe10daa --- /dev/null +++ b/.github/dependabot.yaml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" + groups: + github-actions: + patterns: + - "*" diff --git a/.github/workflows/ghcr_deploy.yml b/.github/workflows/ghcr_deploy.yml index 58cda02c31b7..4045056e3a25 100644 --- a/.github/workflows/ghcr_deploy.yml +++ b/.github/workflows/ghcr_deploy.yml @@ -25,6 +25,11 @@ jobs: if: github.repository == 'BerriAI/litellm' runs-on: ubuntu-latest steps: + - + name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.commit_hash }} - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -41,12 +46,14 @@ jobs: name: Build and push uses: docker/build-push-action@v5 with: + context: . push: true tags: litellm/litellm:${{ github.event.inputs.tag || 'latest' }} - name: Build and push litellm-database image uses: docker/build-push-action@v5 with: + context: . push: true file: Dockerfile.database tags: litellm/litellm-database:${{ github.event.inputs.tag || 'latest' }} @@ -54,6 +61,7 @@ jobs: name: Build and push litellm-spend-logs image uses: docker/build-push-action@v5 with: + context: . push: true file: ./litellm-js/spend-logs/Dockerfile tags: litellm/litellm-spend_logs:${{ github.event.inputs.tag || 'latest' }} @@ -68,6 +76,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.commit_hash }} # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - name: Log in to the Container registry uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 @@ -92,7 +102,7 @@ jobs: - name: Build and push Docker image uses: docker/build-push-action@4976231911ebf5f32aad765192d35f942aa48cb8 with: - context: https://github.com/BerriAI/litellm.git#${{ github.event.inputs.commit_hash}} + context: . push: true tags: ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.release_type }} # if a tag is provided, use that, otherwise use the release tag, and if neither is available, use 'latest' labels: ${{ steps.meta.outputs.labels }} @@ -106,6 +116,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.commit_hash }} - name: Log in to the Container registry uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 @@ -128,7 +140,7 @@ jobs: - name: Build and push Database Docker image uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 with: - context: https://github.com/BerriAI/litellm.git#${{ github.event.inputs.commit_hash}} + context: . file: Dockerfile.database push: true tags: ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.release_type }} @@ -143,6 +155,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.commit_hash }} - name: Log in to the Container registry uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 @@ -165,7 +179,7 @@ jobs: - name: Build and push Database Docker image uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 with: - context: https://github.com/BerriAI/litellm.git#${{ github.event.inputs.commit_hash}} + context: . file: ./litellm-js/spend-logs/Dockerfile push: true tags: ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.release_type }} @@ -176,6 +190,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.commit_hash }} - name: Log in to the Container registry uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 diff --git a/.gitignore b/.gitignore index 69061d62d3bf..b633e1d3d863 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,4 @@ litellm/proxy/_experimental/out/404/index.html litellm/proxy/_experimental/out/model_hub/index.html litellm/proxy/_experimental/out/onboarding/index.html litellm/tests/log.txt +litellm/tests/langfuse.log diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cc41d85f1455..9af3f2e8aac1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,19 @@ repos: +- repo: local + hooks: + - id: mypy + name: mypy + entry: python3 -m mypy --ignore-missing-imports + language: system + types: [python] + files: ^litellm/ + - id: isort + name: isort + entry: isort + language: system + types: [python] + files: litellm/.*\.py + exclude: ^litellm/__init__.py$ - repo: https://github.com/psf/black rev: 24.2.0 hooks: @@ -16,11 +31,10 @@ repos: name: Check if files match entry: python3 ci_cd/check_files_match.py language: system -- repo: local - hooks: - - id: mypy - name: mypy - entry: python3 -m mypy --ignore-missing-imports - language: system - types: [python] - files: ^litellm/ \ No newline at end of file + # - id: check-file-length + # name: Check file length + # entry: python check_file_length.py + # args: ["10000"] # set your desired maximum number of lines + # language: python + # files: litellm/.*\.py + # exclude: ^litellm/tests/ \ No newline at end of file diff --git a/check_file_length.py b/check_file_length.py new file mode 100644 index 000000000000..f23b79add25d --- /dev/null +++ b/check_file_length.py @@ -0,0 +1,28 @@ +import sys + + +def check_file_length(max_lines, filenames): + bad_files = [] + for filename in filenames: + with open(filename, "r") as file: + lines = file.readlines() + if len(lines) > max_lines: + bad_files.append((filename, len(lines))) + return bad_files + + +if __name__ == "__main__": + max_lines = int(sys.argv[1]) + filenames = sys.argv[2:] + + bad_files = check_file_length(max_lines, filenames) + if bad_files: + bad_files.sort( + key=lambda x: x[1], reverse=True + ) # Sort files by length in descending order + for filename, length in bad_files: + print(f"{filename}: {length} lines") + + sys.exit(1) + else: + sys.exit(0) diff --git a/docs/my-website/docs/completion/drop_params.md b/docs/my-website/docs/completion/drop_params.md new file mode 100644 index 000000000000..e79a88e14bba --- /dev/null +++ b/docs/my-website/docs/completion/drop_params.md @@ -0,0 +1,110 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Drop Unsupported Params + +Drop unsupported OpenAI params by your LLM Provider. + +## Quick Start + +```python +import litellm +import os + +# set keys +os.environ["COHERE_API_KEY"] = "co-.." + +litellm.drop_params = True # πŸ‘ˆ KEY CHANGE + +response = litellm.completion( + model="command-r", + messages=[{"role": "user", "content": "Hey, how's it going?"}], + response_format={"key": "value"}, + ) +``` + + +LiteLLM maps all supported openai params by provider + model (e.g. function calling is supported by anthropic on bedrock but not titan). + +See `litellm.get_supported_openai_params("command-r")` [**Code**](https://github.com/BerriAI/litellm/blob/main/litellm/utils.py#L3584) + +If a provider/model doesn't support a particular param, you can drop it. + +## OpenAI Proxy Usage + +```yaml +litellm_settings: + drop_params: true +``` + +## Pass drop_params in `completion(..)` + +Just drop_params when calling specific models + + + + +```python +import litellm +import os + +# set keys +os.environ["COHERE_API_KEY"] = "co-.." + +response = litellm.completion( + model="command-r", + messages=[{"role": "user", "content": "Hey, how's it going?"}], + response_format={"key": "value"}, + drop_params=True + ) +``` + + + +```yaml +- litellm_params: + api_base: my-base + model: openai/my-model + drop_params: true # πŸ‘ˆ KEY CHANGE + model_name: my-model +``` + + + +## Specify params to drop + +To drop specific params when calling a provider (E.g. 'logit_bias' for vllm) + +Use `additional_drop_params` + + + + +```python +import litellm +import os + +# set keys +os.environ["COHERE_API_KEY"] = "co-.." + +response = litellm.completion( + model="command-r", + messages=[{"role": "user", "content": "Hey, how's it going?"}], + response_format={"key": "value"}, + additional_drop_params=["response_format"] + ) +``` + + + +```yaml +- litellm_params: + api_base: my-base + model: openai/my-model + additional_drop_params: ["response_format"] # πŸ‘ˆ KEY CHANGE + model_name: my-model +``` + + + +**additional_drop_params**: List or null - Is a list of openai params you want to drop when making a call to the model. \ No newline at end of file diff --git a/docs/my-website/docs/completion/input.md b/docs/my-website/docs/completion/input.md index 6ad412af8f7a..d3bfce8b0350 100644 --- a/docs/my-website/docs/completion/input.md +++ b/docs/my-website/docs/completion/input.md @@ -67,6 +67,10 @@ By default, LiteLLM raises an exception if the openai param being passed in isn' To drop the param instead, set `litellm.drop_params = True` or `completion(..drop_params=True)`. +This **ONLY DROPS UNSUPPORTED OPENAI PARAMS**. + +LiteLLM assumes any non-openai param is provider specific and passes it in as a kwarg in the request body + ::: ## Input Params @@ -162,7 +166,7 @@ def completion( - `function`: *object* - Required. -- `tool_choice`: *string or object (optional)* - Controls which (if any) function is called by the model. none means the model will not call a function and instead generates a message. auto means the model can pick between generating a message or calling a function. Specifying a particular function via {"type: "function", "function": {"name": "my_function"}} forces the model to call that function. +- `tool_choice`: *string or object (optional)* - Controls which (if any) function is called by the model. none means the model will not call a function and instead generates a message. auto means the model can pick between generating a message or calling a function. Specifying a particular function via `{"type: "function", "function": {"name": "my_function"}}` forces the model to call that function. - `none` is the default when no functions are present. `auto` is the default if functions are present. diff --git a/docs/my-website/docs/debugging/hosted_debugging.md b/docs/my-website/docs/debugging/hosted_debugging.md index 5c98ac6f568c..e69de29bb2d1 100644 --- a/docs/my-website/docs/debugging/hosted_debugging.md +++ b/docs/my-website/docs/debugging/hosted_debugging.md @@ -1,90 +0,0 @@ -import Image from '@theme/IdealImage'; -import QueryParamReader from '../../src/components/queryParamReader.js' - -# [Beta] Monitor Logs in Production - -:::note - -This is in beta. Expect frequent updates, as we improve based on your feedback. - -::: - -LiteLLM provides an integration to let you monitor logs in production. - -πŸ‘‰ Jump to our sample LiteLLM Dashboard: https://admin.litellm.ai/ - - -Dashboard - -## Debug your first logs - - Open In Colab - - - -### 1. Get your LiteLLM Token - -Go to [admin.litellm.ai](https://admin.litellm.ai/) and copy the code snippet with your unique token - -Usage - -### 2. Set up your environment - -**Add it to your .env** - -```python -import os - -os.env["LITELLM_TOKEN"] = "e24c4c06-d027-4c30-9e78-18bc3a50aebb" # replace with your unique token - -``` - -**Turn on LiteLLM Client** -```python -import litellm -litellm.client = True -``` - -### 3. Make a normal `completion()` call -```python -import litellm -from litellm import completion -import os - -# set env variables -os.environ["LITELLM_TOKEN"] = "e24c4c06-d027-4c30-9e78-18bc3a50aebb" # replace with your unique token -os.environ["OPENAI_API_KEY"] = "openai key" - -litellm.use_client = True # enable logging dashboard -messages = [{ "content": "Hello, how are you?","role": "user"}] - -# openai call -response = completion(model="gpt-3.5-turbo", messages=messages) -``` - -Your `completion()` call print with a link to your session dashboard (https://admin.litellm.ai/) - -In the above case it would be: [`admin.litellm.ai/e24c4c06-d027-4c30-9e78-18bc3a50aebb`](https://admin.litellm.ai/e24c4c06-d027-4c30-9e78-18bc3a50aebb) - -Click on your personal dashboard link. Here's how you can find it πŸ‘‡ - -Dashboard - -[πŸ‘‹ Tell us if you need better privacy controls](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version?month=2023-08) - -### 3. Review request log - -Oh! Looks like our request was made successfully. Let's click on it and see exactly what got sent to the LLM provider. - - - - -Ah! So we can see that this request was made to a **Baseten** (see litellm_params > custom_llm_provider) for a model with ID - **7qQNLDB** (see model). The message sent was - `"Hey, how's it going?"` and the response received was - `"As an AI language model, I don't have feelings or emotions, but I can assist you with your queries. How can I assist you today?"` - -Dashboard Log Row - -:::info - -πŸŽ‰ Congratulations! You've successfully debugger your first log! - -::: \ No newline at end of file diff --git a/docs/my-website/docs/observability/athina_integration.md b/docs/my-website/docs/observability/athina_integration.md index 62c889751851..cd1442f35ab9 100644 --- a/docs/my-website/docs/observability/athina_integration.md +++ b/docs/my-website/docs/observability/athina_integration.md @@ -2,6 +2,15 @@ import Image from '@theme/IdealImage'; # Athina + +:::tip + +This is community maintained, Please make an issue if you run into a bug +https://github.com/BerriAI/litellm + +::: + + [Athina](https://athina.ai/) is an evaluation framework and production monitoring platform for your LLM-powered app. Athina is designed to enhance the performance and reliability of AI applications through real-time monitoring, granular analytics, and plug-and-play evaluations. diff --git a/docs/my-website/docs/observability/greenscale_integration.md b/docs/my-website/docs/observability/greenscale_integration.md index 0dd673226e12..49eadc6453dd 100644 --- a/docs/my-website/docs/observability/greenscale_integration.md +++ b/docs/my-website/docs/observability/greenscale_integration.md @@ -1,5 +1,14 @@ # Greenscale - Track LLM Spend and Responsible Usage + +:::tip + +This is community maintained, Please make an issue if you run into a bug +https://github.com/BerriAI/litellm + +::: + + [Greenscale](https://greenscale.ai/) is a production monitoring platform for your LLM-powered app that provides you granular key insights into your GenAI spending and responsible usage. Greenscale only captures metadata to minimize the exposure risk of personally identifiable information (PII). ## Getting Started diff --git a/docs/my-website/docs/observability/helicone_integration.md b/docs/my-website/docs/observability/helicone_integration.md index de89ba8da8c0..f7fd330c3094 100644 --- a/docs/my-website/docs/observability/helicone_integration.md +++ b/docs/my-website/docs/observability/helicone_integration.md @@ -1,4 +1,13 @@ # Helicone Tutorial + +:::tip + +This is community maintained, Please make an issue if you run into a bug +https://github.com/BerriAI/litellm + +::: + + [Helicone](https://helicone.ai/) is an open source observability platform that proxies your OpenAI traffic and provides you key insights into your spend, latency and usage. ## Use Helicone to log requests across all LLM Providers (OpenAI, Azure, Anthropic, Cohere, Replicate, PaLM) diff --git a/docs/my-website/docs/observability/langfuse_integration.md b/docs/my-website/docs/observability/langfuse_integration.md index 07970f599d8a..9703d38a03b1 100644 --- a/docs/my-website/docs/observability/langfuse_integration.md +++ b/docs/my-website/docs/observability/langfuse_integration.md @@ -1,6 +1,6 @@ import Image from '@theme/IdealImage'; -# Langfuse - Logging LLM Input/Output +# πŸ”₯ Langfuse - Logging LLM Input/Output LangFuse is open Source Observability & Analytics for LLM Apps Detailed production traces and a granular view on quality, cost and latency @@ -122,10 +122,12 @@ response = completion( metadata={ "generation_name": "ishaan-test-generation", # set langfuse Generation Name "generation_id": "gen-id22", # set langfuse Generation ID + "parent_observation_id": "obs-id9" # set langfuse Parent Observation ID "version": "test-generation-version" # set langfuse Generation Version "trace_user_id": "user-id2", # set langfuse Trace User ID "session_id": "session-1", # set langfuse Session ID "tags": ["tag1", "tag2"], # set langfuse Tags + "trace_name": "new-trace-name" # set langfuse Trace Name "trace_id": "trace-id22", # set langfuse Trace ID "trace_metadata": {"key": "value"}, # set langfuse Trace Metadata "trace_version": "test-trace-version", # set langfuse Trace Version (if not set, defaults to Generation Version) @@ -147,9 +149,10 @@ print(response) You can also pass `metadata` as part of the request header with a `langfuse_*` prefix: ```shell -curl --location 'http://0.0.0.0:4000/chat/completions' \ - --header 'Content-Type: application/json' \ - --header 'langfuse_trace_id: trace-id22' \ +curl --location --request POST 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'langfuse_trace_id: trace-id2' \ --header 'langfuse_trace_user_id: user-id2' \ --header 'langfuse_trace_metadata: {"key":"value"}' \ --data '{ @@ -190,9 +193,10 @@ The following parameters can be updated on a continuation of a trace by passing #### Generation Specific Parameters -* `generation_id` - Identifier for the generation, auto-generated by default -* `generation_name` - Identifier for the generation, auto-generated by default -* `prompt` - Langfuse prompt object used for the generation, defaults to None +* `generation_id` - Identifier for the generation, auto-generated by default +* `generation_name` - Identifier for the generation, auto-generated by default +* `parent_observation_id` - Identifier for the parent observation, defaults to `None` +* `prompt` - Langfuse prompt object used for the generation, defaults to `None` Any other key value pairs passed into the metadata not listed in the above spec for a `litellm` completion will be added as a metadata key value pair for the generation. diff --git a/docs/my-website/docs/observability/langsmith_integration.md b/docs/my-website/docs/observability/langsmith_integration.md index b115866d5454..c038abd82149 100644 --- a/docs/my-website/docs/observability/langsmith_integration.md +++ b/docs/my-website/docs/observability/langsmith_integration.md @@ -1,6 +1,16 @@ import Image from '@theme/IdealImage'; # Langsmith - Logging LLM Input/Output + + +:::tip + +This is community maintained, Please make an issue if you run into a bug +https://github.com/BerriAI/litellm + +::: + + An all-in-one developer platform for every step of the application lifecycle https://smith.langchain.com/ diff --git a/docs/my-website/docs/observability/logfire_integration.md b/docs/my-website/docs/observability/logfire_integration.md index c1f425f4251a..a2d406f9c63f 100644 --- a/docs/my-website/docs/observability/logfire_integration.md +++ b/docs/my-website/docs/observability/logfire_integration.md @@ -1,6 +1,6 @@ import Image from '@theme/IdealImage'; -# Logfire - Logging LLM Input/Output +# πŸ”₯ Logfire - Logging LLM Input/Output Logfire is open Source Observability & Analytics for LLM Apps Detailed production traces and a granular view on quality, cost and latency @@ -14,10 +14,14 @@ join our [discord](https://discord.gg/wuPM9dRgDw) ## Pre-Requisites -Ensure you have run `pip install logfire` for this integration +Ensure you have installed the following packages to use this integration ```shell -pip install logfire litellm +pip install litellm + +pip install opentelemetry-api==1.25.0 +pip install opentelemetry-sdk==1.25.0 +pip install opentelemetry-exporter-otlp==1.25.0 ``` ## Quick Start @@ -25,8 +29,7 @@ pip install logfire litellm Get your Logfire token from [Logfire](https://logfire.pydantic.dev/) ```python -litellm.success_callback = ["logfire"] -litellm.failure_callback = ["logfire"] # logs errors to logfire +litellm.callbacks = ["logfire"] ``` ```python diff --git a/docs/my-website/docs/observability/lunary_integration.md b/docs/my-website/docs/observability/lunary_integration.md index 9b8e90df7b41..56e74132f789 100644 --- a/docs/my-website/docs/observability/lunary_integration.md +++ b/docs/my-website/docs/observability/lunary_integration.md @@ -1,5 +1,13 @@ # Lunary - Logging and tracing LLM input/output +:::tip + +This is community maintained, Please make an issue if you run into a bug +https://github.com/BerriAI/litellm + +::: + + [Lunary](https://lunary.ai/) is an open-source AI developer platform providing observability, prompt management, and evaluation tools for AI developers.