diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..961aec0 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,12 @@ +# Set default behavior to automatically normalize line endings. +* text=auto + +# Explicitly declare text files you want to always be normalized and converted to native line endings on checkout. +*.py text eol=lf + +# Declare files that will always have CRLF line endings on checkout. +*.bat text eol=crlf + +# Declare files that should never be normalized. +*.jpg -text +*.png -text diff --git a/alpha_codium/litellm/proxy/_types.py b/alpha_codium/litellm/proxy/_types.py new file mode 100644 index 0000000..2ca69cf --- /dev/null +++ b/alpha_codium/litellm/proxy/_types.py @@ -0,0 +1,1721 @@ +import enum +import json +import os +import sys +import uuid +from dataclasses import fields +from datetime import datetime +from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, TypedDict, Union + +from pydantic import BaseModel, Extra, Field +try: + from pydantic import model_validator, Json, ConfigDict +except ImportError: + model_validator = None + Json = None + ConfigDict = None + +from typing_extensions import Annotated + +from litellm.types.router import UpdateRouterConfig +from litellm.types.utils import ProviderField + +if TYPE_CHECKING: + from opentelemetry.trace import Span as _Span + + Span = _Span +else: + Span = Any + + +class LitellmUserRoles(str, enum.Enum): + """ + Admin Roles: + PROXY_ADMIN: admin over the platform + PROXY_ADMIN_VIEW_ONLY: can login, view all own keys, view all spend + + Internal User Roles: + INTERNAL_USER: can login, view/create/delete their own keys, view their spend + INTERNAL_USER_VIEW_ONLY: can login, view their own keys, view their own spend + + + Team Roles: + TEAM: used for JWT auth + + + Customer Roles: + CUSTOMER: External users -> these are customers + + """ + + # Admin Roles + PROXY_ADMIN = "proxy_admin" + PROXY_ADMIN_VIEW_ONLY = "proxy_admin_viewer" + + # Internal User Roles + INTERNAL_USER = "internal_user" + INTERNAL_USER_VIEW_ONLY = "internal_user_viewer" + + # Team Roles + TEAM = "team" + + # Customer Roles - External users of proxy + CUSTOMER = "customer" + + def __str__(self): + return str(self.value) + + @property + def description(self): + """ + Descriptions for the enum values + """ + descriptions = { + "proxy_admin": "admin over litellm proxy, has all permissions", + "proxy_admin_viewer": "view all keys, view all spend", + "internal_user": "view/create/delete their own keys, view their own spend", + "internal_user_viewer": "view their own keys, view their own spend", + "team": "team scope used for JWT auth", + "customer": "customer", + } + return descriptions.get(self.value, "") + + @property + def ui_label(self): + """ + UI labels for the enum values + """ + ui_labels = { + "proxy_admin": "Admin (All Permissions)", + "proxy_admin_viewer": "Admin (View Only)", + "internal_user": "Internal User (Create/Delete/View)", + "internal_user_viewer": "Internal User (View Only)", + "team": "Team", + "customer": "Customer", + } + return ui_labels.get(self.value, "") + + +class LitellmTableNames(str, enum.Enum): + """ + Enum for Table Names used by LiteLLM + """ + + TEAM_TABLE_NAME: str = "LiteLLM_TeamTable" + USER_TABLE_NAME: str = "LiteLLM_UserTable" + KEY_TABLE_NAME: str = "LiteLLM_VerificationToken" + PROXY_MODEL_TABLE_NAME: str = "LiteLLM_ModelTable" + + +AlertType = Literal[ + "llm_exceptions", + "llm_too_slow", + "llm_requests_hanging", + "budget_alerts", + "db_exceptions", + "daily_reports", + "spend_reports", + "cooldown_deployment", + "new_model_added", + "outage_alerts", + "region_outage_alerts", +] + + +def hash_token(token: str): + import hashlib + + # Hash the string using SHA-256 + hashed_token = hashlib.sha256(token.encode()).hexdigest() + + return hashed_token + + +class LiteLLMBase(BaseModel): + """ + Implements default functions, all pydantic objects should have. + """ + + def json(self, **kwargs): + try: + return self.model_dump(**kwargs) # noqa + except Exception as e: + # if using pydantic v1 + return self.dict(**kwargs) + + def fields_set(self): + try: + return self.model_fields_set # noqa + except: + # if using pydantic v1 + return self.__fields_set__ + + model_config = ConfigDict(protected_namespaces=()) + + +class LiteLLM_UpperboundKeyGenerateParams(LiteLLMBase): + """ + Set default upperbound to max budget a key called via `/key/generate` can be. + """ + + max_budget: Optional[float] = None + budget_duration: Optional[str] = None + max_parallel_requests: Optional[int] = None + tpm_limit: Optional[int] = None + rpm_limit: Optional[int] = None + + +class LiteLLMRoutes(enum.Enum): + openai_route_names: List = [ + "chat_completion", + "completion", + "embeddings", + "image_generation", + "audio_transcriptions", + "moderations", + "model_list", # OpenAI /v1/models route + ] + openai_routes: List = [ + # chat completions + "/engines/{model}/chat/completions", + "/openai/deployments/{model}/chat/completions", + "/chat/completions", + "/v1/chat/completions", + # completions + "/engines/{model}/completions", + "/openai/deployments/{model}/completions", + "/completions", + "/v1/completions", + # embeddings + "/engines/{model}/embeddings", + "/openai/deployments/{model}/embeddings", + "/embeddings", + "/v1/embeddings", + # image generation + "/images/generations", + "/v1/images/generations", + # audio transcription + "/audio/transcriptions", + "/v1/audio/transcriptions", + # audio Speech + "/audio/speech", + "/v1/audio/speech", + # moderations + "/moderations", + "/v1/moderations", + # batches + "/v1/batches", + "/batches", + "/v1/batches{batch_id}", + "/batches{batch_id}", + # files + "/v1/files", + "/files", + "/v1/files/{file_id}", + "/files/{file_id}", + "/v1/files/{file_id}/content", + "/files/{file_id}/content", + # assistants-related routes + "/assistants", + "/v1/assistants", + "/v1/assistants/{assistant_id}", + "/assistants/{assistant_id}", + "/threads", + "/v1/threads", + "/threads/{thread_id}", + "/v1/threads/{thread_id}", + "/threads/{thread_id}/messages", + "/v1/threads/{thread_id}/messages", + "/threads/{thread_id}/runs", + "/v1/threads/{thread_id}/runs", + # models + "/models", + "/v1/models", + # token counter + "/utils/token_counter", + ] + + anthropic_routes: List = [ + "/v1/messages", + ] + + info_routes: List = [ + "/key/info", + "/team/info", + "/team/list", + "/user/info", + "/model/info", + "/v2/model/info", + "/v2/key/info", + "/model_group/info", + "/health", + ] + + # NOTE: ROUTES ONLY FOR MASTER KEY - only the Master Key should be able to Reset Spend + master_key_only_routes: List = [ + "/global/spend/reset", + ] + + sso_only_routes: List = [ + "/key/generate", + "/key/update", + "/key/delete", + "/global/spend/logs", + "/global/predict/spend/logs", + "/sso/get/logout_url", + ] + + management_routes: List = [ # key + "/key/generate", + "/key/update", + "/key/delete", + "/key/info", + # user + "/user/new", + "/user/update", + "/user/delete", + "/user/info", + # team + "/team/new", + "/team/update", + "/team/delete", + "/team/list", + "/team/info", + "/team/block", + "/team/unblock", + # model + "/model/new", + "/model/update", + "/model/delete", + "/model/info", + ] + + spend_tracking_routes: List = [ + # spend + "/spend/keys", + "/spend/users", + "/spend/tags", + "/spend/calculate", + "/spend/logs", + ] + + global_spend_tracking_routes: List = [ + # global spend + "/global/spend/logs", + "/global/spend", + "/global/spend/keys", + "/global/spend/teams", + "/global/spend/end_users", + "/global/spend/models", + "/global/predict/spend/logs", + "/global/spend/report", + ] + + public_routes: List = [ + "/routes", + "/", + "/health/liveliness", + "/health/liveness", + "/health/readiness", + "/test", + "/config/yaml", + "/metrics", + ] + + internal_user_routes: List = ( + [ + "/key/generate", + "/key/update", + "/key/delete", + "/key/info", + ] + + spend_tracking_routes + + sso_only_routes + ) + + +# class LiteLLMAllowedRoutes(LiteLLMBase): +# """ +# Defines allowed routes based on key type. + +# Types = ["admin", "team", "user", "unmapped"] +# """ + +# admin_allowed_routes: List[ +# Literal["openai_routes", "info_routes", "management_routes", "spend_tracking_routes", "global_spend_tracking_routes"] +# ] = ["management_routes"] + + +class LiteLLM_JWTAuth(LiteLLMBase): + """ + A class to define the roles and permissions for a LiteLLM Proxy w/ JWT Auth. + + Attributes: + - admin_jwt_scope: The JWT scope required for proxy admin roles. + - admin_allowed_routes: list of allowed routes for proxy admin roles. + - team_jwt_scope: The JWT scope required for proxy team roles. + - team_id_jwt_field: The field in the JWT token that stores the team ID. Default - `client_id`. + - team_allowed_routes: list of allowed routes for proxy team roles. + - user_id_jwt_field: The field in the JWT token that stores the user id (maps to `LiteLLMUserTable`). Use this for internal employees. + - end_user_id_jwt_field: The field in the JWT token that stores the end-user ID (maps to `LiteLLMEndUserTable`). Turn this off by setting to `None`. Enables end-user cost tracking. Use this for external customers. + - public_key_ttl: Default - 600s. TTL for caching public JWT keys. + + See `auth_checks.py` for the specific routes + """ + + admin_jwt_scope: str = "litellm_proxy_admin" + admin_allowed_routes: List[ + Literal[ + "openai_routes", + "info_routes", + "management_routes", + "spend_tracking_routes", + "global_spend_tracking_routes", + ] + ] = [ + "management_routes", + "spend_tracking_routes", + "global_spend_tracking_routes", + "info_routes", + ] + team_id_jwt_field: Optional[str] = None + team_allowed_routes: List[ + Literal["openai_routes", "info_routes", "management_routes"] + ] = ["openai_routes", "info_routes"] + team_id_default: Optional[str] = Field( + default=None, + description="If no team_id given, default permissions/spend-tracking to this team.s", + ) + org_id_jwt_field: Optional[str] = None + user_id_jwt_field: Optional[str] = None + user_id_upsert: bool = Field( + default=False, description="If user doesn't exist, upsert them into the db." + ) + end_user_id_jwt_field: Optional[str] = None + public_key_ttl: float = 600 + + def __init__(self, **kwargs: Any) -> None: + # get the attribute names for this Pydantic model + allowed_keys = self.__annotations__.keys() + + invalid_keys = set(kwargs.keys()) - allowed_keys + + if invalid_keys: + raise ValueError( + f"Invalid arguments provided: {', '.join(invalid_keys)}. Allowed arguments are: {', '.join(allowed_keys)}." + ) + + super().__init__(**kwargs) + + +class LiteLLMPromptInjectionParams(LiteLLMBase): + heuristics_check: bool = False + vector_db_check: bool = False + llm_api_check: bool = False + llm_api_name: Optional[str] = None + llm_api_system_prompt: Optional[str] = None + llm_api_fail_call_string: Optional[str] = None + reject_as_response: Optional[bool] = Field( + default=False, + description="Return rejected request error message as a string to the user. Default behaviour is to raise an exception.", + ) + + @model_validator(mode="before") + @classmethod + def check_llm_api_params(cls, values): + llm_api_check = values.get("llm_api_check") + if llm_api_check is True: + if "llm_api_name" not in values or not values["llm_api_name"]: + raise ValueError( + "If llm_api_check is set to True, llm_api_name must be provided" + ) + if ( + "llm_api_system_prompt" not in values + or not values["llm_api_system_prompt"] + ): + raise ValueError( + "If llm_api_check is set to True, llm_api_system_prompt must be provided" + ) + if ( + "llm_api_fail_call_string" not in values + or not values["llm_api_fail_call_string"] + ): + raise ValueError( + "If llm_api_check is set to True, llm_api_fail_call_string must be provided" + ) + return values + + +######### Request Class Definition ###### +class ProxyChatCompletionRequest(LiteLLMBase): + model: str + messages: List[Dict[str, str]] + temperature: Optional[float] = None + top_p: Optional[float] = None + n: Optional[int] = None + stream: Optional[bool] = None + stop: Optional[List[str]] = None + max_tokens: Optional[int] = None + presence_penalty: Optional[float] = None + frequency_penalty: Optional[float] = None + logit_bias: Optional[Dict[str, float]] = None + user: Optional[str] = None + response_format: Optional[Dict[str, str]] = None + seed: Optional[int] = None + tools: Optional[List[str]] = None + tool_choice: Optional[str] = None + functions: Optional[List[str]] = None # soon to be deprecated + function_call: Optional[str] = None # soon to be deprecated + + # Optional LiteLLM params + caching: Optional[bool] = None + api_base: Optional[str] = None + api_version: Optional[str] = None + api_key: Optional[str] = None + num_retries: Optional[int] = None + context_window_fallback_dict: Optional[Dict[str, str]] = None + fallbacks: Optional[List[str]] = None + metadata: Optional[Dict[str, str]] = {} + deployment_id: Optional[str] = None + request_timeout: Optional[int] = None + + model_config = ConfigDict( + extra="allow" + ) # allow params not defined here, these fall in litellm.completion(**kwargs) + + +class ModelInfoDelete(LiteLLMBase): + id: str + + +class ModelInfo(LiteLLMBase): + id: Optional[str] + mode: Optional[Literal["embedding", "chat", "completion"]] + input_cost_per_token: Optional[float] = 0.0 + output_cost_per_token: Optional[float] = 0.0 + max_tokens: Optional[int] = 2048 # assume 2048 if not set + + # for azure models we need users to specify the base model, one azure you can call deployments - azure/my-random-model + # we look up the base model in model_prices_and_context_window.json + base_model: Optional[ + Literal[ + "gpt-4-1106-preview", + "gpt-4-32k", + "gpt-4", + "gpt-3.5-turbo-16k", + "gpt-3.5-turbo", + "text-embedding-ada-002", + ] + ] + + model_config = ConfigDict(protected_namespaces=(), extra="allow") + + @model_validator(mode="before") + @classmethod + def set_model_info(cls, values): + if values.get("id") is None: + values.update({"id": str(uuid.uuid4())}) + if values.get("mode") is None: + values.update({"mode": None}) + if values.get("input_cost_per_token") is None: + values.update({"input_cost_per_token": None}) + if values.get("output_cost_per_token") is None: + values.update({"output_cost_per_token": None}) + if values.get("max_tokens") is None: + values.update({"max_tokens": None}) + if values.get("base_model") is None: + values.update({"base_model": None}) + return values + + +class ProviderInfo(LiteLLMBase): + name: str + fields: List[ProviderField] + + +class BlockUsers(LiteLLMBase): + user_ids: List[str] # required + + +class ModelParams(LiteLLMBase): + model_name: str + litellm_params: dict + model_info: ModelInfo + + model_config = ConfigDict(protected_namespaces=()) + + @model_validator(mode="before") + @classmethod + def set_model_info(cls, values): + if values.get("model_info") is None: + values.update({"model_info": ModelInfo()}) + return values + + +class GenerateRequestBase(LiteLLMBase): + """ + Overlapping schema between key and user generate/update requests + """ + + models: Optional[list] = [] + spend: Optional[float] = 0 + max_budget: Optional[float] = None + user_id: Optional[str] = None + team_id: Optional[str] = None + max_parallel_requests: Optional[int] = None + metadata: Optional[dict] = {} + tpm_limit: Optional[int] = None + rpm_limit: Optional[int] = None + budget_duration: Optional[str] = None + allowed_cache_controls: Optional[list] = [] + soft_budget: Optional[float] = None + + +class GenerateKeyRequest(GenerateRequestBase): + key_alias: Optional[str] = None + duration: Optional[str] = None + aliases: Optional[dict] = {} + config: Optional[dict] = {} + permissions: Optional[dict] = {} + model_max_budget: Optional[dict] = ( + {} + ) # {"gpt-4": 5.0, "gpt-3.5-turbo": 5.0}, defaults to {} + + model_config = ConfigDict(protected_namespaces=()) + send_invite_email: Optional[bool] = None + + +class GenerateKeyResponse(GenerateKeyRequest): + key: str + key_name: Optional[str] = None + expires: Optional[datetime] + user_id: Optional[str] = None + token_id: Optional[str] = None + + @model_validator(mode="before") + @classmethod + def set_model_info(cls, values): + if values.get("token") is not None: + values.update({"key": values.get("token")}) + dict_fields = [ + "metadata", + "aliases", + "config", + "permissions", + "model_max_budget", + ] + for field in dict_fields: + value = values.get(field) + if value is not None and isinstance(value, str): + try: + values[field] = json.loads(value) + except json.JSONDecodeError: + raise ValueError(f"Field {field} should be a valid dictionary") + + return values + + +class UpdateKeyRequest(GenerateKeyRequest): + # Note: the defaults of all Params here MUST BE NONE + # else they will get overwritten + key: str + duration: Optional[str] = None + spend: Optional[float] = None + metadata: Optional[dict] = None + + +class KeyRequest(LiteLLMBase): + keys: List[str] + + +class LiteLLM_ModelTable(LiteLLMBase): + model_aliases: Optional[str] = None # json dump the dict + created_by: str + updated_by: str + + model_config = ConfigDict(protected_namespaces=()) + + +class NewUserRequest(GenerateKeyRequest): + max_budget: Optional[float] = None + user_email: Optional[str] = None + user_role: Optional[ + Literal[ + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + LitellmUserRoles.INTERNAL_USER, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, + LitellmUserRoles.TEAM, + LitellmUserRoles.CUSTOMER, + ] + ] = None + teams: Optional[list] = None + organization_id: Optional[str] = None + auto_create_key: bool = ( + True # flag used for returning a key as part of the /user/new response + ) + send_invite_email: Optional[bool] = None + + +class NewUserResponse(GenerateKeyResponse): + max_budget: Optional[float] = None + user_email: Optional[str] = None + user_role: Optional[ + Literal[ + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + LitellmUserRoles.INTERNAL_USER, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, + LitellmUserRoles.TEAM, + LitellmUserRoles.CUSTOMER, + ] + ] = None + teams: Optional[list] = None + organization_id: Optional[str] = None + + +class UpdateUserRequest(GenerateRequestBase): + # Note: the defaults of all Params here MUST BE NONE + # else they will get overwritten + user_id: Optional[str] = None + password: Optional[str] = None + user_email: Optional[str] = None + spend: Optional[float] = None + metadata: Optional[dict] = None + user_role: Optional[ + Literal[ + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + LitellmUserRoles.INTERNAL_USER, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, + LitellmUserRoles.TEAM, + LitellmUserRoles.CUSTOMER, + ] + ] = None + max_budget: Optional[float] = None + + @model_validator(mode="before") + @classmethod + def check_user_info(cls, values): + if values.get("user_id") is None and values.get("user_email") is None: + raise ValueError("Either user id or user email must be provided") + return values + + +class DeleteUserRequest(LiteLLMBase): + user_ids: List[str] # required + + +class NewCustomerRequest(LiteLLMBase): + """ + Create a new customer, allocate a budget to them + """ + + user_id: str + alias: Optional[str] = None # human-friendly alias + blocked: bool = False # allow/disallow requests for this end-user + max_budget: Optional[float] = None + budget_id: Optional[str] = None # give either a budget_id or max_budget + allowed_model_region: Optional[Literal["eu"]] = ( + None # require all user requests to use models in this specific region + ) + default_model: Optional[str] = ( + None # if no equivalent model in allowed region - default all requests to this model + ) + + @model_validator(mode="before") + @classmethod + def check_user_info(cls, values): + if values.get("max_budget") is not None and values.get("budget_id") is not None: + raise ValueError("Set either 'max_budget' or 'budget_id', not both.") + + return values + + +class UpdateCustomerRequest(LiteLLMBase): + """ + Update a Customer, use this to update customer budgets etc + + """ + + user_id: str + alias: Optional[str] = None # human-friendly alias + blocked: bool = False # allow/disallow requests for this end-user + max_budget: Optional[float] = None + budget_id: Optional[str] = None # give either a budget_id or max_budget + allowed_model_region: Optional[Literal["eu"]] = ( + None # require all user requests to use models in this specific region + ) + default_model: Optional[str] = ( + None # if no equivalent model in allowed region - default all requests to this model + ) + + +class DeleteCustomerRequest(LiteLLMBase): + """ + Delete multiple Customers + """ + + user_ids: List[str] + + +class Member(LiteLLMBase): + role: Literal["admin", "user"] + user_id: Optional[str] = None + user_email: Optional[str] = None + + @model_validator(mode="before") + @classmethod + def check_user_info(cls, values): + if not isinstance(values, dict): + raise ValueError("input needs to be a dictionary") + if values.get("user_id") is None and values.get("user_email") is None: + raise ValueError("Either user id or user email must be provided") + return values + + +class TeamBase(LiteLLMBase): + team_alias: Optional[str] = None + team_id: Optional[str] = None + organization_id: Optional[str] = None + admins: list = [] + members: list = [] + members_with_roles: List[Member] = [] + metadata: Optional[dict] = None + tpm_limit: Optional[int] = None + rpm_limit: Optional[int] = None + + # Budget fields + max_budget: Optional[float] = None + budget_duration: Optional[str] = None + + models: list = [] + blocked: bool = False + + +class NewTeamRequest(TeamBase): + model_aliases: Optional[dict] = None + + model_config = ConfigDict(protected_namespaces=()) + + +class GlobalEndUsersSpend(LiteLLMBase): + api_key: Optional[str] = None + startTime: Optional[datetime] = None + endTime: Optional[datetime] = None + + +class TeamMemberAddRequest(LiteLLMBase): + team_id: str + member: Union[List[Member], Member] + max_budget_in_team: Optional[float] = None # Users max budget within the team + + def __init__(self, **data): + member_data = data.get("member") + if isinstance(member_data, list): + # If member is a list of dictionaries, convert each dictionary to a Member object + members = [Member(**item) for item in member_data] + # Replace member_data with the list of Member objects + data["member"] = members + elif isinstance(member_data, dict): + # If member is a dictionary, convert it to a single Member object + member = Member(**member_data) + # Replace member_data with the single Member object + data["member"] = member + # Call the superclass __init__ method to initialize the object + super().__init__(**data) + + +class TeamMemberDeleteRequest(LiteLLMBase): + team_id: str + user_id: Optional[str] = None + user_email: Optional[str] = None + + @model_validator(mode="before") + @classmethod + def check_user_info(cls, values): + if values.get("user_id") is None and values.get("user_email") is None: + raise ValueError("Either user id or user email must be provided") + return values + + +class UpdateTeamRequest(LiteLLMBase): + """ + UpdateTeamRequest, used by /team/update when you need to update a team + + team_id: str + team_alias: Optional[str] = None + organization_id: Optional[str] = None + metadata: Optional[dict] = None + tpm_limit: Optional[int] = None + rpm_limit: Optional[int] = None + max_budget: Optional[float] = None + models: Optional[list] = None + blocked: Optional[bool] = None + budget_duration: Optional[str] = None + """ + + team_id: str # required + team_alias: Optional[str] = None + organization_id: Optional[str] = None + metadata: Optional[dict] = None + tpm_limit: Optional[int] = None + rpm_limit: Optional[int] = None + max_budget: Optional[float] = None + models: Optional[list] = None + blocked: Optional[bool] = None + budget_duration: Optional[str] = None + + +class ResetTeamBudgetRequest(LiteLLMBase): + """ + internal type used to reset the budget on a team + used by reset_budget() + + team_id: str + spend: float + budget_reset_at: datetime + """ + + team_id: str + spend: float + budget_reset_at: datetime + updated_at: datetime + + +class DeleteTeamRequest(LiteLLMBase): + team_ids: List[str] # required + + +class BlockTeamRequest(LiteLLMBase): + team_id: str # required + + +class AddTeamCallback(LiteLLMBase): + callback_name: str + callback_type: Literal["success", "failure", "success_and_failure"] + # for now - only supported for langfuse + callback_vars: Dict[ + Literal["langfuse_public_key", "langfuse_secret_key", "langfuse_host"], str + ] + + +class TeamCallbackMetadata(LiteLLMBase): + success_callback: Optional[List[str]] = [] + failure_callback: Optional[List[str]] = [] + # for now - only supported for langfuse + callback_vars: Optional[ + Dict[ + Literal["langfuse_public_key", "langfuse_secret_key", "langfuse_host"], str + ] + ] = {} + + +class LiteLLM_TeamTable(TeamBase): + spend: Optional[float] = None + max_parallel_requests: Optional[int] = None + budget_duration: Optional[str] = None + budget_reset_at: Optional[datetime] = None + model_id: Optional[int] = None + last_refreshed_at: Optional[float] = None + + model_config = ConfigDict(protected_namespaces=()) + + @model_validator(mode="before") + @classmethod + def set_model_info(cls, values): + dict_fields = [ + "metadata", + "aliases", + "config", + "permissions", + "model_max_budget", + "model_aliases", + ] + for field in dict_fields: + value = values.get(field) + if value is not None and isinstance(value, str): + try: + values[field] = json.loads(value) + except json.JSONDecodeError: + raise ValueError(f"Field {field} should be a valid dictionary") + + return values + + +class TeamRequest(LiteLLMBase): + teams: List[str] + + +class LiteLLM_BudgetTable(LiteLLMBase): + """Represents user-controllable params for a LiteLLM_BudgetTable record""" + + soft_budget: Optional[float] = None + max_budget: Optional[float] = None + max_parallel_requests: Optional[int] = None + tpm_limit: Optional[int] = None + rpm_limit: Optional[int] = None + model_max_budget: Optional[dict] = None + budget_duration: Optional[str] = None + + model_config = ConfigDict(protected_namespaces=()) + + +class LiteLLM_TeamMemberTable(LiteLLM_BudgetTable): + """ + Used to track spend of a user_id within a team_id + """ + + spend: Optional[float] = None + user_id: Optional[str] = None + team_id: Optional[str] = None + budget_id: Optional[str] = None + + model_config = ConfigDict(protected_namespaces=()) + + +class NewOrganizationRequest(LiteLLM_BudgetTable): + organization_id: Optional[str] = None + organization_alias: str + models: List = [] + budget_id: Optional[str] = None + + +class LiteLLM_OrganizationTable(LiteLLMBase): + """Represents user-controllable params for a LiteLLM_OrganizationTable record""" + + organization_id: Optional[str] = None + organization_alias: Optional[str] = None + budget_id: str + metadata: Optional[dict] = None + models: List[str] + created_by: str + updated_by: str + + +class NewOrganizationResponse(LiteLLM_OrganizationTable): + organization_id: str + created_at: datetime + updated_at: datetime + + +class OrganizationRequest(LiteLLMBase): + organizations: List[str] + + +class BudgetNew(LiteLLMBase): + budget_id: str = Field(default=None, description="The unique budget id.") + max_budget: Optional[float] = Field( + default=None, + description="Requests will fail if this budget (in USD) is exceeded.", + ) + soft_budget: Optional[float] = Field( + default=None, + description="Requests will NOT fail if this is exceeded. Will fire alerting though.", + ) + max_parallel_requests: Optional[int] = Field( + default=None, description="Max concurrent requests allowed for this budget id." + ) + tpm_limit: Optional[int] = Field( + default=None, description="Max tokens per minute, allowed for this budget id." + ) + rpm_limit: Optional[int] = Field( + default=None, description="Max requests per minute, allowed for this budget id." + ) + budget_duration: Optional[str] = Field( + default=None, + description="Max duration budget should be set for (e.g. '1hr', '1d', '28d')", + ) + + +class BudgetRequest(LiteLLMBase): + budgets: List[str] + + +class BudgetDeleteRequest(LiteLLMBase): + id: str + + +class KeyManagementSystem(enum.Enum): + GOOGLE_KMS = "google_kms" + AZURE_KEY_VAULT = "azure_key_vault" + AWS_SECRET_MANAGER = "aws_secret_manager" + LOCAL = "local" + AWS_KMS = "aws_kms" + + +class KeyManagementSettings(LiteLLMBase): + hosted_keys: List + + +class TeamDefaultSettings(LiteLLMBase): + team_id: str + + model_config = ConfigDict( + extra="allow" + ) # allow params not defined here, these fall in litellm.completion(**kwargs) + + +class DynamoDBArgs(LiteLLMBase): + billing_mode: Literal["PROVISIONED_THROUGHPUT", "PAY_PER_REQUEST"] + read_capacity_units: Optional[int] = None + write_capacity_units: Optional[int] = None + ssl_verify: Optional[bool] = None + region_name: str + user_table_name: str = "LiteLLM_UserTable" + key_table_name: str = "LiteLLM_VerificationToken" + config_table_name: str = "LiteLLM_Config" + spend_table_name: str = "LiteLLM_SpendLogs" + aws_role_name: Optional[str] = None + aws_session_name: Optional[str] = None + aws_web_identity_token: Optional[str] = None + aws_provider_id: Optional[str] = None + aws_policy_arns: Optional[List[str]] = None + aws_policy: Optional[str] = None + aws_duration_seconds: Optional[int] = None + assume_role_aws_role_name: Optional[str] = None + assume_role_aws_session_name: Optional[str] = None + + +class ConfigFieldUpdate(LiteLLMBase): + field_name: str + field_value: Any + config_type: Literal["general_settings"] + + +class ConfigFieldDelete(LiteLLMBase): + config_type: Literal["general_settings"] + field_name: str + + +class ConfigList(LiteLLMBase): + field_name: str + field_type: str + field_description: str + field_value: Any + stored_in_db: Optional[bool] + field_default_value: Any + premium_field: bool = False + + +class ConfigGeneralSettings(LiteLLMBase): + """ + Documents all the fields supported by `general_settings` in config.yaml + """ + + completion_model: Optional[str] = Field( + None, description="proxy level default model for all chat completion calls" + ) + key_management_system: Optional[KeyManagementSystem] = Field( + None, description="key manager to load keys from / decrypt keys with" + ) + use_google_kms: Optional[bool] = Field( + None, description="decrypt keys with google kms" + ) + use_azure_key_vault: Optional[bool] = Field( + None, description="load keys from azure key vault" + ) + master_key: Optional[str] = Field( + None, description="require a key for all calls to proxy" + ) + database_url: Optional[str] = Field( + None, + description="connect to a postgres db - needed for generating temporary keys + tracking spend / key", + ) + database_connection_pool_limit: Optional[int] = Field( + 100, + description="default connection pool for prisma client connecting to postgres db", + ) + database_connection_timeout: Optional[float] = Field( + 60, description="default timeout for a connection to the database" + ) + database_type: Optional[Literal["dynamo_db"]] = Field( + None, description="to use dynamodb instead of postgres db" + ) + database_args: Optional[DynamoDBArgs] = Field( + None, + description="custom args for instantiating dynamodb client - e.g. billing provision", + ) + otel: Optional[bool] = Field( + None, + description="[BETA] OpenTelemetry support - this might change, use with caution.", + ) + custom_auth: Optional[str] = Field( + None, + description="override user_api_key_auth with your own auth script - https://docs.litellm.ai/docs/proxy/virtual_keys#custom-auth", + ) + max_parallel_requests: Optional[int] = Field( + None, + description="maximum parallel requests for each api key", + ) + global_max_parallel_requests: Optional[int] = Field( + None, description="global max parallel requests to allow for a proxy instance." + ) + max_request_size_mb: Optional[int] = Field( + None, + description="max request size in MB, if a request is larger than this size it will be rejected", + ) + max_response_size_mb: Optional[int] = Field( + None, + description="max response size in MB, if a response is larger than this size it will be rejected", + ) + infer_model_from_keys: Optional[bool] = Field( + None, + description="for `/models` endpoint, infers available model based on environment keys (e.g. OPENAI_API_KEY)", + ) + background_health_checks: Optional[bool] = Field( + None, description="run health checks in background" + ) + health_check_interval: int = Field( + 300, description="background health check interval in seconds" + ) + alerting: Optional[List] = Field( + None, + description="List of alerting integrations. Today, just slack - `alerting: ['slack']`", + ) + alert_types: Optional[List[AlertType]] = Field( + None, + description="List of alerting types. By default it is all alerts", + ) + alert_to_webhook_url: Optional[Dict] = Field( + None, + description="Mapping of alert type to webhook url. e.g. `alert_to_webhook_url: {'budget_alerts': 'https://hooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXXXXXXXXX'}`", + ) + alerting_args: Optional[Dict] = Field( + None, description="Controllable params for slack alerting - e.g. ttl in cache." + ) + alerting_threshold: Optional[int] = Field( + None, + description="sends alerts if requests hang for 5min+", + ) + ui_access_mode: Optional[Literal["admin_only", "all"]] = Field( + "all", description="Control access to the Proxy UI" + ) + allowed_routes: Optional[List] = Field( + None, description="Proxy API Endpoints you want users to be able to access" + ) + enable_public_model_hub: bool = Field( + default=False, + description="Public model hub for users to see what models they have access to, supported openai params, etc.", + ) + + +class ConfigYAML(LiteLLMBase): + """ + Documents all the fields supported by the config.yaml + """ + + environment_variables: Optional[dict] = Field( + None, + description="Object to pass in additional environment variables via POST request", + ) + model_list: Optional[List[ModelParams]] = Field( + None, + description="List of supported models on the server, with model-specific configs", + ) + litellm_settings: Optional[dict] = Field( + None, + description="litellm Module settings. See __init__.py for all, example litellm.drop_params=True, litellm.set_verbose=True, litellm.api_base, litellm.cache", + ) + general_settings: Optional[ConfigGeneralSettings] = None + router_settings: Optional[UpdateRouterConfig] = Field( + None, + description="litellm router object settings. See router.py __init__ for all, example router.num_retries=5, router.timeout=5, router.max_retries=5, router.retry_after=5", + ) + + model_config = ConfigDict(protected_namespaces=()) + + +class LiteLLM_VerificationToken(LiteLLMBase): + token: Optional[str] = None + key_name: Optional[str] = None + key_alias: Optional[str] = None + spend: float = 0.0 + max_budget: Optional[float] = None + expires: Optional[str] = None + models: List = [] + aliases: Dict = {} + config: Dict = {} + user_id: Optional[str] = None + team_id: Optional[str] = None + max_parallel_requests: Optional[int] = None + metadata: Dict = {} + tpm_limit: Optional[int] = None + rpm_limit: Optional[int] = None + budget_duration: Optional[str] = None + budget_reset_at: Optional[datetime] = None + allowed_cache_controls: Optional[list] = [] + permissions: Dict = {} + model_spend: Dict = {} + model_max_budget: Dict = {} + soft_budget_cooldown: bool = False + litellm_budget_table: Optional[dict] = None + + org_id: Optional[str] = None # org id for a given key + + model_config = ConfigDict(protected_namespaces=()) + + +class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken): + """ + Combined view of litellm verification token + litellm team table (select values) + """ + + team_spend: Optional[float] = None + team_alias: Optional[str] = None + team_tpm_limit: Optional[int] = None + team_rpm_limit: Optional[int] = None + team_max_budget: Optional[float] = None + team_models: List = [] + team_blocked: bool = False + soft_budget: Optional[float] = None + team_model_aliases: Optional[Dict] = None + team_member_spend: Optional[float] = None + team_metadata: Optional[Dict] = None + + # End User Params + end_user_id: Optional[str] = None + end_user_tpm_limit: Optional[int] = None + end_user_rpm_limit: Optional[int] = None + end_user_max_budget: Optional[float] = None + + # Time stamps + last_refreshed_at: Optional[float] = None # last time joint view was pulled from db + + +class UserAPIKeyAuth( + LiteLLM_VerificationTokenView +): # the expected response object for user api key auth + """ + Return the row in the db + """ + + api_key: Optional[str] = None + user_role: Optional[ + Literal[ + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + LitellmUserRoles.INTERNAL_USER, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, + LitellmUserRoles.TEAM, + LitellmUserRoles.CUSTOMER, + ] + ] = None + allowed_model_region: Optional[Literal["eu"]] = None + parent_otel_span: Optional[Span] = None + + @model_validator(mode="before") + @classmethod + def check_api_key(cls, values): + if values.get("api_key") is not None: + values.update({"token": hash_token(values.get("api_key"))}) + if isinstance(values.get("api_key"), str) and values.get( + "api_key" + ).startswith("sk-"): + values.update({"api_key": hash_token(values.get("api_key"))}) + return values + + class Config: + arbitrary_types_allowed = True + + +class LiteLLM_Config(LiteLLMBase): + param_name: str + param_value: Dict + + +class LiteLLM_UserTable(LiteLLMBase): + user_id: str + max_budget: Optional[float] + spend: float = 0.0 + model_max_budget: Optional[Dict] = {} + model_spend: Optional[Dict] = {} + user_email: Optional[str] + models: list = [] + tpm_limit: Optional[int] = None + rpm_limit: Optional[int] = None + + @model_validator(mode="before") + @classmethod + def set_model_info(cls, values): + if values.get("spend") is None: + values.update({"spend": 0.0}) + if values.get("models") is None: + values.update({"models": []}) + return values + + model_config = ConfigDict(protected_namespaces=()) + + +class LiteLLM_EndUserTable(LiteLLMBase): + user_id: str + blocked: bool + alias: Optional[str] = None + spend: float = 0.0 + allowed_model_region: Optional[Literal["eu"]] = None + default_model: Optional[str] = None + litellm_budget_table: Optional[LiteLLM_BudgetTable] = None + + @model_validator(mode="before") + @classmethod + def set_model_info(cls, values): + if values.get("spend") is None: + values.update({"spend": 0.0}) + return values + + model_config = ConfigDict(protected_namespaces=()) + + +class LiteLLM_SpendLogs(LiteLLMBase): + request_id: str + api_key: str + model: Optional[str] = "" + api_base: Optional[str] = "" + call_type: str + spend: Optional[float] = 0.0 + total_tokens: Optional[int] = 0 + prompt_tokens: Optional[int] = 0 + completion_tokens: Optional[int] = 0 + startTime: Union[str, datetime, None] + endTime: Union[str, datetime, None] + user: Optional[str] = "" + metadata: Optional[Json] = {} + cache_hit: Optional[str] = "False" + cache_key: Optional[str] = None + request_tags: Optional[Json] = None + requester_ip_address: Optional[str] = None + + +class LiteLLM_ErrorLogs(LiteLLMBase): + request_id: Optional[str] = str(uuid.uuid4()) + api_base: Optional[str] = "" + model_group: Optional[str] = "" + litellm_model_name: Optional[str] = "" + model_id: Optional[str] = "" + request_kwargs: Optional[dict] = {} + exception_type: Optional[str] = "" + status_code: Optional[str] = "" + exception_string: Optional[str] = "" + startTime: Union[str, datetime, None] + endTime: Union[str, datetime, None] + + +class LiteLLM_AuditLogs(LiteLLMBase): + id: str + updated_at: datetime + changed_by: str + changed_by_api_key: Optional[str] = None + action: Literal["created", "updated", "deleted"] + table_name: Literal[ + LitellmTableNames.TEAM_TABLE_NAME, + LitellmTableNames.USER_TABLE_NAME, + LitellmTableNames.KEY_TABLE_NAME, + LitellmTableNames.PROXY_MODEL_TABLE_NAME, + ] + object_id: str + before_value: Optional[Json] = None + updated_values: Optional[Json] = None + + +class LiteLLM_SpendLogs_ResponseObject(LiteLLMBase): + response: Optional[List[Union[LiteLLM_SpendLogs, Any]]] = None + + +class TokenCountRequest(LiteLLMBase): + model: str + prompt: Optional[str] = None + messages: Optional[List[dict]] = None + + +class TokenCountResponse(LiteLLMBase): + total_tokens: int + request_model: str + model_used: str + tokenizer_type: str + + +class CallInfo(LiteLLMBase): + """Used for slack budget alerting""" + + spend: float + max_budget: Optional[float] = None + token: Optional[str] = Field(default=None, description="Hashed value of that key") + customer_id: Optional[str] = None + user_id: Optional[str] = None + team_id: Optional[str] = None + team_alias: Optional[str] = None + user_email: Optional[str] = None + key_alias: Optional[str] = None + projected_exceeded_date: Optional[str] = None + projected_spend: Optional[float] = None + + +class WebhookEvent(CallInfo): + event: Literal[ + "budget_crossed", + "threshold_crossed", + "projected_limit_exceeded", + "key_created", + "internal_user_created", + "spend_tracked", + ] + event_group: Literal["internal_user", "key", "team", "proxy", "customer"] + event_message: str # human-readable description of event + + +class SpecialModelNames(enum.Enum): + all_team_models = "all-team-models" + all_proxy_models = "all-proxy-models" + + +class InvitationNew(LiteLLMBase): + user_id: str + + +class InvitationUpdate(LiteLLMBase): + invitation_id: str + is_accepted: bool + + +class InvitationDelete(LiteLLMBase): + invitation_id: str + + +class InvitationModel(LiteLLMBase): + id: str + user_id: str + is_accepted: bool + accepted_at: Optional[datetime] + expires_at: datetime + created_at: datetime + created_by: str + updated_at: datetime + updated_by: str + + +class InvitationClaim(LiteLLMBase): + invitation_link: str + user_id: str + password: str + + +class ConfigFieldInfo(LiteLLMBase): + field_name: str + field_value: Any + + +class CallbackOnUI(LiteLLMBase): + litellm_callback_name: str + litellm_callback_params: Optional[list] + ui_callback_name: str + + +class AllCallbacks(LiteLLMBase): + langfuse: CallbackOnUI = CallbackOnUI( + litellm_callback_name="langfuse", + ui_callback_name="Langfuse", + litellm_callback_params=[ + "LANGFUSE_PUBLIC_KEY", + "LANGFUSE_SECRET_KEY", + ], + ) + + otel: CallbackOnUI = CallbackOnUI( + litellm_callback_name="otel", + ui_callback_name="OpenTelemetry", + litellm_callback_params=[ + "OTEL_EXPORTER", + "OTEL_ENDPOINT", + "OTEL_HEADERS", + ], + ) + + s3: CallbackOnUI = CallbackOnUI( + litellm_callback_name="s3", + ui_callback_name="s3 Bucket (AWS)", + litellm_callback_params=[ + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_REGION_NAME", + ], + ) + + openmeter: CallbackOnUI = CallbackOnUI( + litellm_callback_name="openmeter", + ui_callback_name="OpenMeter", + litellm_callback_params=[ + "OPENMETER_API_ENDPOINT", + "OPENMETER_API_KEY", + ], + ) + + custom_callback_api: CallbackOnUI = CallbackOnUI( + litellm_callback_name="custom_callback_api", + litellm_callback_params=["GENERIC_LOGGER_ENDPOINT"], + ui_callback_name="Custom Callback API", + ) + + datadog: CallbackOnUI = CallbackOnUI( + litellm_callback_name="datadog", + litellm_callback_params=["DD_API_KEY", "DD_SITE"], + ui_callback_name="Datadog", + ) + + +class SpendLogsMetadata(TypedDict): + """ + Specific metadata k,v pairs logged to spendlogs for easier cost tracking + """ + + user_api_key: Optional[str] + user_api_key_alias: Optional[str] + user_api_key_team_id: Optional[str] + user_api_key_user_id: Optional[str] + user_api_key_team_alias: Optional[str] + spend_logs_metadata: Optional[ + dict + ] # special param to log k,v pairs to spendlogs for a call + requester_ip_address: Optional[str] + + +class SpendLogsPayload(TypedDict): + request_id: str + call_type: str + api_key: str + spend: float + total_tokens: int + prompt_tokens: int + completion_tokens: int + startTime: datetime + endTime: datetime + completionStartTime: Optional[datetime] + model: str + model_id: Optional[str] + model_group: Optional[str] + api_base: str + user: str + metadata: str # json str + cache_hit: str + cache_key: str + request_tags: str # json str + team_id: Optional[str] + end_user: Optional[str] + requester_ip_address: Optional[str] + + +class SpanAttributes(str, enum.Enum): + # Note: We've taken this from opentelemetry-semantic-conventions-ai + # I chose to not add a new dependency to litellm for this + + # Semantic Conventions for LLM requests, this needs to be removed after + # OpenTelemetry Semantic Conventions support Gen AI. + # Issue at https://github.com/open-telemetry/opentelemetry-python/issues/3868 + # Refer to https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/llm-spans.md + + LLM_SYSTEM = "gen_ai.system" + LLM_REQUEST_MODEL = "gen_ai.request.model" + LLM_REQUEST_MAX_TOKENS = "gen_ai.request.max_tokens" + LLM_REQUEST_TEMPERATURE = "gen_ai.request.temperature" + LLM_REQUEST_TOP_P = "gen_ai.request.top_p" + LLM_PROMPTS = "gen_ai.prompt" + LLM_COMPLETIONS = "gen_ai.completion" + LLM_RESPONSE_MODEL = "gen_ai.response.model" + LLM_USAGE_COMPLETION_TOKENS = "gen_ai.usage.completion_tokens" + LLM_USAGE_PROMPT_TOKENS = "gen_ai.usage.prompt_tokens" + LLM_TOKEN_TYPE = "gen_ai.token.type" + # To be added + # LLM_RESPONSE_FINISH_REASON = "gen_ai.response.finish_reasons" + # LLM_RESPONSE_ID = "gen_ai.response.id" + + # LLM + LLM_REQUEST_TYPE = "llm.request.type" + LLM_USAGE_TOTAL_TOKENS = "llm.usage.total_tokens" + LLM_USAGE_TOKEN_TYPE = "llm.usage.token_type" + LLM_USER = "llm.user" + LLM_HEADERS = "llm.headers" + LLM_TOP_K = "llm.top_k" + LLM_IS_STREAMING = "llm.is_streaming" + LLM_FREQUENCY_PENALTY = "llm.frequency_penalty" + LLM_PRESENCE_PENALTY = "llm.presence_penalty" + LLM_CHAT_STOP_SEQUENCES = "llm.chat.stop_sequences" + LLM_REQUEST_FUNCTIONS = "llm.request.functions" + LLM_REQUEST_REPETITION_PENALTY = "llm.request.repetition_penalty" + LLM_RESPONSE_FINISH_REASON = "llm.response.finish_reason" + LLM_RESPONSE_STOP_REASON = "llm.response.stop_reason" + LLM_CONTENT_COMPLETION_CHUNK = "llm.content.completion.chunk" + + # OpenAI + LLM_OPENAI_RESPONSE_SYSTEM_FINGERPRINT = "gen_ai.openai.system_fingerprint" + LLM_OPENAI_API_BASE = "gen_ai.openai.api_base" + LLM_OPENAI_API_VERSION = "gen_ai.openai.api_version" + LLM_OPENAI_API_TYPE = "gen_ai.openai.api_type" + + +class ManagementEndpointLoggingPayload(LiteLLMBase): + route: str + request_data: dict + response: Optional[dict] = None + exception: Optional[Any] = None + start_time: Optional[datetime] = None + end_time: Optional[datetime] = None + + +class ProxyException(Exception): + # NOTE: DO NOT MODIFY THIS + # This is used to map exactly to OPENAI Exceptions + def __init__( + self, + message: str, + type: str, + param: Optional[str], + code: Optional[int], + headers: Optional[Dict[str, str]] = None, + ): + self.message = message + self.type = type + self.param = param + self.code = code + if headers is not None: + for k, v in headers.items(): + if not isinstance(v, str): + headers[k] = str(v) + self.headers = headers or {} + + # rules for proxyExceptions + # Litellm router.py returns "No healthy deployment available" when there are no deployments available + # Should map to 429 errors https://github.com/BerriAI/litellm/issues/2487 + if ( + "No healthy deployment available" in self.message + or "No deployments available" in self.message + ): + self.code = 429 + + def to_dict(self) -> dict: + """Converts the ProxyException instance to a dictionary.""" + return { + "message": self.message, + "type": self.type, + "param": self.param, + "code": self.code, + } + + +class CommonProxyErrors(str, enum.Enum): + db_not_connected_error = "DB not connected" + no_llm_router = "No models configured on proxy" + not_allowed_access = "Admin-only endpoint. Not allowed to access this." + not_premium_user = "You must be a LiteLLM Enterprise user to use this feature. If you have a license please set `LITELLM_LICENSE` in your env. If you want to obtain a license meet with us here: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat" + + +class SpendCalculateRequest(LiteLLMBase): + model: Optional[str] = None + messages: Optional[List] = None + completion_response: Optional[dict] = None + + +class ProxyErrorTypes(str, enum.Enum): + budget_exceeded = "budget_exceeded" + expired_key = "expired_key" + auth_error = "auth_error" + internal_server_error = "internal_server_error" + bad_request_error = "bad_request_error" diff --git a/requirements.txt b/requirements.txt index 9db1f41..1c324d1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,5 +33,5 @@ pysnooper loguru numpy retry - +pydantic>=2.8.2 # uninstall ipython to catch breakpoints on debug with sandbox==false