-
Notifications
You must be signed in to change notification settings - Fork 295
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: Token rate limit on specified routes (#65)
- Configurable token rate limit - Fix startup info - More elegant chat log output
- Loading branch information
Showing
14 changed files
with
364 additions
and
250 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,89 @@ | ||
| from rich import print | ||
| from rich.panel import Panel | ||
| from rich.table import Table | ||
|
|
||
|
|
||
| def print_startup_info(base_url, route_prefix, api_key, fwd_key, /, style, **kwargs): | ||
| """ | ||
| Prints the startup information of the application. | ||
| """ | ||
| try: | ||
| from dotenv import load_dotenv | ||
|
|
||
| load_dotenv(".env") | ||
| except Exception: | ||
| ... | ||
| route_prefix = route_prefix or "/" | ||
| if not isinstance(api_key, str): | ||
| api_key = True if len(api_key) else False | ||
| if not isinstance(fwd_key, str): | ||
| fwd_key = True if len(fwd_key) else False | ||
| table = Table(title="", box=None, width=50) | ||
|
|
||
| matrcs = { | ||
| "base url": { | ||
| 'value': base_url, | ||
| }, | ||
| "route prefix": { | ||
| 'value': route_prefix, | ||
| }, | ||
| "api keys": { | ||
| 'value': str(api_key), | ||
| }, | ||
| "forward keys": { | ||
| 'value': str(fwd_key), | ||
| 'style': "#62E883" if fwd_key or not api_key else "red", | ||
| }, | ||
| } | ||
| table.add_column("", justify='left', width=10) | ||
| table.add_column("", justify='left') | ||
| for key, value in matrcs.items(): | ||
| table.add_row(key, value['value'], style=value.get('style', style)) | ||
| for key, value in kwargs.items(): | ||
| table.add_row(key, str(value), style=style) | ||
|
|
||
| print(Panel(table, title="🤗 openai-forward is ready to serve! ", expand=False)) | ||
|
|
||
|
|
||
| def print_rate_limit_info( | ||
| strategy: str, | ||
| global_req_rate_limit: str, | ||
| req_rate_limit: dict, | ||
| token_rate_limit: dict, | ||
| **kwargs, | ||
| ): | ||
| """ | ||
| Print rate limit information. | ||
| Args: | ||
| strategy (str): The strategy used for rate limiting. | ||
| global_req_rate_limit (str): The global request rate limit. | ||
| req_rate_limit (dict): A dictionary of request rate limit. | ||
| token_rate_limit (dict): A dictionary of token rate limit. | ||
| **kwargs: Other limits info. | ||
| Returns: | ||
| None | ||
| """ | ||
| table = Table(title="", box=None, width=50) | ||
| table.add_column("") | ||
| table.add_column("", justify='left') | ||
| if strategy: | ||
| table.add_row("strategy", strategy, style='#7CD9FF') | ||
|
|
||
| if global_req_rate_limit: | ||
| table.add_row( | ||
| "global rate limit", f"{global_req_rate_limit} (req)", style='#C5FF95' | ||
| ) | ||
| for key, value in req_rate_limit.items(): | ||
| table.add_row(key, f"{value} (req)", style='#C5FF95') | ||
|
|
||
| for key, value in token_rate_limit.items(): | ||
| if isinstance(value, float): | ||
| value = f"{value:.4f} s/token" | ||
| table.add_row(key, f"{value} (token)", style='#C5FF95') | ||
|
|
||
| for key, value in kwargs.items(): | ||
| table.add_row(key, str(value), style='#C5FF95') | ||
|
|
||
| print(Panel(table, title="⏱️ Rate Limit configuration", expand=False)) |
Oops, something went wrong.