Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update boto3 dependency to version 1.28.57, refactor bedrock client initialization and remove troubleshooting guide from documentation. #497

Merged
merged 2 commits into from
Sep 30, 2023
Merged

Update boto3 dependency to version 1.28.57, refactor bedrock client initialization and remove troubleshooting guide from documentation. #497

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4f67261
Update boto3 dependency to version 1.28.57, refactor bedrock client i…
coconut49 Sep 30, 2023
ad5eddb
moved boto3 out of dependency
coconut49 Sep 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions .circleci/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,7 @@ jobs:
pip install -q google-generativeai
pip install openai[datalib]
pip install -Uq chromadb==0.3.29
pip install https://github.com/BerriAI/litellm/raw/main/cookbook/bedrock_resources/boto3-1.28.21-py3-none-any.whl
pip install https://github.com/BerriAI/litellm/raw/main/cookbook/bedrock_resources/botocore-1.31.21-py3-none-any.whl
pip install "boto3>=1.28.57"
- save_cache:
paths:
- ./venv
Expand Down
Binary file removed cookbook/bedrock_resources/boto3-1.28.21-py3-none-any.whl
View file
Open in desktop
Binary file not shown.
Binary file removed cookbook/bedrock_resources/boto3-1.28.21.tar.gz
View file
Open in desktop
Binary file not shown.
Binary file removed cookbook/bedrock_resources/botocore-1.31.21-py3-none-any.whl
View file
Open in desktop
Binary file not shown.
Binary file removed cookbook/bedrock_resources/botocore-1.31.21.tar.gz
View file
Open in desktop
Binary file not shown.
10 changes: 0 additions & 10 deletions docs/my-website/docs/providers/bedrock.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,13 +75,3 @@ for chunk in response:
}
}
```

## Troubleshooting
If creating a boto3 bedrock client fails with `Unknown service: 'bedrock'`
Try re installing boto3 using the following commands
```shell
pip install https://github.com/BerriAI/litellm/raw/main/cookbook/bedrock_resources/boto3-1.28.21-py3-none-any.whl
pip install https://github.com/BerriAI/litellm/raw/main/cookbook/bedrock_resources/botocore-1.31.21-py3-none-any.whl
```

See Page 26 on [Amazon Bedrock User Guide](https://d2eo22ngex1n9g.cloudfront.net/Documentation/BedrockUserGuide.pdf)
111 changes: 48 additions & 63 deletions litellm/llms/bedrock.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,8 @@
import os
import json
from enum import Enum
import requests
import time
from typing import Callable
from litellm.utils import ModelResponse, get_secret
import sys

class BedrockError(Exception):
def __init__(self, status_code, message):
Expand All @@ -15,38 +12,24 @@ def __init__(self, status_code, message):
self.message
) # Call the base class constructor with the parameters it needs


class AnthropicConstants(Enum):
HUMAN_PROMPT = "\n\nHuman:"
AI_PROMPT = "\n\nAssistant:"


def init_bedrock_client(region_name):
import sys
import boto3
import subprocess
try:
client = boto3.client(
service_name="bedrock",
region_name=region_name,
endpoint_url=f'https://bedrock.{region_name}.amazonaws.com'
)
except Exception as e:
try:
command1 = "python3 -m pip install https://github.com/BerriAI/litellm/raw/main/cookbook/bedrock_resources/boto3-1.28.21-py3-none-any.whl"
subprocess.run(command1, shell=True, check=True)
# Command 2: Install boto3 from URL
command2 = "python3 -m pip install https://github.com/BerriAI/litellm/raw/main/cookbook/bedrock_resources/botocore-1.31.21-py3-none-any.whl"
subprocess.run(command2, shell=True, check=True)

import boto3
client = boto3.client(
service_name="bedrock",
region_name=region_name,
endpoint_url=f'https://bedrock.{region_name}.amazonaws.com'
)
except Exception as e:
raise e

client = boto3.client(
service_name="bedrock-runtime",
region_name=region_name,
endpoint_url=f'https://bedrock-runtime.{region_name}.amazonaws.com'
)
Comment on lines +24 to +28
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The invoke_model method only exists in the bedrock-runtime, see the boto/boto3#3881

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if possible can you run the bedrock tests in test_completion.py with these changes for bedrock and send a screenshot that they work.

If not I can test them tomorrow

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The Titan test fails because I don't have access, but the rest pass. Please refer to the screenshots. Thank you for your prompt assistance!
image

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

looks awesome - what is that tool you're using to run pytest. looks super cool

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

😊 It's PyCharm with the following config:
image


return client


def convert_messages_to_prompt(messages, provider):
# handle anthropic prompts using anthropic constants
if provider == "anthropic":
Expand Down Expand Up @@ -80,30 +63,31 @@ def convert_messages_to_prompt(messages, provider):
prompt += f"{message['content']}"
return prompt


"""
BEDROCK AUTH Keys/Vars
os.environ['AWS_ACCESS_KEY_ID'] = ""
os.environ['AWS_SECRET_ACCESS_KEY'] = ""
"""


# set os.environ['AWS_REGION_NAME'] = <your-region_name>

def completion(
model: str,
messages: list,
model_response: ModelResponse,
print_verbose: Callable,
encoding,
logging_obj,
optional_params=None,
stream=False,
litellm_params=None,
logger_fn=None,
model: str,
messages: list,
model_response: ModelResponse,
print_verbose: Callable,
encoding,
logging_obj,
optional_params=None,
stream=False,
litellm_params=None,
logger_fn=None,
):

region_name = (
get_secret("AWS_REGION_NAME") or
"us-west-2" # default to us-west-2 if user not specified
get_secret("AWS_REGION_NAME") or
"us-west-2" # default to us-west-2 if user not specified
)

client = init_bedrock_client(region_name)
Expand All @@ -119,48 +103,48 @@ def completion(
elif provider == "ai21":
data = json.dumps({
"prompt": prompt,
})
})

else: # amazon titan
else: # amazon titan
data = json.dumps({
"inputText": prompt,
"inputText": prompt,
"textGenerationConfig": optional_params,
})
## LOGGING
})
## LOGGING
logging_obj.pre_call(
input=prompt,
api_key="",
additional_args={"complete_input_dict": data},
)
input=prompt,
api_key="",
additional_args={"complete_input_dict": data},
)

## COMPLETION CALL
accept = 'application/json'
contentType = 'application/json'
if stream == True:
response = client.invoke_model_with_response_stream(
body=data,
modelId=model,
accept=accept,
body=data,
modelId=model,
accept=accept,
contentType=contentType
)
response = response.get('body')
return response

response = client.invoke_model(
body=data,
modelId=model,
accept=accept,
body=data,
modelId=model,
accept=accept,
contentType=contentType
)
response_body = json.loads(response.get('body').read())

## LOGGING
logging_obj.post_call(
input=prompt,
api_key="",
original_response=response,
additional_args={"complete_input_dict": data},
)
input=prompt,
api_key="",
original_response=response,
additional_args={"complete_input_dict": data},
)
print_verbose(f"raw model_response: {response}")
## RESPONSE OBJECT
outputText = "default"
Expand All @@ -169,7 +153,7 @@ def completion(
elif provider == "anthropic":
outputText = response_body['completion']
model_response["finish_reason"] = response_body["stop_reason"]
else: # amazon titan
else: # amazon titan
outputText = response_body.get('results')[0].get('outputText')
if "error" in outputText:
raise BedrockError(
Expand All @@ -185,7 +169,7 @@ def completion(
## CALCULATING USAGE - baseten charges on time, not tokens - have some mapping of cost here.
prompt_tokens = len(
encoding.encode(prompt)
)
)
completion_tokens = len(
encoding.encode(model_response["choices"][0]["message"]["content"])
)
Expand All @@ -199,6 +183,7 @@ def completion(
}
return model_response


def embedding():
# logic for parsing in - calling - parsing out model embedding calls
pass