From 5e413b20cbeef0a1e56d8fc31c3ae1da719e2d12 Mon Sep 17 00:00:00 2001 From: Jayr Castro Date: Fri, 10 May 2024 15:49:11 +0800 Subject: [PATCH] added new models and GPT Vision with tools calling --- serverless_openai/apis.py | 95 ++++++++++++++- serverless_openai/helpers.py | 4 +- setup.py | 2 +- test.ipynb | 221 +++++++++++++++++++++++++++++++++-- 4 files changed, 307 insertions(+), 15 deletions(-) diff --git a/serverless_openai/apis.py b/serverless_openai/apis.py index 0a7238c..ed8dbb7 100644 --- a/serverless_openai/apis.py +++ b/serverless_openai/apis.py @@ -37,7 +37,7 @@ class Config: def chat_completion( self, messages: Messages, - model: Union[TextCompletionModels, str] = TextCompletionModels.gpt4_0125, + model: Union[TextCompletionModels, str] = TextCompletionModels.gpt4_turbo, tries: int = 5, timeout: int = 500, temperature: Optional[float] = 1, @@ -187,7 +187,7 @@ def dall_e( def vision( self, messages: VisionMessage, - model: Union[VisionModels, str] = VisionModels.gpt4_vision, + model: Union[VisionModels, str] = VisionModels.gpt4_turbo, tries: int = 5, timeout: int = 500, temperature: Optional[float] = 1, @@ -232,11 +232,11 @@ def vision( def vision_longimage( self, messages: VisionMessage, - model: Union[VisionModels, str] = VisionModels.gpt4_vision, + model: Union[VisionModels, str] = VisionModels.gpt4_turbo, tries: int = 5, timeout: int = 500, temperature: Optional[float] = 1, - max_tokens: Optional[int] = 1024, + max_tokens: Optional[int] = 1024 ) -> OpenAIResults: if isinstance(messages.image, list): @@ -297,6 +297,93 @@ def vision_longimage( except Exception as e: print("ERROR:", e) return OpenAIResults(result=False, result_json=res) + + def vision_tools( + self, + messages: VisionMessage, + model: Union[VisionModels, str] = VisionModels.gpt4_turbo, + tries: int = 5, + timeout: int = 500, + temperature: Optional[float] = 1, + max_tokens: Optional[int] = 1024, + tools: Optional[List[dict]] = None, + tool_choice: Optional[str] = None, + response_format: Optional[dict] = {"type": "json_object"}, + ) -> OpenAIResults: + + if isinstance(messages.image, list): + img_b64_list = [] + for img in messages.image: + try: + TypeAdapter(HttpUrl).validate_python(img) + img_b64_list.append(img) + except: + if 'data:image/jpeg;base64' in img: + image_np = b64_to_np(img) + else: + image_np = urlimage_to_np(img) + img_b64_list.extend(crop_image(image_np)) + + elif isinstance(messages.image, str): + if 'data:image/jpeg;base64' in messages.image: + image_np = b64_to_np(messages.image) + else: + image_np = urlimage_to_np(messages.image) + img_b64_list = crop_image(image_np) + + elif isinstance(messages.image, np.ndarray): + image_np = messages.image + img_b64_list = crop_image(image_np) + + newm = [ + { + "role": messages.role, + "content": [ + { + "type": "text", + "text": messages.text + }, + ] + } + ] + for b64 in img_b64_list: + newm[0]['content'].append( + { + "type": "image_url", + "image_url": { + "url": b64, + "detail": "high" + } + } + ) + + data = { + "model": model, + "messages": newm, + "temperature": temperature, + } + + if max_tokens: + data['max_tokens'] = max_tokens + + if tool_choice: + data["response_format"] = response_format + data["tools"] = tools + data["tool_choice"] = {"type": "function", "function": {"name": tool_choice}} + + results = {} + for _ in range(tries): + try: + results = requests.post(self.completion_url, headers=self.headers, json=data, timeout=timeout).json() + if 'choices' in results: + res = results['choices'][0]['message'] + res_json = json.loads(res['tool_calls'][0]['function']['arguments'], strict=False) + return OpenAIResults(result=res_json, result_json=results) + else: + print("RESULTS:", results) + except Exception as e: + print("ERROR:", e) + return OpenAIResults(result=False, result_json=results) def embeddings( self, diff --git a/serverless_openai/helpers.py b/serverless_openai/helpers.py index 30c6360..79196d3 100644 --- a/serverless_openai/helpers.py +++ b/serverless_openai/helpers.py @@ -25,7 +25,8 @@ class Roles(str, ExtendedEnum): class TextCompletionModels(str, ExtendedEnum): gpt4_1106 : str = "gpt-4-1106-preview" gpt4 : str = "gpt-4" - gpt4_turbo : str = "gpt-4-turbo-preview" + gpt4_turbo : str = "gpt-4-turbo" + gpt4_turbo_prev : str = "gpt-4-turbo-preview" gpt4_0125 : str = "gpt-4-0125-preview" gpt35_turbo_0125: str = "gpt-3.5-turbo-0125" gpt35_turbo_1106 : str = "gpt-3.5-turbo-1106" @@ -45,6 +46,7 @@ class ImageCreationModels(str, ExtendedEnum): class VisionModels(str, ExtendedEnum): gpt4_vision : str = "gpt-4-vision-preview" + gpt4_turbo : str = "gpt-4-turbo" class VisionMessage(BaseModel): text: str diff --git a/setup.py b/setup.py index 7093384..bfd696e 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -VERSION = '1.3.7' +VERSION = '1.3.8' DESCRIPTION = "A package for using Openai in serverless environment" LONG_DESCRIPTION = 'A package for using Openai with scraping and etc. in serverless application such as AWS Lambda and GCP Cloud Function' diff --git a/test.ipynb b/test.ipynb index 2909874..67efba8 100644 --- a/test.ipynb +++ b/test.ipynb @@ -235,9 +235,117 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "To create a database schema as illustrated in the diagram, the MySQL code below defines each table and sets up foreign keys as appropriate:\n", + "\n", + "```sql\n", + "-- Create dim_region table\n", + "CREATE TABLE dim_region (\n", + " id INT PRIMARY KEY,\n", + " region VARCHAR(255)\n", + ");\n", + "\n", + "-- Create dim_country table\n", + "CREATE TABLE dim_country (\n", + " id INT PRIMARY KEY,\n", + " country VARCHAR(255),\n", + " ctry_cd VARCHAR(255),\n", + " region_id INT,\n", + " pop_id INT,\n", + " FOREIGN KEY (region_id) REFERENCES dim_region(id)\n", + ");\n", + "\n", + "-- Create dim_population table\n", + "CREATE TABLE dim_population (\n", + " id INT PRIMARY KEY,\n", + " pop_1m VARCHAR(255)\n", + ");\n", + "\n", + "-- Create dim_virus_family table\n", + "CREATE TABLE dim_virus_family (\n", + " id INT PRIMARY KEY,\n", + " name VARCHAR(255)\n", + ");\n", + "\n", + "-- Create dim_virus table\n", + "CREATE TABLE dim_virus (\n", + " id INT PRIMARY KEY,\n", + " name VARCHAR(255),\n", + " family_id INT,\n", + " type_id INT,\n", + " infect_rate FLOAT,\n", + " FOREIGN KEY (family_id) REFERENCES dim_virus_family(id)\n", + ");\n", + "\n", + "-- Create dim_transmission table\n", + "CREATE TABLE dim_transmission (\n", + " id INT PRIMARY KEY,\n", + " type VARCHAR(255)\n", + ");\n", + "\n", + "-- Create dim_year table\n", + "CREATE TABLE dim_year (\n", + " id INT PRIMARY KEY,\n", + " year INT\n", + ");\n", + "\n", + "-- Create dim_month table\n", + "CREATE TABLE dim_month (\n", + " id INT PRIMARY KEY,\n", + " name VARCHAR(255)\n", + ");\n", + "\n", + "-- Create dim_day table\n", + "CREATE TABLE dim_day (\n", + " id INT PRIMARY KEY,\n", + " name VARCHAR(255)\n", + ");\n", + "\n", + "-- Create dim_dates table\n", + "CREATE TABLE dim_dates (\n", + " id INT PRIMARY KEY,\n", + " year_id INT,\n", + " month_id INT,\n", + " day_id INT,\n", + " start_dt DATE,\n", + " FOREIGN KEY (year_id) REFERENCES dim_year(id),\n", + " FOREIGN KEY (month_id) REFERENCES dim_month(id),\n", + " FOREIGN KEY (day_id) REFERENCES dim_day(id)\n", + ");\n", + "\n", + "-- Create fact_pandemic table\n", + "CREATE TABLE fact_pandemic (\n", + " dates_id INT,\n", + " virus_id INT,\n", + " location_id INT,\n", + " cases_cnt INT,\n", + " new_cases_per1m INT,\n", + " recovered_cnt INT,\n", + " death_cnt INT,\n", + " PRIMARY KEY (dates_id, virus_id, location_id),\n", + " FOREIGN KEY (dates_id) REFERENCES dim_dates(id),\n", + " FOREIGN KEY (virus_id) REFERENCES dim_virus(id),\n", + " FOREIGN KEY (location_id) REFERENCES dim_country(id)\n", + ");\n", + "```\n", + "\n", + "### Additional Notes:\n", + "1. **Data Types**: Adjust the data types if necessary, especially for IDs, which could be `AUTO_INCREMENT` if you expect the database to generate unique IDs automatically.\n", + "2. **Cascade Operations**: Consider using `ON DELETE` and `ON UPDATE` rules for the foreign keys if your database needs to handle deletions and updates of related records.\n", + "3. **Indexes**: Depending on your queries, add appropriate indexes to improve performance, especially on foreign keys and frequently queried fields.\n", + "4. **Integrity**: Ensure there's an operational database schema versioning, rollbacks, or migration strategy to manage structural changes smoothly.\n", + "\n", + "This schema setup will create a relational database that mirrors the relationships and entities outlined in your diagram. Adjust further as per specific performance or business requirements.\n", + "{'id': 'chatcmpl-9NFGKIj33ZkVP8EZ10R164hEErjiq', 'object': 'chat.completion', 'created': 1715327240, 'model': 'gpt-4-turbo-2024-04-09', 'choices': [{'index': 0, 'message': {'role': 'assistant', 'content': \"To create a database schema as illustrated in the diagram, the MySQL code below defines each table and sets up foreign keys as appropriate:\\n\\n```sql\\n-- Create dim_region table\\nCREATE TABLE dim_region (\\n id INT PRIMARY KEY,\\n region VARCHAR(255)\\n);\\n\\n-- Create dim_country table\\nCREATE TABLE dim_country (\\n id INT PRIMARY KEY,\\n country VARCHAR(255),\\n ctry_cd VARCHAR(255),\\n region_id INT,\\n pop_id INT,\\n FOREIGN KEY (region_id) REFERENCES dim_region(id)\\n);\\n\\n-- Create dim_population table\\nCREATE TABLE dim_population (\\n id INT PRIMARY KEY,\\n pop_1m VARCHAR(255)\\n);\\n\\n-- Create dim_virus_family table\\nCREATE TABLE dim_virus_family (\\n id INT PRIMARY KEY,\\n name VARCHAR(255)\\n);\\n\\n-- Create dim_virus table\\nCREATE TABLE dim_virus (\\n id INT PRIMARY KEY,\\n name VARCHAR(255),\\n family_id INT,\\n type_id INT,\\n infect_rate FLOAT,\\n FOREIGN KEY (family_id) REFERENCES dim_virus_family(id)\\n);\\n\\n-- Create dim_transmission table\\nCREATE TABLE dim_transmission (\\n id INT PRIMARY KEY,\\n type VARCHAR(255)\\n);\\n\\n-- Create dim_year table\\nCREATE TABLE dim_year (\\n id INT PRIMARY KEY,\\n year INT\\n);\\n\\n-- Create dim_month table\\nCREATE TABLE dim_month (\\n id INT PRIMARY KEY,\\n name VARCHAR(255)\\n);\\n\\n-- Create dim_day table\\nCREATE TABLE dim_day (\\n id INT PRIMARY KEY,\\n name VARCHAR(255)\\n);\\n\\n-- Create dim_dates table\\nCREATE TABLE dim_dates (\\n id INT PRIMARY KEY,\\n year_id INT,\\n month_id INT,\\n day_id INT,\\n start_dt DATE,\\n FOREIGN KEY (year_id) REFERENCES dim_year(id),\\n FOREIGN KEY (month_id) REFERENCES dim_month(id),\\n FOREIGN KEY (day_id) REFERENCES dim_day(id)\\n);\\n\\n-- Create fact_pandemic table\\nCREATE TABLE fact_pandemic (\\n dates_id INT,\\n virus_id INT,\\n location_id INT,\\n cases_cnt INT,\\n new_cases_per1m INT,\\n recovered_cnt INT,\\n death_cnt INT,\\n PRIMARY KEY (dates_id, virus_id, location_id),\\n FOREIGN KEY (dates_id) REFERENCES dim_dates(id),\\n FOREIGN KEY (virus_id) REFERENCES dim_virus(id),\\n FOREIGN KEY (location_id) REFERENCES dim_country(id)\\n);\\n```\\n\\n### Additional Notes:\\n1. **Data Types**: Adjust the data types if necessary, especially for IDs, which could be `AUTO_INCREMENT` if you expect the database to generate unique IDs automatically.\\n2. **Cascade Operations**: Consider using `ON DELETE` and `ON UPDATE` rules for the foreign keys if your database needs to handle deletions and updates of related records.\\n3. **Indexes**: Depending on your queries, add appropriate indexes to improve performance, especially on foreign keys and frequently queried fields.\\n4. **Integrity**: Ensure there's an operational database schema versioning, rollbacks, or migration strategy to manage structural changes smoothly.\\n\\nThis schema setup will create a relational database that mirrors the relationships and entities outlined in your diagram. Adjust further as per specific performance or business requirements.\"}, 'logprobs': None, 'finish_reason': 'stop'}], 'usage': {'prompt_tokens': 787, 'completion_tokens': 652, 'total_tokens': 1439}, 'system_fingerprint': 'fp_0737e0dfd9'}\n" + ] + } + ], "source": [ "from serverless_openai import OpenAIAPI, VisionMessage\n", "import os\n", @@ -266,23 +374,23 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "You've provided several images that represent different contexts, so I'll address each individually.\n", + "The images you provided show different examples of digital materials and scenarios:\n", "\n", - "1. The first two images depict screens from a website belonging to Zero Company. The text on the site explains that they are a performance marketing agency offering services related to digital marketing. They describe their approach, expertise, and list some of the services they provide, such as Google Ads Management, Social Media Campaigns, and YouTube Advertising Services. The site also features testimonials from clients, and it indicates that the company is recognized as a Google Premier Partner. The third image from this set showcases the footer of the website with additional navigation links and contact information for the company.\n", + "1. **Zero Company Marketing Website**: This image is a screenshot of a digital marketing agency’s website named Zero Company. The web page highlights services and credentials of the company, including its position as a Google Premier Partner, and the different digital advertising services it offers such as Google Ads Management, Social Media Campaigns, and YouTube Advertising. The page also features customer testimonials and the effectiveness of their marketing strategies by showing significant numbers of clicks, conversions, and impressions they've achieved.\n", "\n", - "2. The fourth image represents a layout design for a print advertisement featuring a pickup truck with a heavy-duty rack. It's a professional mockup of a design project that includes a primary poster, color swatches, textures representative of the product materials, and a secondary smaller image. This type of layout is likely used in marketing materials or as part of a graphic designer's portfolio.\n", + "2. **Print Marketing Material**: This design layout features promotional material for a heavy-duty truck rack. It's presented as a part of a mockup that includes the original graphic design and some design elements such as color palettes and textures. This kind of layout is typically used in product brochures or digital advertisements.\n", "\n", - "3. The fifth image captures a real-life scene where an individual is taking a photo of a sunset landscape with an overpass using an older model mobile phone. The focus is on the phone and the individual's hand, with the landscape and sunset reflected in the phone's screen, representing the concept of capturing moments and the juxtaposition of technology with the natural environment.\n", + "3. **Photographing a Road Bridge at Sunset**: The third image captures a person using an older model of a mobile phone to take a photo of a bridge at sunset. This image illustrates a simple moment of photography possibly capturing the vanishing perspective of the road under the bridge during the twilight hours.\n", "\n", - "Please let me know if you need any more details about these images.\n", - "{'id': 'chatcmpl-9H8bd05z0d8ehrq3irAD1h3LQhv1s', 'object': 'chat.completion', 'created': 1713871685, 'model': 'gpt-4-1106-vision-preview', 'choices': [{'index': 0, 'message': {'role': 'assistant', 'content': \"You've provided several images that represent different contexts, so I'll address each individually.\\n\\n1. The first two images depict screens from a website belonging to Zero Company. The text on the site explains that they are a performance marketing agency offering services related to digital marketing. They describe their approach, expertise, and list some of the services they provide, such as Google Ads Management, Social Media Campaigns, and YouTube Advertising Services. The site also features testimonials from clients, and it indicates that the company is recognized as a Google Premier Partner. The third image from this set showcases the footer of the website with additional navigation links and contact information for the company.\\n\\n2. The fourth image represents a layout design for a print advertisement featuring a pickup truck with a heavy-duty rack. It's a professional mockup of a design project that includes a primary poster, color swatches, textures representative of the product materials, and a secondary smaller image. This type of layout is likely used in marketing materials or as part of a graphic designer's portfolio.\\n\\n3. The fifth image captures a real-life scene where an individual is taking a photo of a sunset landscape with an overpass using an older model mobile phone. The focus is on the phone and the individual's hand, with the landscape and sunset reflected in the phone's screen, representing the concept of capturing moments and the juxtaposition of technology with the natural environment.\\n\\nPlease let me know if you need any more details about these images.\"}, 'logprobs': None, 'finish_reason': 'stop'}], 'usage': {'prompt_tokens': 5876, 'completion_tokens': 293, 'total_tokens': 6169}, 'system_fingerprint': None}\n" + "Each of these images represents a different aspect of graphic design and digital marketing, showcasing how both physical and digital designs are used to communicate messages and capture moments.\n", + "{'id': 'chatcmpl-9NFH9Fh3OngARaAJTixsMUP5NrfZk', 'object': 'chat.completion', 'created': 1715327291, 'model': 'gpt-4-turbo-2024-04-09', 'choices': [{'index': 0, 'message': {'role': 'assistant', 'content': \"The images you provided show different examples of digital materials and scenarios:\\n\\n1. **Zero Company Marketing Website**: This image is a screenshot of a digital marketing agency’s website named Zero Company. The web page highlights services and credentials of the company, including its position as a Google Premier Partner, and the different digital advertising services it offers such as Google Ads Management, Social Media Campaigns, and YouTube Advertising. The page also features customer testimonials and the effectiveness of their marketing strategies by showing significant numbers of clicks, conversions, and impressions they've achieved.\\n\\n2. **Print Marketing Material**: This design layout features promotional material for a heavy-duty truck rack. It's presented as a part of a mockup that includes the original graphic design and some design elements such as color palettes and textures. This kind of layout is typically used in product brochures or digital advertisements.\\n\\n3. **Photographing a Road Bridge at Sunset**: The third image captures a person using an older model of a mobile phone to take a photo of a bridge at sunset. This image illustrates a simple moment of photography possibly capturing the vanishing perspective of the road under the bridge during the twilight hours.\\n\\nEach of these images represents a different aspect of graphic design and digital marketing, showcasing how both physical and digital designs are used to communicate messages and capture moments.\"}, 'logprobs': None, 'finish_reason': 'stop'}], 'usage': {'prompt_tokens': 5876, 'completion_tokens': 264, 'total_tokens': 6140}, 'system_fingerprint': 'fp_0737e0dfd9'}\n" ] } ], @@ -308,6 +416,101 @@ "print(res.result_json)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Test Vision with tools calling" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from serverless_openai import OpenAIAPI, VisionMessage\n", + "import os\n", + "\n", + "# Initialize OpenAIAPI with API Key\n", + "openai_api_key = os.getenv(\"openai_api_key\")\n", + "# openai_api_key = 'API_KEY_HERE'\n", + "ai = OpenAIAPI(api_key=openai_api_key)\n", + "\n", + "tool_name = \"details_function\"\n", + "tools = [{\n", + " \"type\": \"function\",\n", + " \"function\": {\n", + " \"name\": tool_name,\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"in_depth_details\": {\n", + " \"type\": \"array\",\n", + " \"description\": \"In-depth details of the images provided by the user\",\n", + " \"items\": {\n", + " \"type\": \"string\"\n", + " }\n", + " }\n", + " },\n", + " \"required\": [\"in_depth_details\"]\n", + " }\n", + " }\n", + "\n", + "}]\n", + "\n", + "# url = \"images/test2.png\"\n", + "url2 = \"images/test.png\"\n", + "url3 = \"https://aicmo-bucket.s3.amazonaws.com/vision-cmo/8688398Image_created_with_a_mobile_phone.png\"\n", + "# vm = VisionMessage(text=\"What is this?\", image=[url, url2, url3])\n", + "vm = VisionMessage(text=\"Give in-depth details of the image, return results in JSON format\", image=[url3, url2])\n", + "res = ai.vision_tools(messages=vm, tools=tools, tool_choice=tool_name)\n", + "\n", + "# Actual result\n", + "print(res.result)\n", + "\n", + "# Other details from the result in Dictionary format\n", + "print(res.result_json)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "This image shows a perspective of a road leading straight ahead under a bridge, taken during twilight. The photo captures the moment through another smaller mobile device held in a person's hand, which includes part of their black sleeve. The surrounding environment appears to be open and quiet with signs of recent construction, as indicated by the presence of barriers and freshly poured concrete.\n" + ] + } + ], + "source": [ + "print(res.result['in_depth_details'][0])" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['An image of a graphic design project featuring a rugged truck equipped with a heavy-duty rack presented in a poster format. The composition includes various elements related to design presentation such as color palettes, material swatches, and sectional views of the rack.',\n", + " 'An image capturing a person holding up a smartphone to photograph a sunset scene over a road leading to a bridge. The perspective is from behind the person, focusing on the phone screen showing the image being captured and the actual scene ahead.']" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "res.result['in_depth_details']" + ] + }, { "cell_type": "markdown", "metadata": {},