Initial working version

ToothyDev · Sep 13, 2024 · 637d131 · 637d131
1 parent 13698c5
commit 637d131
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 3 deletions.
diff --git a/cogs/socials.py b/cogs/socials.py
@@ -237,16 +237,19 @@ async def gpt(self, ctx: discord.ApplicationContext, text: str):
         messages = await ctx.channel.history(limit=50).flatten()
         messages.reverse()
         input_history = [{"role": "system", "content": utils.SYSTEM_PROMPT}]
+        alt_text = None
         for message in messages:
             if message.author != self.bot.user:
+                if message.attachments and message.attachments[0].content_type.startswith("image"):
+                    alt_text = f"\nThe user attached an image to the message: {await utils.analyse_image(message.attachments[0].url)}"
                 if message.content is None or message.content == "":
                     input_history.append(
                         {"role": "user", "name": message.author.display_name,
-                         "content": f"{message.author.display_name} ({utils.get_gender(message.author)}) said: <image or other type of message>"})
+                         "content": f"{message.author.display_name} ({utils.get_gender(message.author)}) sent a file. {alt_text}"})
                 else:
                     input_history.append(
                         {"role": "user", "name": message.author.display_name,
-                         "content": f"{message.author.display_name} ({utils.get_gender(message.author)}) said: {message.content}"})
+                         "content": f"{message.author.display_name} ({utils.get_gender(message.author)}) said: {message.content} {alt_text}"})
                 continue
             try:
                 usermessage = message.content.split("\n")[0]  # Get the first line of the message. the user prompt
@@ -258,7 +261,7 @@ async def gpt(self, ctx: discord.ApplicationContext, text: str):
                 continue
             input_history.append(
                 {"role": "user", "name": ctx.guild.get_member(message.interaction_metadata.user.id).display_name,
-                 "content": f"{ctx.guild.get_member(message.interaction_metadata.user.id).display_name} ({utils.get_gender(ctx.guild.get_member(message.interaction_metadata.user.id))}) said: {usermessage[12:]}"})
+                 "content": f"{ctx.guild.get_member(message.interaction_metadata.user.id).display_name} ({utils.get_gender(ctx.guild.get_member(message.interaction_metadata.user.id))}) said: {usermessage[12:]} {alt_text}"})
             input_history.append({"role": "assistant", "content": botmsg[9:]})
         input_history.append(
             {"role": "user", "name": ctx.author.display_name,

diff --git a/utils/api_helpers.py b/utils/api_helpers.py
@@ -14,6 +14,30 @@ async def generate_from_history(history: list[dict]) -> str:
     return chat_completion.choices[0].message.content
 
 
+async def analyse_image(image_url: str) -> str:
+    client = AsyncGroq(api_key=config.groq_api_key)
+    image_completion = await client.chat.completions.create(
+        messages=[
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text",
+                     "text": "Describe this image. Your output will be used to describe this image to a 'blind' LLM."
+                             "Keep yourself short."},
+                    {
+                        "type": "image_url",
+                        "image_url": {
+                            "url": image_url,
+                        },
+                    },
+                ],
+            }
+        ],
+        model="llava-v1.5-7b-4096-preview",
+    )
+    return image_completion.choices[0].message.content
+
+
 async def generate_single(prompt: str) -> str:
     client = AsyncGroq(api_key=config.groq_api_key)
     chat_completion = await client.chat.completions.create(messages=[{"role": "user", "content": prompt}],