add qwen-vl-max-0809 for understand videos

api/core/model_runtime/model_providers/tongyi/llm/llm.py

@@ -1,5 +1,6 @@
 import base64
 import os
+import re
 import tempfile
 import uuid
 from collections.abc import Generator
@@ -483,9 +484,15 @@ def _convert_prompt_messages_to_tongyi_messages(self, prompt_messages: list[Prom
                                 # convert image base64 data to file in /tmp
                                 image_url = self._save_base64_image_to_file(message_content.data)
 
-                            sub_message_dict = {
-                                "image": image_url
-                            }
+                            is_video_url = re.search(r'\.(mp4|webm|ogg|mov|avi|mkv|flv|wmv)$', image_url, re.IGNORECASE)
+                            if is_video_url:
+                                sub_message_dict = {
+                                    "video": image_url
+                                }
+                            else:
+                                sub_message_dict = {
+                                    "image": image_url
+                                }
                             sub_messages.append(sub_message_dict)
 
                     # resort sub_messages to ensure text is always at last

api/core/model_runtime/model_providers/tongyi/llm/qwen-vl-max-0809.yaml

@@ -0,0 +1,47 @@
+model: qwen-vl-max-0809
+label:
+  en_US: qwen-vl-max-0809
+model_type: llm
+features:
+  - vision
+  - agent-thought
+model_properties:
+  mode: chat
+  context_size: 32768
+parameter_rules:
+  - name: top_p
+    use_template: top_p
+    type: float
+    default: 0.8
+    min: 0.1
+    max: 0.9
+    help:
+      zh_Hans: 生成过程中核采样方法概率阈值，例如，取值为0.8时，仅保留概率加起来大于等于0.8的最可能token的最小集合作为候选集。取值范围为（0,1.0)，取值越大，生成的随机性越高；取值越低，生成的确定性越高。
+      en_US: The probability threshold of the kernel sampling method during the generation process. For example, when the value is 0.8, only the smallest set of the most likely tokens with a sum of probabilities greater than or equal to 0.8 is retained as the candidate set. The value range is (0,1.0). The larger the value, the higher the randomness generated; the lower the value, the higher the certainty generated.
+  - name: top_k
+    type: int
+    min: 0
+    max: 99
+    label:
+      zh_Hans: 取样数量
+      en_US: Top k
+    help:
+      zh_Hans: 生成时，采样候选集的大小。例如，取值为50时，仅将单次生成中得分最高的50个token组成随机采样的候选集。取值越大，生成的随机性越高；取值越小，生成的确定性越高。
+      en_US: The size of the sample candidate set when generated. For example, when the value is 50, only the 50 highest-scoring tokens in a single generation form a randomly sampled candidate set. The larger the value, the higher the randomness generated; the smaller the value, the higher the certainty generated.
+  - name: seed
+    required: false
+    type: int
+    default: 1234
+    label:
+      zh_Hans: 随机种子
+      en_US: Random seed
+    help:
+      zh_Hans: 生成时使用的随机数种子，用户控制模型生成内容的随机性。支持无符号64位整数，默认值为 1234。在使用seed时，模型将尽可能生成相同或相似的结果，但目前不保证每次生成的结果完全相同。
+      en_US: The random number seed used when generating, the user controls the randomness of the content generated by the model. Supports unsigned 64-bit integers, default value is 1234. When using seed, the model will try its best to generate the same or similar results, but there is currently no guarantee that the results will be exactly the same every time.
+  - name: response_format
+    use_template: response_format
+pricing:
+  input: '0.02'
+  output: '0.02'
+  unit: '0.001'
+  currency: RMB

web/app/components/base/image-gallery/index.tsx

@@ -4,6 +4,7 @@ import React, { useState } from 'react'
 import s from './style.module.css'
 import cn from '@/utils/classnames'
 import ImagePreview from '@/app/components/base/image-uploader/image-preview'
+import { isVideoLink } from '@/app/components/base/image-uploader/utils'
 
 type Props = {
   srcs: string[]
@@ -38,16 +39,18 @@ const ImageGallery: FC<Props> = ({
     <div className={cn(s[`img-${imgNum}`], 'flex flex-wrap')}>
       {/* TODO: support preview */}
       {srcs.map((src, index) => (
-        // eslint-disable-next-line @next/next/no-img-element
-        <img
-          key={index}
-          className={s.item}
-          style={imgStyle}
-          src={src}
-          alt=''
-          onClick={() => setImagePreviewUrl(src)}
-          onError={e => e.currentTarget.remove()}
-        />
+
+        isVideoLink(src)
+          ? <video key={index} className={s.item} style={imgStyle} src={src} controls />
+          : <img
+            key={index}
+            className={s.item}
+            style={imgStyle}
+            src={src}
+            alt=''
+            onClick={() => setImagePreviewUrl(src)}
+            onError={e => e.currentTarget.remove()}
+          />
       ))}
       {
         imagePreviewUrl && (

web/app/components/base/image-uploader/image-list.tsx

@@ -5,13 +5,15 @@ import {
   RiCloseLine,
   RiLoader2Line,
 } from '@remixicon/react'
+import videoThumbImg from './video-thumb.png'
 import cn from '@/utils/classnames'
 import { RefreshCcw01 } from '@/app/components/base/icons/src/vender/line/arrows'
 import { AlertTriangle } from '@/app/components/base/icons/src/vender/solid/alertsAndFeedback'
 import Tooltip from '@/app/components/base/tooltip'
 import type { ImageFile } from '@/types/app'
 import { TransferMethod } from '@/types/app'
 import ImagePreview from '@/app/components/base/image-uploader/image-preview'
+import { isVideoLink } from '@/app/components/base/image-uploader/utils'
 
 type ImageListProps = {
   list: ImageFile[]
@@ -101,17 +103,22 @@ const ImageList: FC<ImageListProps> = ({
             onLoad={() => handleImageLinkLoadSuccess(item)}
             onError={() => handleImageLinkLoadError(item)}
             src={
-              item.type === TransferMethod.remote_url
-                ? item.url
-                : item.base64Url
-            }
-            onClick={() =>
-              item.progress === 100
-              && setImagePreviewUrl(
-                (item.type === TransferMethod.remote_url
+              isVideoLink(item.url)
+                ? videoThumbImg.src
+                : item.type === TransferMethod.remote_url
                   ? item.url
-                  : item.base64Url) as string,
-              )
+                  : item.base64Url
+            }
+            onClick={
+              isVideoLink(item.url)
+                ? undefined
+                : () =>
+                  item.progress === 100
+                  && setImagePreviewUrl(
+                    (item.type === TransferMethod.remote_url
+                      ? item.url
+                      : item.base64Url) as string,
+                  )
             }
           />
           {!readonly && (

web/app/components/base/image-uploader/utils.ts

@@ -34,3 +34,7 @@ export const imageUpload: ImageUpload = ({
       onErrorCallback()
     })
 }
+
+export const isVideoLink = (url: string): boolean => {
+  return /\.(mp4|mov|avi|mkv|flv|wmv)$/i.test(url)
+}