Merge commit '8d9a45908397b3f3c0d56e86160bb97b5611b79b'

* commit '8d9a45908397b3f3c0d56e86160bb97b5611b79b': (240 commits)
  fix: remove obsoleted 'version' elements in compose files (langgenius#5553)
  chore: add a secondary confirmation dialog when the user delete the tool (langgenius#5634)
  fix: ro-RO is not a valid language (langgenius#5635)
  FR: langgenius#4048 - Add color customization to the chatbot  (langgenius#4885)
  feat: tracing fe (langgenius#5487)
  feat: add ops trace (langgenius#5483)
  chore: cleanup test_delete_by_document_id method in opensearch vdb test (langgenius#5619)
  chore(pyproject.toml): Add Ruff formatter config. (langgenius#5627)
  Fix link to documentation of nodes (langgenius#5623)
  feat: undo/redo for workflow editor (langgenius#3927)
  fix: populate app configs to system environment variables (langgenius#5590)
  fix: DuckDuckGo image search tool error (langgenius#5606)
  delete the deprecated method  (langgenius#5612)
  fix: context icon in chat (langgenius#5604)
  fix: Modify the incorrect configuration name for Google storage (langgenius#5595)
  feat: update issue template (langgenius#5592)
  chore: fix typo in config descriptions (langgenius#5585)
  fix: add support for FILE type in ToolParameterConverter (langgenius#5578)
  Fix/single run panel show parent scrollbar (langgenius#5574)
  refactor: extract cors configs into dify config and cleanup the config class (langgenius#5507)
  ...

# Conflicts:
#	.github/workflows/build-push.yml
#	api/Dockerfile
#	api/core/helper/code_executor/code_executor.py
#	api/requirements.txt

.devcontainer/post_start_command.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-cd api && pip install -r requirements.txt
+poetry install -C api

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -14,13 +14,15 @@ body:
           required: true
         - label: I confirm that I am using English to submit this report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)).
           required: true
+        - label: "请务必使用英文提交 Issue，否则会被关闭。谢谢！:）"
+          required: true
         - label: "Please do not modify this template :) and fill in all the required fields."
           required: true
 
   - type: input
     attributes:
       label: Dify version
-      placeholder: 0.3.21
+      placeholder: 0.6.11
       description: See about section in Dify console
     validations:
       required: true
@@ -40,7 +42,7 @@ body:
   - type: textarea
     attributes:
       label: Steps to reproduce
-      description: We highly suggest including screenshots and a bug report log.
+      description: We highly suggest including screenshots and a bug report log. Please use the right markdown syntax for code blocks.
       placeholder: Having detailed steps helps us reproduce the bug.
     validations:
       required: true

.github/ISSUE_TEMPLATE/document_issue.yml

@@ -12,6 +12,8 @@ body:
           required: true
         - label: I confirm that I am using English to submit report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)).
           required: true
+        - label: "请务必使用英文提交 Issue，否则会被关闭。谢谢！:）"
+          required: true
         - label: "Please do not modify this template :) and fill in all the required fields."
           required: true
   - type: textarea

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -12,35 +12,25 @@ body:
           required: true
         - label: I confirm that I am using English to submit this report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)).
           required: true
+        - label: "请务必使用英文提交 Issue，否则会被关闭。谢谢！:）"
+          required: true
         - label: "Please do not modify this template :) and fill in all the required fields."
           required: true
   - type: textarea
     attributes:
-      label: 1. Is this request related to a challenge you're experiencing?
+      label: 1. Is this request related to a challenge you're experiencing? Tell me about your story.
       placeholder: Please describe the specific scenario or problem you're facing as clearly as possible. For instance "I was trying to use [feature] for [specific task], and [what happened]... It was frustrating because...."
     validations:
       required: true
   - type: textarea
     attributes:
-      label: 2. Describe the feature you'd like to see
-      placeholder: Think about what you want to achieve and how this feature will help you. Sketches, flow diagrams, or any visual representation will be a major plus.
-    validations:
-      required: true
-  - type: textarea
-    attributes:
-      label: 3. How will this feature improve your workflow or experience?
-      placeholder: Tell us how this change will benefit your work. This helps us prioritize based on user impact.
-    validations:
-      required: true
-  - type: textarea
-    attributes:
-      label: 4. Additional context or comments
+      label: 2. Additional context or comments
       placeholder: (Any other information, comments, documentations, links, or screenshots that would provide more clarity. This is the place to add anything else not covered above.)
     validations:
       required: false
   - type: checkboxes
     attributes:
-      label: 5. Can you help us with this feature?
+      label: 3. Can you help us with this feature?
       description: Let us know! This is not a commitment, but a starting point for collaboration.
       options:
         - label: I am interested in contributing to this feature.

.github/ISSUE_TEMPLATE/translation_issue.yml

@@ -12,6 +12,8 @@ body:
           required: true
         - label: I confirm that I am using English to submit this report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)).
           required: true
+        - label: "请务必使用英文提交 Issue，否则会被关闭。谢谢！:）"
+          required: true
         - label: "Please do not modify this template :) and fill in all the required fields."
           required: true
   - type: input

.github/workflows/api-tests.yml

@@ -26,26 +26,34 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Install Poetry
+        uses: abatilo/actions-poetry@v3
+
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: 'pip'
+          cache: 'poetry'
           cache-dependency-path: |
-            ./api/requirements.txt
-            ./api/requirements-dev.txt
+            api/pyproject.toml
+            api/poetry.lock
+
+      - name: Poetry check
+        run: |
+          poetry check -C api --lock
+          poetry show -C api
 
       - name: Install dependencies
-        run: pip install -r ./api/requirements.txt -r ./api/requirements-dev.txt
+        run: poetry install -C api --with dev
 
       - name: Run Unit tests
-        run: dev/pytest/pytest_unit_tests.sh
+        run: poetry run -C api bash dev/pytest/pytest_unit_tests.sh
 
       - name: Run ModelRuntime
-        run: dev/pytest/pytest_model_runtime.sh
+        run: poetry run -C api bash dev/pytest/pytest_model_runtime.sh
 
       - name: Run Tool
-        run: dev/pytest/pytest_tools.sh
+        run: poetry run -C api bash dev/pytest/pytest_tools.sh
 
       - name: Set up Sandbox
         uses: hoverkraft-tech/[email protected]
@@ -57,9 +65,9 @@ jobs:
             ssrf_proxy
 
       - name: Run Workflow
-        run: dev/pytest/pytest_workflow.sh
+        run: poetry run -C api bash dev/pytest/pytest_workflow.sh
 
-      - name: Set up Vector Stores (Weaviate, Qdrant, PGVector, Milvus, PgVecto-RS)
+      - name: Set up Vector Stores (Weaviate, Qdrant, PGVector, Milvus, PgVecto-RS, Chroma)
         uses: hoverkraft-tech/[email protected]
         with:
           compose-file: |
@@ -68,6 +76,7 @@ jobs:
             docker/docker-compose.milvus.yaml
             docker/docker-compose.pgvecto-rs.yaml
             docker/docker-compose.pgvector.yaml
+            docker/docker-compose.chroma.yaml
           services: |
             weaviate
             qdrant
@@ -76,6 +85,7 @@ jobs:
             milvus-standalone
             pgvecto-rs
             pgvector
+            chroma
 
       - name: Test Vector Stores
-        run: dev/pytest/pytest_vdb.sh
+        run: poetry run -C api bash dev/pytest/pytest_vdb.sh

.github/workflows/build-push.yml

@@ -7,6 +7,10 @@ on:
   release:
     types: [published]
 
+concurrency:
+  group: build-push-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 env:
   DIFY_WEB_IMAGE_NAME: ${{ vars.DIFY_WEB_IMAGE_NAME }}
   DIFY_API_IMAGE_NAME: ${{ vars.DIFY_API_IMAGE_NAME }}
@@ -17,19 +21,35 @@ env:
   KUBECONFIG: ${{ secrets.KUBECONFIG }}
 
 jobs:
-  build-and-push:
-    runs-on: ubuntu-latest
+  build:
+    runs-on: ${{ matrix.platform == 'linux/arm64' && 'arm64_runner' || 'ubuntu-latest' }}
     if: github.event.pull_request.draft == false
     strategy:
       matrix:
         include:
-          - service_name: "web"
-            image_name_env: "DIFY_WEB_IMAGE_NAME"
-            context: "web"
-          - service_name: "api"
+          - service_name: "build-api-amd64"
+            image_name_env: "DIFY_API_IMAGE_NAME"
+            context: "api"
+            platform: linux/amd64
+          - service_name: "build-api-arm64"
             image_name_env: "DIFY_API_IMAGE_NAME"
             context: "api"
+            platform: linux/arm64
+          - service_name: "build-web-amd64"
+            image_name_env: "DIFY_WEB_IMAGE_NAME"
+            context: "web"
+            platform: linux/amd64
+          - service_name: "build-web-arm64"
+            image_name_env: "DIFY_WEB_IMAGE_NAME"
+            context: "web"
+            platform: linux/arm64
+
     steps:
+      - name: Prepare
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
+
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
 
@@ -51,7 +71,66 @@ jobs:
           username: ${{ env.username }}
           password: ${{ env.password }}
 
-      - name: Extract metadata (tags, labels) for Docker
+      - name: Extract metadata for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env[matrix.image_name_env] }}
+
+      - name: Build Docker image
+        id: build
+        uses: docker/build-push-action@v6
+        with:
+          context: "{{defaultContext}}:${{ matrix.context }}"
+          platforms: ${{ matrix.platform }}
+          build-args: COMMIT_SHA=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }}
+          labels: ${{ steps.meta.outputs.labels }}
+          outputs: type=image,name=${{ env[matrix.image_name_env] }},push-by-digest=true,name-canonical=true,push=true
+          cache-from: type=gha,scope=${{ matrix.service_name }}
+          cache-to: type=gha,mode=max,scope=${{ matrix.service_name }}
+
+      - name: Export digest
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+
+      - name: Upload digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-${{ matrix.context }}-${{ env.PLATFORM_PAIR }}
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  create-manifest:
+    needs: build
+    runs-on: ubuntu-latest
+    if: github.repository == 'langgenius/dify'
+    strategy:
+      matrix:
+        include:
+          - service_name: "merge-api-images"
+            image_name_env: "DIFY_API_IMAGE_NAME"
+            context: "api"
+          - service_name: "merge-web-images"
+            image_name_env: "DIFY_WEB_IMAGE_NAME"
+            context: "web"
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp/digests
+          pattern: digests-${{ matrix.context }}-*
+          merge-multiple: true
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ env.DOCKERHUB_USER }}
+          password: ${{ env.DOCKERHUB_TOKEN }}
+
+      - name: Extract metadata for Docker
         id: meta
         uses: docker/metadata-action@v5
         with:
@@ -90,3 +169,12 @@ jobs:
           sudo chmod 644 kubeconfig.yaml
           kubectl -n ${{ env.NAMESPACE }} set image deployment/dify-${{ matrix.service_name }} ${{ matrix.service_name }}=${{ env[matrix.image_name_env] }}:${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }}
           
+      - name: Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env[matrix.image_name_env] }}@sha256:%s ' *)
+
+      - name: Inspect image
+        run: |
+          docker buildx imagetools inspect ${{ env[matrix.image_name_env] }}:${{ steps.meta.outputs.version }}

.github/workflows/db-migration-test.yml

@@ -23,24 +23,29 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Install Poetry
+        uses: abatilo/actions-poetry@v3
+
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: 'pip'
+          cache: 'poetry'
           cache-dependency-path: |
-            ./api/requirements.txt
+            api/pyproject.toml
+            api/poetry.lock
 
       - name: Install dependencies
-        run: pip install -r ./api/requirements.txt
+        run: poetry install -C api
 
-      - name: Set up Middleware
+      - name: Set up Middlewares
         uses: hoverkraft-tech/[email protected]
         with:
           compose-file: |
             docker/docker-compose.middleware.yaml
           services: |
             db
+            redis
 
       - name: Prepare configs
         run: |
@@ -50,4 +55,4 @@ jobs:
       - name: Run DB Migration
         run: |
           cd api
-          flask db upgrade
+          poetry run python -m flask upgrade-db

.github/workflows/style.yml

@@ -24,6 +24,9 @@ jobs:
         with:
           files: api/**
 
+      - name: Install Poetry
+        uses: abatilo/actions-poetry@v3
+
       - name: Set up Python
         uses: actions/setup-python@v5
         if: steps.changed-files.outputs.any_changed == 'true'
@@ -32,15 +35,15 @@ jobs:
 
       - name: Python dependencies
         if: steps.changed-files.outputs.any_changed == 'true'
-        run: pip install ruff dotenv-linter
+        run: poetry install -C api --only lint
 
       - name: Ruff check
         if: steps.changed-files.outputs.any_changed == 'true'
-        run: ruff check --preview ./api
+        run: poetry run -C api ruff check ./api
 
       - name: Dotenv check
         if: steps.changed-files.outputs.any_changed == 'true'
-        run: dotenv-linter ./api/.env.example ./web/.env.example
+        run: poetry run -C api dotenv-linter ./api/.env.example ./web/.env.example
 
       - name: Lint hints
         if: failure()
@@ -96,7 +99,8 @@ jobs:
             **.sh
             **.yaml
             **.yml
-            Dockerfile
+            **Dockerfile
+            dev/**
 
       - name: Super-linter
         uses: super-linter/super-linter/slim@v6
@@ -109,7 +113,8 @@ jobs:
           IGNORE_GITIGNORED_FILES: true
           VALIDATE_BASH: true
           VALIDATE_BASH_EXEC: true
-          VALIDATE_GITHUB_ACTIONS: true
+          # FIXME: temporarily disabled until api-docker.yaml's run script is fixed for shellcheck
+          # VALIDATE_GITHUB_ACTIONS: true
           VALIDATE_DOCKERFILE_HADOLINT: true
           VALIDATE_XML: true
           VALIDATE_YAML: true