From 9b7fdadce499e782ebf5450bd1b055a4e43956c1 Mon Sep 17 00:00:00 2001 From: rerorero Date: Tue, 18 Jun 2024 14:08:40 +0900 Subject: [PATCH 1/3] fix: wrong token usage in iteration node for streaming result (#5336) --- api/core/app/entities/task_entities.py | 5 +++-- .../app/task_pipeline/workflow_iteration_cycle_manage.py | 8 +++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/api/core/app/entities/task_entities.py b/api/core/app/entities/task_entities.py index cb023ae9fe6759..af93399a24a1a6 100644 --- a/api/core/app/entities/task_entities.py +++ b/api/core/app/entities/task_entities.py @@ -367,7 +367,7 @@ class Data(BaseModel): class IterationNodeCompletedStreamResponse(StreamResponse): """ - NodeStartStreamResponse entity + NodeCompletedStreamResponse entity """ class Data(BaseModel): """ @@ -385,6 +385,7 @@ class Data(BaseModel): error: Optional[str] = None elapsed_time: float total_tokens: int + execution_metadata: Optional[dict] = None finished_at: int steps: int @@ -545,4 +546,4 @@ class Data(BaseModel): total_tokens: int = 0 node_data: BaseNodeData - current_iterations: dict[str, Data] = None \ No newline at end of file + current_iterations: dict[str, Data] = None diff --git a/api/core/app/task_pipeline/workflow_iteration_cycle_manage.py b/api/core/app/task_pipeline/workflow_iteration_cycle_manage.py index 19bb7788c9e3f5..69af81d02691f8 100644 --- a/api/core/app/task_pipeline/workflow_iteration_cycle_manage.py +++ b/api/core/app/task_pipeline/workflow_iteration_cycle_manage.py @@ -95,6 +95,9 @@ def _handle_iteration_to_stream_response(self, task_id: str, event: QueueIterati error=None, elapsed_time=time.perf_counter() - current_iteration.started_at, total_tokens=current_iteration.total_tokens, + execution_metadata={ + 'total_tokens': current_iteration.total_tokens, + }, finished_at=int(time.time()), steps=current_iteration.current_index ) @@ -276,7 +279,10 @@ def _handle_iteration_exception(self, task_id: str, error: str) -> Generator[Ite error=error, elapsed_time=time.perf_counter() - current_iteration.started_at, total_tokens=current_iteration.total_tokens, + execution_metadata={ + 'total_tokens': current_iteration.total_tokens, + }, finished_at=int(time.time()), steps=current_iteration.current_index ) - ) \ No newline at end of file + ) From 5f0ce5811aeedfbed0100e95f3a04a2ac1d078d8 Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Tue, 18 Jun 2024 13:26:01 +0800 Subject: [PATCH 2/3] feat: add `flask upgrade-db` command for running db upgrade with redis lock (#5333) --- .github/workflows/db-migration-test.yml | 5 +++-- api/commands.py | 26 ++++++++++++++++++++++++- api/docker/entrypoint.sh | 2 +- api/migrations/README | 1 - 4 files changed, 29 insertions(+), 5 deletions(-) diff --git a/.github/workflows/db-migration-test.yml b/.github/workflows/db-migration-test.yml index 2f059bea9d7aae..64e8eb291c466e 100644 --- a/.github/workflows/db-migration-test.yml +++ b/.github/workflows/db-migration-test.yml @@ -38,13 +38,14 @@ jobs: - name: Install dependencies run: poetry install -C api - - name: Set up Middleware + - name: Set up Middlewares uses: hoverkraft-tech/compose-action@v2.0.0 with: compose-file: | docker/docker-compose.middleware.yaml services: | db + redis - name: Prepare configs run: | @@ -54,4 +55,4 @@ jobs: - name: Run DB Migration run: | cd api - poetry run python -m flask db upgrade + poetry run python -m flask upgrade-db diff --git a/api/commands.py b/api/commands.py index 7a8d8000a0f443..f3e0769134f137 100644 --- a/api/commands.py +++ b/api/commands.py @@ -1,5 +1,6 @@ import base64 import json +import logging import secrets from typing import Optional @@ -12,6 +13,7 @@ from core.rag.datasource.vdb.vector_type import VectorType from core.rag.models.document import Document from extensions.ext_database import db +from extensions.ext_redis import redis_client from libs.helper import email as email_validate from libs.password import hash_password, password_pattern, valid_password from libs.rsa import generate_key_pair @@ -553,6 +555,28 @@ def create_tenant(email: str, language: Optional[str] = None): 'Account: {}\nPassword: {}'.format(email, new_password), fg='green')) +@click.command('upgrade-db', help='upgrade the database') +def upgrade_db(): + click.echo('Preparing database migration...') + lock = redis_client.lock(name='db_upgrade_lock', timeout=60) + if lock.acquire(blocking=False): + try: + click.echo(click.style('Start database migration.', fg='green')) + + # run db migration + import flask_migrate + flask_migrate.upgrade() + + click.echo(click.style('Database migration successful!', fg='green')) + + except Exception as e: + logging.exception(f'Database migration failed, error: {e}') + finally: + lock.release() + else: + click.echo('Database migration skipped') + + def register_commands(app): app.cli.add_command(reset_password) app.cli.add_command(reset_email) @@ -561,4 +585,4 @@ def register_commands(app): app.cli.add_command(convert_to_agent_apps) app.cli.add_command(add_qdrant_doc_id_index) app.cli.add_command(create_tenant) - + app.cli.add_command(upgrade_db) diff --git a/api/docker/entrypoint.sh b/api/docker/entrypoint.sh index 2dad52f8f6d2bf..7b1f1dfc03c679 100755 --- a/api/docker/entrypoint.sh +++ b/api/docker/entrypoint.sh @@ -4,7 +4,7 @@ set -e if [[ "${MIGRATION_ENABLED}" == "true" ]]; then echo "Running migrations" - flask db upgrade + flask upgrade-db fi if [[ "${MODE}" == "worker" ]]; then diff --git a/api/migrations/README b/api/migrations/README index 220678df7ab06e..0e048441597444 100644 --- a/api/migrations/README +++ b/api/migrations/README @@ -1,2 +1 @@ Single-database configuration for Flask. - From c7d378555a191adeea726bce5b2dcbae83123415 Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Tue, 18 Jun 2024 13:27:03 +0800 Subject: [PATCH 3/3] chore: set build system to Poetry and remove unnecessary settings with package mode disabled (#5263) --- api/pyproject.toml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/api/pyproject.toml b/api/pyproject.toml index 4f6f0651b382e8..158a8e85b70fff 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -1,6 +1,10 @@ [project] requires-python = ">=3.10" +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + [tool.ruff] exclude = [ ] @@ -78,17 +82,10 @@ MOCK_SWITCH = "true" CODE_MAX_STRING_LENGTH = "80000" CODE_EXECUTION_ENDPOINT="http://127.0.0.1:8194" CODE_EXECUTION_API_KEY="dify-sandbox" - FIRECRAWL_API_KEY = "fc-" - - [tool.poetry] name = "dify-api" -version = "0.6.11" -description = "" -authors = ["Dify "] -readme = "README.md" package-mode = false [tool.poetry.dependencies]