From 67bb428630f21c1f3b7b9f70357901cb9bea0edf Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Fri, 14 Jun 2024 11:44:31 +0800 Subject: [PATCH] nit (+1 squashed commit) Squashed commits: [69fe305c8] appconfigs --- api/app.py | 15 +- api/config.py | 215 +-------- api/configs/__init__.py | 0 api/configs/app_configs.py | 42 ++ api/configs/deploy/__init__.py | 16 + api/configs/enterprise/__init__.py | 18 + api/configs/extra/__init__.py | 12 + api/configs/extra/notion_configs.py | 33 ++ api/configs/extra/sentry_configs.py | 23 + api/configs/feature/__init__.py | 420 ++++++++++++++++++ api/configs/middleware/__init__.py | 43 ++ api/configs/middleware/redis_configs.py | 38 ++ api/configs/packaging/__init__.py | 17 + .../console/workspace/tool_providers.py | 2 +- api/extensions/ext_compress.py | 2 +- api/extensions/ext_redis.py | 12 +- api/migrations/README | 1 + api/poetry.lock | 32 +- api/pyproject.toml | 2 +- api/requirements.txt | 1 + .../unit_tests/settings/test_app_settings.py | 46 ++ 21 files changed, 765 insertions(+), 225 deletions(-) create mode 100644 api/configs/__init__.py create mode 100644 api/configs/app_configs.py create mode 100644 api/configs/deploy/__init__.py create mode 100644 api/configs/enterprise/__init__.py create mode 100644 api/configs/extra/__init__.py create mode 100644 api/configs/extra/notion_configs.py create mode 100644 api/configs/extra/sentry_configs.py create mode 100644 api/configs/feature/__init__.py create mode 100644 api/configs/middleware/__init__.py create mode 100644 api/configs/middleware/redis_configs.py create mode 100644 api/configs/packaging/__init__.py create mode 100644 api/tests/unit_tests/settings/test_app_settings.py diff --git a/api/app.py b/api/app.py index a36750410e156d..8957265219beb6 100644 --- a/api/app.py +++ b/api/app.py @@ -1,5 +1,7 @@ import os +from configs.app_configs import DifyConfigs + if not os.environ.get("DEBUG") or os.environ.get("DEBUG").lower() != 'true': from gevent import monkey @@ -74,10 +76,19 @@ class DifyApp(Flask): # Application Factory Function # ---------------------------- +def create_flask_app_with_configs() -> Flask: + """ + create a raw flask app + with configs loaded from .env file + """ + dify_app = DifyApp(__name__) + dify_app.config.from_object(Config()) + dify_app.config.from_mapping(DifyConfigs().dict()) + return dify_app + def create_app() -> Flask: - app = DifyApp(__name__) - app.config.from_object(Config()) + app = create_flask_app_with_configs() app.secret_key = app.config['SECRET_KEY'] diff --git a/api/config.py b/api/config.py index 3c62501f2eaeaf..7efe9c5bc1e70e 100644 --- a/api/config.py +++ b/api/config.py @@ -5,49 +5,26 @@ dotenv.load_dotenv() DEFAULTS = { - 'EDITION': 'SELF_HOSTED', 'DB_USERNAME': 'postgres', 'DB_PASSWORD': '', 'DB_HOST': 'localhost', 'DB_PORT': '5432', 'DB_DATABASE': 'dify', 'DB_CHARSET': '', - 'REDIS_HOST': 'localhost', - 'REDIS_PORT': '6379', - 'REDIS_DB': '0', - 'REDIS_USE_SSL': 'False', - 'OAUTH_REDIRECT_PATH': '/console/api/oauth/authorize', - 'OAUTH_REDIRECT_INDEX_PATH': '/', - 'CONSOLE_WEB_URL': 'https://cloud.dify.ai', - 'CONSOLE_API_URL': 'https://cloud.dify.ai', - 'SERVICE_API_URL': 'https://api.dify.ai', - 'APP_WEB_URL': 'https://udify.app', - 'FILES_URL': '', - 'FILES_ACCESS_TIMEOUT': 300, 'S3_USE_AWS_MANAGED_IAM': 'False', 'S3_ADDRESS_STYLE': 'auto', - 'STORAGE_TYPE': 'local', - 'STORAGE_LOCAL_PATH': 'storage', - 'CHECK_UPDATE_URL': 'https://updates.dify.ai', - 'DEPLOY_ENV': 'PRODUCTION', 'SQLALCHEMY_DATABASE_URI_SCHEME': 'postgresql', 'SQLALCHEMY_POOL_SIZE': 30, 'SQLALCHEMY_MAX_OVERFLOW': 10, 'SQLALCHEMY_POOL_RECYCLE': 3600, 'SQLALCHEMY_POOL_PRE_PING': 'False', 'SQLALCHEMY_ECHO': 'False', - 'SENTRY_TRACES_SAMPLE_RATE': 1.0, - 'SENTRY_PROFILES_SAMPLE_RATE': 1.0, 'WEAVIATE_GRPC_ENABLED': 'True', 'WEAVIATE_BATCH_SIZE': 100, 'QDRANT_CLIENT_TIMEOUT': 20, 'QDRANT_GRPC_ENABLED': 'False', 'QDRANT_GRPC_PORT': '6334', 'CELERY_BACKEND': 'database', - 'LOG_LEVEL': 'INFO', - 'LOG_FILE': '', - 'LOG_FORMAT': '%(asctime)s.%(msecs)03d %(levelname)s [%(threadName)s] [%(filename)s:%(lineno)d] - %(message)s', - 'LOG_DATEFORMAT': '%Y-%m-%d %H:%M:%S', 'HOSTED_OPENAI_QUOTA_LIMIT': 200, 'HOSTED_OPENAI_TRIAL_ENABLED': 'False', 'HOSTED_OPENAI_TRIAL_MODELS': 'gpt-3.5-turbo,gpt-3.5-turbo-1106,gpt-3.5-turbo-instruct,gpt-3.5-turbo-16k,gpt-3.5-turbo-16k-0613,gpt-3.5-turbo-0613,gpt-3.5-turbo-0125,text-davinci-003', @@ -62,31 +39,7 @@ 'HOSTED_MODERATION_PROVIDERS': '', 'HOSTED_FETCH_APP_TEMPLATES_MODE': 'remote', 'HOSTED_FETCH_APP_TEMPLATES_REMOTE_DOMAIN': 'https://tmpl.dify.ai', - 'CLEAN_DAY_SETTING': 30, - 'UPLOAD_FILE_SIZE_LIMIT': 15, - 'UPLOAD_FILE_BATCH_LIMIT': 5, - 'UPLOAD_IMAGE_FILE_SIZE_LIMIT': 10, - 'OUTPUT_MODERATION_BUFFER_SIZE': 300, - 'MULTIMODAL_SEND_IMAGE_FORMAT': 'base64', - 'INVITE_EXPIRY_HOURS': 72, - 'BILLING_ENABLED': 'False', - 'CAN_REPLACE_LOGO': 'False', - 'MODEL_LB_ENABLED': 'False', - 'ETL_TYPE': 'dify', - 'KEYWORD_STORE': 'jieba', - 'BATCH_UPLOAD_LIMIT': 20, - 'CODE_EXECUTION_ENDPOINT': 'http://sandbox:8194', - 'CODE_EXECUTION_API_KEY': 'dify-sandbox', - 'TOOL_ICON_CACHE_MAX_AGE': 3600, 'MILVUS_DATABASE': 'default', - 'KEYWORD_DATA_SOURCE_TYPE': 'database', - 'INNER_API': 'False', - 'ENTERPRISE_ENABLED': 'False', - 'INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH': 1000, - 'WORKFLOW_MAX_EXECUTION_STEPS': 500, - 'WORKFLOW_MAX_EXECUTION_TIME': 1200, - 'WORKFLOW_CALL_MAX_DEPTH': 5, - 'APP_MAX_EXECUTION_TIME': 1200, } @@ -114,65 +67,14 @@ class Config: """Application configuration class.""" def __init__(self): - # ------------------------ - # General Configurations. - # ------------------------ - self.CURRENT_VERSION = "0.6.11" - self.COMMIT_SHA = get_env('COMMIT_SHA') - self.EDITION = get_env('EDITION') - self.DEPLOY_ENV = get_env('DEPLOY_ENV') self.TESTING = False - self.LOG_LEVEL = get_env('LOG_LEVEL') - self.LOG_FILE = get_env('LOG_FILE') - self.LOG_FORMAT = get_env('LOG_FORMAT') - self.LOG_DATEFORMAT = get_env('LOG_DATEFORMAT') - self.API_COMPRESSION_ENABLED = get_bool_env('API_COMPRESSION_ENABLED') - - # The backend URL prefix of the console API. - # used to concatenate the login authorization callback or notion integration callback. - self.CONSOLE_API_URL = get_env('CONSOLE_API_URL') - - # The front-end URL prefix of the console web. - # used to concatenate some front-end addresses and for CORS configuration use. - self.CONSOLE_WEB_URL = get_env('CONSOLE_WEB_URL') - - # WebApp Url prefix. - # used to display WebAPP API Base Url to the front-end. - self.APP_WEB_URL = get_env('APP_WEB_URL') - - # Service API Url prefix. - # used to display Service API Base Url to the front-end. - self.SERVICE_API_URL = get_env('SERVICE_API_URL') - - # File preview or download Url prefix. - # used to display File preview or download Url to the front-end or as Multi-model inputs; - # Url is signed and has expiration time. - self.FILES_URL = get_env('FILES_URL') if get_env('FILES_URL') else self.CONSOLE_API_URL - - # File Access Time specifies a time interval in seconds for the file to be accessed. - # The default value is 300 seconds. - self.FILES_ACCESS_TIMEOUT = int(get_env('FILES_ACCESS_TIMEOUT')) - - # Your App secret key will be used for securely signing the session cookie - # Make sure you are changing this key for your deployment with a strong key. - # You can generate a strong key using `openssl rand -base64 42`. - # Alternatively you can set it with `SECRET_KEY` environment variable. - self.SECRET_KEY = get_env('SECRET_KEY') - - # Enable or disable the inner API. - self.INNER_API = get_bool_env('INNER_API') - # The inner API key is used to authenticate the inner API. - self.INNER_API_KEY = get_env('INNER_API_KEY') # cors settings self.CONSOLE_CORS_ALLOW_ORIGINS = get_cors_allow_origins( - 'CONSOLE_CORS_ALLOW_ORIGINS', self.CONSOLE_WEB_URL) + 'CONSOLE_CORS_ALLOW_ORIGINS', get_env('CONSOLE_WEB_URL')) self.WEB_API_CORS_ALLOW_ORIGINS = get_cors_allow_origins( 'WEB_API_CORS_ALLOW_ORIGINS', '*') - # check update url - self.CHECK_UPDATE_URL = get_env('CHECK_UPDATE_URL') - # ------------------------ # Database Configurations. # ------------------------ @@ -195,16 +97,6 @@ def __init__(self): self.SQLALCHEMY_ECHO = get_bool_env('SQLALCHEMY_ECHO') - # ------------------------ - # Redis Configurations. - # ------------------------ - self.REDIS_HOST = get_env('REDIS_HOST') - self.REDIS_PORT = get_env('REDIS_PORT') - self.REDIS_USERNAME = get_env('REDIS_USERNAME') - self.REDIS_PASSWORD = get_env('REDIS_PASSWORD') - self.REDIS_DB = get_env('REDIS_DB') - self.REDIS_USE_SSL = get_bool_env('REDIS_USE_SSL') - # ------------------------ # Celery worker Configurations. # ------------------------ @@ -212,19 +104,8 @@ def __init__(self): self.CELERY_BACKEND = get_env('CELERY_BACKEND') self.CELERY_RESULT_BACKEND = 'db+{}'.format(self.SQLALCHEMY_DATABASE_URI) \ if self.CELERY_BACKEND == 'database' else self.CELERY_BROKER_URL - self.BROKER_USE_SSL = self.CELERY_BROKER_URL.startswith('rediss://') + self.BROKER_USE_SSL = self.CELERY_BROKER_URL.startswith('rediss://') if self.CELERY_BROKER_URL else False - # ------------------------ - # Code Execution Sandbox Configurations. - # ------------------------ - self.CODE_EXECUTION_ENDPOINT = get_env('CODE_EXECUTION_ENDPOINT') - self.CODE_EXECUTION_API_KEY = get_env('CODE_EXECUTION_API_KEY') - - # ------------------------ - # File Storage Configurations. - # ------------------------ - self.STORAGE_TYPE = get_env('STORAGE_TYPE') - self.STORAGE_LOCAL_PATH = get_env('STORAGE_LOCAL_PATH') # S3 Storage settings self.S3_USE_AWS_MANAGED_IAM = get_bool_env('S3_USE_AWS_MANAGED_IAM') @@ -264,8 +145,6 @@ def __init__(self): # Vector Store Configurations. # Currently, only support: qdrant, milvus, zilliz, weaviate, relyt, pgvector # ------------------------ - self.VECTOR_STORE = get_env('VECTOR_STORE') - self.KEYWORD_STORE = get_env('KEYWORD_STORE') # qdrant settings self.QDRANT_URL = get_env('QDRANT_URL') @@ -295,7 +174,6 @@ def __init__(self): self.RELYT_PASSWORD = get_env('RELYT_PASSWORD') self.RELYT_DATABASE = get_env('RELYT_DATABASE') - # tencent settings self.TENCENT_VECTOR_DB_URL = get_env('TENCENT_VECTOR_DB_URL') self.TENCENT_VECTOR_DB_API_KEY = get_env('TENCENT_VECTOR_DB_API_KEY') @@ -334,85 +212,9 @@ def __init__(self): self.CHROMA_AUTH_PROVIDER = get_env('CHROMA_AUTH_PROVIDER') self.CHROMA_AUTH_CREDENTIALS = get_env('CHROMA_AUTH_CREDENTIALS') - # ------------------------ - # Mail Configurations. - # ------------------------ - self.MAIL_TYPE = get_env('MAIL_TYPE') - self.MAIL_DEFAULT_SEND_FROM = get_env('MAIL_DEFAULT_SEND_FROM') - self.RESEND_API_KEY = get_env('RESEND_API_KEY') - self.RESEND_API_URL = get_env('RESEND_API_URL') - # SMTP settings - self.SMTP_SERVER = get_env('SMTP_SERVER') - self.SMTP_PORT = get_env('SMTP_PORT') - self.SMTP_USERNAME = get_env('SMTP_USERNAME') - self.SMTP_PASSWORD = get_env('SMTP_PASSWORD') - self.SMTP_USE_TLS = get_bool_env('SMTP_USE_TLS') - self.SMTP_OPPORTUNISTIC_TLS = get_bool_env('SMTP_OPPORTUNISTIC_TLS') - - # ------------------------ - # Workspace Configurations. - # ------------------------ - self.INVITE_EXPIRY_HOURS = int(get_env('INVITE_EXPIRY_HOURS')) - - # ------------------------ - # Sentry Configurations. - # ------------------------ - self.SENTRY_DSN = get_env('SENTRY_DSN') - self.SENTRY_TRACES_SAMPLE_RATE = float(get_env('SENTRY_TRACES_SAMPLE_RATE')) - self.SENTRY_PROFILES_SAMPLE_RATE = float(get_env('SENTRY_PROFILES_SAMPLE_RATE')) - - # ------------------------ - # Business Configurations. - # ------------------------ - - # multi model send image format, support base64, url, default is base64 - self.MULTIMODAL_SEND_IMAGE_FORMAT = get_env('MULTIMODAL_SEND_IMAGE_FORMAT') - - # Dataset Configurations. - self.CLEAN_DAY_SETTING = get_env('CLEAN_DAY_SETTING') - - # File upload Configurations. - self.UPLOAD_FILE_SIZE_LIMIT = int(get_env('UPLOAD_FILE_SIZE_LIMIT')) - self.UPLOAD_FILE_BATCH_LIMIT = int(get_env('UPLOAD_FILE_BATCH_LIMIT')) - self.UPLOAD_IMAGE_FILE_SIZE_LIMIT = int(get_env('UPLOAD_IMAGE_FILE_SIZE_LIMIT')) - self.BATCH_UPLOAD_LIMIT = get_env('BATCH_UPLOAD_LIMIT') - - # RAG ETL Configurations. - self.ETL_TYPE = get_env('ETL_TYPE') - self.UNSTRUCTURED_API_URL = get_env('UNSTRUCTURED_API_URL') - self.UNSTRUCTURED_API_KEY = get_env('UNSTRUCTURED_API_KEY') - self.KEYWORD_DATA_SOURCE_TYPE = get_env('KEYWORD_DATA_SOURCE_TYPE') - - # Indexing Configurations. - self.INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH = get_env('INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH') - - # Tool Configurations. - self.TOOL_ICON_CACHE_MAX_AGE = get_env('TOOL_ICON_CACHE_MAX_AGE') - - self.WORKFLOW_MAX_EXECUTION_STEPS = int(get_env('WORKFLOW_MAX_EXECUTION_STEPS')) - self.WORKFLOW_MAX_EXECUTION_TIME = int(get_env('WORKFLOW_MAX_EXECUTION_TIME')) - self.WORKFLOW_CALL_MAX_DEPTH = int(get_env('WORKFLOW_CALL_MAX_DEPTH')) - self.APP_MAX_EXECUTION_TIME = int(get_env('APP_MAX_EXECUTION_TIME')) - - # Moderation in app Configurations. - self.OUTPUT_MODERATION_BUFFER_SIZE = int(get_env('OUTPUT_MODERATION_BUFFER_SIZE')) - - # Notion integration setting - self.NOTION_CLIENT_ID = get_env('NOTION_CLIENT_ID') - self.NOTION_CLIENT_SECRET = get_env('NOTION_CLIENT_SECRET') - self.NOTION_INTEGRATION_TYPE = get_env('NOTION_INTEGRATION_TYPE') - self.NOTION_INTERNAL_SECRET = get_env('NOTION_INTERNAL_SECRET') - self.NOTION_INTEGRATION_TOKEN = get_env('NOTION_INTEGRATION_TOKEN') - # ------------------------ # Platform Configurations. # ------------------------ - self.GITHUB_CLIENT_ID = get_env('GITHUB_CLIENT_ID') - self.GITHUB_CLIENT_SECRET = get_env('GITHUB_CLIENT_SECRET') - self.GOOGLE_CLIENT_ID = get_env('GOOGLE_CLIENT_ID') - self.GOOGLE_CLIENT_SECRET = get_env('GOOGLE_CLIENT_SECRET') - self.OAUTH_REDIRECT_PATH = get_env('OAUTH_REDIRECT_PATH') - self.HOSTED_OPENAI_API_KEY = get_env('HOSTED_OPENAI_API_KEY') self.HOSTED_OPENAI_API_BASE = get_env('HOSTED_OPENAI_API_BASE') self.HOSTED_OPENAI_API_ORGANIZATION = get_env('HOSTED_OPENAI_API_ORGANIZATION') @@ -443,16 +245,3 @@ def __init__(self): # fetch app templates mode, remote, builtin, db(only for dify SaaS), default: remote self.HOSTED_FETCH_APP_TEMPLATES_MODE = get_env('HOSTED_FETCH_APP_TEMPLATES_MODE') self.HOSTED_FETCH_APP_TEMPLATES_REMOTE_DOMAIN = get_env('HOSTED_FETCH_APP_TEMPLATES_REMOTE_DOMAIN') - - # Model Load Balancing Configurations. - self.MODEL_LB_ENABLED = get_bool_env('MODEL_LB_ENABLED') - - # Platform Billing Configurations. - self.BILLING_ENABLED = get_bool_env('BILLING_ENABLED') - - # ------------------------ - # Enterprise feature Configurations. - # **Before using, please contact business@dify.ai by email to inquire about licensing matters.** - # ------------------------ - self.ENTERPRISE_ENABLED = get_bool_env('ENTERPRISE_ENABLED') - self.CAN_REPLACE_LOGO = get_bool_env('CAN_REPLACE_LOGO') diff --git a/api/configs/__init__.py b/api/configs/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/api/configs/app_configs.py b/api/configs/app_configs.py new file mode 100644 index 00000000000000..96352e43d0e2ab --- /dev/null +++ b/api/configs/app_configs.py @@ -0,0 +1,42 @@ +from pydantic_settings import BaseSettings, SettingsConfigDict + +from configs.deploy import DeploymentConfigs +from configs.enterprise import EnterpriseFeatureConfigs +from configs.extra import ExtraServiceConfigs +from configs.feature import FeatureConfigs +from configs.middleware import MiddlewareConfigs +from configs.packaging import PackagingInfo + + +class DifyConfigs( + # based on pydantic-settings + BaseSettings, + + # Packaging info + PackagingInfo, + + # Deployment configs + DeploymentConfigs, + + # Feature configs + FeatureConfigs, + + # Middleware configs + MiddlewareConfigs, + + # Extra service configs + ExtraServiceConfigs, + + # Enterprise feature configs + # **Before using, please contact business@dify.ai by email to inquire about licensing matters.** + EnterpriseFeatureConfigs, +): + + model_config = SettingsConfigDict( + # read from dotenv format config file + env_file='.env', + env_file_encoding='utf-8', + + # ignore extra attributes + extra='ignore', + ) diff --git a/api/configs/deploy/__init__.py b/api/configs/deploy/__init__.py new file mode 100644 index 00000000000000..9a1c1b84d7cfb7 --- /dev/null +++ b/api/configs/deploy/__init__.py @@ -0,0 +1,16 @@ +from pydantic import BaseModel, Field + + +class DeploymentConfigs(BaseModel): + """ + Deployment configs + """ + EDITION: str = Field( + description='deployment edition', + default='SELF_HOSTED', + ) + + DEPLOY_ENV: str = Field( + description='deployment environment, default to PRODUCTION.', + default='PRODUCTION', + ) diff --git a/api/configs/enterprise/__init__.py b/api/configs/enterprise/__init__.py new file mode 100644 index 00000000000000..37164ba2e3177f --- /dev/null +++ b/api/configs/enterprise/__init__.py @@ -0,0 +1,18 @@ +from pydantic import BaseModel, Field + + +class EnterpriseFeatureConfigs(BaseModel): + """ + Enterprise feature configs. + **Before using, please contact business@dify.ai by email to inquire about licensing matters.** + """ + ENTERPRISE_ENABLED: bool = Field( + description='whether to enable enterprise features.' + 'Before using, please contact business@dify.ai by email to inquire about licensing matters.', + default=False, + ) + + CAN_REPLACE_LOGO: bool = Field( + description='whether to allow replacing enterprise logo.', + default=False, + ) diff --git a/api/configs/extra/__init__.py b/api/configs/extra/__init__.py new file mode 100644 index 00000000000000..f248fcb47a4961 --- /dev/null +++ b/api/configs/extra/__init__.py @@ -0,0 +1,12 @@ +from pydantic import BaseModel + +from configs.extra.notion_configs import NotionConfigs +from configs.extra.sentry_configs import SentryConfigs + + +class ExtraServiceConfigs( + # place the configs in alphabet order + NotionConfigs, + SentryConfigs, +): + pass diff --git a/api/configs/extra/notion_configs.py b/api/configs/extra/notion_configs.py new file mode 100644 index 00000000000000..da96b3c8c57296 --- /dev/null +++ b/api/configs/extra/notion_configs.py @@ -0,0 +1,33 @@ +from typing import Optional + +from pydantic import BaseModel, Field + + +class NotionConfigs(BaseModel): + """ + Notion integration configs + """ + NOTION_CLIENT_ID: Optional[str] = Field( + description='Notion client ID', + default=None, + ) + + NOTION_CLIENT_SECRET: Optional[str] = Field( + description='Notion client secret key', + default=None, + ) + + NOTION_INTEGRATION_TYPE: Optional[str] = Field( + description='Notion integration type, default to None, available values: internal.', + default=None, + ) + + NOTION_INTERNAL_SECRET: Optional[str] = Field( + description='Notion internal secret key', + default=None, + ) + + NOTION_INTEGRATION_TOKEN: Optional[str] = Field( + description='Notion integration token', + default=None, + ) diff --git a/api/configs/extra/sentry_configs.py b/api/configs/extra/sentry_configs.py new file mode 100644 index 00000000000000..7f19b347d4afaf --- /dev/null +++ b/api/configs/extra/sentry_configs.py @@ -0,0 +1,23 @@ +from typing import Optional + +from pydantic import BaseModel, Field, PositiveFloat + + +class SentryConfigs(BaseModel): + """ + Sentry configs + """ + SENTRY_DSN: Optional[str] = Field( + description='Sentry DSN', + default=None, + ) + + SENTRY_TRACES_SAMPLE_RATE: PositiveFloat = Field( + description='Sentry trace sample rate', + default=1.0, + ) + + SENTRY_PROFILES_SAMPLE_RATE: PositiveFloat = Field( + description='Sentry profiles sample rate', + default=1.0, + ) diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py new file mode 100644 index 00000000000000..f880ab6f502bd4 --- /dev/null +++ b/api/configs/feature/__init__.py @@ -0,0 +1,420 @@ +from typing import Optional + +from pydantic import AliasChoices, BaseModel, Field, NonNegativeInt, PositiveInt + + +class SecurityConfigs(BaseModel): + """ + Secret Key configs + """ + SECRET_KEY: Optional[str] = Field( + description='Your App secret key will be used for securely signing the session cookie' + 'Make sure you are changing this key for your deployment with a strong key.' + 'You can generate a strong key using `openssl rand -base64 42`.' + 'Alternatively you can set it with `SECRET_KEY` environment variable.', + default=None, + ) + + +class AppExecutionConfigs(BaseModel): + """ + App Execution configs + """ + APP_MAX_EXECUTION_TIME: PositiveInt = Field( + description='execution timeout in seconds for app execution', + default=1200, + ) + + +class CodeExecutionSandboxConfigs(BaseModel): + """ + Code Execution Sandbox configs + """ + CODE_EXECUTION_ENDPOINT: str = Field( + description='whether to enable HTTP response compression of gzip', + default='http://sandbox:8194', + ) + + CODE_EXECUTION_API_KEY: str = Field( + description='API key for code execution service', + default='dify-sandbox', + ) + + +class EndpointConfigs(BaseModel): + """ + Module URL configs + """ + CONSOLE_API_URL: str = Field( + description='The backend URL prefix of the console API.' + 'used to concatenate the login authorization callback or notion integration callback.', + default='https://cloud.dify.ai', + ) + + CONSOLE_WEB_URL: str = Field( + description='The front-end URL prefix of the console web.' + 'used to concatenate some front-end addresses and for CORS configuration use.', + default='https://cloud.dify.ai', + ) + + SERVICE_API_URL: str = Field( + description='Service API Url prefix.' + 'used to display Service API Base Url to the front-end.', + default='https://api.dify.ai', + ) + + APP_WEB_URL: str = Field( + description='WebApp Url prefix.' + 'used to display WebAPP API Base Url to the front-end.', + default='https://udify.app', + ) + + +class FileAccessConfigs(BaseModel): + """ + File Access configs + """ + FILES_URL: str = Field( + description='File preview or download Url prefix.' + ' used to display File preview or download Url to the front-end or as Multi-model inputs;' + 'Url is signed and has expiration time.', + validation_alias=AliasChoices('FILES_URL', 'CONSOLE_API_URL'), + alias_priority=1, + default='https://cloud.dify.ai', + ) + + FILES_ACCESS_TIMEOUT: int = Field( + description='timeout in seconds for file accessing', + default=300, + ) + + +class FileUploadConfigs(BaseModel): + """ + File Uploading configs + """ + UPLOAD_FILE_SIZE_LIMIT: NonNegativeInt = Field( + description='size limit in Megabytes for uploading files', + default=15, + ) + + UPLOAD_FILE_BATCH_LIMIT: NonNegativeInt = Field( + description='batch size limit for uploading files', + default=5, + ) + + UPLOAD_IMAGE_FILE_SIZE_LIMIT: NonNegativeInt = Field( + description='image file size limit in Megabytes for uploading files', + default=10, + ) + + BATCH_UPLOAD_LIMIT: NonNegativeInt = Field( + description='', # todo: to be clarified + default=20, + ) + + +class HttpConfigs(BaseModel): + """ + HTTP configs + """ + API_COMPRESSION_ENABLED: bool = Field( + description='whether to enable HTTP response compression of gzip', + default=False, + ) + + +class InnerAPIConfigs(BaseModel): + """ + Inner API configs + """ + INNER_API: bool = Field( + description='whether to enable the inner API', + default=False, + ) + + INNER_API_KEY: Optional[str] = Field( + description='The inner API key is used to authenticate the inner API', + default=None, + ) + + +class LoggingConfigs(BaseModel): + """ + Logging configs + """ + + LOG_LEVEL: str = Field( + description='Log output level, default to INFO.' + 'It is recommended to set it to ERROR for production.', + default='INFO', + ) + + LOG_FILE: Optional[str] = Field( + description='logging output file path', + default=None, + ) + + LOG_FORMAT: str = Field( + description='log format', + default='%(asctime)s.%(msecs)03d %(levelname)s [%(threadName)s] [%(filename)s:%(lineno)d] - %(message)s', + ) + + LOG_DATEFORMAT: Optional[str] = Field( + description='log date format', + default=None, + ) + + +class ModelLoadBalanceConfigs(BaseModel): + """ + Model load balance configs + """ + MODEL_LB_ENABLED: bool = Field( + description='whether to enable model load balancing', + default=False, + ) + + +class BillingConfigs(BaseModel): + """ + Platform Billing Configurations + """ + BILLING_ENABLED: bool = Field( + description='whether to enable billing', + default=False, + ) + + +class UpdateConfigs(BaseModel): + """ + Update configs + """ + CHECK_UPDATE_URL: str = Field( + description='url for checking updates', + default='https://updates.dify.ai', + ) + + +class WorkflowConfigs(BaseModel): + """ + Workflow feature configs + """ + + WORKFLOW_MAX_EXECUTION_STEPS: PositiveInt = Field( + description='max execution steps in single workflow execution', + default=500, + ) + + WORKFLOW_MAX_EXECUTION_TIME: PositiveInt = Field( + description='max execution time in seconds in single workflow execution', + default=1200, + ) + + WORKFLOW_CALL_MAX_DEPTH: PositiveInt = Field( + description='max depth of calling in single workflow execution', + default=5, + ) + + +class OAuthConfigs(BaseModel): + """ + oauth configs + """ + OAUTH_REDIRECT_PATH: str = Field( + description='redirect path for OAuth', + default='/console/api/oauth/authorize', + ) + + GITHUB_CLIENT_ID: Optional[str] = Field( + description='GitHub client id for OAuth', + default=None, + ) + + GITHUB_CLIENT_SECRET: Optional[str] = Field( + description='GitHub client secret key for OAuth', + default=None, + ) + + GOOGLE_CLIENT_ID: Optional[str] = Field( + description='Google client id for OAuth', + default=None, + ) + + GOOGLE_CLIENT_SECRET: Optional[str] = Field( + description='Google client secret key for OAuth', + default=None, + ) + + +class ModerationConfigs(BaseModel): + """ + Moderation in app configs. + """ + + # todo: to be clarified in usage and unit + OUTPUT_MODERATION_BUFFER_SIZE: PositiveInt = Field( + description='buffer size for moderation', + default=300, + ) + + +class ToolConfigs(BaseModel): + """ + Tool configs + """ + + TOOL_ICON_CACHE_MAX_AGE: PositiveInt = Field( + description='max age in seconds for tool icon caching', + default=3600, + ) + + +class MailConfigs(BaseModel): + """ + Mail Configurations + """ + + MAIL_TYPE: Optional[str] = Field( + description='Mail provider type name, default to None, availabile values are `smtp` and `resend`.', + default=None, + ) + + MAIL_DEFAULT_SEND_FROM: Optional[str] = Field( + description='default email address for sending from ', + default=None, + ) + + RESEND_API_KEY: Optional[str] = Field( + description='API key for Resend', + default=None, + ) + + RESEND_API_URL: Optional[str] = Field( + description='API URL for Resend', + default=None, + ) + + SMTP_SERVER: Optional[str] = Field( + description='smtp server host', + default=None, + ) + + SMTP_PORT: Optional[int] = Field( + description='smtp server port', + default=None, + ) + + SMTP_USERNAME: Optional[str] = Field( + description='smtp server username', + default=None, + ) + + SMTP_PASSWORD: Optional[str] = Field( + description='smtp server password', + default=None, + ) + + SMTP_USE_TLS: bool = Field( + description='whether to use TLS connection to smtp server', + default=False, + ) + + SMTP_OPPORTUNISTIC_TLS: bool = Field( + description='whether to use opportunistic TLS connection to smtp server', + default=False, + ) + + +class RagEtlConfigs(BaseModel): + """ + RAG ETL Configurations. + """ + + ETL_TYPE: str = Field( + description='RAG ETL type name, default to `dify`, available values are `dify` and `Unstructured`. ', + default='dify', + ) + + KEYWORD_DATA_SOURCE_TYPE: str = Field( + description='source type for keyword data, default to `database`, available values are `database` .', + default='database', + ) + + UNSTRUCTURED_API_URL: Optional[str] = Field( + description='API URL for Unstructured', + default=None, + ) + + UNSTRUCTURED_API_KEY: Optional[str] = Field( + description='API key for Unstructured', + default=None, + ) + + +class DataSetConfigs(BaseModel): + """ + Dataset configs + """ + + CLEAN_DAY_SETTING: PositiveInt = Field( + description='interval in days for cleaning up dataset', + default=30, + ) + + +class WorkspaceConfigs(BaseModel): + """ + Workspace configs + """ + + INVITE_EXPIRY_HOURS: PositiveInt = Field( + description='workspaces invitation expiration in hours', + default=72, + ) + + +class IndexingConfigs(BaseModel): + """ + Indexing configs. + """ + + INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: PositiveInt = Field( + description='max segmentation token length for indexing', + default=1000, + ) + + +class ImageFormatConfigs(BaseModel): + MULTIMODAL_SEND_IMAGE_FORMAT: str = Field( + description='multi model send image format, support base64, url, default is base64', + default='base64', + ) + + +class FeatureConfigs( + # place the configs in alphabet order + AppExecutionConfigs, + BillingConfigs, + CodeExecutionSandboxConfigs, + DataSetConfigs, + EndpointConfigs, + FileAccessConfigs, + FileUploadConfigs, + HttpConfigs, + ImageFormatConfigs, + InnerAPIConfigs, + IndexingConfigs, + LoggingConfigs, + MailConfigs, + ModelLoadBalanceConfigs, + ModerationConfigs, + OAuthConfigs, + RagEtlConfigs, + SecurityConfigs, + ToolConfigs, + UpdateConfigs, + WorkflowConfigs, + WorkspaceConfigs, +): + pass diff --git a/api/configs/middleware/__init__.py b/api/configs/middleware/__init__.py new file mode 100644 index 00000000000000..23fc620b78dbdb --- /dev/null +++ b/api/configs/middleware/__init__.py @@ -0,0 +1,43 @@ +from typing import Optional + +from pydantic import BaseModel, Field + +from configs.middleware.redis_configs import RedisConfigs + + +class StorageConfigs(BaseModel): + STORAGE_TYPE: str = Field( + description='storage type,' + ' default to `local`,' + ' available values are `local`, `s3`, `azure-blob`, `aliyun-oss`, `google-storage`.', + default='local', + ) + + STORAGE_LOCAL_PATH: str = Field( + description='local storage path', + default='storage', + ) + + +class VectorStoreConfigs(BaseModel): + VECTOR_STORE: Optional[str] = Field( + description='vector store type', + default=None, + ) + + +class KeywordStoreConfigs(BaseModel): + KEYWORD_STORE: str = Field( + description='keyword store type', + default='jieba', + ) + + +class MiddlewareConfigs( + # place the configs in alphabet order + KeywordStoreConfigs, + RedisConfigs, + StorageConfigs, + VectorStoreConfigs, +): + pass diff --git a/api/configs/middleware/redis_configs.py b/api/configs/middleware/redis_configs.py new file mode 100644 index 00000000000000..9f98e7cabfefa7 --- /dev/null +++ b/api/configs/middleware/redis_configs.py @@ -0,0 +1,38 @@ +from typing import Optional + +from pydantic import BaseModel, Field, NonNegativeInt, PositiveInt + + +class RedisConfigs(BaseModel): + """ + Redis configs + """ + REDIS_HOST: str = Field( + description='Redis host', + default='localhost', + ) + + REDIS_PORT: PositiveInt = Field( + description='Redis port', + default=6379, + ) + + REDIS_USERNAME: Optional[str] = Field( + description='Redis username', + default=None, + ) + + REDIS_PASSWORD: Optional[str] = Field( + description='Redis password', + default=None, + ) + + REDIS_DB: NonNegativeInt = Field( + description='Redis database id, default to 0', + default=0, + ) + + REDIS_USE_SSL: bool = Field( + description='whether to use SSL for Redis connection', + default=False, + ) diff --git a/api/configs/packaging/__init__.py b/api/configs/packaging/__init__.py new file mode 100644 index 00000000000000..95ccb850ed3b40 --- /dev/null +++ b/api/configs/packaging/__init__.py @@ -0,0 +1,17 @@ +from pydantic import BaseModel, Field + + +class PackagingInfo(BaseModel): + """ + Packaging build information + """ + + CURRENT_VERSION: str = Field( + description='Dify version', + default='0.6.11', + ) + + COMMIT_SHA: str = Field( + description="SHA-1 checksum of the git commit used to build the app", + default='', + ) diff --git a/api/controllers/console/workspace/tool_providers.py b/api/controllers/console/workspace/tool_providers.py index a911e9b2cb3e54..6e3f78d4e2e55b 100644 --- a/api/controllers/console/workspace/tool_providers.py +++ b/api/controllers/console/workspace/tool_providers.py @@ -104,7 +104,7 @@ class ToolBuiltinProviderIconApi(Resource): @setup_required def get(self, provider): icon_bytes, mimetype = BuiltinToolManageService.get_builtin_tool_provider_icon(provider) - icon_cache_max_age = int(current_app.config.get('TOOL_ICON_CACHE_MAX_AGE')) + icon_cache_max_age = current_app.config.get('TOOL_ICON_CACHE_MAX_AGE') return send_file(io.BytesIO(icon_bytes), mimetype=mimetype, max_age=icon_cache_max_age) class ToolApiProviderAddApi(Resource): diff --git a/api/extensions/ext_compress.py b/api/extensions/ext_compress.py index 4a349d37b41613..1dbaffcfb0dc27 100644 --- a/api/extensions/ext_compress.py +++ b/api/extensions/ext_compress.py @@ -2,7 +2,7 @@ def init_app(app: Flask): - if app.config.get('API_COMPRESSION_ENABLED', False): + if app.config.get('API_COMPRESSION_ENABLED'): from flask_compress import Compress app.config['COMPRESS_MIMETYPES'] = [ diff --git a/api/extensions/ext_redis.py b/api/extensions/ext_redis.py index c758ccb7dfa8c0..23d7768d4d0f5a 100644 --- a/api/extensions/ext_redis.py +++ b/api/extensions/ext_redis.py @@ -6,15 +6,15 @@ def init_app(app): connection_class = Connection - if app.config.get('REDIS_USE_SSL', False): + if app.config.get('REDIS_USE_SSL'): connection_class = SSLConnection redis_client.connection_pool = redis.ConnectionPool(**{ - 'host': app.config.get('REDIS_HOST', 'localhost'), - 'port': app.config.get('REDIS_PORT', 6379), - 'username': app.config.get('REDIS_USERNAME', None), - 'password': app.config.get('REDIS_PASSWORD', None), - 'db': app.config.get('REDIS_DB', 0), + 'host': app.config.get('REDIS_HOST'), + 'port': app.config.get('REDIS_PORT'), + 'username': app.config.get('REDIS_USERNAME'), + 'password': app.config.get('REDIS_PASSWORD'), + 'db': app.config.get('REDIS_DB'), 'encoding': 'utf-8', 'encoding_errors': 'strict', 'decode_responses': False diff --git a/api/migrations/README b/api/migrations/README index 0e048441597444..220678df7ab06e 100644 --- a/api/migrations/README +++ b/api/migrations/README @@ -1 +1,2 @@ Single-database configuration for Flask. + diff --git a/api/poetry.lock b/api/poetry.lock index fe8e1ebb28e8a7..3f12090df75b2a 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -5857,6 +5857,25 @@ phonenumbers = ["phonenumbers (>=8,<9)"] pycountry = ["pycountry (>=23)"] python-ulid = ["python-ulid (>=1,<2)", "python-ulid (>=1,<3)"] +[[package]] +name = "pydantic-settings" +version = "2.3.3" +description = "Settings management using Pydantic" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic_settings-2.3.3-py3-none-any.whl", hash = "sha256:e4ed62ad851670975ec11285141db888fd24947f9440bd4380d7d8788d4965de"}, + {file = "pydantic_settings-2.3.3.tar.gz", hash = "sha256:87fda838b64b5039b970cd47c3e8a1ee460ce136278ff672980af21516f6e6ce"}, +] + +[package.dependencies] +pydantic = ">=2.7.0" +python-dotenv = ">=0.21.0" + +[package.extras] +toml = ["tomli (>=2.0.1)"] +yaml = ["pyyaml (>=6.0.1)"] + [[package]] name = "pydub" version = "0.25.1" @@ -6414,6 +6433,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -6421,8 +6441,16 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -6439,6 +6467,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -6446,6 +6475,7 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -8944,4 +8974,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "21360e271c46e0368b8e3bd26287caca73145a73ee73287669f91e7eac6f05b9" +content-hash = "79b4019178c3ce29897ad574e7645ba8213d572b5cb7ef8ebd32d0c9c7296aff" diff --git a/api/pyproject.toml b/api/pyproject.toml index a83d98b43842df..36202cd5ee8832 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -170,6 +170,7 @@ lxml = "5.1.0" xlrd = "~2.0.1" pydantic = "~2.7.4" pydantic_extra_types = "~2.8.1" +pydantic-settings = "~2.3.3" pgvecto-rs = "0.1.4" firecrawl-py = "0.0.5" oss2 = "2.18.5" @@ -202,4 +203,3 @@ optional = true [tool.poetry.group.lint.dependencies] ruff = "~0.4.8" dotenv-linter = "~0.5.0" - diff --git a/api/requirements.txt b/api/requirements.txt index 7ab636e226e604..7c6d61217d9933 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -77,6 +77,7 @@ azure-identity==1.16.1 lxml==5.1.0 pydantic~=2.7.4 pydantic_extra_types~=2.8.1 +pydantic-settings~=2.3.3 pgvecto-rs==0.1.4 tcvectordb==1.3.2 firecrawl-py==0.0.5 diff --git a/api/tests/unit_tests/settings/test_app_settings.py b/api/tests/unit_tests/settings/test_app_settings.py new file mode 100644 index 00000000000000..de4fd57db6edee --- /dev/null +++ b/api/tests/unit_tests/settings/test_app_settings.py @@ -0,0 +1,46 @@ +import pytest +from flask import Flask + +from config import Config +from configs.app_configs import DifyConfigs + + +def test_dify_configs_undefined_entry(): + # load dotenv file with pydantic-settings + settings = DifyConfigs() + + # entries not defined in app settings + with pytest.raises(TypeError): + # TypeError: 'AppSettings' object is not subscriptable + assert settings['LOG_LEVEL'] == 'INFO' + + +def test_dify_configs(): + # load dotenv file with pydantic-settings + settings = DifyConfigs() + + # constant values + assert settings.COMMIT_SHA == '' + + # default values + assert settings.EDITION == 'SELF_HOSTED' + assert settings.API_COMPRESSION_ENABLED is False + + +def test_flask_configs(): + flask_app = Flask('app') + flask_app.config.from_object(Config()) + flask_app.config.from_mapping(DifyConfigs().dict()) + config = flask_app.config + + # configs read from dotenv directly + assert config['LOG_LEVEL'] == 'INFO' + + # configs read from pydantic-settings + assert config['COMMIT_SHA'] == '' + assert config['EDITION'] == 'SELF_HOSTED' + assert config['API_COMPRESSION_ENABLED'] is False + + assert config['CONSOLE_API_URL'] == 'https://cloud.dify.ai' + # fallback to alias choices value as CONSOLE_API_URL + assert config['FILES_URL'] == 'https://cloud.dify.ai'