diff --git a/api/app.py b/api/app.py index a36750410e156d..4a52835d730d8c 100644 --- a/api/app.py +++ b/api/app.py @@ -1,5 +1,7 @@ import os +from configs.app_configs import AppConfigs + if not os.environ.get("DEBUG") or os.environ.get("DEBUG").lower() != 'true': from gevent import monkey @@ -25,7 +27,6 @@ from config import Config # DO NOT REMOVE BELOW -from events import event_handlers from extensions import ( ext_celery, ext_code_based_extension, @@ -42,7 +43,6 @@ from extensions.ext_database import db from extensions.ext_login import login_manager from libs.passport import PassportService -from models import account, dataset, model, source, task, tool, tools, web from services.account_service import AccountService # DO NOT REMOVE ABOVE @@ -74,10 +74,19 @@ class DifyApp(Flask): # Application Factory Function # ---------------------------- +def create_flask_app() -> Flask: + """ + create a raw flask app + with configs loaded from .env file + """ + dify_app = DifyApp(__name__) + dify_app.config.from_object(Config()) + dify_app.config.from_mapping(AppConfigs().dict()) + return dify_app + def create_app() -> Flask: - app = DifyApp(__name__) - app.config.from_object(Config()) + app = create_flask_app() app.secret_key = app.config['SECRET_KEY'] diff --git a/api/config.py b/api/config.py index 0e7cef3286e1f7..42a43d20aeaa70 100644 --- a/api/config.py +++ b/api/config.py @@ -5,7 +5,6 @@ dotenv.load_dotenv() DEFAULTS = { - 'EDITION': 'SELF_HOSTED', 'DB_USERNAME': 'postgres', 'DB_PASSWORD': '', 'DB_HOST': 'localhost', @@ -18,18 +17,12 @@ 'REDIS_USE_SSL': 'False', 'OAUTH_REDIRECT_PATH': '/console/api/oauth/authorize', 'OAUTH_REDIRECT_INDEX_PATH': '/', - 'CONSOLE_WEB_URL': 'https://cloud.dify.ai', - 'CONSOLE_API_URL': 'https://cloud.dify.ai', - 'SERVICE_API_URL': 'https://api.dify.ai', - 'APP_WEB_URL': 'https://udify.app', 'FILES_URL': '', 'FILES_ACCESS_TIMEOUT': 300, 'S3_USE_AWS_MANAGED_IAM': 'False', 'S3_ADDRESS_STYLE': 'auto', 'STORAGE_TYPE': 'local', 'STORAGE_LOCAL_PATH': 'storage', - 'CHECK_UPDATE_URL': 'https://updates.dify.ai', - 'DEPLOY_ENV': 'PRODUCTION', 'SQLALCHEMY_DATABASE_URI_SCHEME': 'postgresql', 'SQLALCHEMY_POOL_SIZE': 30, 'SQLALCHEMY_MAX_OVERFLOW': 10, @@ -44,10 +37,6 @@ 'QDRANT_GRPC_ENABLED': 'False', 'QDRANT_GRPC_PORT': '6334', 'CELERY_BACKEND': 'database', - 'LOG_LEVEL': 'INFO', - 'LOG_FILE': '', - 'LOG_FORMAT': '%(asctime)s.%(msecs)03d %(levelname)s [%(threadName)s] [%(filename)s:%(lineno)d] - %(message)s', - 'LOG_DATEFORMAT': '%Y-%m-%d %H:%M:%S', 'HOSTED_OPENAI_QUOTA_LIMIT': 200, 'HOSTED_OPENAI_TRIAL_ENABLED': 'False', 'HOSTED_OPENAI_TRIAL_MODELS': 'gpt-3.5-turbo,gpt-3.5-turbo-1106,gpt-3.5-turbo-instruct,gpt-3.5-turbo-16k,gpt-3.5-turbo-16k-0613,gpt-3.5-turbo-0613,gpt-3.5-turbo-0125,text-davinci-003', @@ -114,51 +103,13 @@ class Config: """Application configuration class.""" def __init__(self): - # ------------------------ - # General Configurations. - # ------------------------ - self.CURRENT_VERSION = "0.6.11" - self.COMMIT_SHA = get_env('COMMIT_SHA') - self.EDITION = get_env('EDITION') - self.DEPLOY_ENV = get_env('DEPLOY_ENV') self.TESTING = False - self.LOG_LEVEL = get_env('LOG_LEVEL') - self.LOG_FILE = get_env('LOG_FILE') - self.LOG_FORMAT = get_env('LOG_FORMAT') - self.LOG_DATEFORMAT = get_env('LOG_DATEFORMAT') - self.API_COMPRESSION_ENABLED = get_bool_env('API_COMPRESSION_ENABLED') - - # The backend URL prefix of the console API. - # used to concatenate the login authorization callback or notion integration callback. - self.CONSOLE_API_URL = get_env('CONSOLE_API_URL') - - # The front-end URL prefix of the console web. - # used to concatenate some front-end addresses and for CORS configuration use. - self.CONSOLE_WEB_URL = get_env('CONSOLE_WEB_URL') - # WebApp Url prefix. - # used to display WebAPP API Base Url to the front-end. - self.APP_WEB_URL = get_env('APP_WEB_URL') - - # Service API Url prefix. - # used to display Service API Base Url to the front-end. - self.SERVICE_API_URL = get_env('SERVICE_API_URL') - - # File preview or download Url prefix. - # used to display File preview or download Url to the front-end or as Multi-model inputs; - # Url is signed and has expiration time. - self.FILES_URL = get_env('FILES_URL') if get_env('FILES_URL') else self.CONSOLE_API_URL # File Access Time specifies a time interval in seconds for the file to be accessed. # The default value is 300 seconds. self.FILES_ACCESS_TIMEOUT = int(get_env('FILES_ACCESS_TIMEOUT')) - # Your App secret key will be used for securely signing the session cookie - # Make sure you are changing this key for your deployment with a strong key. - # You can generate a strong key using `openssl rand -base64 42`. - # Alternatively you can set it with `SECRET_KEY` environment variable. - self.SECRET_KEY = get_env('SECRET_KEY') - # Enable or disable the inner API. self.INNER_API = get_bool_env('INNER_API') # The inner API key is used to authenticate the inner API. @@ -166,13 +117,10 @@ def __init__(self): # cors settings self.CONSOLE_CORS_ALLOW_ORIGINS = get_cors_allow_origins( - 'CONSOLE_CORS_ALLOW_ORIGINS', self.CONSOLE_WEB_URL) + 'CONSOLE_CORS_ALLOW_ORIGINS', get_env('CONSOLE_WEB_URL')) self.WEB_API_CORS_ALLOW_ORIGINS = get_cors_allow_origins( 'WEB_API_CORS_ALLOW_ORIGINS', '*') - # check update url - self.CHECK_UPDATE_URL = get_env('CHECK_UPDATE_URL') - # ------------------------ # Database Configurations. # ------------------------ @@ -212,7 +160,7 @@ def __init__(self): self.CELERY_BACKEND = get_env('CELERY_BACKEND') self.CELERY_RESULT_BACKEND = 'db+{}'.format(self.SQLALCHEMY_DATABASE_URI) \ if self.CELERY_BACKEND == 'database' else self.CELERY_BROKER_URL - self.BROKER_USE_SSL = self.CELERY_BROKER_URL.startswith('rediss://') + self.BROKER_USE_SSL = self.CELERY_BROKER_URL.startswith('rediss://') if self.CELERY_BROKER_URL else False # ------------------------ # Code Execution Sandbox Configurations. diff --git a/api/configs/app_configs.py b/api/configs/app_configs.py new file mode 100644 index 00000000000000..913da033ac2cae --- /dev/null +++ b/api/configs/app_configs.py @@ -0,0 +1,26 @@ +from pydantic_settings import BaseSettings, SettingsConfigDict + +from configs.build_info import BuildInfo +from configs.deployment_configs import DeploymentConfigs +from configs.endpoint_configs import EndpointConfigs +from configs.http_configs import HttpConfigs +from configs.logging_configs import LoggingConfigs +from configs.security_configs import SecurityConfigs + + +class AppConfigs( + # pydantic-settings + BaseSettings, + + # app configs + BuildInfo, + DeploymentConfigs, + EndpointConfigs, + HttpConfigs, + LoggingConfigs, + SecurityConfigs, +): + # read from dotenv format config file + model_config = SettingsConfigDict( + env_file='.env', + env_file_encoding='utf-8') diff --git a/api/configs/build_info.py b/api/configs/build_info.py new file mode 100644 index 00000000000000..99e6188b91a23a --- /dev/null +++ b/api/configs/build_info.py @@ -0,0 +1,17 @@ +from pydantic import BaseModel, Field + + +class BuildInfo(BaseModel): + """ + Build information + """ + + CURRENT_VERSION: str = Field( + description='Dify version', + default='0.6.11', + ) + + COMMIT_SHA: str = Field( + description="SHA-1 checksum of the git commit used to build the app", + default='', + ) diff --git a/api/configs/deployment_configs.py b/api/configs/deployment_configs.py new file mode 100644 index 00000000000000..9a1c1b84d7cfb7 --- /dev/null +++ b/api/configs/deployment_configs.py @@ -0,0 +1,16 @@ +from pydantic import BaseModel, Field + + +class DeploymentConfigs(BaseModel): + """ + Deployment configs + """ + EDITION: str = Field( + description='deployment edition', + default='SELF_HOSTED', + ) + + DEPLOY_ENV: str = Field( + description='deployment environment, default to PRODUCTION.', + default='PRODUCTION', + ) diff --git a/api/configs/endpoint_configs.py b/api/configs/endpoint_configs.py new file mode 100644 index 00000000000000..682a3c505b4efd --- /dev/null +++ b/api/configs/endpoint_configs.py @@ -0,0 +1,39 @@ +from pydantic import AliasChoices, BaseModel, Field + + +class EndpointConfigs(BaseModel): + """ + Module URL configs + """ + CONSOLE_API_URL: str = Field( + description='The backend URL prefix of the console API.' + 'used to concatenate the login authorization callback or notion integration callback.', + default='https://cloud.dify.ai', + ) + + CONSOLE_WEB_URL: str = Field( + description='The front-end URL prefix of the console web.' + 'used to concatenate some front-end addresses and for CORS configuration use.', + default='https://cloud.dify.ai', + ) + + SERVICE_API_URL: str = Field( + description='Service API Url prefix.' + 'used to display Service API Base Url to the front-end.', + default='https://api.dify.ai', + ) + + APP_WEB_URL: str = Field( + description='WebApp Url prefix.' + 'used to display WebAPP API Base Url to the front-end.', + default='https://udify.app', + ) + + FILES_URL: str = Field( + description='File preview or download Url prefix.' + ' used to display File preview or download Url to the front-end or as Multi-model inputs;' + 'Url is signed and has expiration time.', + validation_alias=AliasChoices('FILES_URL', 'CONSOLE_API_URL'), + alias_priority=1, + default='https://cloud.dify.ai', + ) diff --git a/api/configs/http_configs.py b/api/configs/http_configs.py new file mode 100644 index 00000000000000..f03bbb4f033a09 --- /dev/null +++ b/api/configs/http_configs.py @@ -0,0 +1,11 @@ +from pydantic import BaseModel, Field + + +class HttpConfigs(BaseModel): + """ + HTTP configs + """ + API_COMPRESSION_ENABLED: bool = Field( + description='whether to enable HTTP response compression of gzip', + default=False, + ) diff --git a/api/configs/logging_configs.py b/api/configs/logging_configs.py new file mode 100644 index 00000000000000..6ac14cbb5a7e4d --- /dev/null +++ b/api/configs/logging_configs.py @@ -0,0 +1,28 @@ +from pydantic import BaseModel, Field + + +class LoggingConfigs(BaseModel): + """ + Logging configs + """ + + LOG_LEVEL: str = Field( + description='Log output level, default to INFO.' + 'It is recommended to set it to ERROR for production.', + default='INFO', + ) + + LOG_FILE: str = Field( + description='logging output file path', + default='', + ) + + LOG_FORMAT: str = Field( + description='log format', + default='%(asctime)s.%(msecs)03d %(levelname)s [%(threadName)s] [%(filename)s:%(lineno)d] - %(message)s', + ) + + LOG_DATEFORMAT: str = Field( + description='log date format', + default='', + ) diff --git a/api/configs/security_configs.py b/api/configs/security_configs.py new file mode 100644 index 00000000000000..bf6f4735511bfc --- /dev/null +++ b/api/configs/security_configs.py @@ -0,0 +1,16 @@ +from typing import Optional + +from pydantic import BaseModel, Field + + +class SecurityConfigs(BaseModel): + """ + Secret Key configs + """ + SECRET_KEY: Optional[str] = Field( + description='Your App secret key will be used for securely signing the session cookie' + 'Make sure you are changing this key for your deployment with a strong key.' + 'You can generate a strong key using `openssl rand -base64 42`.' + 'Alternatively you can set it with `SECRET_KEY` environment variable.', + default=None, + ) diff --git a/api/configs/update_configs.py b/api/configs/update_configs.py new file mode 100644 index 00000000000000..92af4f356a81c4 --- /dev/null +++ b/api/configs/update_configs.py @@ -0,0 +1,11 @@ +from pydantic import BaseModel, Field + + +class HttpConfigs(BaseModel): + """ + HTTP configs + """ + CHECK_UPDATE_URL: str = Field( + description='url for checking updates', + default='https://updates.dify.ai', + ) diff --git a/api/extensions/ext_compress.py b/api/extensions/ext_compress.py index 4a349d37b41613..1dbaffcfb0dc27 100644 --- a/api/extensions/ext_compress.py +++ b/api/extensions/ext_compress.py @@ -2,7 +2,7 @@ def init_app(app: Flask): - if app.config.get('API_COMPRESSION_ENABLED', False): + if app.config.get('API_COMPRESSION_ENABLED'): from flask_compress import Compress app.config['COMPRESS_MIMETYPES'] = [ diff --git a/api/poetry.lock b/api/poetry.lock index 6d716a15c2beba..bde401469e1b64 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -5823,6 +5823,25 @@ phonenumbers = ["phonenumbers (>=8,<9)"] pycountry = ["pycountry (>=23)"] python-ulid = ["python-ulid (>=1,<2)", "python-ulid (>=1,<3)"] +[[package]] +name = "pydantic-settings" +version = "2.3.3" +description = "Settings management using Pydantic" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic_settings-2.3.3-py3-none-any.whl", hash = "sha256:e4ed62ad851670975ec11285141db888fd24947f9440bd4380d7d8788d4965de"}, + {file = "pydantic_settings-2.3.3.tar.gz", hash = "sha256:87fda838b64b5039b970cd47c3e8a1ee460ce136278ff672980af21516f6e6ce"}, +] + +[package.dependencies] +pydantic = ">=2.7.0" +python-dotenv = ">=0.21.0" + +[package.extras] +toml = ["tomli (>=2.0.1)"] +yaml = ["pyyaml (>=6.0.1)"] + [[package]] name = "pydub" version = "0.25.1" @@ -8921,4 +8940,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "e967aa4b61dc7c40f2f50eb325038da1dc0ff633d8f778e7a7560bdabce744dc" +content-hash = "d5cf8a44ffb11a3c15b19895657fa13aba02757cb1157b3259a7fdec4e7a140d" diff --git a/api/pyproject.toml b/api/pyproject.toml index b56556a62b00fb..1ab133c19e9a1f 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -172,6 +172,7 @@ lxml = "5.1.0" xlrd = "~2.0.1" pydantic = "~2.7.4" pydantic_extra_types = "~2.8.1" +pydantic-settings = "~2.3.3" pgvecto-rs = "0.1.4" firecrawl-py = "0.0.5" oss2 = "2.18.5" diff --git a/api/requirements.txt b/api/requirements.txt index a6a1d8c5cedc33..20a30356983791 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -77,6 +77,7 @@ azure-identity==1.16.1 lxml==5.1.0 pydantic~=2.7.4 pydantic_extra_types~=2.8.1 +pydantic-settings~=2.3.3 pgvecto-rs==0.1.4 tcvectordb==1.3.2 firecrawl-py==0.0.5 diff --git a/api/tests/unit_tests/settings/test_app_settings.py b/api/tests/unit_tests/settings/test_app_settings.py new file mode 100644 index 00000000000000..85b031a11e06d2 --- /dev/null +++ b/api/tests/unit_tests/settings/test_app_settings.py @@ -0,0 +1,46 @@ +import pytest +from flask import Flask + +from config import Config +from configs.app_configs import AppConfigs + + +def test_app_settings_undefined_entry(): + # load dotenv file with pydantic-settings + settings = AppConfigs() + + # entries not defined in app settings + with pytest.raises(TypeError): + # TypeError: 'AppSettings' object is not subscriptable + assert settings['LOG_LEVEL'] == 'INFO' + + +def test_app_settings(): + # load dotenv file with pydantic-settings + settings = AppConfigs() + + # constant values + assert settings.COMMIT_SHA == '' + + # default values + assert settings.EDITION == 'SELF_HOSTED' + assert settings.API_COMPRESSION_ENABLED is False + + +def test_flask_configs(): + flask_app = Flask('app') + flask_app.config.from_object(Config()) + flask_app.config.from_mapping(AppConfigs().dict()) + config = flask_app.config + + # configs read from dotenv directly + assert config['LOG_LEVEL'] == 'INFO' + + # configs read from pydantic-settings + assert config['COMMIT_SHA'] == '' + assert config['EDITION'] == 'SELF_HOSTED' + assert config['API_COMPRESSION_ENABLED'] is False + + assert config['CONSOLE_API_URL'] == 'https://cloud.dify.ai' + # fallback to alias choices value as CONSOLE_API_URL + assert config['FILES_URL'] == 'https://cloud.dify.ai'