diff --git a/api/configs/app_config.py b/api/configs/app_config.py index 494e256442134a..ff8c77de4836d3 100644 --- a/api/configs/app_config.py +++ b/api/configs/app_config.py @@ -1,4 +1,3 @@ -from pydantic import Field, computed_field from pydantic_settings import SettingsConfigDict from configs.deploy import DeploymentConfig @@ -24,8 +23,6 @@ class DifyConfig( # **Before using, please contact business@dify.ai by email to inquire about licensing matters.** EnterpriseFeatureConfig, ): - DEBUG: bool = Field(default=False, description='whether to enable debug mode.') - model_config = SettingsConfigDict( # read from dotenv format config file env_file='.env', @@ -35,33 +32,7 @@ class DifyConfig( extra='ignore', ) - CODE_MAX_NUMBER: int = 9223372036854775807 - CODE_MIN_NUMBER: int = -9223372036854775808 - CODE_MAX_DEPTH: int = 5 - CODE_MAX_PRECISION: int = 20 - CODE_MAX_STRING_LENGTH: int = 80000 - CODE_MAX_STRING_ARRAY_LENGTH: int = 30 - CODE_MAX_OBJECT_ARRAY_LENGTH: int = 30 - CODE_MAX_NUMBER_ARRAY_LENGTH: int = 1000 - - HTTP_REQUEST_MAX_CONNECT_TIMEOUT: int = 300 - HTTP_REQUEST_MAX_READ_TIMEOUT: int = 600 - HTTP_REQUEST_MAX_WRITE_TIMEOUT: int = 600 - HTTP_REQUEST_NODE_MAX_BINARY_SIZE: int = 1024 * 1024 * 10 - - @computed_field - def HTTP_REQUEST_NODE_READABLE_MAX_BINARY_SIZE(self) -> str: - return f'{self.HTTP_REQUEST_NODE_MAX_BINARY_SIZE / 1024 / 1024:.2f}MB' - - HTTP_REQUEST_NODE_MAX_TEXT_SIZE: int = 1024 * 1024 - - @computed_field - def HTTP_REQUEST_NODE_READABLE_MAX_TEXT_SIZE(self) -> str: - return f'{self.HTTP_REQUEST_NODE_MAX_TEXT_SIZE / 1024 / 1024:.2f}MB' - - SSRF_PROXY_HTTP_URL: str | None = None - SSRF_PROXY_HTTPS_URL: str | None = None - - MODERATION_BUFFER_SIZE: int = Field(default=300, description='The buffer size for moderation.') - - MAX_VARIABLE_SIZE: int = Field(default=5 * 1024, description='The maximum size of a variable. default is 5KB.') + # Before adding any config, + # please consider to arrange it in the proper config group of existed or added + # for better readability and maintainability. + # Thanks for your concentration and consideration. diff --git a/api/configs/deploy/__init__.py b/api/configs/deploy/__init__.py index 219b315784323e..c99e3d21d247d7 100644 --- a/api/configs/deploy/__init__.py +++ b/api/configs/deploy/__init__.py @@ -11,6 +11,11 @@ class DeploymentConfig(BaseSettings): default='langgenius/dify', ) + DEBUG: bool = Field( + description='whether to enable debug mode.', + default=False, + ) + TESTING: bool = Field( description='', default=False, diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py index 27de2c04613af1..7f36abf7a6c0a4 100644 --- a/api/configs/feature/__init__.py +++ b/api/configs/feature/__init__.py @@ -1,6 +1,6 @@ from typing import Optional -from pydantic import AliasChoices, Field, NonNegativeInt, PositiveInt, computed_field +from pydantic import AliasChoices, Field, NegativeInt, NonNegativeInt, PositiveInt, computed_field from pydantic_settings import BaseSettings from configs.feature.hosted_service import HostedServiceConfig @@ -52,6 +52,46 @@ class CodeExecutionSandboxConfig(BaseSettings): default='dify-sandbox', ) + CODE_MAX_NUMBER: PositiveInt = Field( + description='max depth for code execution', + default=9223372036854775807, + ) + + CODE_MIN_NUMBER: NegativeInt = Field( + description='', + default=-9223372036854775807, + ) + + CODE_MAX_DEPTH: PositiveInt = Field( + description='max depth for code execution', + default=5, + ) + + CODE_MAX_PRECISION: PositiveInt = Field( + description='max precision digits for float type in code execution', + default=20, + ) + + CODE_MAX_STRING_LENGTH: PositiveInt = Field( + description='max string length for code execution', + default=80000, + ) + + CODE_MAX_STRING_ARRAY_LENGTH: PositiveInt = Field( + description='', + default=30, + ) + + CODE_MAX_OBJECT_ARRAY_LENGTH: PositiveInt = Field( + description='', + default=30, + ) + + CODE_MAX_NUMBER_ARRAY_LENGTH: PositiveInt = Field( + description='', + default=1000, + ) + class EndpointConfig(BaseSettings): """ @@ -157,6 +197,41 @@ def CONSOLE_CORS_ALLOW_ORIGINS(self) -> list[str]: def WEB_API_CORS_ALLOW_ORIGINS(self) -> list[str]: return self.inner_WEB_API_CORS_ALLOW_ORIGINS.split(',') + HTTP_REQUEST_MAX_CONNECT_TIMEOUT: NonNegativeInt = Field( + description='', + default=300, + ) + + HTTP_REQUEST_MAX_READ_TIMEOUT: NonNegativeInt = Field( + description='', + default=600, + ) + + HTTP_REQUEST_MAX_WRITE_TIMEOUT: NonNegativeInt = Field( + description='', + default=600, + ) + + HTTP_REQUEST_NODE_MAX_BINARY_SIZE: PositiveInt = Field( + description='', + default=10 * 1024 * 1024, + ) + + HTTP_REQUEST_NODE_MAX_TEXT_SIZE: PositiveInt = Field( + description='', + default=1 * 1024 * 1024, + ) + + SSRF_PROXY_HTTP_URL: Optional[str] = Field( + description='HTTP URL for SSRF proxy', + default=None, + ) + + SSRF_PROXY_HTTPS_URL: Optional[str] = Field( + description='HTTPS URL for SSRF proxy', + default=None, + ) + class InnerAPIConfig(BaseSettings): """ @@ -255,6 +330,11 @@ class WorkflowConfig(BaseSettings): default=5, ) + MAX_VARIABLE_SIZE: PositiveInt = Field( + description='The maximum size in bytes of a variable. default to 5KB.', + default=5 * 1024, + ) + class OAuthConfig(BaseSettings): """ @@ -291,8 +371,7 @@ class ModerationConfig(BaseSettings): Moderation in app configs. """ - # todo: to be clarified in usage and unit - OUTPUT_MODERATION_BUFFER_SIZE: PositiveInt = Field( + MODERATION_BUFFER_SIZE: PositiveInt = Field( description='buffer size for moderation', default=300, ) @@ -444,7 +523,6 @@ class CeleryBeatConfig(BaseSettings): class PositionConfig(BaseSettings): - POSITION_PROVIDER_PINS: str = Field( description='The heads of model providers', default='', diff --git a/api/core/workflow/nodes/code/code_node.py b/api/core/workflow/nodes/code/code_node.py index 71adf18a51c134..17554d3db4574b 100644 --- a/api/core/workflow/nodes/code/code_node.py +++ b/api/core/workflow/nodes/code/code_node.py @@ -11,15 +11,6 @@ from core.workflow.nodes.code.entities import CodeNodeData from models.workflow import WorkflowNodeExecutionStatus -MAX_NUMBER = dify_config.CODE_MAX_NUMBER -MIN_NUMBER = dify_config.CODE_MIN_NUMBER -MAX_PRECISION = dify_config.CODE_MAX_PRECISION -MAX_DEPTH = dify_config.CODE_MAX_DEPTH -MAX_STRING_LENGTH = dify_config.CODE_MAX_STRING_LENGTH -MAX_STRING_ARRAY_LENGTH = dify_config.CODE_MAX_STRING_ARRAY_LENGTH -MAX_OBJECT_ARRAY_LENGTH = dify_config.CODE_MAX_OBJECT_ARRAY_LENGTH -MAX_NUMBER_ARRAY_LENGTH = dify_config.CODE_MAX_NUMBER_ARRAY_LENGTH - class CodeNode(BaseNode): _node_data_cls = CodeNodeData @@ -97,8 +88,9 @@ def _check_string(self, value: str, variable: str) -> str: else: raise ValueError(f"Output variable `{variable}` must be a string") - if len(value) > MAX_STRING_LENGTH: - raise ValueError(f'The length of output variable `{variable}` must be less than {MAX_STRING_LENGTH} characters') + if len(value) > dify_config.CODE_MAX_STRING_ARRAY_LENGTH: + raise ValueError(f'The length of output variable `{variable}` must be' + f' less than {dify_config.CODE_MAX_STRING_ARRAY_LENGTH} characters') return value.replace('\x00', '') @@ -115,13 +107,15 @@ def _check_number(self, value: Union[int, float], variable: str) -> Union[int, f else: raise ValueError(f"Output variable `{variable}` must be a number") - if value > MAX_NUMBER or value < MIN_NUMBER: - raise ValueError(f'Output variable `{variable}` is out of range, it must be between {MIN_NUMBER} and {MAX_NUMBER}.') + if value > dify_config.CODE_MAX_NUMBER or value < dify_config.CODE_MIN_NUMBER: + raise ValueError(f'Output variable `{variable}` is out of range,' + f' it must be between {dify_config.CODE_MIN_NUMBER} and {dify_config.CODE_MAX_NUMBER}.') if isinstance(value, float): # raise error if precision is too high - if len(str(value).split('.')[1]) > MAX_PRECISION: - raise ValueError(f'Output variable `{variable}` has too high precision, it must be less than {MAX_PRECISION} digits.') + if len(str(value).split('.')[1]) > dify_config.CODE_MAX_PRECISION: + raise ValueError(f'Output variable `{variable}` has too high precision,' + f' it must be less than {dify_config.CODE_MAX_PRECISION} digits.') return value @@ -134,8 +128,8 @@ def _transform_result(self, result: dict, output_schema: Optional[dict[str, Code :param output_schema: output schema :return: """ - if depth > MAX_DEPTH: - raise ValueError("Depth limit reached, object too deep.") + if depth > dify_config.CODE_MAX_DEPTH: + raise ValueError(f"Depth limit ${dify_config.CODE_MAX_DEPTH} reached, object too deep.") transformed_result = {} if output_schema is None: @@ -235,9 +229,10 @@ def _transform_result(self, result: dict, output_schema: Optional[dict[str, Code f'Output {prefix}{dot}{output_name} is not an array, got {type(result.get(output_name))} instead.' ) else: - if len(result[output_name]) > MAX_NUMBER_ARRAY_LENGTH: + if len(result[output_name]) > dify_config.CODE_MAX_NUMBER_ARRAY_LENGTH: raise ValueError( - f'The length of output variable `{prefix}{dot}{output_name}` must be less than {MAX_NUMBER_ARRAY_LENGTH} elements.' + f'The length of output variable `{prefix}{dot}{output_name}` must be' + f' less than {dify_config.CODE_MAX_NUMBER_ARRAY_LENGTH} elements.' ) transformed_result[output_name] = [ @@ -257,9 +252,10 @@ def _transform_result(self, result: dict, output_schema: Optional[dict[str, Code f'Output {prefix}{dot}{output_name} is not an array, got {type(result.get(output_name))} instead.' ) else: - if len(result[output_name]) > MAX_STRING_ARRAY_LENGTH: + if len(result[output_name]) > dify_config.CODE_MAX_STRING_ARRAY_LENGTH: raise ValueError( - f'The length of output variable `{prefix}{dot}{output_name}` must be less than {MAX_STRING_ARRAY_LENGTH} elements.' + f'The length of output variable `{prefix}{dot}{output_name}` must be' + f' less than {dify_config.CODE_MAX_STRING_ARRAY_LENGTH} elements.' ) transformed_result[output_name] = [ @@ -279,9 +275,10 @@ def _transform_result(self, result: dict, output_schema: Optional[dict[str, Code f'Output {prefix}{dot}{output_name} is not an array, got {type(result.get(output_name))} instead.' ) else: - if len(result[output_name]) > MAX_OBJECT_ARRAY_LENGTH: + if len(result[output_name]) > dify_config.CODE_MAX_OBJECT_ARRAY_LENGTH: raise ValueError( - f'The length of output variable `{prefix}{dot}{output_name}` must be less than {MAX_OBJECT_ARRAY_LENGTH} elements.' + f'The length of output variable `{prefix}{dot}{output_name}` must be' + f' less than {dify_config.CODE_MAX_OBJECT_ARRAY_LENGTH} elements.' ) for i, value in enumerate(result[output_name]): diff --git a/api/core/workflow/nodes/http_request/http_executor.py b/api/core/workflow/nodes/http_request/http_executor.py index db18bd00b2d9e1..d16bff58bda12b 100644 --- a/api/core/workflow/nodes/http_request/http_executor.py +++ b/api/core/workflow/nodes/http_request/http_executor.py @@ -18,11 +18,6 @@ ) from core.workflow.utils.variable_template_parser import VariableTemplateParser -MAX_BINARY_SIZE = dify_config.HTTP_REQUEST_NODE_MAX_BINARY_SIZE -READABLE_MAX_BINARY_SIZE = dify_config.HTTP_REQUEST_NODE_READABLE_MAX_BINARY_SIZE -MAX_TEXT_SIZE = dify_config.HTTP_REQUEST_NODE_MAX_TEXT_SIZE -READABLE_MAX_TEXT_SIZE = dify_config.HTTP_REQUEST_NODE_READABLE_MAX_TEXT_SIZE - class HttpExecutorResponse: headers: dict[str, str] @@ -237,16 +232,14 @@ def _validate_and_parse_response(self, response: httpx.Response) -> HttpExecutor else: raise ValueError(f'Invalid response type {type(response)}') - if executor_response.is_file: - if executor_response.size > MAX_BINARY_SIZE: - raise ValueError( - f'File size is too large, max size is {READABLE_MAX_BINARY_SIZE}, but current size is {executor_response.readable_size}.' - ) - else: - if executor_response.size > MAX_TEXT_SIZE: - raise ValueError( - f'Text size is too large, max size is {READABLE_MAX_TEXT_SIZE}, but current size is {executor_response.readable_size}.' - ) + threshold_size = dify_config.HTTP_REQUEST_NODE_MAX_BINARY_SIZE if executor_response.is_file \ + else dify_config.HTTP_REQUEST_NODE_MAX_TEXT_SIZE + if executor_response.size > threshold_size: + raise ValueError( + f'{"File" if executor_response.is_file else "Text"} size is too large,' + f' max size is {threshold_size / 1024 / 1024:.2f} MB,' + f' but current size is {executor_response.readable_size}.' + ) return executor_response diff --git a/api/poetry.lock b/api/poetry.lock index a68eadefcfa21a..0a8919a30aa3c3 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -6372,13 +6372,13 @@ semver = ["semver (>=3.0.2)"] [[package]] name = "pydantic-settings" -version = "2.3.4" +version = "2.4.0" description = "Settings management using Pydantic" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic_settings-2.3.4-py3-none-any.whl", hash = "sha256:11ad8bacb68a045f00e4f862c7a718c8a9ec766aa8fd4c32e39a0594b207b53a"}, - {file = "pydantic_settings-2.3.4.tar.gz", hash = "sha256:c5802e3d62b78e82522319bbc9b8f8ffb28ad1c988a99311d04f2a6051fca0a7"}, + {file = "pydantic_settings-2.4.0-py3-none-any.whl", hash = "sha256:bb6849dc067f1687574c12a639e231f3a6feeed0a12d710c1382045c5db1c315"}, + {file = "pydantic_settings-2.4.0.tar.gz", hash = "sha256:ed81c3a0f46392b4d7c0a565c05884e6e54b3456e6f0fe4d8814981172dc9a88"}, ] [package.dependencies] @@ -6386,6 +6386,7 @@ pydantic = ">=2.7.0" python-dotenv = ">=0.21.0" [package.extras] +azure-key-vault = ["azure-identity (>=1.16.0)", "azure-keyvault-secrets (>=4.8.0)"] toml = ["tomli (>=2.0.1)"] yaml = ["pyyaml (>=6.0.1)"] @@ -9633,4 +9634,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "69c20af8ecacced3cca092662223a1511acaf65cb2616a5a1e38b498223463e0" +content-hash = "d7336115709114c2a4ff09b392f717e9c3547ae82b6a111d0c885c7a44269f02" diff --git a/api/pyproject.toml b/api/pyproject.toml index 8e050c4101fc9a..47b638573b971a 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -162,7 +162,7 @@ pandas = { version = "~2.2.2", extras = ["performance", "excel"] } psycopg2-binary = "~2.9.6" pycryptodome = "3.19.1" pydantic = "~2.8.2" -pydantic-settings = "~2.3.4" +pydantic-settings = "~2.4.0" pydantic_extra_types = "~2.9.0" pyjwt = "~2.8.0" pypdfium2 = "~4.17.0"