From ecd7266736c75d1b3f9efd18e99b36f6bc61b945 Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Fri, 23 Aug 2024 17:48:19 +0800 Subject: [PATCH 01/11] update --- api/configs/app_config.py | 11 ----------- api/configs/feature/__init__.py | 12 ++++++++++-- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/api/configs/app_config.py b/api/configs/app_config.py index 494e256442134a..9978ec3b15eadb 100644 --- a/api/configs/app_config.py +++ b/api/configs/app_config.py @@ -35,15 +35,6 @@ class DifyConfig( extra='ignore', ) - CODE_MAX_NUMBER: int = 9223372036854775807 - CODE_MIN_NUMBER: int = -9223372036854775808 - CODE_MAX_DEPTH: int = 5 - CODE_MAX_PRECISION: int = 20 - CODE_MAX_STRING_LENGTH: int = 80000 - CODE_MAX_STRING_ARRAY_LENGTH: int = 30 - CODE_MAX_OBJECT_ARRAY_LENGTH: int = 30 - CODE_MAX_NUMBER_ARRAY_LENGTH: int = 1000 - HTTP_REQUEST_MAX_CONNECT_TIMEOUT: int = 300 HTTP_REQUEST_MAX_READ_TIMEOUT: int = 600 HTTP_REQUEST_MAX_WRITE_TIMEOUT: int = 600 @@ -62,6 +53,4 @@ def HTTP_REQUEST_NODE_READABLE_MAX_TEXT_SIZE(self) -> str: SSRF_PROXY_HTTP_URL: str | None = None SSRF_PROXY_HTTPS_URL: str | None = None - MODERATION_BUFFER_SIZE: int = Field(default=300, description='The buffer size for moderation.') - MAX_VARIABLE_SIZE: int = Field(default=5 * 1024, description='The maximum size of a variable. default is 5KB.') diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py index 27de2c04613af1..af88f073a1b3b5 100644 --- a/api/configs/feature/__init__.py +++ b/api/configs/feature/__init__.py @@ -52,6 +52,15 @@ class CodeExecutionSandboxConfig(BaseSettings): default='dify-sandbox', ) + CODE_MAX_NUMBER: int = 9223372036854775807 + CODE_MIN_NUMBER: int = -9223372036854775808 + CODE_MAX_DEPTH: int = 5 + CODE_MAX_PRECISION: int = 20 + CODE_MAX_STRING_LENGTH: int = 80000 + CODE_MAX_STRING_ARRAY_LENGTH: int = 30 + CODE_MAX_OBJECT_ARRAY_LENGTH: int = 30 + CODE_MAX_NUMBER_ARRAY_LENGTH: int = 1000 + class EndpointConfig(BaseSettings): """ @@ -291,8 +300,7 @@ class ModerationConfig(BaseSettings): Moderation in app configs. """ - # todo: to be clarified in usage and unit - OUTPUT_MODERATION_BUFFER_SIZE: PositiveInt = Field( + MODERATION_BUFFER_SIZE: PositiveInt = Field( description='buffer size for moderation', default=300, ) From bba8952c1e4d4afc71b29ded10bb1a9380dfe496 Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Fri, 23 Aug 2024 17:51:52 +0800 Subject: [PATCH 02/11] update --- api/configs/app_config.py | 1 - api/configs/feature/__init__.py | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/api/configs/app_config.py b/api/configs/app_config.py index 9978ec3b15eadb..d4eddf4890a957 100644 --- a/api/configs/app_config.py +++ b/api/configs/app_config.py @@ -53,4 +53,3 @@ def HTTP_REQUEST_NODE_READABLE_MAX_TEXT_SIZE(self) -> str: SSRF_PROXY_HTTP_URL: str | None = None SSRF_PROXY_HTTPS_URL: str | None = None - MAX_VARIABLE_SIZE: int = Field(default=5 * 1024, description='The maximum size of a variable. default is 5KB.') diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py index af88f073a1b3b5..ecf90a1ae21b26 100644 --- a/api/configs/feature/__init__.py +++ b/api/configs/feature/__init__.py @@ -264,6 +264,11 @@ class WorkflowConfig(BaseSettings): default=5, ) + MAX_VARIABLE_SIZE: PositiveInt = Field( + description='The maximum size in bytes of a variable. default to 5KB.', + default=5 * 1024, + ) + class OAuthConfig(BaseSettings): """ From 829f956bfc9db0ecb985ce51053d7449eef02d46 Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Fri, 23 Aug 2024 18:07:52 +0800 Subject: [PATCH 03/11] update --- api/configs/app_config.py | 25 +++---------------- api/configs/deploy/__init__.py | 5 ++++ api/configs/feature/__init__.py | 18 ++++++++++++- .../nodes/http_request/http_executor.py | 15 +++++------ 4 files changed, 32 insertions(+), 31 deletions(-) diff --git a/api/configs/app_config.py b/api/configs/app_config.py index d4eddf4890a957..ff8c77de4836d3 100644 --- a/api/configs/app_config.py +++ b/api/configs/app_config.py @@ -1,4 +1,3 @@ -from pydantic import Field, computed_field from pydantic_settings import SettingsConfigDict from configs.deploy import DeploymentConfig @@ -24,8 +23,6 @@ class DifyConfig( # **Before using, please contact business@dify.ai by email to inquire about licensing matters.** EnterpriseFeatureConfig, ): - DEBUG: bool = Field(default=False, description='whether to enable debug mode.') - model_config = SettingsConfigDict( # read from dotenv format config file env_file='.env', @@ -35,21 +32,7 @@ class DifyConfig( extra='ignore', ) - HTTP_REQUEST_MAX_CONNECT_TIMEOUT: int = 300 - HTTP_REQUEST_MAX_READ_TIMEOUT: int = 600 - HTTP_REQUEST_MAX_WRITE_TIMEOUT: int = 600 - HTTP_REQUEST_NODE_MAX_BINARY_SIZE: int = 1024 * 1024 * 10 - - @computed_field - def HTTP_REQUEST_NODE_READABLE_MAX_BINARY_SIZE(self) -> str: - return f'{self.HTTP_REQUEST_NODE_MAX_BINARY_SIZE / 1024 / 1024:.2f}MB' - - HTTP_REQUEST_NODE_MAX_TEXT_SIZE: int = 1024 * 1024 - - @computed_field - def HTTP_REQUEST_NODE_READABLE_MAX_TEXT_SIZE(self) -> str: - return f'{self.HTTP_REQUEST_NODE_MAX_TEXT_SIZE / 1024 / 1024:.2f}MB' - - SSRF_PROXY_HTTP_URL: str | None = None - SSRF_PROXY_HTTPS_URL: str | None = None - + # Before adding any config, + # please consider to arrange it in the proper config group of existed or added + # for better readability and maintainability. + # Thanks for your concentration and consideration. diff --git a/api/configs/deploy/__init__.py b/api/configs/deploy/__init__.py index 219b315784323e..c99e3d21d247d7 100644 --- a/api/configs/deploy/__init__.py +++ b/api/configs/deploy/__init__.py @@ -11,6 +11,11 @@ class DeploymentConfig(BaseSettings): default='langgenius/dify', ) + DEBUG: bool = Field( + description='whether to enable debug mode.', + default=False, + ) + TESTING: bool = Field( description='', default=False, diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py index ecf90a1ae21b26..7669f3b19184d2 100644 --- a/api/configs/feature/__init__.py +++ b/api/configs/feature/__init__.py @@ -166,6 +166,23 @@ def CONSOLE_CORS_ALLOW_ORIGINS(self) -> list[str]: def WEB_API_CORS_ALLOW_ORIGINS(self) -> list[str]: return self.inner_WEB_API_CORS_ALLOW_ORIGINS.split(',') + HTTP_REQUEST_MAX_CONNECT_TIMEOUT: int = 300 + HTTP_REQUEST_MAX_READ_TIMEOUT: int = 600 + HTTP_REQUEST_MAX_WRITE_TIMEOUT: int = 600 + HTTP_REQUEST_NODE_MAX_BINARY_SIZE: int = 1024 * 1024 * 10 + HTTP_REQUEST_NODE_MAX_TEXT_SIZE: int = 1024 * 1024 + + @computed_field + def HTTP_REQUEST_NODE_READABLE_MAX_BINARY_SIZE(self) -> str: + return f'{self.HTTP_REQUEST_NODE_MAX_BINARY_SIZE / 1024 / 1024:.2f}MB' + + @computed_field + def HTTP_REQUEST_NODE_READABLE_MAX_TEXT_SIZE(self) -> str: + return f'{self.HTTP_REQUEST_NODE_MAX_TEXT_SIZE / 1024 / 1024:.2f}MB' + + SSRF_PROXY_HTTP_URL: Optional[str] = None + SSRF_PROXY_HTTPS_URL: Optional[str] = None + class InnerAPIConfig(BaseSettings): """ @@ -457,7 +474,6 @@ class CeleryBeatConfig(BaseSettings): class PositionConfig(BaseSettings): - POSITION_PROVIDER_PINS: str = Field( description='The heads of model providers', default='', diff --git a/api/core/workflow/nodes/http_request/http_executor.py b/api/core/workflow/nodes/http_request/http_executor.py index db18bd00b2d9e1..e2d8ed3e2a4637 100644 --- a/api/core/workflow/nodes/http_request/http_executor.py +++ b/api/core/workflow/nodes/http_request/http_executor.py @@ -18,11 +18,6 @@ ) from core.workflow.utils.variable_template_parser import VariableTemplateParser -MAX_BINARY_SIZE = dify_config.HTTP_REQUEST_NODE_MAX_BINARY_SIZE -READABLE_MAX_BINARY_SIZE = dify_config.HTTP_REQUEST_NODE_READABLE_MAX_BINARY_SIZE -MAX_TEXT_SIZE = dify_config.HTTP_REQUEST_NODE_MAX_TEXT_SIZE -READABLE_MAX_TEXT_SIZE = dify_config.HTTP_REQUEST_NODE_READABLE_MAX_TEXT_SIZE - class HttpExecutorResponse: headers: dict[str, str] @@ -238,14 +233,16 @@ def _validate_and_parse_response(self, response: httpx.Response) -> HttpExecutor raise ValueError(f'Invalid response type {type(response)}') if executor_response.is_file: - if executor_response.size > MAX_BINARY_SIZE: + if executor_response.size > dify_config.HTTP_REQUEST_NODE_MAX_BINARY_SIZE: raise ValueError( - f'File size is too large, max size is {READABLE_MAX_BINARY_SIZE}, but current size is {executor_response.readable_size}.' + f'File size is too large, max size is {dify_config.HTTP_REQUEST_NODE_READABLE_MAX_TEXT_SIZE},' + f' but current size is {executor_response.readable_size}.' ) else: - if executor_response.size > MAX_TEXT_SIZE: + if executor_response.size > dify_config.HTTP_REQUEST_NODE_MAX_TEXT_SIZE: raise ValueError( - f'Text size is too large, max size is {READABLE_MAX_TEXT_SIZE}, but current size is {executor_response.readable_size}.' + f'Text size is too large, max size is {dify_config.HTTP_REQUEST_NODE_READABLE_MAX_TEXT_SIZE},' + f' but current size is {executor_response.readable_size}.' ) return executor_response From 5814cf022a091ea206240f6d609ed959fa164c0f Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Fri, 23 Aug 2024 18:09:40 +0800 Subject: [PATCH 04/11] update --- api/configs/feature/__init__.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py index 7669f3b19184d2..541951f23c5127 100644 --- a/api/configs/feature/__init__.py +++ b/api/configs/feature/__init__.py @@ -180,8 +180,15 @@ def HTTP_REQUEST_NODE_READABLE_MAX_BINARY_SIZE(self) -> str: def HTTP_REQUEST_NODE_READABLE_MAX_TEXT_SIZE(self) -> str: return f'{self.HTTP_REQUEST_NODE_MAX_TEXT_SIZE / 1024 / 1024:.2f}MB' - SSRF_PROXY_HTTP_URL: Optional[str] = None - SSRF_PROXY_HTTPS_URL: Optional[str] = None + SSRF_PROXY_HTTP_URL: Optional[str] = Field( + description='HTTP URL for SSRF proxy', + default=None, + ) + + SSRF_PROXY_HTTPS_URL: Optional[str] = Field( + description='HTTPS URL for SSRF proxy', + default=None, + ) class InnerAPIConfig(BaseSettings): From d5b39cf85fb5cc6bbb254fbe9b50775636c514ef Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Fri, 23 Aug 2024 18:13:21 +0800 Subject: [PATCH 05/11] update --- api/core/workflow/nodes/http_request/http_executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/core/workflow/nodes/http_request/http_executor.py b/api/core/workflow/nodes/http_request/http_executor.py index e2d8ed3e2a4637..11aad7aac2c4be 100644 --- a/api/core/workflow/nodes/http_request/http_executor.py +++ b/api/core/workflow/nodes/http_request/http_executor.py @@ -235,7 +235,7 @@ def _validate_and_parse_response(self, response: httpx.Response) -> HttpExecutor if executor_response.is_file: if executor_response.size > dify_config.HTTP_REQUEST_NODE_MAX_BINARY_SIZE: raise ValueError( - f'File size is too large, max size is {dify_config.HTTP_REQUEST_NODE_READABLE_MAX_TEXT_SIZE},' + f'File size is too large, max size is {dify_config.HTTP_REQUEST_NODE_MAX_BINARY_SIZE},' f' but current size is {executor_response.readable_size}.' ) else: From 475744f7f149bcba35fc3692989f9ed1acb6f711 Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Fri, 23 Aug 2024 21:26:00 +0800 Subject: [PATCH 06/11] update --- api/configs/feature/__init__.py | 36 +++++++++++++++---- api/core/workflow/nodes/code/code_node.py | 43 +++++++++++------------ 2 files changed, 50 insertions(+), 29 deletions(-) diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py index 541951f23c5127..b1e9eed42fbb20 100644 --- a/api/configs/feature/__init__.py +++ b/api/configs/feature/__init__.py @@ -54,12 +54,36 @@ class CodeExecutionSandboxConfig(BaseSettings): CODE_MAX_NUMBER: int = 9223372036854775807 CODE_MIN_NUMBER: int = -9223372036854775808 - CODE_MAX_DEPTH: int = 5 - CODE_MAX_PRECISION: int = 20 - CODE_MAX_STRING_LENGTH: int = 80000 - CODE_MAX_STRING_ARRAY_LENGTH: int = 30 - CODE_MAX_OBJECT_ARRAY_LENGTH: int = 30 - CODE_MAX_NUMBER_ARRAY_LENGTH: int = 1000 + + CODE_MAX_DEPTH: PositiveInt = Field( + description='max depth for code execution', + default=5, + ) + + CODE_MAX_PRECISION: PositiveInt = Field( + description='max precision digits for float type in code execution', + default=20, + ) + + CODE_MAX_STRING_LENGTH: PositiveInt = Field( + description='max string length for code execution', + default=80000, + ) + + CODE_MAX_STRING_ARRAY_LENGTH: PositiveInt = Field( + description='', + default=30, + ) + + CODE_MAX_OBJECT_ARRAY_LENGTH: PositiveInt = Field( + description='', + default=30, + ) + + CODE_MAX_NUMBER_ARRAY_LENGTH: PositiveInt = Field( + description='', + default=1000, + ) class EndpointConfig(BaseSettings): diff --git a/api/core/workflow/nodes/code/code_node.py b/api/core/workflow/nodes/code/code_node.py index 71adf18a51c134..17554d3db4574b 100644 --- a/api/core/workflow/nodes/code/code_node.py +++ b/api/core/workflow/nodes/code/code_node.py @@ -11,15 +11,6 @@ from core.workflow.nodes.code.entities import CodeNodeData from models.workflow import WorkflowNodeExecutionStatus -MAX_NUMBER = dify_config.CODE_MAX_NUMBER -MIN_NUMBER = dify_config.CODE_MIN_NUMBER -MAX_PRECISION = dify_config.CODE_MAX_PRECISION -MAX_DEPTH = dify_config.CODE_MAX_DEPTH -MAX_STRING_LENGTH = dify_config.CODE_MAX_STRING_LENGTH -MAX_STRING_ARRAY_LENGTH = dify_config.CODE_MAX_STRING_ARRAY_LENGTH -MAX_OBJECT_ARRAY_LENGTH = dify_config.CODE_MAX_OBJECT_ARRAY_LENGTH -MAX_NUMBER_ARRAY_LENGTH = dify_config.CODE_MAX_NUMBER_ARRAY_LENGTH - class CodeNode(BaseNode): _node_data_cls = CodeNodeData @@ -97,8 +88,9 @@ def _check_string(self, value: str, variable: str) -> str: else: raise ValueError(f"Output variable `{variable}` must be a string") - if len(value) > MAX_STRING_LENGTH: - raise ValueError(f'The length of output variable `{variable}` must be less than {MAX_STRING_LENGTH} characters') + if len(value) > dify_config.CODE_MAX_STRING_ARRAY_LENGTH: + raise ValueError(f'The length of output variable `{variable}` must be' + f' less than {dify_config.CODE_MAX_STRING_ARRAY_LENGTH} characters') return value.replace('\x00', '') @@ -115,13 +107,15 @@ def _check_number(self, value: Union[int, float], variable: str) -> Union[int, f else: raise ValueError(f"Output variable `{variable}` must be a number") - if value > MAX_NUMBER or value < MIN_NUMBER: - raise ValueError(f'Output variable `{variable}` is out of range, it must be between {MIN_NUMBER} and {MAX_NUMBER}.') + if value > dify_config.CODE_MAX_NUMBER or value < dify_config.CODE_MIN_NUMBER: + raise ValueError(f'Output variable `{variable}` is out of range,' + f' it must be between {dify_config.CODE_MIN_NUMBER} and {dify_config.CODE_MAX_NUMBER}.') if isinstance(value, float): # raise error if precision is too high - if len(str(value).split('.')[1]) > MAX_PRECISION: - raise ValueError(f'Output variable `{variable}` has too high precision, it must be less than {MAX_PRECISION} digits.') + if len(str(value).split('.')[1]) > dify_config.CODE_MAX_PRECISION: + raise ValueError(f'Output variable `{variable}` has too high precision,' + f' it must be less than {dify_config.CODE_MAX_PRECISION} digits.') return value @@ -134,8 +128,8 @@ def _transform_result(self, result: dict, output_schema: Optional[dict[str, Code :param output_schema: output schema :return: """ - if depth > MAX_DEPTH: - raise ValueError("Depth limit reached, object too deep.") + if depth > dify_config.CODE_MAX_DEPTH: + raise ValueError(f"Depth limit ${dify_config.CODE_MAX_DEPTH} reached, object too deep.") transformed_result = {} if output_schema is None: @@ -235,9 +229,10 @@ def _transform_result(self, result: dict, output_schema: Optional[dict[str, Code f'Output {prefix}{dot}{output_name} is not an array, got {type(result.get(output_name))} instead.' ) else: - if len(result[output_name]) > MAX_NUMBER_ARRAY_LENGTH: + if len(result[output_name]) > dify_config.CODE_MAX_NUMBER_ARRAY_LENGTH: raise ValueError( - f'The length of output variable `{prefix}{dot}{output_name}` must be less than {MAX_NUMBER_ARRAY_LENGTH} elements.' + f'The length of output variable `{prefix}{dot}{output_name}` must be' + f' less than {dify_config.CODE_MAX_NUMBER_ARRAY_LENGTH} elements.' ) transformed_result[output_name] = [ @@ -257,9 +252,10 @@ def _transform_result(self, result: dict, output_schema: Optional[dict[str, Code f'Output {prefix}{dot}{output_name} is not an array, got {type(result.get(output_name))} instead.' ) else: - if len(result[output_name]) > MAX_STRING_ARRAY_LENGTH: + if len(result[output_name]) > dify_config.CODE_MAX_STRING_ARRAY_LENGTH: raise ValueError( - f'The length of output variable `{prefix}{dot}{output_name}` must be less than {MAX_STRING_ARRAY_LENGTH} elements.' + f'The length of output variable `{prefix}{dot}{output_name}` must be' + f' less than {dify_config.CODE_MAX_STRING_ARRAY_LENGTH} elements.' ) transformed_result[output_name] = [ @@ -279,9 +275,10 @@ def _transform_result(self, result: dict, output_schema: Optional[dict[str, Code f'Output {prefix}{dot}{output_name} is not an array, got {type(result.get(output_name))} instead.' ) else: - if len(result[output_name]) > MAX_OBJECT_ARRAY_LENGTH: + if len(result[output_name]) > dify_config.CODE_MAX_OBJECT_ARRAY_LENGTH: raise ValueError( - f'The length of output variable `{prefix}{dot}{output_name}` must be less than {MAX_OBJECT_ARRAY_LENGTH} elements.' + f'The length of output variable `{prefix}{dot}{output_name}` must be' + f' less than {dify_config.CODE_MAX_OBJECT_ARRAY_LENGTH} elements.' ) for i, value in enumerate(result[output_name]): From 399f8ca3c37ecd3b764771366c73c7c26dde64b4 Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Fri, 23 Aug 2024 21:36:56 +0800 Subject: [PATCH 07/11] update --- api/configs/feature/__init__.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py index b1e9eed42fbb20..fd9f3d06cca4bd 100644 --- a/api/configs/feature/__init__.py +++ b/api/configs/feature/__init__.py @@ -1,6 +1,6 @@ from typing import Optional -from pydantic import AliasChoices, Field, NonNegativeInt, PositiveInt, computed_field +from pydantic import AliasChoices, Field, NegativeInt, NonNegativeInt, PositiveInt, computed_field from pydantic_settings import BaseSettings from configs.feature.hosted_service import HostedServiceConfig @@ -52,8 +52,15 @@ class CodeExecutionSandboxConfig(BaseSettings): default='dify-sandbox', ) - CODE_MAX_NUMBER: int = 9223372036854775807 - CODE_MIN_NUMBER: int = -9223372036854775808 + CODE_MAX_NUMBER: PositiveInt =Field( + description='max depth for code execution', + default=-9223372036854775807, + ) + + CODE_MIN_NUMBER: NegativeInt =Field( + description='', + default=-9223372036854775807, + ) CODE_MAX_DEPTH: PositiveInt = Field( description='max depth for code execution', From 340190ee65c81411452331f12183e5adbe1a038f Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Fri, 23 Aug 2024 21:41:47 +0800 Subject: [PATCH 08/11] update --- api/configs/feature/__init__.py | 33 +++++++++++++++---- .../nodes/http_request/http_executor.py | 2 +- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py index fd9f3d06cca4bd..b32da10ec90263 100644 --- a/api/configs/feature/__init__.py +++ b/api/configs/feature/__init__.py @@ -52,12 +52,12 @@ class CodeExecutionSandboxConfig(BaseSettings): default='dify-sandbox', ) - CODE_MAX_NUMBER: PositiveInt =Field( + CODE_MAX_NUMBER: PositiveInt = Field( description='max depth for code execution', default=-9223372036854775807, ) - CODE_MIN_NUMBER: NegativeInt =Field( + CODE_MIN_NUMBER: NegativeInt = Field( description='', default=-9223372036854775807, ) @@ -197,11 +197,30 @@ def CONSOLE_CORS_ALLOW_ORIGINS(self) -> list[str]: def WEB_API_CORS_ALLOW_ORIGINS(self) -> list[str]: return self.inner_WEB_API_CORS_ALLOW_ORIGINS.split(',') - HTTP_REQUEST_MAX_CONNECT_TIMEOUT: int = 300 - HTTP_REQUEST_MAX_READ_TIMEOUT: int = 600 - HTTP_REQUEST_MAX_WRITE_TIMEOUT: int = 600 - HTTP_REQUEST_NODE_MAX_BINARY_SIZE: int = 1024 * 1024 * 10 - HTTP_REQUEST_NODE_MAX_TEXT_SIZE: int = 1024 * 1024 + HTTP_REQUEST_MAX_CONNECT_TIMEOUT: NonNegativeInt = Field( + description='', + default=300, + ) + + HTTP_REQUEST_MAX_READ_TIMEOUT: NonNegativeInt = Field( + description='', + default=600, + ) + + HTTP_REQUEST_MAX_WRITE_TIMEOUT: NonNegativeInt = Field( + description='', + default=600, + ) + + HTTP_REQUEST_NODE_MAX_BINARY_SIZE: PositiveInt = Field( + description='', + default=1024 * 1024 * 10, + ) + + HTTP_REQUEST_NODE_MAX_TEXT_SIZE: PositiveInt = Field( + description='', + default=1024 * 1024 * 1, + ) @computed_field def HTTP_REQUEST_NODE_READABLE_MAX_BINARY_SIZE(self) -> str: diff --git a/api/core/workflow/nodes/http_request/http_executor.py b/api/core/workflow/nodes/http_request/http_executor.py index 11aad7aac2c4be..ecb2062d4555a3 100644 --- a/api/core/workflow/nodes/http_request/http_executor.py +++ b/api/core/workflow/nodes/http_request/http_executor.py @@ -235,7 +235,7 @@ def _validate_and_parse_response(self, response: httpx.Response) -> HttpExecutor if executor_response.is_file: if executor_response.size > dify_config.HTTP_REQUEST_NODE_MAX_BINARY_SIZE: raise ValueError( - f'File size is too large, max size is {dify_config.HTTP_REQUEST_NODE_MAX_BINARY_SIZE},' + f'File size is too large, max size is {dify_config.HTTP_REQUEST_NODE_READABLE_MAX_BINARY_SIZE},' f' but current size is {executor_response.readable_size}.' ) else: From c0cf4a25b7a2de6a0671048dafe5da3214d5d784 Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Fri, 23 Aug 2024 21:49:18 +0800 Subject: [PATCH 09/11] update --- api/configs/feature/__init__.py | 12 ++--------- .../nodes/http_request/http_executor.py | 20 ++++++++----------- 2 files changed, 10 insertions(+), 22 deletions(-) diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py index b32da10ec90263..c83b4410ce70c4 100644 --- a/api/configs/feature/__init__.py +++ b/api/configs/feature/__init__.py @@ -214,22 +214,14 @@ def WEB_API_CORS_ALLOW_ORIGINS(self) -> list[str]: HTTP_REQUEST_NODE_MAX_BINARY_SIZE: PositiveInt = Field( description='', - default=1024 * 1024 * 10, + default=10 * 1024 * 1024, ) HTTP_REQUEST_NODE_MAX_TEXT_SIZE: PositiveInt = Field( description='', - default=1024 * 1024 * 1, + default=1 * 1024 * 1024, ) - @computed_field - def HTTP_REQUEST_NODE_READABLE_MAX_BINARY_SIZE(self) -> str: - return f'{self.HTTP_REQUEST_NODE_MAX_BINARY_SIZE / 1024 / 1024:.2f}MB' - - @computed_field - def HTTP_REQUEST_NODE_READABLE_MAX_TEXT_SIZE(self) -> str: - return f'{self.HTTP_REQUEST_NODE_MAX_TEXT_SIZE / 1024 / 1024:.2f}MB' - SSRF_PROXY_HTTP_URL: Optional[str] = Field( description='HTTP URL for SSRF proxy', default=None, diff --git a/api/core/workflow/nodes/http_request/http_executor.py b/api/core/workflow/nodes/http_request/http_executor.py index ecb2062d4555a3..d16bff58bda12b 100644 --- a/api/core/workflow/nodes/http_request/http_executor.py +++ b/api/core/workflow/nodes/http_request/http_executor.py @@ -232,18 +232,14 @@ def _validate_and_parse_response(self, response: httpx.Response) -> HttpExecutor else: raise ValueError(f'Invalid response type {type(response)}') - if executor_response.is_file: - if executor_response.size > dify_config.HTTP_REQUEST_NODE_MAX_BINARY_SIZE: - raise ValueError( - f'File size is too large, max size is {dify_config.HTTP_REQUEST_NODE_READABLE_MAX_BINARY_SIZE},' - f' but current size is {executor_response.readable_size}.' - ) - else: - if executor_response.size > dify_config.HTTP_REQUEST_NODE_MAX_TEXT_SIZE: - raise ValueError( - f'Text size is too large, max size is {dify_config.HTTP_REQUEST_NODE_READABLE_MAX_TEXT_SIZE},' - f' but current size is {executor_response.readable_size}.' - ) + threshold_size = dify_config.HTTP_REQUEST_NODE_MAX_BINARY_SIZE if executor_response.is_file \ + else dify_config.HTTP_REQUEST_NODE_MAX_TEXT_SIZE + if executor_response.size > threshold_size: + raise ValueError( + f'{"File" if executor_response.is_file else "Text"} size is too large,' + f' max size is {threshold_size / 1024 / 1024:.2f} MB,' + f' but current size is {executor_response.readable_size}.' + ) return executor_response From 05d86997ff066372d5faee39aca5728606f48326 Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Fri, 23 Aug 2024 22:06:31 +0800 Subject: [PATCH 10/11] fix --- api/configs/feature/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py index c83b4410ce70c4..7f36abf7a6c0a4 100644 --- a/api/configs/feature/__init__.py +++ b/api/configs/feature/__init__.py @@ -54,7 +54,7 @@ class CodeExecutionSandboxConfig(BaseSettings): CODE_MAX_NUMBER: PositiveInt = Field( description='max depth for code execution', - default=-9223372036854775807, + default=9223372036854775807, ) CODE_MIN_NUMBER: NegativeInt = Field( From 635dc7ba633e33eca3f48596355e4110a494036a Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Fri, 23 Aug 2024 22:22:33 +0800 Subject: [PATCH 11/11] update --- api/poetry.lock | 9 +++++---- api/pyproject.toml | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/api/poetry.lock b/api/poetry.lock index a68eadefcfa21a..0a8919a30aa3c3 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -6372,13 +6372,13 @@ semver = ["semver (>=3.0.2)"] [[package]] name = "pydantic-settings" -version = "2.3.4" +version = "2.4.0" description = "Settings management using Pydantic" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic_settings-2.3.4-py3-none-any.whl", hash = "sha256:11ad8bacb68a045f00e4f862c7a718c8a9ec766aa8fd4c32e39a0594b207b53a"}, - {file = "pydantic_settings-2.3.4.tar.gz", hash = "sha256:c5802e3d62b78e82522319bbc9b8f8ffb28ad1c988a99311d04f2a6051fca0a7"}, + {file = "pydantic_settings-2.4.0-py3-none-any.whl", hash = "sha256:bb6849dc067f1687574c12a639e231f3a6feeed0a12d710c1382045c5db1c315"}, + {file = "pydantic_settings-2.4.0.tar.gz", hash = "sha256:ed81c3a0f46392b4d7c0a565c05884e6e54b3456e6f0fe4d8814981172dc9a88"}, ] [package.dependencies] @@ -6386,6 +6386,7 @@ pydantic = ">=2.7.0" python-dotenv = ">=0.21.0" [package.extras] +azure-key-vault = ["azure-identity (>=1.16.0)", "azure-keyvault-secrets (>=4.8.0)"] toml = ["tomli (>=2.0.1)"] yaml = ["pyyaml (>=6.0.1)"] @@ -9633,4 +9634,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "69c20af8ecacced3cca092662223a1511acaf65cb2616a5a1e38b498223463e0" +content-hash = "d7336115709114c2a4ff09b392f717e9c3547ae82b6a111d0c885c7a44269f02" diff --git a/api/pyproject.toml b/api/pyproject.toml index 8e050c4101fc9a..47b638573b971a 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -162,7 +162,7 @@ pandas = { version = "~2.2.2", extras = ["performance", "excel"] } psycopg2-binary = "~2.9.6" pycryptodome = "3.19.1" pydantic = "~2.8.2" -pydantic-settings = "~2.3.4" +pydantic-settings = "~2.4.0" pydantic_extra_types = "~2.9.0" pyjwt = "~2.8.0" pypdfium2 = "~4.17.0"