Chore/remove python dependencies selector (#7494)

api/core/helper/code_executor/code_executor.py

@@ -1,15 +1,13 @@
 import logging
-import time
 from enum import Enum
 from threading import Lock
-from typing import Literal, Optional
+from typing import Optional
 
-from httpx import Timeout, get, post
+from httpx import Timeout, post
 from pydantic import BaseModel
 from yarl import URL
 
 from configs import dify_config
-from core.helper.code_executor.entities import CodeDependency
 from core.helper.code_executor.javascript.javascript_transformer import NodeJsTemplateTransformer
 from core.helper.code_executor.jinja2.jinja2_transformer import Jinja2TemplateTransformer
 from core.helper.code_executor.python3.python3_transformer import Python3TemplateTransformer
@@ -66,8 +64,7 @@ class CodeExecutor:
     def execute_code(cls, 
                      language: CodeLanguage, 
                      preload: str, 
-                     code: str, 
-                     dependencies: Optional[list[CodeDependency]] = None) -> str:
+                     code: str) -> str:
         """
         Execute code
         :param language: code language
@@ -87,9 +84,6 @@ def execute_code(cls,
             'enable_network': True
         }
 
-        if dependencies:
-            data['dependencies'] = [dependency.model_dump() for dependency in dependencies]
-
         try:
             response = post(str(url), json=data, headers=headers, timeout=CODE_EXECUTION_TIMEOUT)
             if response.status_code == 503:
@@ -119,7 +113,7 @@ def execute_code(cls,
         return response.data.stdout or ''
 
     @classmethod
-    def execute_workflow_code_template(cls, language: CodeLanguage, code: str, inputs: dict, dependencies: Optional[list[CodeDependency]] = None) -> dict:
+    def execute_workflow_code_template(cls, language: CodeLanguage, code: str, inputs: dict) -> dict:
         """
         Execute code
         :param language: code language
@@ -131,67 +125,12 @@ def execute_workflow_code_template(cls, language: CodeLanguage, code: str, input
         if not template_transformer:
             raise CodeExecutionException(f'Unsupported language {language}')
 
-        runner, preload, dependencies = template_transformer.transform_caller(code, inputs, dependencies)
+        runner, preload = template_transformer.transform_caller(code, inputs)
 
         try:
-            response = cls.execute_code(language, preload, runner, dependencies)
+            response = cls.execute_code(language, preload, runner)
         except CodeExecutionException as e:
             raise e
 
         return template_transformer.transform_response(response)
-
-    @classmethod
-    def list_dependencies(cls, language: str) -> list[CodeDependency]:
-        if language not in cls.supported_dependencies_languages:
-            return []
-
-        with cls.dependencies_cache_lock:
-            if language in cls.dependencies_cache:
-                # check expiration
-                dependencies = cls.dependencies_cache[language]
-                if dependencies['expiration'] > time.time():
-                    return dependencies['data']
-                # remove expired cache
-                del cls.dependencies_cache[language]
-
-        dependencies = cls._get_dependencies(language)
-        with cls.dependencies_cache_lock:
-            cls.dependencies_cache[language] = {
-                'data': dependencies,
-                'expiration': time.time() + 60
-            }
-
-        return dependencies
-
-    @classmethod
-    def _get_dependencies(cls, language: Literal['python3']) -> list[CodeDependency]:
-        """
-        List dependencies
-        """
-        url = URL(CODE_EXECUTION_ENDPOINT) / 'v1' / 'sandbox' / 'dependencies'
-
-        headers = {
-            'X-Api-Key': CODE_EXECUTION_API_KEY
-        }
-
-        running_language = cls.code_language_to_running_language.get(language)
-        if isinstance(running_language, Enum):
-            running_language = running_language.value
-
-        data = {
-            'language': running_language,
-        }
-
-        try:
-            response = get(str(url), params=data, headers=headers, timeout=CODE_EXECUTION_TIMEOUT)
-            if response.status_code != 200:
-                raise Exception(f'Failed to list dependencies, got status code {response.status_code}, please check if the sandbox service is running')
-            response = response.json()
-            dependencies = response.get('data', {}).get('dependencies', [])
-            return [
-                CodeDependency(**dependency) for dependency in dependencies
-                if dependency.get('name') not in Python3TemplateTransformer.get_standard_packages()
-            ]
-        except Exception as e:
-            logger.exception(f'Failed to list dependencies: {e}')
-            return []
+

api/core/helper/code_executor/code_node_provider.py

@@ -2,8 +2,6 @@
 
 from pydantic import BaseModel
 
-from core.helper.code_executor.code_executor import CodeExecutor
-
 
 class CodeNodeProvider(BaseModel):
     @staticmethod
@@ -23,10 +21,6 @@ def get_default_code(cls) -> str:
         """
         pass
 
-    @classmethod
-    def get_default_available_packages(cls) -> list[dict]:
-        return [p.model_dump() for p in CodeExecutor.list_dependencies(cls.get_language())]
-
     @classmethod
     def get_default_config(cls) -> dict:
         return {
@@ -50,6 +44,5 @@ def get_default_config(cls) -> dict:
                         "children": None
                     }
                 }
-            },
-            "available_dependencies": cls.get_default_available_packages(),
+            }
         }

api/core/helper/code_executor/jinja2/jinja2_formatter.py

@@ -3,7 +3,7 @@
 
 class Jinja2Formatter:
     @classmethod
-    def format(cls, template: str, inputs: str) -> str:
+    def format(cls, template: str, inputs: dict) -> str:
         """
         Format template
         :param template: template

api/core/helper/code_executor/jinja2/jinja2_transformer.py

@@ -1,14 +1,9 @@
 from textwrap import dedent
 
-from core.helper.code_executor.python3.python3_transformer import Python3TemplateTransformer
 from core.helper.code_executor.template_transformer import TemplateTransformer
 
 
 class Jinja2TemplateTransformer(TemplateTransformer):
-    @classmethod
-    def get_standard_packages(cls) -> set[str]:
-        return {'jinja2'} | Python3TemplateTransformer.get_standard_packages()
-
     @classmethod
     def transform_response(cls, response: str) -> dict:
         """

api/core/helper/code_executor/python3/python3_transformer.py

@@ -4,30 +4,6 @@
 
 
 class Python3TemplateTransformer(TemplateTransformer):
-    @classmethod
-    def get_standard_packages(cls) -> set[str]:
-        return {
-            'base64',
-            'binascii',
-            'collections',
-            'datetime',
-            'functools',
-            'hashlib',
-            'hmac',
-            'itertools',
-            'json',
-            'math',
-            'operator',
-            'os',
-            'random',
-            're',
-            'string',
-            'sys',
-            'time',
-            'traceback',
-            'uuid',
-        }
-
     @classmethod
     def get_runner_script(cls) -> str:
         runner_script = dedent(f"""

api/core/helper/code_executor/template_transformer.py

@@ -2,9 +2,6 @@
 import re
 from abc import ABC, abstractmethod
 from base64 import b64encode
-from typing import Optional
-
-from core.helper.code_executor.entities import CodeDependency
 
 
 class TemplateTransformer(ABC):
@@ -13,12 +10,7 @@ class TemplateTransformer(ABC):
     _result_tag: str = '<<RESULT>>'
 
     @classmethod
-    def get_standard_packages(cls) -> set[str]:
-        return set()
-
-    @classmethod
-    def transform_caller(cls, code: str, inputs: dict,
-                         dependencies: Optional[list[CodeDependency]] = None) -> tuple[str, str, list[CodeDependency]]:
+    def transform_caller(cls, code: str, inputs: dict) -> tuple[str, str]:
         """
         Transform code to python runner
         :param code: code
@@ -28,14 +20,7 @@ def transform_caller(cls, code: str, inputs: dict,
         runner_script = cls.assemble_runner_script(code, inputs)
         preload_script = cls.get_preload_script()
 
-        packages = dependencies or []
-        standard_packages = cls.get_standard_packages()
-        for package in standard_packages:
-            if package not in packages:
-                packages.append(CodeDependency(name=package, version=''))
-        packages = list({dep.name: dep for dep in packages if dep.name}.values())
-
-        return runner_script, preload_script, packages
+        return runner_script, preload_script
 
     @classmethod
     def extract_result_str_from_response(cls, response: str) -> str:

api/core/workflow/nodes/code/code_node.py

@@ -67,7 +67,6 @@ def _run(self, variable_pool: VariablePool) -> NodeRunResult:
                 language=code_language,
                 code=code,
                 inputs=variables,
-                dependencies=node_data.dependencies
             )
 
             # Transform result

api/core/workflow/nodes/code/entities.py

@@ -3,7 +3,6 @@
 from pydantic import BaseModel
 
 from core.helper.code_executor.code_executor import CodeLanguage
-from core.helper.code_executor.entities import CodeDependency
 from core.workflow.entities.base_node_data_entities import BaseNodeData
 from core.workflow.entities.variable_entities import VariableSelector
 
@@ -16,8 +15,12 @@ class Output(BaseModel):
         type: Literal['string', 'number', 'object', 'array[string]', 'array[number]', 'array[object]']
         children: Optional[dict[str, 'Output']] = None
 
+    class Dependency(BaseModel):
+        name: str
+        version: str
+
     variables: list[VariableSelector]
     code_language: Literal[CodeLanguage.PYTHON3, CodeLanguage.JAVASCRIPT]
     code: str
     outputs: dict[str, Output]
-    dependencies: Optional[list[CodeDependency]] = None
+    dependencies: Optional[list[Dependency]] = None

api/tests/integration_tests/workflow/nodes/__mock/code_executor.py

@@ -6,14 +6,13 @@
 from jinja2 import Template
 
 from core.helper.code_executor.code_executor import CodeExecutor, CodeLanguage
-from core.helper.code_executor.entities import CodeDependency
 
 MOCK = os.getenv('MOCK_SWITCH', 'false') == 'true'
 
 class MockedCodeExecutor:
     @classmethod
     def invoke(cls, language: Literal['python3', 'javascript', 'jinja2'], 
-               code: str, inputs: dict, dependencies: Optional[list[CodeDependency]] = None) -> dict:
+               code: str, inputs: dict) -> dict:
         # invoke directly
         match language:
             case CodeLanguage.PYTHON3:
@@ -24,6 +23,8 @@ def invoke(cls, language: Literal['python3', 'javascript', 'jinja2'],
                 return {
                     "result": Template(code).render(inputs)
                 }
+            case _:
+                raise Exception("Language not supported")
 
 @pytest.fixture
 def setup_code_executor_mock(request, monkeypatch: MonkeyPatch):

api/tests/integration_tests/workflow/nodes/code_executor/test_code_javascript.py

@@ -28,14 +28,6 @@ def test_javascript_with_code_template():
         inputs={'arg1': 'Hello', 'arg2': 'World'})
     assert result == {'result': 'HelloWorld'}
 
-
-def test_javascript_list_default_available_packages():
-    packages = JavascriptCodeProvider.get_default_available_packages()
-
-    # no default packages available for javascript
-    assert len(packages) == 0
-
-
 def test_javascript_get_runner_script():
     runner_script = NodeJsTemplateTransformer.get_runner_script()
     assert runner_script.count(NodeJsTemplateTransformer._code_placeholder) == 1

api/tests/integration_tests/workflow/nodes/code_executor/test_code_python3.py

@@ -29,15 +29,6 @@ def test_python3_with_code_template():
     assert result == {'result': 'HelloWorld'}
 
 
-def test_python3_list_default_available_packages():
-    packages = Python3CodeProvider.get_default_available_packages()
-    assert len(packages) > 0
-    assert {'requests', 'httpx'}.issubset(p['name'] for p in packages)
-
-    # check JSON serializable
-    assert len(str(json.dumps(packages))) > 0
-
-
 def test_python3_get_runner_script():
     runner_script = Python3TemplateTransformer.get_runner_script()
     assert runner_script.count(Python3TemplateTransformer._code_placeholder) == 1