refactor: extract storage provider configs into dify configs (#5443)

api/config.py

@@ -9,8 +9,6 @@
     'DB_PORT': '5432',
     'DB_DATABASE': 'dify',
     'DB_CHARSET': '',
-    'S3_USE_AWS_MANAGED_IAM': 'False',
-    'S3_ADDRESS_STYLE': 'auto',
     'SQLALCHEMY_DATABASE_URI_SCHEME': 'postgresql',
     'SQLALCHEMY_POOL_SIZE': 30,
     'SQLALCHEMY_MAX_OVERFLOW': 10,
@@ -100,41 +98,6 @@ def __init__(self):
             if self.CELERY_BACKEND == 'database' else self.CELERY_BROKER_URL
         self.BROKER_USE_SSL = self.CELERY_BROKER_URL.startswith('rediss://') if self.CELERY_BROKER_URL else False
 
-
-        # S3 Storage settings
-        self.S3_USE_AWS_MANAGED_IAM = get_bool_env('S3_USE_AWS_MANAGED_IAM')
-        self.S3_ENDPOINT = get_env('S3_ENDPOINT')
-        self.S3_BUCKET_NAME = get_env('S3_BUCKET_NAME')
-        self.S3_ACCESS_KEY = get_env('S3_ACCESS_KEY')
-        self.S3_SECRET_KEY = get_env('S3_SECRET_KEY')
-        self.S3_REGION = get_env('S3_REGION')
-        self.S3_ADDRESS_STYLE = get_env('S3_ADDRESS_STYLE')
-
-        # Azure Blob Storage settings
-        self.AZURE_BLOB_ACCOUNT_NAME = get_env('AZURE_BLOB_ACCOUNT_NAME')
-        self.AZURE_BLOB_ACCOUNT_KEY = get_env('AZURE_BLOB_ACCOUNT_KEY')
-        self.AZURE_BLOB_CONTAINER_NAME = get_env('AZURE_BLOB_CONTAINER_NAME')
-        self.AZURE_BLOB_ACCOUNT_URL = get_env('AZURE_BLOB_ACCOUNT_URL')
-
-        # Aliyun Storage settings
-        self.ALIYUN_OSS_BUCKET_NAME = get_env('ALIYUN_OSS_BUCKET_NAME')
-        self.ALIYUN_OSS_ACCESS_KEY = get_env('ALIYUN_OSS_ACCESS_KEY')
-        self.ALIYUN_OSS_SECRET_KEY = get_env('ALIYUN_OSS_SECRET_KEY')
-        self.ALIYUN_OSS_ENDPOINT = get_env('ALIYUN_OSS_ENDPOINT')
-        self.ALIYUN_OSS_REGION = get_env('ALIYUN_OSS_REGION')
-        self.ALIYUN_OSS_AUTH_VERSION = get_env('ALIYUN_OSS_AUTH_VERSION')
-
-        # Google Cloud Storage settings
-        self.GOOGLE_STORAGE_BUCKET_NAME = get_env('GOOGLE_STORAGE_BUCKET_NAME')
-        self.GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64 = get_env('GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64')
-
-        # Tencent Cos Storage settings
-        self.TENCENT_COS_BUCKET_NAME = get_env('TENCENT_COS_BUCKET_NAME')
-        self.TENCENT_COS_REGION = get_env('TENCENT_COS_REGION')
-        self.TENCENT_COS_SECRET_ID = get_env('TENCENT_COS_SECRET_ID')
-        self.TENCENT_COS_SECRET_KEY = get_env('TENCENT_COS_SECRET_KEY')
-        self.TENCENT_COS_SCHEME = get_env('TENCENT_COS_SCHEME')
-
         # ------------------------
         # Platform Configurations.
         # ------------------------

api/configs/middleware/__init__.py

@@ -3,6 +3,11 @@
 from pydantic import BaseModel, Field
 
 from configs.middleware.redis_config import RedisConfig
+from configs.middleware.storage.aliyun_oss_storage_config import AliyunOSSStorageConfig
+from configs.middleware.storage.amazon_s3_storage_config import S3StorageConfig
+from configs.middleware.storage.azure_blob_storage_config import AzureBlobStorageConfig
+from configs.middleware.storage.google_cloud_storage_config import GoogleCloudStorageConfig
+from configs.middleware.storage.tencent_cos_storage_config import TencentCloudCOSStorageConfig
 from configs.middleware.vdb.chroma_configs import ChromaConfigs
 from configs.middleware.vdb.milvus_configs import MilvusConfigs
 from configs.middleware.vdb.opensearch_configs import OpenSearchConfigs
@@ -48,20 +53,27 @@ class MiddlewareConfig(
     # place the configs in alphabet order
     KeywordStoreConfigs,
     RedisConfig,
+
+    # configs of storage and storage providers
     StorageConfigs,
+    AliyunOSSStorageConfig,
+    AzureBlobStorageConfig,
+    GoogleCloudStorageConfig,
+    TencentCloudCOSStorageConfig,
+    S3StorageConfig,
 
     # configs of vdb and vdb providers
     VectorStoreConfigs,
     ChromaConfigs,
     MilvusConfigs,
     OpenSearchConfigs,
+    OracleConfigs,
     PGVectorConfigs,
     PGVectoRSConfigs,
     QdrantConfigs,
     RelytConfigs,
     TencentVectorDBConfigs,
     TiDBVectorConfigs,
     WeaviateConfigs,
-    OracleConfigs,
 ):
     pass

api/configs/middleware/storage/aliyun_oss_storage_config.py

@@ -0,0 +1,39 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class AliyunOSSStorageConfig(BaseModel):
+    """
+    Aliyun storage configs
+    """
+
+    ALIYUN_OSS_BUCKET_NAME: Optional[str] = Field(
+        description='Aliyun storage ',
+        default=None,
+    )
+
+    ALIYUN_OSS_ACCESS_KEY: Optional[str] = Field(
+        description='Aliyun storage access key',
+        default=None,
+    )
+
+    ALIYUN_OSS_SECRET_KEY: Optional[str] = Field(
+        description='Aliyun storage secret key',
+        default=None,
+    )
+
+    ALIYUN_OSS_ENDPOINT: Optional[str] = Field(
+        description='Aliyun storage endpoint URL',
+        default=None,
+    )
+
+    ALIYUN_OSS_REGION: Optional[str] = Field(
+        description='Aliyun storage region',
+        default=None,
+    )
+
+    ALIYUN_OSS_AUTH_VERSION: Optional[str] = Field(
+        description='Aliyun storage authentication version',
+        default=None,
+    )

api/configs/middleware/storage/amazon_s3_storage_config.py

@@ -0,0 +1,44 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class S3StorageConfig(BaseModel):
+    """
+    S3 storage configs
+    """
+
+    S3_ENDPOINT: Optional[str] = Field(
+        description='S3 storage endpoint',
+        default=None,
+    )
+
+    S3_REGION: Optional[str] = Field(
+        description='S3 storage region',
+        default=None,
+    )
+
+    S3_BUCKET_NAME: Optional[str] = Field(
+        description='S3 storage bucket name',
+        default=None,
+    )
+
+    S3_ACCESS_KEY: Optional[str] = Field(
+        description='S3 storage access key',
+        default=None,
+    )
+
+    S3_SECRET_KEY: Optional[str] = Field(
+        description='S3 storage secret key',
+        default=None,
+    )
+
+    S3_ADDRESS_STYLE: str = Field(
+        description='S3 storage address style',
+        default='auto',
+    )
+
+    S3_USE_AWS_MANAGED_IAM: bool = Field(
+        description='whether to use aws managed IAM for S3',
+        default=False,
+    )

api/configs/middleware/storage/azure_blob_storage_config.py

@@ -0,0 +1,29 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class AzureBlobStorageConfig(BaseModel):
+    """
+    Azure Blob storage configs
+    """
+
+    AZURE_BLOB_ACCOUNT_NAME: Optional[str] = Field(
+        description='Azure Blob account name',
+        default=None,
+    )
+
+    AZURE_BLOB_ACCOUNT_KEY: Optional[str] = Field(
+        description='Azure Blob account key',
+        default=None,
+    )
+
+    AZURE_BLOB_CONTAINER_NAME: Optional[str] = Field(
+        description='Azure Blob container name',
+        default=None,
+    )
+
+    AZURE_BLOB_ACCOUNT_URL: Optional[str] = Field(
+        description='Azure Blob account url',
+        default=None,
+    )

api/configs/middleware/storage/google_cloud_storage_config.py

@@ -0,0 +1,19 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class GoogleCloudStorageConfig(BaseModel):
+    """
+    Google Cloud storage configs
+    """
+
+    GOOGLE_STORAGE_BUCKET_NAME: Optional[str] = Field(
+        description='Google Cloud storage bucket name',
+        default=None,
+    )
+
+    GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64: Optional[str] = Field(
+        description='Google Cloud storage service account json base64',
+        default=None,
+    )

api/configs/middleware/storage/tencent_cos_storage_config.py

@@ -0,0 +1,34 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class TencentCloudCOSStorageConfig(BaseModel):
+    """
+    Tencent Cloud COS storage configs
+    """
+
+    TENCENT_COS_BUCKET_NAME: Optional[str] = Field(
+        description='Tencent Cloud COS bucket name',
+        default=None,
+    )
+
+    TENCENT_COS_REGION: Optional[str] = Field(
+        description='Tencent Cloud COS region',
+        default=None,
+    )
+
+    TENCENT_COS_SECRET_ID: Optional[str] = Field(
+        description='Tencent Cloud COS secret id',
+        default=None,
+    )
+
+    TENCENT_COS_SECRET_KEY: Optional[str] = Field(
+        description='Tencent Cloud COS secret key',
+        default=None,
+    )
+
+    TENCENT_COS_SCHEME: Optional[str] = Field(
+        description='Tencent Cloud COS scheme',
+        default=None,
+    )