Skip to content

Commit

Permalink
resolved conflicts
Browse files Browse the repository at this point in the history
  • Loading branch information
@roadgoat19
roadgoat19 committed Aug 29, 2024
1 parent 7b817be commit 9d575c5
Show file tree
Hide file tree
Showing 8 changed files with 597 additions and 472 deletions.
3 changes: 2 additions & 1 deletion .github/workflows/api-tests.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ jobs:
- name: Run Workflow
run: poetry run -C api bash dev/pytest/pytest_workflow.sh

- name: Set up Vector Stores (Weaviate, Couchbase, Qdrant, PGVector, Milvus, PgVecto-RS, Chroma)
- name: Set up Vector Stores (Weaviate, Qdrant, PGVector, Milvus, PgVecto-RS, Chroma, MyScale, ElasticSearch, Couchbase)
uses: hoverkraft-tech/[email protected]
with:
compose-file: |
Expand All @@ -91,5 +91,6 @@ jobs:
pgvecto-rs
pgvector
chroma
elasticsearch
- name: Test Vector Stores
run: poetry run -C api bash dev/pytest/pytest_vdb.sh
3 changes: 2 additions & 1 deletion .github/workflows/expose_service_ports.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@ yq eval '.services.chroma.ports += ["8000:8000"]' -i docker/docker-compose.yaml
yq eval '.services["milvus-standalone"].ports += ["19530:19530"]' -i docker/docker-compose.yaml
yq eval '.services.pgvector.ports += ["5433:5432"]' -i docker/docker-compose.yaml
yq eval '.services["pgvecto-rs"].ports += ["5431:5432"]' -i docker/docker-compose.yaml
yq eval '.services["elasticsearch"].ports += ["9200:9200"]' -i docker/docker-compose.yaml
yq eval '.services.couchbase-server.ports += ["8091-8096:8091-8096"]' -i docker/docker-compose.yaml
yq eval '.services.couchbase-server.ports += ["11210:11210"]' -i docker/docker-compose.yaml

echo "Ports exposed for sandbox, weaviate, qdrant, couchbase, chroma, milvus, pgvector, pgvecto-rs."
echo "Ports exposed for sandbox, weaviate, qdrant, chroma, milvus, pgvector, pgvecto-rs, elasticsearch, couchbase"
421 changes: 222 additions & 199 deletions api/commands.py
View file

Large diffs are not rendered by default.

101 changes: 52 additions & 49 deletions api/configs/middleware/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from configs.middleware.vdb.analyticdb_config import AnalyticdbConfig
from configs.middleware.vdb.chroma_config import ChromaConfig
from configs.middleware.vdb.couchbase_config import CouchbaseConfig
from configs.middleware.vdb.elasticsearch_config import ElasticsearchConfig
from configs.middleware.vdb.milvus_config import MilvusConfig
from configs.middleware.vdb.myscale_config import MyScaleConfig
from configs.middleware.vdb.opensearch_config import OpenSearchConfig
Expand All @@ -29,144 +30,147 @@

class StorageConfig(BaseSettings):
STORAGE_TYPE: str = Field(
description='storage type,'
' default to `local`,'
' available values are `local`, `s3`, `azure-blob`, `aliyun-oss`, `google-storage`.',
default='local',
description="storage type,"
" default to `local`,"
" available values are `local`, `s3`, `azure-blob`, `aliyun-oss`, `google-storage`.",
default="local",
)

STORAGE_LOCAL_PATH: str = Field(
description='local storage path',
default='storage',
description="local storage path",
default="storage",
)


class VectorStoreConfig(BaseSettings):
VECTOR_STORE: Optional[str] = Field(
description='vector store type',
description="vector store type",
default=None,
)


class KeywordStoreConfig(BaseSettings):
KEYWORD_STORE: str = Field(
description='keyword store type',
default='jieba',
description="keyword store type",
default="jieba",
)


class DatabaseConfig:
DB_HOST: str = Field(
description='db host',
default='localhost',
description="db host",
default="localhost",
)

DB_PORT: PositiveInt = Field(
description='db port',
description="db port",
default=5432,
)

DB_USERNAME: str = Field(
description='db username',
default='postgres',
description="db username",
default="postgres",
)

DB_PASSWORD: str = Field(
description='db password',
default='',
description="db password",
default="",
)

DB_DATABASE: str = Field(
description='db database',
default='dify',
description="db database",
default="dify",
)

DB_CHARSET: str = Field(
description='db charset',
default='',
description="db charset",
default="",
)

DB_EXTRAS: str = Field(
description='db extras options. Example: keepalives_idle=60&keepalives=1',
default='',
description="db extras options. Example: keepalives_idle=60&keepalives=1",
default="",
)

SQLALCHEMY_DATABASE_URI_SCHEME: str = Field(
description='db uri scheme',
default='postgresql',
description="db uri scheme",
default="postgresql",
)

@computed_field
@property
def SQLALCHEMY_DATABASE_URI(self) -> str:
db_extras = (
f"{self.DB_EXTRAS}&client_encoding={self.DB_CHARSET}"
if self.DB_CHARSET
else self.DB_EXTRAS
f"{self.DB_EXTRAS}&client_encoding={self.DB_CHARSET}" if self.DB_CHARSET else self.DB_EXTRAS
).strip("&")
db_extras = f"?{db_extras}" if db_extras else ""
return (f"{self.SQLALCHEMY_DATABASE_URI_SCHEME}://"
f"{quote_plus(self.DB_USERNAME)}:{quote_plus(self.DB_PASSWORD)}@{self.DB_HOST}:{self.DB_PORT}/{self.DB_DATABASE}"
f"{db_extras}")
return (
f"{self.SQLALCHEMY_DATABASE_URI_SCHEME}://"
f"{quote_plus(self.DB_USERNAME)}:{quote_plus(self.DB_PASSWORD)}@{self.DB_HOST}:{self.DB_PORT}/{self.DB_DATABASE}"
f"{db_extras}"
)

SQLALCHEMY_POOL_SIZE: NonNegativeInt = Field(
description='pool size of SqlAlchemy',
description="pool size of SqlAlchemy",
default=30,
)

SQLALCHEMY_MAX_OVERFLOW: NonNegativeInt = Field(
description='max overflows for SqlAlchemy',
description="max overflows for SqlAlchemy",
default=10,
)

SQLALCHEMY_POOL_RECYCLE: NonNegativeInt = Field(
description='SqlAlchemy pool recycle',
description="SqlAlchemy pool recycle",
default=3600,
)

SQLALCHEMY_POOL_PRE_PING: bool = Field(
description='whether to enable pool pre-ping in SqlAlchemy',
description="whether to enable pool pre-ping in SqlAlchemy",
default=False,
)

SQLALCHEMY_ECHO: bool | str = Field(
description='whether to enable SqlAlchemy echo',
description="whether to enable SqlAlchemy echo",
default=False,
)

@computed_field
@property
def SQLALCHEMY_ENGINE_OPTIONS(self) -> dict[str, Any]:
return {
'pool_size': self.SQLALCHEMY_POOL_SIZE,
'max_overflow': self.SQLALCHEMY_MAX_OVERFLOW,
'pool_recycle': self.SQLALCHEMY_POOL_RECYCLE,
'pool_pre_ping': self.SQLALCHEMY_POOL_PRE_PING,
'connect_args': {'options': '-c timezone=UTC'},
"pool_size": self.SQLALCHEMY_POOL_SIZE,
"max_overflow": self.SQLALCHEMY_MAX_OVERFLOW,
"pool_recycle": self.SQLALCHEMY_POOL_RECYCLE,
"pool_pre_ping": self.SQLALCHEMY_POOL_PRE_PING,
"connect_args": {"options": "-c timezone=UTC"},
}


class CeleryConfig(DatabaseConfig):
CELERY_BACKEND: str = Field(
description='Celery backend, available values are `database`, `redis`',
default='database',
description="Celery backend, available values are `database`, `redis`",
default="database",
)

CELERY_BROKER_URL: Optional[str] = Field(
description='CELERY_BROKER_URL',
description="CELERY_BROKER_URL",
default=None,
)

@computed_field
@property
def CELERY_RESULT_BACKEND(self) -> str | None:
return 'db+{}'.format(self.SQLALCHEMY_DATABASE_URI) \
if self.CELERY_BACKEND == 'database' else self.CELERY_BROKER_URL
return (
"db+{}".format(self.SQLALCHEMY_DATABASE_URI)
if self.CELERY_BACKEND == "database"
else self.CELERY_BROKER_URL
)

@computed_field
@property
def BROKER_USE_SSL(self) -> bool:
return self.CELERY_BROKER_URL.startswith('rediss://') if self.CELERY_BROKER_URL else False
return self.CELERY_BROKER_URL.startswith("rediss://") if self.CELERY_BROKER_URL else False


class MiddlewareConfig(
Expand All @@ -175,7 +179,6 @@ class MiddlewareConfig(
DatabaseConfig,
KeywordStoreConfig,
RedisConfig,

# configs of storage and storage providers
StorageConfig,
AliyunOSSStorageConfig,
Expand All @@ -184,7 +187,6 @@ class MiddlewareConfig(
TencentCloudCOSStorageConfig,
S3StorageConfig,
OCIStorageConfig,

# configs of vdb and vdb providers
VectorStoreConfig,
AnalyticdbConfig,
Expand All @@ -200,6 +202,7 @@ class MiddlewareConfig(
TencentVectorDBConfig,
TiDBVectorConfig,
WeaviateConfig,
CouchbaseConfig
CouchbaseConfig,
ElasticsearchConfig
):
pass
Loading

0 comments on commit 9d575c5

Please sign in to comment.