Seperate case_id with description

- Change case.get()() to case.case_cls() - Add case.case_name property - Rename cases Signed-off-by: yangxuan <[email protected]>
zilliztech · Jun 21, 2023 · 59e67ca · 59e67ca
1 parent ca85a28
commit 59e67ca
Show file tree

Hide file tree

Showing 8 changed files with 231 additions and 233 deletions.
diff --git a/tests/test_models.py b/tests/test_models.py
@@ -24,7 +24,7 @@ def test_test_result(self):
                 db=DB.Milvus,
                 db_config=DB.Milvus.config(),
                 db_case_config=DB.Milvus.case_config_cls(index=IndexType.Flat)(),
-                case_config=CaseConfig(case_id=CaseType.PerformanceLZero),
+                case_config=CaseConfig(case_id=CaseType.Performance10M),
             ),
             metrics=Metric(),
         )

diff --git a/vectordb_bench/backend/assembler.py b/vectordb_bench/backend/assembler.py
@@ -1,4 +1,4 @@
-from .cases import type2case, CaseLabel
+from .cases import CaseLabel
 from .task_runner import CaseRunner, RunningStatus, TaskRunner
 from ..models import TaskConfig
 from ..backend.clients import EmptyDBCaseConfig
@@ -11,7 +11,7 @@
 class Assembler:
     @classmethod
     def assemble(cls, run_id , task: TaskConfig) -> CaseRunner:
-        c_cls = type2case.get(task.case_config.case_id)
+        c_cls = task.case_config.case_id.case_cls
 
         c = c_cls()
         if type(task.db_case_config) != EmptyDBCaseConfig:

diff --git a/vectordb_bench/backend/cases.py b/vectordb_bench/backend/cases.py
@@ -13,40 +13,38 @@
 
 class CaseType(Enum):
     """
-    Value will be displayed in UI
-
     Example:
-        >>> c = CaseType.CapacitySDim.get()()
+        >>> case_cls = CaseType.CapacityDim128.case_cls
         >>> assert c is not None
-        >>> c.name
+        >>> CaseType.CapacityDim128.case_name
         "Capacity Test (128 Dim Repeated)"
-        >>> c.description
-        ""
     """
 
-    CapacitySDim = "Capacity Test (128 Dim Repeated)"
-    CapacityLDim = "Capacity Test (960 Dim Repeated)"
-
-    Performance100M = "Search Performance Test (100M Dataset, 768 Dim)"
-    PerformanceLZero = "Search Performance Test (10M Dataset, 768 Dim)"
-    PerformanceMZero = "Search Performance Test (1M Dataset, 768 Dim)"
-
-    PerformanceLLow = (
-        "Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)"
-    )
-    PerformanceMLow = (
-        "Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)"
-    )
-    PerformanceLHigh = (
-        "Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)"
-    )
-    PerformanceMHigh = (
-        "Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)"
-    )
-
-    def get(self) -> Case:
+    CapacityDim128 = 1
+    CapacityDim960 = 2
+
+    Performance100M = 3
+    Performance10M = 4
+    Performance1M = 5
+
+    Performance10M1P = 6
+    Performance1M1P = 7
+    Performance10M99P = 8
+    Performance1M99P = 9
+
+    Custom = 100
+
+    @property
+    def case_cls(self, custom_configs: dict | None = None) -> Case:
         return type2case.get(self)
 
+    @property
+    def case_name(self) -> str:
+        c = self.case_cls
+        if c is not None:
+            return c().name
+        raise ValueError("Case unsupported")
+
 
 class CaseLabel(Enum):
     Load = auto()
@@ -57,7 +55,7 @@ class Case(BaseModel):
     """Undifined case
 
     Fields:
-        case_id(CaseType): default 11 case type plus one custom cases.
+        case_id(CaseType): default 9 case type plus one custom cases.
         label(CaseLabel): performance or load.
         dataset(DataSet): dataset for this case runner.
         filter_rate(float | None): one of 99% | 1% | None
@@ -86,75 +84,75 @@ def filters(self) -> dict | None:
 
 class CapacityCase(Case, BaseModel):
     label: CaseLabel = CaseLabel.Load
-    filter_rate: float | int | None = None
+    filter_rate: float | None = None
 
 
 class PerformanceCase(Case, BaseModel):
     label: CaseLabel = CaseLabel.Performance
-    filter_rate: float | int | None = None
+    filter_rate: float | None = None
 
 
-class CapacityLDimCase(CapacityCase):
-    case_id: CaseType = CaseType.CapacityLDim
+class CapacityDim960(CapacityCase):
+    case_id: CaseType = CaseType.CapacityDim960
     dataset: ds.DataSet = ds.get(ds.Name.GIST, ds.Label.SMALL)
     name: str = "Capacity Test (960 Dim Repeated)"
     description: str = """This case tests the vector database's loading capacity by repeatedly inserting large-dimension vectors (GIST 100K vectors, <b>960 dimensions</b>) until it is fully loaded.
 Number of inserted vectors will be reported."""
 
 
-class CapacitySDimCase(CapacityCase):
-    case_id: CaseType = CaseType.CapacitySDim
+class CapacityDim128(CapacityCase):
+    case_id: CaseType = CaseType.CapacityDim128
     dataset: ds.DataSet = ds.get(ds.Name.SIFT, ds.Label.SMALL)
     name: str = "Capacity Test (128 Dim Repeated)"
     description: str = """This case tests the vector database's loading capacity by repeatedly inserting small-dimension vectors (SIFT 100K vectors, <b>128 dimensions</b>) until it is fully loaded.
 Number of inserted vectors will be reported."""
 
 
-class PerformanceLZero(PerformanceCase):
-    case_id: CaseType = CaseType.PerformanceLZero
+class Performance10M(PerformanceCase):
+    case_id: CaseType = CaseType.Performance10M
     dataset: ds.DataSet = ds.get(ds.Name.Cohere, ds.Label.LARGE)
     name: str = "Search Performance Test (10M Dataset, 768 Dim)"
     description: str = """This case tests the search performance of a vector database with a large dataset (<b>Cohere 10M vectors</b>, 768 dimensions) at varying parallel levels.
 Results will show index building time, recall, and maximum QPS."""
 
 
-class PerformanceMZero(PerformanceCase):
-    case_id: CaseType = CaseType.PerformanceMZero
+class Performance1M(PerformanceCase):
+    case_id: CaseType = CaseType.Performance1M
     dataset: ds.DataSet = ds.get(ds.Name.Cohere, ds.Label.MEDIUM)
     name: str = "Search Performance Test (1M Dataset, 768 Dim)"
     description: str = """This case tests the search performance of a vector database with a medium dataset (<b>Cohere 1M vectors</b>, 768 dimensions) at varying parallel levels.
 Results will show index building time, recall, and maximum QPS."""
 
 
-class PerformanceLLow(PerformanceCase):
-    case_id: CaseType = CaseType.PerformanceLLow
+class Performance10M1P(PerformanceCase):
+    case_id: CaseType = CaseType.Performance10M1P
     filter_rate: float | int | None = 0.01
     dataset: ds.DataSet = ds.get(ds.Name.Cohere, ds.Label.LARGE)
     name: str = "Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)"
     description: str = """This case tests the search performance of a vector database with a large dataset (<b>Cohere 10M vectors</b>, 768 dimensions) under a low filtering rate (<b>1% vectors</b>), at varying parallel levels.
 Results will show index building time, recall, and maximum QPS."""
 
 
-class PerformanceMLow(PerformanceCase):
-    case_id: CaseType = CaseType.PerformanceMLow
+class Performance1M1P(PerformanceCase):
+    case_id: CaseType = CaseType.Performance1M1P
     filter_rate: float | int | None = 0.01
     dataset: ds.DataSet = ds.get(ds.Name.Cohere, ds.Label.MEDIUM)
     name: str = "Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)"
     description: str = """This case tests the search performance of a vector database with a medium dataset (<b>Cohere 1M vectors</b>, 768 dimensions) under a low filtering rate (<b>1% vectors</b>), at varying parallel levels.
 Results will show index building time, recall, and maximum QPS."""
 
 
-class PerformanceLHigh(PerformanceCase):
-    case_id: CaseType = CaseType.PerformanceLHigh
+class Performance10M99P(PerformanceCase):
+    case_id: CaseType = CaseType.Performance10M99P
     filter_rate: float | int | None = 0.99
     dataset: ds.DataSet = ds.get(ds.Name.Cohere, ds.Label.LARGE)
     name: str = "Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)"
     description: str = """This case tests the search performance of a vector database with a large dataset (<b>Cohere 10M vectors</b>, 768 dimensions) under a high filtering rate (<b>99% vectors</b>), at varying parallel levels.
 Results will show index building time, recall, and maximum QPS."""
 
 
-class PerformanceMHigh(PerformanceCase):
-    case_id: CaseType = CaseType.PerformanceMHigh
+class Performance1M99P(PerformanceCase):
+    case_id: CaseType = CaseType.Performance1M99P
     filter_rate: float | int | None = 0.99
     dataset: ds.DataSet = ds.get(ds.Name.Cohere, ds.Label.MEDIUM)
     name: str = "Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)"
@@ -173,15 +171,15 @@ class Performance100M(PerformanceCase):
 
 
 type2case = {
-    CaseType.CapacityLDim: CapacityLDimCase,
-    CaseType.CapacitySDim: CapacitySDimCase,
+    CaseType.CapacityDim960: CapacityDim960,
+    CaseType.CapacityDim128: CapacityDim128,
 
     CaseType.Performance100M: Performance100M,
-    CaseType.PerformanceLZero: PerformanceLZero,
-    CaseType.PerformanceMZero: PerformanceMZero,
+    CaseType.Performance10M: Performance10M,
+    CaseType.Performance1M: Performance1M,
 
-    CaseType.PerformanceLLow: PerformanceLLow,
-    CaseType.PerformanceMLow: PerformanceMLow,
-    CaseType.PerformanceLHigh: PerformanceLHigh,
-    CaseType.PerformanceMHigh: PerformanceMHigh,
+    CaseType.Performance10M1P: Performance10M1P,
+    CaseType.Performance1M1P: Performance1M1P,
+    CaseType.Performance10M99P: Performance10M99P,
+    CaseType.Performance1M99P: Performance1M99P,
 }
diff --git a/vectordb_bench/frontend/components/check_results/data.py b/vectordb_bench/frontend/components/check_results/data.py
@@ -58,7 +58,7 @@ def mergeTasks(tasks: list[CaseResult]):
             db = metricInfo["db"]
             db_label = metricInfo["db_label"]
             label = metricInfo["label"]
-            case_name = case_id.get()().name
+            case_name = case_id.case_name
             if label == ResultLabel.NORMAL:
                 mergedTasks.append(
                     {

diff --git a/vectordb_bench/frontend/components/check_results/filters.py b/vectordb_bench/frontend/components/check_results/filters.py
@@ -57,7 +57,7 @@ def getShowDbsAndCases(result: list[CaseResult], st) -> tuple[list[str], list[Ca
     allDbNames = list(set({res.task_config.db_name for res in result}))
     allDbNames.sort()
     allCasesSet = set({res.task_config.case_config.case_id for res in result})
-    allCases: list[Case] = [case.get()() for case in CASE_LIST if case in allCasesSet]
+    allCases: list[Case] = [case.case_cls() for case in CASE_LIST if case in allCasesSet]
 
     # DB Filter
     dbFilterContainer = st.container()

diff --git a/vectordb_bench/frontend/components/run_test/caseSelector.py b/vectordb_bench/frontend/components/run_test/caseSelector.py
@@ -33,7 +33,7 @@ def caseSelector(st, activedDbList):
 
 
 def caseItem(st, allCaseConfigs, case: CaseType, activedDbList):
-    selected = st.checkbox(case.get()().name)
+    selected = st.checkbox(case.casename)
     st.markdown(
         f"<div style='color: #1D2939; margin: -8px 0 20px {CHECKBOX_INDENT}px; font-size: 14px;'>{case.get()().description}</div>",
         unsafe_allow_html=True,

diff --git a/vectordb_bench/frontend/const/dbCaseConfigs.py b/vectordb_bench/frontend/const/dbCaseConfigs.py
@@ -14,17 +14,17 @@
 DIVIDER = "DIVIDER"
 CASE_LIST_WITH_DIVIDER = [
     CaseType.Performance100M,
-    CaseType.PerformanceLZero,
-    CaseType.PerformanceMZero,
+    CaseType.Performance10M,
+    CaseType.Performance1M,
     DIVIDER,
-    CaseType.PerformanceLLow,
-    CaseType.PerformanceMLow,
+    CaseType.Performance10M1P,
+    CaseType.Performance1M1P,
     DIVIDER,
-    CaseType.PerformanceLHigh,
-    CaseType.PerformanceMHigh,
+    CaseType.Performance10M99P,
+    CaseType.Performance1M99P,
     DIVIDER,
-    CaseType.CapacityLDim,
-    CaseType.CapacitySDim,
+    CaseType.CapacityDim960,
+    CaseType.CapacityDim128,
 ]
 
 CASE_LIST = [item for item in CASE_LIST_WITH_DIVIDER if isinstance(item, CaseType)]
@@ -225,36 +225,36 @@ class CaseConfigInput(BaseModel):
 
 CASE_CONFIG_MAP = {
     DB.Milvus: {
-        CaseType.CapacityLDim: MilvusLoadConfig,
-        CaseType.CapacitySDim: MilvusLoadConfig,
+        CaseType.CapacityDim960: MilvusLoadConfig,
+        CaseType.CapacityDim128: MilvusLoadConfig,
         CaseType.Performance100M: MilvusPerformanceConfig,
-        CaseType.PerformanceLZero: MilvusPerformanceConfig,
-        CaseType.PerformanceMZero: MilvusPerformanceConfig,
-        CaseType.PerformanceLLow: MilvusPerformanceConfig,
-        CaseType.PerformanceMLow: MilvusPerformanceConfig,
-        CaseType.PerformanceLHigh: MilvusPerformanceConfig,
-        CaseType.PerformanceMHigh: MilvusPerformanceConfig,
+        CaseType.Performance10M: MilvusPerformanceConfig,
+        CaseType.Performance1M: MilvusPerformanceConfig,
+        CaseType.Performance10M1P: MilvusPerformanceConfig,
+        CaseType.Performance1M1P: MilvusPerformanceConfig,
+        CaseType.Performance10M99P: MilvusPerformanceConfig,
+        CaseType.Performance1M99P: MilvusPerformanceConfig,
     },
     DB.WeaviateCloud: {
-        CaseType.CapacityLDim: WeaviateLoadConfig,
-        CaseType.CapacitySDim: WeaviateLoadConfig,
+        CaseType.CapacityDim960: WeaviateLoadConfig,
+        CaseType.CapacityDim128: WeaviateLoadConfig,
         CaseType.Performance100M: WeaviatePerformanceConfig,
-        CaseType.PerformanceLZero: WeaviatePerformanceConfig,
-        CaseType.PerformanceMZero: WeaviatePerformanceConfig,
-        CaseType.PerformanceLLow: WeaviatePerformanceConfig,
-        CaseType.PerformanceMLow: WeaviatePerformanceConfig,
-        CaseType.PerformanceLHigh: WeaviatePerformanceConfig,
-        CaseType.PerformanceMHigh: WeaviatePerformanceConfig,
+        CaseType.Performance10M: WeaviatePerformanceConfig,
+        CaseType.Performance1M: WeaviatePerformanceConfig,
+        CaseType.Performance10M1P: WeaviatePerformanceConfig,
+        CaseType.Performance1M1P: WeaviatePerformanceConfig,
+        CaseType.Performance10M99P: WeaviatePerformanceConfig,
+        CaseType.Performance1M99P: WeaviatePerformanceConfig,
     },
     DB.ElasticCloud: {
-        CaseType.CapacityLDim: ESLoadingConfig,
-        CaseType.CapacitySDim: ESLoadingConfig,
+        CaseType.CapacityDim960: ESLoadingConfig,
+        CaseType.CapacityDim128: ESLoadingConfig,
         CaseType.Performance100M: ESPerformanceConfig,
-        CaseType.PerformanceLZero: ESPerformanceConfig,
-        CaseType.PerformanceMZero: ESPerformanceConfig,
-        CaseType.PerformanceLLow: ESPerformanceConfig,
-        CaseType.PerformanceMLow: ESPerformanceConfig,
-        CaseType.PerformanceLHigh: ESPerformanceConfig,
-        CaseType.PerformanceMHigh: ESPerformanceConfig,
+        CaseType.Performance10M: ESPerformanceConfig,
+        CaseType.Performance1M: ESPerformanceConfig,
+        CaseType.Performance10M1P: ESPerformanceConfig,
+        CaseType.Performance1M1P: ESPerformanceConfig,
+        CaseType.Performance10M99P: ESPerformanceConfig,
+        CaseType.Performance1M99P: ESPerformanceConfig,
     },
 }