feat(db): improve support for index creation (#79)

* fix: indexes in collection * fix: add eval * fix: add eval in correct place * fix: remove value * fix: remove name and other fields * feat: changes * support other mongo fields too * fix: options in create_index * fix: spellings * minor changes * docs: update docs of IndexConfig * test: fix existing tests * fix: corner case of already existing indexes * refactor: flake8 * test: increase coverage * 0.5.0 Automatically generated by python-semantic-release * refactor: address review comments * fix examples; simplify code Co-authored-by: semantic-release <semantic-release> Co-authored-by: Alex Kanitz <[email protected]>
elixir-cloud-aai · Aug 9, 2020 · 3002953 · 3002953
1 parent c3194b7
commit 3002953
Show file tree

Hide file tree

Showing 5 changed files with 47 additions and 55 deletions.
diff --git a/foca/__init__.py b/foca/__init__.py
@@ -1 +1 @@
-__version__ = '0.4.0'
+__version__ = '0.5.0'
diff --git a/foca/database/register_mongodb.py b/foca/database/register_mongodb.py
@@ -52,11 +52,17 @@ def register_mongodb(
                         f"Added database collection '{coll_name}'."
                     )
 
-                    # Add indices
+                    # Add indexes
                     if coll_conf.indexes is not None:
+                        # Remove already created indexes if any
+                        coll_conf.client.drop_indexes()
                         for index in coll_conf.indexes:
                             if index.keys is not None:
-                                coll_conf.client.create_index(**index.dict())
+                                coll_conf.client.create_index(
+                                    index.keys, **index.options)
+                        logger.info(
+                            f"Indexes created for collection '{coll_name}'."
+                        )
 
     return conf
 

diff --git a/foca/models/config.py b/foca/models/config.py
@@ -7,7 +7,7 @@
 import operator
 from pathlib import Path
 import os
-from typing import (Any, Dict, List, Optional, Tuple, Union)
+from typing import (Any, Dict, List, Optional, Union)
 
 from pydantic import (BaseModel, Field, validator)  # pylint: disable=E0611
 import pymongo
@@ -733,64 +733,45 @@ class IndexConfig(FOCABaseConfig):
     """Model for configuring indexes for a MongoDB collection.
 
     Args:
-        keys: A list of key-direction tuples indicating the field to be indexed
+        keys: A key-direction dictionary indicating the field to be indexed
             and the sort order of that index. The sort order must be a valid
-            MongoDB index specifier, one of `pymongo.ASCENDING`,
-            `pymongo.DESCENDING`, `pymongo.GEO2D` etc. or their corresponding
-            values `1`, `-1`, `'2d'`, respectively; cf.:
+            MongoDB index specifier, one of the corresponding values of
+            `pymongo.ASCENDING`,`pymongo.DESCENDING`, `pymongo.GEO2D` etc. cf.:
             https://api.mongodb.com/python/current/api/pymongo/collection.html
-        name: Custom name to use for the index. If `None` is provided, a name
-            will be generated.
-        unique: Whether a uniqueness constraint shall be created on the index.
-        background: Whether the index shall be created in the background.
-        sparse: Whether documents that lack the indexed field shall be omitted
-            from the index.
+        options: A dictionary of any additional index creation options. cf.:
+            https://api.mongodb.com/python/1.9/api/pymongo/collection.html
 
     Attributes:
-        keys: A list of key-direction tuples indicating the field to be indexed
+        keys: A key-direction dictionary indicating the field to be indexed
             and the sort order of that index. The sort order must be a valid
-            MongoDB index specifier, one of `pymongo.ASCENDING`,
-            `pymongo.DESCENDING`, `pymongo.GEO2D` etc. or their corresponding
-            values `1`, `-1`, `'2d'`, respectively; cf.:
+            MongoDB index specifier, one of the corresponding values of
+            `pymongo.ASCENDING`,`pymongo.DESCENDING`, `pymongo.GEO2D` etc. cf.:
             https://api.mongodb.com/python/current/api/pymongo/collection.html
-        name: Custom name to use for the index. If `None` is provided, a name
-            will be generated.
-        unique: Whether a uniqueness constraint shall be created on the index.
-        background: Whether the index shall be created in the background.
-        sparse: Whether documents that lack the indexed field shall be omitted
-            from the index.
+        options: A dictionary of any additional index creation options. cf.:
+            https://api.mongodb.com/python/1.9/api/pymongo/collection.html
 
     Raises:
         pydantic.ValidationError: The class was instantianted with an illegal
             data type.
 
     Example:
         >>> IndexConfig(
-        ...     keys=[('last_name', pymongo.DESCENDING)],
-        ...     unique=True,
-        ...     sparse=False,
+        ...     keys={'name': -1, 'id': 1},
+        ...     options={'unique': True, 'sparse': False}
         ... )
-        IndexConfig(keys=[('last_name', -1)], name=None, unique=True, backgrou\
-nd=False, sparse=False)
+        IndexConfig(keys=[('name', -1), ('id', 1)], options={'unique': True, '\
+sparse': False})
     """
-    keys: Optional[List[Tuple[str, PymongoDirectionEnum]]] = None
-    name: Optional[str] = None
-    unique: Optional[bool] = False
-    background: Optional[bool] = False
-    sparse: Optional[bool] = False
+    keys: Optional[Dict] = None
+    options: Dict = dict()
 
     @validator('keys', always=True, allow_reuse=True)
     def store_enum_value(cls, v):  # pylint: disable=E0213
-        """Store value of enumerator, rather than enumerator object."""
+        """Convert dict values of keys into list of tuples"""
         if not v:
-            return v
+            return None
         else:
-            new_v = []
-            for item in v:
-                tmp_list = list(item)
-                tmp_list[1] = tmp_list[1].value
-                new_v.append(tuple(tmp_list))
-            return new_v
+            return [tuple([key, val]) for key, val in v.items()]
 
 
 class CollectionConfig(FOCABaseConfig):
@@ -812,10 +793,10 @@ class CollectionConfig(FOCABaseConfig):
 
     Example:
         >>> CollectionConfig(
-        ...     indexes=[IndexConfig(keys=[('last_name', 1)])],
+        ...     indexes=[IndexConfig(keys={'last_name': 1})],
         ... )
-        CollectionConfig(indexes=[IndexConfig(keys=[('last_name', 1)], name=No\
-ne, unique=False, background=False, sparse=False)], client=None)
+        CollectionConfig(indexes=[IndexConfig(keys=[('last_name', 1)], options\
+={})], client=None)}, client=None)
     """
     indexes: Optional[List[IndexConfig]] = None
     client: Optional[pymongo.collection.Collection] = None
@@ -844,13 +825,12 @@ class DBConfig(FOCABaseConfig):
         >>> DBConfig(
         ...     collections={
         ...         'my_collection': CollectionConfig(
-        ...             indexes=[IndexConfig(keys=[('last_name', 1)])],
+        ...             indexes=[IndexConfig(keys={'last_name': 1})],
         ...         ),
         ...     },
         ... )
         DBConfig(collections={'my_collection': CollectionConfig(indexes=[Index\
-Config(keys=[('last_name', 1)], name=None, unique=False, background=False, spa\
-rse=False)], client=None)}, client=None)
+Config(keys=[('last_name', 1)], options={})], client=None)}, client=None)
     """
     collections: Optional[Dict[str, CollectionConfig]] = None
     client: Optional[pymongo.database.Database] = None

diff --git a/tests/database/test_register_mongodb.py b/tests/database/test_register_mongodb.py
@@ -32,8 +32,8 @@
         'collections': {
             'my_collection': {
                 'indexes': [{
-                    'keys': [('indexed_field', 1)],
-                    'sparse': False
+                    'keys': {'indexed_field': 1},
+                    'options': {'sparse': False}
                 }]
             }
         }
@@ -107,6 +107,10 @@ def test_register_mongodb_cust_collections(monkeypatch):
         'pymongo.collection.Collection.create_index',
         lambda *args, **kwargs: None,
     )
+    monkeypatch.setattr(
+        'pymongo.collection.Collection.drop_indexes',
+        lambda *args, **kwargs: None,
+    )
     app = Flask(__name__)
     res = register_mongodb(
         app=app,

diff --git a/tests/models/test_config.py b/tests/models/test_config.py
@@ -33,11 +33,13 @@
 MODULE_PATCH_NOT_NESTED = 'some.path.EXCEPTIONS_NOT_NESTED'
 MODULE_PATCH_NOT_EXC = 'some.path.EXCEPTIONS_NOT_EXC'
 INDEX_CONFIG = {
-    'keys': [('last_name', -1)],
-    'name': 'indexLastName',
-    'unique': True,
-    'background': False,
-    'sparse': False,
+    'keys': {'last_name': -1},
+    'options': {
+        'name': 'indexLastName',
+        'unique': True,
+        'background': False,
+        'sparse': False
+    }
 }
 COLLECTION_CONFIG = {
     'indexes': [INDEX_CONFIG],