slow queries (#207)

* addressed slow queries on initialization of the database; added several minor bulk import optimizations
cve-search · Nov 21, 2023 · 022e452 · 022e452
1 parent 4ac13dc
commit 022e452
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 12 deletions.
diff --git a/CveXplore/database/maintenance/DownloadHandler.py b/CveXplore/database/maintenance/DownloadHandler.py
@@ -162,6 +162,8 @@ def _db_bulk_writer(self, batch: list):
         try:
             if self.feed_type.lower() == "epss":
                 self.database["cves"].bulk_write(batch, ordered=False)
+            elif self.feed_type.lower() == "cves" or self.feed_type.lower() == "cpe":
+                self.database[self.feed_type.lower()].insert_many(batch, ordered=False)
             else:
                 self.database[self.feed_type.lower()].bulk_write(batch, ordered=False)
         except BulkWriteError as err:

diff --git a/CveXplore/database/maintenance/Sources_process.py b/CveXplore/database/maintenance/Sources_process.py
@@ -243,13 +243,13 @@ def process_downloads(self, sites: list = None):
     def update(self, **kwargs):
         self.logger.info("CPE database update started")
 
+        self.process_downloads()
+
         # if collection is non-existent; assume it's not an update
         if self.feed_type.lower() not in self.getTableNames():
             DatabaseIndexer().create_indexes(collection=self.feed_type.lower())
             self.is_update = False
 
-        self.process_downloads()
-
         self.logger.info("Finished CPE database update")
 
         return self.last_modified
@@ -265,10 +265,10 @@ def populate(self, **kwargs):
 
         self.dropCollection(self.feed_type.lower())
 
-        DatabaseIndexer().create_indexes(collection=self.feed_type.lower())
-
         self.process_downloads()
 
+        DatabaseIndexer().create_indexes(collection=self.feed_type.lower())
+
         self.logger.info("Finished CPE database population")
 
         return self.last_modified
@@ -797,13 +797,13 @@ def process_downloads(self, sites: list = None):
     def update(self):
         self.logger.info("CVE database update started")
 
+        self.process_downloads()
+
         # if collection is non-existent; assume it's not an update
         if self.feed_type.lower() not in self.getTableNames():
             DatabaseIndexer().create_indexes(collection=self.feed_type.lower())
             self.is_update = False
 
-        self.process_downloads()
-
         self.logger.info("Finished CVE database update")
 
         return self.last_modified
@@ -825,10 +825,10 @@ def populate(self):
 
         self.dropCollection(self.feed_type.lower())
 
-        DatabaseIndexer().create_indexes(collection=self.feed_type.lower())
-
         self.process_downloads()
 
+        DatabaseIndexer().create_indexes(collection=self.feed_type.lower())
+
         self.logger.info("Finished CVE database population")
 
         return self.last_modified
@@ -1127,6 +1127,7 @@ def __init__(self):
                 MongoAddIndex(
                     index=[("padded_version", ASCENDING)], name="padded_version"
                 ),
+                MongoAddIndex(index=[("lastModified", ASCENDING)], name="lastModified"),
             ],
             "cpeother": [
                 MongoUniqueIndex(index=[("id", ASCENDING)], name="id", unique=True)

diff --git a/CveXplore/database/maintenance/api_handlers.py b/CveXplore/database/maintenance/api_handlers.py
@@ -32,10 +32,11 @@ def process_item(self, item: dict):
                     doc=item,
                 ).entry
             else:
-                return DatabaseAction(
-                    action=DatabaseAction.actions.InsertOne,
-                    doc=item,
-                ).entry
+                # return DatabaseAction(
+                #     action=DatabaseAction.actions.InsertOne,
+                #     doc=item,
+                # ).entry
+                return item
 
     @abstractmethod
     def process_the_item(self, *args):