added error logging on request #205

CveXplore/api/nvd_nist/nvd_nist_api.py

@@ -233,7 +233,7 @@ def get_count(
             if not isinstance(ret_data, Response):
                 return ret_data["totalResults"]
             else:
-                raise ApiDataRetrievalFailed
+                raise ApiDataRetrievalFailed(resource)
         except Exception:
             raise ApiMaxRetryError
 
@@ -455,7 +455,7 @@ async def fetch(self, session: aiohttp.ClientSession, url: str):
                         if "message" in response.headers:
                             self.logger.debug(response.headers["message"])
                     if response.status == 403:
-                        self.logger.debug(f"Request forbidden by administrative rules")
+                        self.logger.error(f"Request forbidden by administrative rules")
                     raise ApiDataRetrievalFailed(url)
         except ApiDataError:
             raise

CveXplore/database/maintenance/Sources_process.py

@@ -149,12 +149,17 @@ def process_downloads(self, sites: list = None):
                             entry, desc=f"Processing batch", leave=False
                         ):
                             if not isinstance(data_list, ApiDataRetrievalFailed):
-                                processed_items = [
-                                    self.process_item(item)
-                                    for item in data_list["products"]
-                                ]
-                                self._db_bulk_writer(processed_items)
-                                pbar.update(len(data_list["products"]))
+                                if not isinstance(data_list, Exception):
+                                    processed_items = [
+                                        self.process_item(item)
+                                        for item in data_list["products"]
+                                    ]
+                                    self._db_bulk_writer(processed_items)
+                                    pbar.update(len(data_list["products"]))
+                                else:
+                                    self.logger.error(
+                                        f"Retrieval of api data resulted in an Exception: {data_list}..."
+                                    )
                             else:
                                 self.logger.error(
                                     f"Retrieval of api data on url: {data_list.args[0]} failed...."
@@ -212,12 +217,17 @@ def process_downloads(self, sites: list = None):
                             entry, desc=f"Processing batch", leave=False
                         ):
                             if not isinstance(data_list, ApiDataRetrievalFailed):
-                                processed_items = [
-                                    self.process_item(item)
-                                    for item in data_list["products"]
-                                ]
-                                self._db_bulk_writer(processed_items)
-                                pbar.update(len(data_list["products"]))
+                                if not isinstance(data_list, Exception):
+                                    processed_items = [
+                                        self.process_item(item)
+                                        for item in data_list["products"]
+                                    ]
+                                    self._db_bulk_writer(processed_items)
+                                    pbar.update(len(data_list["products"]))
+                                else:
+                                    self.logger.error(
+                                        f"Retrieval of api data resulted in an Exception: {data_list}..."
+                                    )
                             else:
                                 self.logger.error(
                                     f"Retrieval of api data on url: {data_list.args[0]} failed...."
@@ -697,12 +707,17 @@ def process_downloads(self, sites: list = None):
                             entry, desc=f"Processing batch", leave=False
                         ):
                             if not isinstance(data_list, ApiDataRetrievalFailed):
-                                processed_items = [
-                                    self.process_item(item)
-                                    for item in data_list["vulnerabilities"]
-                                ]
-                                self._db_bulk_writer(processed_items)
-                                pbar.update(len(data_list["vulnerabilities"]))
+                                if not isinstance(data_list, Exception):
+                                    processed_items = [
+                                        self.process_item(item)
+                                        for item in data_list["vulnerabilities"]
+                                    ]
+                                    self._db_bulk_writer(processed_items)
+                                    pbar.update(len(data_list["vulnerabilities"]))
+                                else:
+                                    self.logger.error(
+                                        f"Retrieval of api data resulted in an Exception: {data_list}..."
+                                    )
                             else:
                                 self.logger.error(
                                     f"Retrieval of api data on url: {data_list.args[0]} failed...."
@@ -756,12 +771,17 @@ def process_downloads(self, sites: list = None):
                             entry, desc=f"Processing batch", leave=False
                         ):
                             if not isinstance(data_list, ApiDataRetrievalFailed):
-                                processed_items = [
-                                    self.process_item(item)
-                                    for item in data_list["vulnerabilities"]
-                                ]
-                                self._db_bulk_writer(processed_items)
-                                pbar.update(len(data_list["vulnerabilities"]))
+                                if not isinstance(data_list, Exception):
+                                    processed_items = [
+                                        self.process_item(item)
+                                        for item in data_list["vulnerabilities"]
+                                    ]
+                                    self._db_bulk_writer(processed_items)
+                                    pbar.update(len(data_list["vulnerabilities"]))
+                                else:
+                                    self.logger.error(
+                                        f"Retrieval of api data resulted in an Exception: {data_list}..."
+                                    )
                             else:
                                 self.logger.error(
                                     f"Retrieval of api data on url: {data_list.args[0]} failed...."