Allow to generate configuration despite some errors (#69)

schemas/qwc-config-generator.json

@@ -133,6 +133,10 @@
         },
         "use_default_map_thumbnail": {
           "description": "Whether to use the default mapthumb (mapthumbs/default.jpg) instead of generating the thumbnail via GetMap if no custom thumbnail is provided. Default: false"
+        },
+        "ignore_errors": {
+          "description": "Ignore errors during generation to allow creating configuration files despite some errors. Default: false",
+          "type": "boolean"
         }
       },
       "required": [

src/config_generator/capabilities_reader.py

@@ -66,7 +66,7 @@ def read_wms_service_capabilities(self, url, service_name, item):
             )
 
             if response.status_code != requests.codes.ok:
-                self.logger.critical(
+                self.logger.error(
                     "Could not get WMS GetProjectSettings from %s:\n%s" %
                     (full_url, response.content)
                 )
@@ -223,7 +223,7 @@ def read_wms_service_capabilities(self, url, service_name, item):
 
             return capabilities
         except Exception as e:
-            self.logger.critical(
+            self.logger.error(
                 "Could not parse WMS GetProjectSettings from %s:\n%s" %
                 (full_url, e)
             )
@@ -547,7 +547,7 @@ def read_wfs_service_capabilities(self, url, service_name, item):
             )
 
             if response.status_code != requests.codes.ok:
-                self.logger.critical(
+                self.logger.error(
                     "Could not get WFS GetCapabilities from %s:\n%s" %
                     (full_url, response.content)
                 )

src/config_generator/config_generator.py

@@ -146,11 +146,11 @@ def __init__(self, config, logger, config_file_dir):
                     config["themesConfig"] = json.load(f)
             except:
                 msg = "Failed to read themes configuration %s" % themes_config
-                self.logger.error(msg)
+                self.logger.critical(msg)
                 raise Exception(msg)
         elif not isinstance(themes_config, dict):
             msg = "Missing or invalid themes configuration in tenantConfig.json"
-            self.logger.error(msg)
+            self.logger.critical(msg)
             raise Exception(msg)
 
         if config.get('template', None):
@@ -172,11 +172,11 @@ def __init__(self, config, logger, config_file_dir):
                                 config_template["themesConfig"] = json.load(f)
                         except:
                             msg = "Failed to read themes configuration %s" % themes_config_template_path
-                            self.logger.error(msg)
+                            self.logger.critical(msg)
                             raise Exception(msg)
                     elif not isinstance(themes_config_template, dict):
                         msg = "No themes configuration in templated tenantConfig.json"
-                        self.logger.debug(msg)
+                        self.logger.critical(msg)
                         raise Exception(msg)
 
                     config_services = dict(map(lambda entry: (entry["name"], entry), config.get("services", [])))
@@ -270,8 +270,7 @@ def __init__(self, config, logger, config_file_dir):
                 "Could not load JSON schema versions from %s:\n%s" %
                 (schema_versions_path, e)
             )
-            self.logger.error(msg)
-            raise Exception(msg)
+            self.logger.warn(msg)
 
         # lookup for JSON schema URLs by service name
         self.schema_urls = {}
@@ -352,7 +351,7 @@ def __init__(self, config, logger, config_file_dir):
                 )
                 os.mkdir(self.tenant_path)
         except Exception as e:
-            self.logger.error("Could not create tenant dir:\n%s" % e)
+            self.logger.critical("Could not create tenant dir:\n%s" % e)
 
     def service_config(self, service):
         """Return any additional service config for service.
@@ -369,13 +368,13 @@ def write_configs(self):
         for service_config in self.config.get('services', []):
             self.write_service_config(service_config['name'])
 
-        for log in self.logger.log_entries():
-            if log["level"] == self.logger.LEVEL_CRITICAL:
-                self.logger.critical(
-                    "A critical error occurred while processing the configuration.")
-                self.logger.critical(
-                    "The configuration files were not updated!")
-                return False
+        criticals, errors = self.check_for_errors()
+        if criticals or (not self.config.get("config").get("ignore_errors", False) and errors):
+            self.logger.critical(
+                "A critical error occurred while processing the configuration.")
+            self.logger.critical(
+                "The configuration files were not updated!")
+            return False
 
         for file_name in os.listdir(os.path.join(self.temp_tenant_path)):
             file_path = os.path.join(self.temp_tenant_path, file_name)
@@ -388,6 +387,8 @@ def write_configs(self):
         self.logger.info(
             '<b style="color: green">The generation of the configuration files was successful</b>')
         self.logger.info('<b style="color: green">Configuration files were updated!</b>')
+        if errors:
+            self.logger.warn('Some errors occured and have been ignored, please check the logs to resolve some problems in configuration or projects.')
         return True
 
     def write_service_config(self, service):
@@ -470,13 +471,13 @@ def write_permissions(self):
 
         self.write_json_file(permissions, 'permissions.json')
 
-        for log in self.logger.log_entries():
-            if log["level"] == self.logger.LEVEL_CRITICAL:
-                self.logger.critical(
-                    "A critical error occurred while processing the configuration.")
-                self.logger.critical(
-                    "The permission files were not updated!")
-                return False
+        criticals, errors = self.check_for_errors()
+        if criticals or (not self.config.get("config").get("ignore_errors", False) and errors):
+            self.logger.critical(
+                "A critical error occurred while processing the configuration.")
+            self.logger.critical(
+                "The permission files were not updated!")
+            return False
 
         copyfile(
             os.path.join(self.temp_tenant_path, 'permissions.json'),
@@ -486,6 +487,8 @@ def write_permissions(self):
         self.logger.info(
             '<b style="color: green">The generation of the permission files was successful</b>')
         self.logger.info('<b style="color: green">Permission files were updated!</b>')
+        if errors:
+            self.logger.warn('Some errors occured and have been ignored, please check the logs to resolve some problems in configuration or projects.')
         return True
 
     def write_json_file(self, config, filename):
@@ -501,7 +504,7 @@ def write_json_file(self, config, filename):
                     config, sort_keys=False, ensure_ascii=False, indent=2
                 ).encode('utf8'))
         except Exception as e:
-            self.logger.error(
+            self.logger.critical(
                 "Could not write '%s' config file:\n%s" % (filename, e)
             )
 
@@ -514,7 +517,7 @@ def cleanup_temp_dir(self):
                 )
                 rmtree(self.temp_config_path)
         except Exception as e:
-            self.logger.error("Could not remove temp config dir:\n%s" % e)
+            self.logger.warn("Could not remove temp config dir:\n%s" % e)
 
     def validate_schema(self, config, schema_url):
         """Validate config against its JSON schema.
@@ -545,14 +548,14 @@ def validate_schema(self, config, schema_url):
             try:
                 response = requests.get(schema_url)
             except Exception as e:
-                self.logger.error(
+                self.logger.warn(
                     "Could not download JSON schema from %s:\n%s" %
                     (schema_url, str(e))
                 )
                 return False
 
             if response.status_code != requests.codes.ok:
-                self.logger.error(
+                self.logger.warn(
                     "Could not download JSON schema from %s:\n%s" %
                     (schema_url, response.text)
                 )
@@ -562,7 +565,7 @@ def validate_schema(self, config, schema_url):
             try:
                 schema = json.loads(response.text)
             except Exception as e:
-                self.logger.error("Could not parse JSON schema:\n%s" % e)
+                self.logger.warn("Could not parse JSON schema:\n%s" % e)
                 return False
 
         # validate against schema
@@ -668,7 +671,7 @@ def search_qgs_projects(self, generator_config, themes_config):
             self.logger.info(
                 "<b>Searching for projects files in %s</b>" % qgis_projects_scan_base_dir)
         else:
-            self.logger.error(
+            self.logger.warn(
                 "The qgis_projects_scan_base_dir sub directory" +
                 " does not exist: " + qgis_projects_scan_base_dir)
             return
@@ -944,3 +947,12 @@ def collect_layers(self, layer):
                 layers += self.collect_layers(sublayer)
 
         return layers
+
+    def check_for_errors(self):
+        """Check if logs contain CRITICAL or ERROR level messages
+
+        Return number of CRITICAL and ERROR messages.
+        """
+        criticals = [log_entry for log_entry in self.logger.log_entries() if log_entry.get('level', self.logger.LEVEL_INFO) == self.logger.LEVEL_CRITICAL]
+        errors = [log_entry for log_entry in self.logger.log_entries() if log_entry.get('level', self.logger.LEVEL_INFO) == self.logger.LEVEL_ERROR]
+        return (len(criticals), len(errors))

src/config_generator/map_viewer_config.py

@@ -609,11 +609,11 @@ def get_thumbnail(self, cfg_item, service_name, capabilities, assets_dir):
             )
 
             if response.status_code != requests.codes.ok:
-                self.logger.critical(
+                self.logger.error(
                     "ERROR generating thumbnail for WMS %s:\n%s" %
                     (service_name, response.content)
                 )
-                return
+                return 'img/mapthumbs/default.jpg'
 
             document = response.content
 

src/config_generator/qgs_reader.py

@@ -64,7 +64,7 @@ def read(self):
                 row = result.mappings().fetchone()
                 conn.close()
                 if not row:
-                    self.logger.critical("Could not find QGS project '%s'" % qgs_filename)
+                    self.logger.error("Could not find QGS project '%s'" % qgs_filename)
                     return False
 
                 qgz = zipfile.ZipFile(io.BytesIO(row['content']))
@@ -79,7 +79,7 @@ def read(self):
                 qgs_filename = self.map_prefix + self.qgs_ext
                 self.qgs_path = os.path.join(self.qgs_resources_path, qgs_filename)
                 if not os.path.exists(self.qgs_path):
-                    self.logger.critical("Could not find QGS project '%s'" % qgs_filename)
+                    self.logger.error("Could not find QGS project '%s'" % qgs_filename)
                     return False
 
                 if self.qgs_ext == ".qgz":
@@ -96,7 +96,7 @@ def read(self):
                     tree = ElementTree.parse(self.qgs_path)
 
             if tree is None or tree.getroot().tag != 'qgis':
-                self.logger.critical("'%s' is not a QGS file" % qgs_filename)
+                self.logger.error("'%s' is not a QGS file" % qgs_filename)
                 return False
             self.root = tree.getroot()
             self.logger.info("Read '%s'" % qgs_filename)

src/server.py

@@ -61,6 +61,8 @@ def generate_configs():
             level = entry["level"].upper()
             if level == "CRITICAL":
                 log_output += '<b style="color: red">CRITICAL: %s</b>\n' % str(entry["msg"])
+            elif level == "ERROR":
+                log_output += '<span style="color: red">ERROR: %s</span>\n' % str(entry["msg"])
             elif level == "WARNING":
                 log_output += '<span style="color: orange">WARNING: %s</span>\n' % str(entry["msg"])
             else: