diff --git a/schemas/qwc-config-generator.json b/schemas/qwc-config-generator.json index 1989282..ed749c2 100644 --- a/schemas/qwc-config-generator.json +++ b/schemas/qwc-config-generator.json @@ -133,6 +133,10 @@ }, "use_default_map_thumbnail": { "description": "Whether to use the default mapthumb (mapthumbs/default.jpg) instead of generating the thumbnail via GetMap if no custom thumbnail is provided. Default: false" + }, + "ignore_errors": { + "description": "Ignore errors during generation to allow creating configuration files despite some errors. Default: false", + "type": "boolean" } }, "required": [ diff --git a/src/config_generator/capabilities_reader.py b/src/config_generator/capabilities_reader.py index 40fe642..66e83d1 100644 --- a/src/config_generator/capabilities_reader.py +++ b/src/config_generator/capabilities_reader.py @@ -66,7 +66,7 @@ def read_wms_service_capabilities(self, url, service_name, item): ) if response.status_code != requests.codes.ok: - self.logger.critical( + self.logger.error( "Could not get WMS GetProjectSettings from %s:\n%s" % (full_url, response.content) ) @@ -223,7 +223,7 @@ def read_wms_service_capabilities(self, url, service_name, item): return capabilities except Exception as e: - self.logger.critical( + self.logger.error( "Could not parse WMS GetProjectSettings from %s:\n%s" % (full_url, e) ) @@ -547,7 +547,7 @@ def read_wfs_service_capabilities(self, url, service_name, item): ) if response.status_code != requests.codes.ok: - self.logger.critical( + self.logger.error( "Could not get WFS GetCapabilities from %s:\n%s" % (full_url, response.content) ) diff --git a/src/config_generator/config_generator.py b/src/config_generator/config_generator.py index 5dd3e60..c475f05 100644 --- a/src/config_generator/config_generator.py +++ b/src/config_generator/config_generator.py @@ -146,11 +146,11 @@ def __init__(self, config, logger, config_file_dir): config["themesConfig"] = json.load(f) except: msg = "Failed to read themes configuration %s" % themes_config - self.logger.error(msg) + self.logger.critical(msg) raise Exception(msg) elif not isinstance(themes_config, dict): msg = "Missing or invalid themes configuration in tenantConfig.json" - self.logger.error(msg) + self.logger.critical(msg) raise Exception(msg) if config.get('template', None): @@ -172,11 +172,11 @@ def __init__(self, config, logger, config_file_dir): config_template["themesConfig"] = json.load(f) except: msg = "Failed to read themes configuration %s" % themes_config_template_path - self.logger.error(msg) + self.logger.critical(msg) raise Exception(msg) elif not isinstance(themes_config_template, dict): msg = "No themes configuration in templated tenantConfig.json" - self.logger.debug(msg) + self.logger.critical(msg) raise Exception(msg) config_services = dict(map(lambda entry: (entry["name"], entry), config.get("services", []))) @@ -270,8 +270,7 @@ def __init__(self, config, logger, config_file_dir): "Could not load JSON schema versions from %s:\n%s" % (schema_versions_path, e) ) - self.logger.error(msg) - raise Exception(msg) + self.logger.warn(msg) # lookup for JSON schema URLs by service name self.schema_urls = {} @@ -352,7 +351,7 @@ def __init__(self, config, logger, config_file_dir): ) os.mkdir(self.tenant_path) except Exception as e: - self.logger.error("Could not create tenant dir:\n%s" % e) + self.logger.critical("Could not create tenant dir:\n%s" % e) def service_config(self, service): """Return any additional service config for service. @@ -369,13 +368,13 @@ def write_configs(self): for service_config in self.config.get('services', []): self.write_service_config(service_config['name']) - for log in self.logger.log_entries(): - if log["level"] == self.logger.LEVEL_CRITICAL: - self.logger.critical( - "A critical error occurred while processing the configuration.") - self.logger.critical( - "The configuration files were not updated!") - return False + criticals, errors = self.check_for_errors() + if criticals or (not self.config.get("config").get("ignore_errors", False) and errors): + self.logger.critical( + "A critical error occurred while processing the configuration.") + self.logger.critical( + "The configuration files were not updated!") + return False for file_name in os.listdir(os.path.join(self.temp_tenant_path)): file_path = os.path.join(self.temp_tenant_path, file_name) @@ -388,6 +387,8 @@ def write_configs(self): self.logger.info( 'The generation of the configuration files was successful') self.logger.info('Configuration files were updated!') + if errors: + self.logger.warn('Some errors occured and have been ignored, please check the logs to resolve some problems in configuration or projects.') return True def write_service_config(self, service): @@ -470,13 +471,13 @@ def write_permissions(self): self.write_json_file(permissions, 'permissions.json') - for log in self.logger.log_entries(): - if log["level"] == self.logger.LEVEL_CRITICAL: - self.logger.critical( - "A critical error occurred while processing the configuration.") - self.logger.critical( - "The permission files were not updated!") - return False + criticals, errors = self.check_for_errors() + if criticals or (not self.config.get("config").get("ignore_errors", False) and errors): + self.logger.critical( + "A critical error occurred while processing the configuration.") + self.logger.critical( + "The permission files were not updated!") + return False copyfile( os.path.join(self.temp_tenant_path, 'permissions.json'), @@ -486,6 +487,8 @@ def write_permissions(self): self.logger.info( 'The generation of the permission files was successful') self.logger.info('Permission files were updated!') + if errors: + self.logger.warn('Some errors occured and have been ignored, please check the logs to resolve some problems in configuration or projects.') return True def write_json_file(self, config, filename): @@ -501,7 +504,7 @@ def write_json_file(self, config, filename): config, sort_keys=False, ensure_ascii=False, indent=2 ).encode('utf8')) except Exception as e: - self.logger.error( + self.logger.critical( "Could not write '%s' config file:\n%s" % (filename, e) ) @@ -514,7 +517,7 @@ def cleanup_temp_dir(self): ) rmtree(self.temp_config_path) except Exception as e: - self.logger.error("Could not remove temp config dir:\n%s" % e) + self.logger.warn("Could not remove temp config dir:\n%s" % e) def validate_schema(self, config, schema_url): """Validate config against its JSON schema. @@ -545,14 +548,14 @@ def validate_schema(self, config, schema_url): try: response = requests.get(schema_url) except Exception as e: - self.logger.error( + self.logger.warn( "Could not download JSON schema from %s:\n%s" % (schema_url, str(e)) ) return False if response.status_code != requests.codes.ok: - self.logger.error( + self.logger.warn( "Could not download JSON schema from %s:\n%s" % (schema_url, response.text) ) @@ -562,7 +565,7 @@ def validate_schema(self, config, schema_url): try: schema = json.loads(response.text) except Exception as e: - self.logger.error("Could not parse JSON schema:\n%s" % e) + self.logger.warn("Could not parse JSON schema:\n%s" % e) return False # validate against schema @@ -668,7 +671,7 @@ def search_qgs_projects(self, generator_config, themes_config): self.logger.info( "Searching for projects files in %s" % qgis_projects_scan_base_dir) else: - self.logger.error( + self.logger.warn( "The qgis_projects_scan_base_dir sub directory" + " does not exist: " + qgis_projects_scan_base_dir) return @@ -944,3 +947,12 @@ def collect_layers(self, layer): layers += self.collect_layers(sublayer) return layers + + def check_for_errors(self): + """Check if logs contain CRITICAL or ERROR level messages + + Return number of CRITICAL and ERROR messages. + """ + criticals = [log_entry for log_entry in self.logger.log_entries() if log_entry.get('level', self.logger.LEVEL_INFO) == self.logger.LEVEL_CRITICAL] + errors = [log_entry for log_entry in self.logger.log_entries() if log_entry.get('level', self.logger.LEVEL_INFO) == self.logger.LEVEL_ERROR] + return (len(criticals), len(errors)) diff --git a/src/config_generator/map_viewer_config.py b/src/config_generator/map_viewer_config.py index 12cf989..6a5edf1 100644 --- a/src/config_generator/map_viewer_config.py +++ b/src/config_generator/map_viewer_config.py @@ -609,11 +609,11 @@ def get_thumbnail(self, cfg_item, service_name, capabilities, assets_dir): ) if response.status_code != requests.codes.ok: - self.logger.critical( + self.logger.error( "ERROR generating thumbnail for WMS %s:\n%s" % (service_name, response.content) ) - return + return 'img/mapthumbs/default.jpg' document = response.content diff --git a/src/config_generator/qgs_reader.py b/src/config_generator/qgs_reader.py index 25b1712..4409f41 100644 --- a/src/config_generator/qgs_reader.py +++ b/src/config_generator/qgs_reader.py @@ -64,7 +64,7 @@ def read(self): row = result.mappings().fetchone() conn.close() if not row: - self.logger.critical("Could not find QGS project '%s'" % qgs_filename) + self.logger.error("Could not find QGS project '%s'" % qgs_filename) return False qgz = zipfile.ZipFile(io.BytesIO(row['content'])) @@ -79,7 +79,7 @@ def read(self): qgs_filename = self.map_prefix + self.qgs_ext self.qgs_path = os.path.join(self.qgs_resources_path, qgs_filename) if not os.path.exists(self.qgs_path): - self.logger.critical("Could not find QGS project '%s'" % qgs_filename) + self.logger.error("Could not find QGS project '%s'" % qgs_filename) return False if self.qgs_ext == ".qgz": @@ -96,7 +96,7 @@ def read(self): tree = ElementTree.parse(self.qgs_path) if tree is None or tree.getroot().tag != 'qgis': - self.logger.critical("'%s' is not a QGS file" % qgs_filename) + self.logger.error("'%s' is not a QGS file" % qgs_filename) return False self.root = tree.getroot() self.logger.info("Read '%s'" % qgs_filename) diff --git a/src/server.py b/src/server.py index 2616255..6ed2dd2 100644 --- a/src/server.py +++ b/src/server.py @@ -61,6 +61,8 @@ def generate_configs(): level = entry["level"].upper() if level == "CRITICAL": log_output += 'CRITICAL: %s\n' % str(entry["msg"]) + elif level == "ERROR": + log_output += 'ERROR: %s\n' % str(entry["msg"]) elif level == "WARNING": log_output += 'WARNING: %s\n' % str(entry["msg"]) else: