From 897d3153af1a961f29ad68b4e4c4f897d919ddb6 Mon Sep 17 00:00:00 2001 From: Sandro Mani Date: Thu, 5 Sep 2024 18:14:18 +0200 Subject: [PATCH] Implement capabilities cache triggered by use_cached_project_metadata=1 in request query --- src/config_generator/capabilities_reader.py | 170 +++++++++++++------- src/config_generator/config_generator.py | 7 +- src/config_generator/theme_reader.py | 8 +- src/config_generator_cli.py | 4 +- src/server.py | 16 +- 5 files changed, 135 insertions(+), 70 deletions(-) diff --git a/src/config_generator/capabilities_reader.py b/src/config_generator/capabilities_reader.py index 66e83d1..0ca2118 100644 --- a/src/config_generator/capabilities_reader.py +++ b/src/config_generator/capabilities_reader.py @@ -2,6 +2,7 @@ from urllib.parse import urljoin, urlparse from xml.etree import ElementTree +import os import re import requests @@ -12,13 +13,17 @@ class CapabilitiesReader(): Load and parse WMS GetProjectSettings.and WFS Capabilities """ - def __init__(self, generator_config, logger): + def __init__(self, generator_config, logger, use_cached_project_metadata, cache_dir): """Constructor :param obj generator_config: ConfigGenerator config :param Logger logger: Logger + :param bool use_cached_project_metadata: Whether to use cached project metadata if available + :param str cache_dir: Project metadata cache directory """ self.logger = logger + self.use_cached_project_metadata = use_cached_project_metadata + self.cache_dir = cache_dir # get default QGIS server URL from ConfigGenerator config self.default_qgis_server_url = generator_config.get( @@ -54,29 +59,46 @@ def read_wms_service_capabilities(self, url, service_name, item): self.logger.warning( "WMS URL is longer than 2000 characters!") - response = requests.get( - full_url, - params={ - 'SERVICE': 'WMS', - 'VERSION': '1.3.0', - 'REQUEST': 'GetProjectSettings', - 'CLEARCACHE': '1' - }, - timeout=self.project_settings_read_timeout - ) + document = None + cache_file = os.path.join(self.cache_dir, full_url.replace('/', '_').replace(':', '_') + "WMS_GetProjectSettings") + if self.use_cached_project_metadata: + try: + with open(cache_file) as fh: + document = fh.read() + self.logger.info("Using cached WMS GetProjectSettings for %s" % full_url) + except: + pass - if response.status_code != requests.codes.ok: - self.logger.error( - "Could not get WMS GetProjectSettings from %s:\n%s" % - (full_url, response.content) + if not document: + response = requests.get( + full_url, + params={ + 'SERVICE': 'WMS', + 'VERSION': '1.3.0', + 'REQUEST': 'GetProjectSettings', + 'CLEARCACHE': '1' + }, + timeout=self.project_settings_read_timeout ) - return {} - self.logger.info( - "Downloaded WMS GetProjectSettings from %s" % full_url - ) + if response.status_code != requests.codes.ok: + self.logger.error( + "Could not get WMS GetProjectSettings from %s:\n%s" % + (full_url, response.content) + ) + return {} + + self.logger.info( + "Downloaded WMS GetProjectSettings from %s" % full_url + ) - document = response.content + document = response.content + try: + os.makedirs(os.path.dirname(cache_file), exist_ok=True) + with open(cache_file, "w") as fh: + fh.write(document.decode('utf-8')) + except: + pass # parse WMS GetProjectSettings XML ElementTree.register_namespace('', 'http://www.opengis.net/wms') @@ -535,29 +557,46 @@ def read_wfs_service_capabilities(self, url, service_name, item): self.logger.warning( "WFS URL is longer than 2000 characters!") - response = requests.get( - full_url, - params={ - 'SERVICE': 'WFS', - 'VERSION': '1.1.0', - 'REQUEST': 'GetCapabilities', - 'CLEARCACHE': '1' - }, - timeout=self.project_settings_read_timeout - ) + document = None + cache_file = os.path.join(self.cache_dir, full_url.replace('/', '_').replace(':', '_') + "_WFS_GetCapabilities") + if self.use_cached_project_metadata: + try: + with open(cache_file) as fh: + document = fh.read() + self.logger.info("Using cached WFS GetCapabilities for %s" % full_url) + except: + pass - if response.status_code != requests.codes.ok: - self.logger.error( - "Could not get WFS GetCapabilities from %s:\n%s" % - (full_url, response.content) + if not document: + response = requests.get( + full_url, + params={ + 'SERVICE': 'WFS', + 'VERSION': '1.1.0', + 'REQUEST': 'GetCapabilities', + 'CLEARCACHE': '1' + }, + timeout=self.project_settings_read_timeout ) - return {} - self.logger.info( - "Downloaded WFS GetCapabilities from %s" % full_url - ) + if response.status_code != requests.codes.ok: + self.logger.error( + "Could not get WFS GetCapabilities from %s:\n%s" % + (full_url, response.content) + ) + return {} - document = response.content + self.logger.info( + "Downloaded WFS GetCapabilities from %s" % full_url + ) + + document = response.content + try: + os.makedirs(os.path.dirname(cache_file), exist_ok=True) + with open(cache_file, "w") as fh: + fh.write(document.decode('utf-8')) + except: + pass # parse WFS Capabilities XML ElementTree.register_namespace('', 'http://www.opengis.net/wfs') @@ -666,28 +705,45 @@ def collect_wfs_layers_attributes(self, full_url): :param str full_url: WFS URL """ try: - response = requests.get( - full_url, - params={ - 'SERVICE': 'WFS', - 'VERSION': '1.1.0', - 'REQUEST': 'DescribeFeatureType' - }, - timeout=self.project_settings_read_timeout - ) + document = None + cache_file = os.path.join(self.cache_dir, full_url.replace('/', '_').replace(':', '_') + "_WFS_DescribeFeatureType") + if self.use_cached_project_metadata: + try: + with open(cache_file) as fh: + document = fh.read() + self.logger.info("Using cached WFS DescribeFeatureType for %s" % full_url) + except: + pass - if response.status_code != requests.codes.ok: - self.logger.error( - "Could not get WFS DescribeFeatureType from %s:\n%s" % - (full_url, response.content) + if not document: + response = requests.get( + full_url, + params={ + 'SERVICE': 'WFS', + 'VERSION': '1.1.0', + 'REQUEST': 'DescribeFeatureType' + }, + timeout=self.project_settings_read_timeout ) - return {} - self.logger.info( - "Downloaded WFS DescribeFeatureType from %s" % full_url - ) + if response.status_code != requests.codes.ok: + self.logger.error( + "Could not get WFS DescribeFeatureType from %s:\n%s" % + (full_url, response.content) + ) + return {} - document = response.content + self.logger.info( + "Downloaded WFS DescribeFeatureType from %s" % full_url + ) + + document = response.content + try: + os.makedirs(os.path.dirname(cache_file), exist_ok=True) + with open(cache_file, "w") as fh: + fh.write(document.decode('utf-8')) + except: + pass # parse WFS Capabilities XML ElementTree.register_namespace('', 'http://www.w3.org/2001/XMLSchema') diff --git a/src/config_generator/config_generator.py b/src/config_generator/config_generator.py index e1db4a5..edb05f9 100644 --- a/src/config_generator/config_generator.py +++ b/src/config_generator/config_generator.py @@ -126,11 +126,13 @@ class ConfigGenerator(): from a tenantConfig.json and QWC ConfigDB. """ - def __init__(self, config, logger, config_file_dir): + def __init__(self, config, logger, config_file_dir, use_cached_project_metadata): """Constructor :param obj config: ConfigGenerator config :param Logger logger: Logger + :param bool use_cached_project_metadata: Whether to use cached project metadata if available + :param str cache_dir: Project metadata cache directory """ self.logger = Logger(logger) @@ -250,7 +252,8 @@ def __init__(self, config, logger, config_file_dir): # load metadata for all QWC2 theme items self.theme_reader = ThemeReader( - generator_config, config["themesConfig"], self.logger, print_layouts + generator_config, config["themesConfig"], self.logger, print_layouts, + use_cached_project_metadata, os.path.join(self.config_path, "__capabilities_cache") ) # lookup for additional service configs by name diff --git a/src/config_generator/theme_reader.py b/src/config_generator/theme_reader.py index 112a1f4..7644895 100644 --- a/src/config_generator/theme_reader.py +++ b/src/config_generator/theme_reader.py @@ -11,13 +11,15 @@ class ThemeReader(): Reads project metadata for all theme items in the QWC2 theme configuration. """ - def __init__(self, generator_config, themes_config, logger, print_layouts): + def __init__(self, generator_config, themes_config, logger, print_layouts, use_cached_project_metadata, cache_dir): """Constructor :param obj generator_config: ConfigGenerator config :param dict themes_config: themes config :param Logger logger: Logger - :param list print_layouts Found print layouts + :param list print_layouts: Found print layouts + :param bool use_cached_project_metadata: Whether to use cached project metadata if available + :param str cache_dir: Project metadata cache directory """ self.config = generator_config self.logger = logger @@ -31,7 +33,7 @@ def __init__(self, generator_config, themes_config, logger, print_layouts): # lookup for services names by URL: {: } self.service_name_lookup = {} - self.capabilities_reader = CapabilitiesReader(generator_config, logger) + self.capabilities_reader = CapabilitiesReader(generator_config, logger, use_cached_project_metadata, cache_dir) self.qgis_project_extension = generator_config.get( 'qgis_project_extension', '.qgs') diff --git a/src/config_generator_cli.py b/src/config_generator_cli.py index 8156bc9..78f32cc 100644 --- a/src/config_generator_cli.py +++ b/src/config_generator_cli.py @@ -36,7 +36,7 @@ def timestamp(self): # parse arguments parser = argparse.ArgumentParser() parser.add_argument( - 'config_file', help="Path to ConfigGenerator config file" + 'config_file', help="Path to ConfigGenerator config file", ) parser.add_argument( "command", choices=['all', 'service_configs', 'permissions'], @@ -57,7 +57,7 @@ def timestamp(self): logger = Logger() # create ConfigGenerator -generator = ConfigGenerator(config, logger, os.path.dirname(args.config_file)) +generator = ConfigGenerator(config, logger, os.path.dirname(args.config_file), False) if args.command == 'all': generator.write_configs() generator.write_permissions() diff --git a/src/server.py b/src/server.py index 6ed2dd2..bdfb92f 100644 --- a/src/server.py +++ b/src/server.py @@ -15,7 +15,7 @@ ).rstrip('/') + '/' -def config_generator(tenant): +def config_generator(tenant, use_cached_project_metadata): """Create a ConfigGenerator instance. :param str tenant: Tenant ID @@ -40,7 +40,7 @@ def config_generator(tenant): raise Exception(msg) # create ConfigGenerator - return ConfigGenerator(config, app.logger, config_file_dir) + return ConfigGenerator(config, app.logger, config_file_dir, use_cached_project_metadata) # routes @@ -51,7 +51,8 @@ def generate_configs(): try: # create ConfigGenerator tenant = request.args.get("tenant") - generator = config_generator(tenant) + use_cached_project_metadata = str(request.args.get("use_cached_project_metadata", "")).lower() in ["1","true"] + generator = config_generator(tenant, use_cached_project_metadata) generator.write_configs() generator.write_permissions() generator.cleanup_temp_dir() @@ -79,7 +80,8 @@ def maps(): try: # get maps from ConfigGenerator tenant = request.args.get('tenant') - generator = config_generator(tenant) + use_cached_project_metadata = str(request.args.get("use_cached_project_metadata", "")).lower() in ["1","true"] + generator = config_generator(tenant, use_cached_project_metadata) maps = generator.maps() generator.cleanup_temp_dir() @@ -94,7 +96,8 @@ def map_details(map_name): try: # get maps from ConfigGenerator tenant = request.args.get('tenant') - generator = config_generator(tenant) + use_cached_project_metadata = str(request.args.get("use_cached_project_metadata", "")).lower() in ["1","true"] + generator = config_generator(tenant, use_cached_project_metadata) map_details = generator.map_details(map_name) generator.cleanup_temp_dir() @@ -112,7 +115,8 @@ def resources(): try: # get maps from ConfigGenerator tenant = request.args.get('tenant') - generator = config_generator(tenant) + use_cached_project_metadata = str(request.args.get("use_cached_project_metadata", "")).lower() in ["1","true"] + generator = config_generator(tenant, use_cached_project_metadata) maps = generator.maps() for map_name in maps: maps_details.append(generator.map_details(