Implement capabilities cache triggered by use_cached_project_metadata…

…=1 in request query

src/config_generator/capabilities_reader.py

@@ -2,6 +2,7 @@
 from urllib.parse import urljoin, urlparse
 from xml.etree import ElementTree
 
+import os
 import re
 import requests
 
@@ -12,13 +13,17 @@ class CapabilitiesReader():
     Load and parse WMS GetProjectSettings.and WFS Capabilities
     """
 
-    def __init__(self, generator_config, logger):
+    def __init__(self, generator_config, logger, use_cached_project_metadata, cache_dir):
         """Constructor
 
         :param obj generator_config: ConfigGenerator config
         :param Logger logger: Logger
+        :param bool use_cached_project_metadata: Whether to use cached project metadata if available
+        :param str cache_dir: Project metadata cache directory
         """
         self.logger = logger
+        self.use_cached_project_metadata = use_cached_project_metadata
+        self.cache_dir = cache_dir
 
         # get default QGIS server URL from ConfigGenerator config
         self.default_qgis_server_url = generator_config.get(
@@ -54,29 +59,46 @@ def read_wms_service_capabilities(self, url, service_name, item):
                 self.logger.warning(
                     "WMS URL is longer than 2000 characters!")
 
-            response = requests.get(
-                full_url,
-                params={
-                    'SERVICE': 'WMS',
-                    'VERSION': '1.3.0',
-                    'REQUEST': 'GetProjectSettings',
-                    'CLEARCACHE': '1'
-                },
-                timeout=self.project_settings_read_timeout
-            )
+            document = None
+            cache_file = os.path.join(self.cache_dir, full_url.replace('/', '_').replace(':', '_') + "WMS_GetProjectSettings")
+            if self.use_cached_project_metadata:
+                try:
+                    with open(cache_file) as fh:
+                        document = fh.read()
+                        self.logger.info("Using cached WMS GetProjectSettings for %s" % full_url)
+                except:
+                    pass
 
-            if response.status_code != requests.codes.ok:
-                self.logger.error(
-                    "Could not get WMS GetProjectSettings from %s:\n%s" %
-                    (full_url, response.content)
+            if not document:
+                response = requests.get(
+                    full_url,
+                    params={
+                        'SERVICE': 'WMS',
+                        'VERSION': '1.3.0',
+                        'REQUEST': 'GetProjectSettings',
+                        'CLEARCACHE': '1'
+                    },
+                    timeout=self.project_settings_read_timeout
                 )
-                return {}
 
-            self.logger.info(
-                "Downloaded WMS GetProjectSettings from %s" % full_url
-            )
+                if response.status_code != requests.codes.ok:
+                    self.logger.error(
+                        "Could not get WMS GetProjectSettings from %s:\n%s" %
+                        (full_url, response.content)
+                    )
+                    return {}
+
+                self.logger.info(
+                    "Downloaded WMS GetProjectSettings from %s" % full_url
+                )
 
-            document = response.content
+                document = response.content
+                try:
+                    os.makedirs(os.path.dirname(cache_file), exist_ok=True)
+                    with open(cache_file, "w") as fh:
+                        fh.write(document.decode('utf-8'))
+                except:
+                    pass
 
             # parse WMS GetProjectSettings XML
             ElementTree.register_namespace('', 'http://www.opengis.net/wms')
@@ -535,29 +557,46 @@ def read_wfs_service_capabilities(self, url, service_name, item):
                 self.logger.warning(
                     "WFS URL is longer than 2000 characters!")
 
-            response = requests.get(
-                full_url,
-                params={
-                    'SERVICE': 'WFS',
-                    'VERSION': '1.1.0',
-                    'REQUEST': 'GetCapabilities',
-                    'CLEARCACHE': '1'
-                },
-                timeout=self.project_settings_read_timeout
-            )
+            document = None
+            cache_file = os.path.join(self.cache_dir, full_url.replace('/', '_').replace(':', '_') + "_WFS_GetCapabilities")
+            if self.use_cached_project_metadata:
+                try:
+                    with open(cache_file) as fh:
+                        document = fh.read()
+                        self.logger.info("Using cached WFS GetCapabilities for %s" % full_url)
+                except:
+                    pass
 
-            if response.status_code != requests.codes.ok:
-                self.logger.error(
-                    "Could not get WFS GetCapabilities from %s:\n%s" %
-                    (full_url, response.content)
+            if not document:
+                response = requests.get(
+                    full_url,
+                    params={
+                        'SERVICE': 'WFS',
+                        'VERSION': '1.1.0',
+                        'REQUEST': 'GetCapabilities',
+                        'CLEARCACHE': '1'
+                    },
+                    timeout=self.project_settings_read_timeout
                 )
-                return {}
 
-            self.logger.info(
-                "Downloaded WFS GetCapabilities from %s" % full_url
-            )
+                if response.status_code != requests.codes.ok:
+                    self.logger.error(
+                        "Could not get WFS GetCapabilities from %s:\n%s" %
+                        (full_url, response.content)
+                    )
+                    return {}
 
-            document = response.content
+                self.logger.info(
+                    "Downloaded WFS GetCapabilities from %s" % full_url
+                )
+
+                document = response.content
+                try:
+                    os.makedirs(os.path.dirname(cache_file), exist_ok=True)
+                    with open(cache_file, "w") as fh:
+                        fh.write(document.decode('utf-8'))
+                except:
+                    pass
 
             # parse WFS Capabilities XML
             ElementTree.register_namespace('', 'http://www.opengis.net/wfs')
@@ -666,28 +705,45 @@ def collect_wfs_layers_attributes(self, full_url):
         :param str full_url: WFS URL
         """
         try:
-            response = requests.get(
-                full_url,
-                params={
-                    'SERVICE': 'WFS',
-                    'VERSION': '1.1.0',
-                    'REQUEST': 'DescribeFeatureType'
-                },
-                timeout=self.project_settings_read_timeout
-            )
+            document = None
+            cache_file = os.path.join(self.cache_dir, full_url.replace('/', '_').replace(':', '_') + "_WFS_DescribeFeatureType")
+            if self.use_cached_project_metadata:
+                try:
+                    with open(cache_file) as fh:
+                        document = fh.read()
+                        self.logger.info("Using cached WFS DescribeFeatureType for %s" % full_url)
+                except:
+                    pass
 
-            if response.status_code != requests.codes.ok:
-                self.logger.error(
-                    "Could not get WFS DescribeFeatureType from %s:\n%s" %
-                    (full_url, response.content)
+            if not document:
+                response = requests.get(
+                    full_url,
+                    params={
+                        'SERVICE': 'WFS',
+                        'VERSION': '1.1.0',
+                        'REQUEST': 'DescribeFeatureType'
+                    },
+                    timeout=self.project_settings_read_timeout
                 )
-                return {}
 
-            self.logger.info(
-                "Downloaded WFS DescribeFeatureType from %s" % full_url
-            )
+                if response.status_code != requests.codes.ok:
+                    self.logger.error(
+                        "Could not get WFS DescribeFeatureType from %s:\n%s" %
+                        (full_url, response.content)
+                    )
+                    return {}
 
-            document = response.content
+                self.logger.info(
+                    "Downloaded WFS DescribeFeatureType from %s" % full_url
+                )
+
+                document = response.content
+                try:
+                    os.makedirs(os.path.dirname(cache_file), exist_ok=True)
+                    with open(cache_file, "w") as fh:
+                        fh.write(document.decode('utf-8'))
+                except:
+                    pass
 
             # parse WFS Capabilities XML
             ElementTree.register_namespace('', 'http://www.w3.org/2001/XMLSchema')

src/config_generator/config_generator.py

@@ -126,11 +126,13 @@ class ConfigGenerator():
     from a tenantConfig.json and QWC ConfigDB.
     """
 
-    def __init__(self, config, logger, config_file_dir):
+    def __init__(self, config, logger, config_file_dir, use_cached_project_metadata):
         """Constructor
 
         :param obj config: ConfigGenerator config
         :param Logger logger: Logger
+        :param bool use_cached_project_metadata: Whether to use cached project metadata if available
+        :param str cache_dir: Project metadata cache directory
         """
         self.logger = Logger(logger)
 
@@ -250,7 +252,8 @@ def __init__(self, config, logger, config_file_dir):
 
         # load metadata for all QWC2 theme items
         self.theme_reader = ThemeReader(
-            generator_config, config["themesConfig"], self.logger, print_layouts
+            generator_config, config["themesConfig"], self.logger, print_layouts,
+            use_cached_project_metadata, os.path.join(self.config_path, "__capabilities_cache")
         )
 
         # lookup for additional service configs by name

src/config_generator/theme_reader.py

@@ -11,13 +11,15 @@ class ThemeReader():
     Reads project metadata for all theme items in the QWC2 theme configuration.
     """
 
-    def __init__(self, generator_config, themes_config, logger, print_layouts):
+    def __init__(self, generator_config, themes_config, logger, print_layouts, use_cached_project_metadata, cache_dir):
         """Constructor
 
         :param obj generator_config: ConfigGenerator config
         :param dict themes_config: themes config
         :param Logger logger: Logger
-        :param list print_layouts Found print layouts
+        :param list print_layouts: Found print layouts
+        :param bool use_cached_project_metadata: Whether to use cached project metadata if available
+        :param str cache_dir: Project metadata cache directory
         """
         self.config = generator_config
         self.logger = logger
@@ -31,7 +33,7 @@ def __init__(self, generator_config, themes_config, logger, print_layouts):
         # lookup for services names by URL: {<url>: <service_name>}
         self.service_name_lookup = {}
 
-        self.capabilities_reader = CapabilitiesReader(generator_config, logger)
+        self.capabilities_reader = CapabilitiesReader(generator_config, logger, use_cached_project_metadata, cache_dir)
 
         self.qgis_project_extension = generator_config.get(
             'qgis_project_extension', '.qgs')

src/config_generator_cli.py

@@ -36,7 +36,7 @@ def timestamp(self):
 # parse arguments
 parser = argparse.ArgumentParser()
 parser.add_argument(
-    'config_file', help="Path to ConfigGenerator config file"
+    'config_file', help="Path to ConfigGenerator config file",
 )
 parser.add_argument(
     "command", choices=['all', 'service_configs', 'permissions'],
@@ -57,7 +57,7 @@ def timestamp(self):
 logger = Logger()
 
 # create ConfigGenerator
-generator = ConfigGenerator(config, logger, os.path.dirname(args.config_file))
+generator = ConfigGenerator(config, logger, os.path.dirname(args.config_file), False)
 if args.command == 'all':
     generator.write_configs()
     generator.write_permissions()

src/server.py

@@ -15,7 +15,7 @@
 ).rstrip('/') + '/'
 
 
-def config_generator(tenant):
+def config_generator(tenant, use_cached_project_metadata):
     """Create a ConfigGenerator instance.
 
     :param str tenant: Tenant ID
@@ -40,7 +40,7 @@ def config_generator(tenant):
         raise Exception(msg)
 
     # create ConfigGenerator
-    return ConfigGenerator(config, app.logger, config_file_dir)
+    return ConfigGenerator(config, app.logger, config_file_dir, use_cached_project_metadata)
 
 
 # routes
@@ -51,7 +51,8 @@ def generate_configs():
     try:
         # create ConfigGenerator
         tenant = request.args.get("tenant")
-        generator = config_generator(tenant)
+        use_cached_project_metadata = str(request.args.get("use_cached_project_metadata", "")).lower() in ["1","true"]
+        generator = config_generator(tenant, use_cached_project_metadata)
         generator.write_configs()
         generator.write_permissions()
         generator.cleanup_temp_dir()
@@ -79,7 +80,8 @@ def maps():
     try:
         # get maps from ConfigGenerator
         tenant = request.args.get('tenant')
-        generator = config_generator(tenant)
+        use_cached_project_metadata = str(request.args.get("use_cached_project_metadata", "")).lower() in ["1","true"]
+        generator = config_generator(tenant, use_cached_project_metadata)
         maps = generator.maps()
         generator.cleanup_temp_dir()
 
@@ -94,7 +96,8 @@ def map_details(map_name):
     try:
         # get maps from ConfigGenerator
         tenant = request.args.get('tenant')
-        generator = config_generator(tenant)
+        use_cached_project_metadata = str(request.args.get("use_cached_project_metadata", "")).lower() in ["1","true"]
+        generator = config_generator(tenant, use_cached_project_metadata)
         map_details = generator.map_details(map_name)
         generator.cleanup_temp_dir()
 
@@ -112,7 +115,8 @@ def resources():
     try:
         # get maps from ConfigGenerator
         tenant = request.args.get('tenant')
-        generator = config_generator(tenant)
+        use_cached_project_metadata = str(request.args.get("use_cached_project_metadata", "")).lower() in ["1","true"]
+        generator = config_generator(tenant, use_cached_project_metadata)
         maps = generator.maps()
         for map_name in maps:
             maps_details.append(generator.map_details(