Add option to save scanned projects and groups in themes configuration (

#67)

schemas/qwc-config-generator.json

@@ -87,6 +87,10 @@
           "description": "Option to group scanned projects by directory in themes configuration. Default: false",
           "type": "boolean"
         },
+        "save_scanned_projects_in_config": {
+          "description": "Option to save scanned projects and groups in themes configuration. Requires write access to INPUT_CONFIG_PATH for config generator service. Default: false",
+          "type": "boolean"
+        },
         "permissions_default_allow": {
           "description": "Set whether resources are permitted or restricted by default. Example: true",
           "type": "boolean"

src/config_generator/config_generator.py

@@ -7,7 +7,7 @@
 import re
 
 from collections import OrderedDict
-from datetime import datetime
+from datetime import datetime, UTC
 from pathlib import Path
 from shutil import move, copyfile, rmtree
 from urllib.parse import urljoin, urlparse
@@ -633,6 +633,7 @@ def search_qgs_projects(self, generator_config, themes_config):
         qgis_projects_scan_base_dir = generator_config.get(
             'qgis_projects_scan_base_dir')
         group_scanned_projects_by_dir = generator_config.get('group_scanned_projects_by_dir', False)
+        save_scanned_projects_in_config = generator_config.get('save_scanned_projects_in_config', False)
         qwc_base_dir = generator_config.get("qwc2_base_dir")
         qgis_project_extension = generator_config.get(
             'qgis_project_extension', '.qgs')
@@ -656,11 +657,8 @@ def search_qgs_projects(self, generator_config, themes_config):
         # collect existing item urls
         items = themes.get(
             "items", [])
-        wms_urls = []
         has_default = False
         for item in items:
-            if item.get("url"):
-                wms_urls.append(item["url"])
             if item.get("default", False):
                 has_default = True
 
@@ -708,30 +706,87 @@ def search_qgs_projects(self, generator_config, themes_config):
                 theme_item["mapCrs"] = themes_config.get(
                     "defaultMapCrs")
 
-                if theme_item["url"] not in wms_urls:
-                    if not has_default:
-                        theme_item["default"] = True
-                        has_default = True
-                    # Add theme to items or group
-                    if group_scanned_projects_by_dir and (item.parent != base_path):
-                        if list(filter(lambda group: group["title"] == item.parent.name, groups)):
-                            item_group = list(filter(lambda group: group["title"] == item.parent.name, groups))[0]
-                        else:
-                            for group in groups:
-                                if list(filter(lambda g: g["title"] == item.parent.name, group["groups"])):
-                                    item_group = list(filter(lambda g: g["title"] == item.parent.name, group["groups"]))[0]
-                        if not list(filter(lambda item: item["url"] == wmspath, item_group["items"])):
-                            self.logger.info(f"Adding project {item.stem} to group {item.parent.name}")
-                            item_group["items"].append(theme_item)
-                        else: self.logger.info(f"Project {item.stem} already exists in group {item.parent.name}")
+                if not has_default:
+                    theme_item["default"] = True
+                    has_default = True
+                # Add theme to items or group
+                if group_scanned_projects_by_dir and (item.parent != base_path):
+                    if list(filter(lambda group: group["title"] == item.parent.name, groups)):
+                        item_group = list(filter(lambda group: group["title"] == item.parent.name, groups))[0]
                     else:
+                        for group in groups:
+                            if list(filter(lambda g: g["title"] == item.parent.name, group["groups"])):
+                                item_group = list(filter(lambda g: g["title"] == item.parent.name, group["groups"]))[0]
+                    if not list(filter(lambda item: item["url"] == wmspath, item_group["items"])):
+                        self.logger.info(f"Adding project {item.stem} to group {item.parent.name}")
+                        item_group["items"].append(theme_item)
+                    else: self.logger.info(f"Project {item.stem} already exists in group {item.parent.name}")
+                else:
+                    # Search for theme if it already exists in items
+                    if not list(filter(lambda item: item["url"] == wmspath, items)):
                         self.logger.info(f"Adding project {item.stem}")
                         items.append(theme_item)
-                else:
-                    self.logger.info(f"Skipping project {item.name}")
+                    else:
+                        self.logger.info(f"Skipping project {item.name}")
         themes["groups"] = groups
         themes["items"] = items
 
+        if save_scanned_projects_in_config:
+            # Save themes_config in file to save scanned themes and groups
+            base_themes_config = self.config.get("themesConfig", None)
+            config_in_path = os.environ.get(
+                'INPUT_CONFIG_PATH', 'config-in/'
+            )
+            config_file_dir = os.path.join(config_in_path, self.tenant)
+            baksuffix = "%s.bak" % datetime.now(UTC).strftime("-%Y%m%d-%H%M%S")
+            if isinstance(base_themes_config, str):
+                themes_config_path = base_themes_config
+                try:
+                    if not os.path.isabs(themes_config_path):
+                        themes_config_path = os.path.join(config_file_dir, themes_config_path)
+                    with open(themes_config_path) as f:
+                        base_themes_config = json.load(f)
+
+                    if base_themes_config != themes_config:
+                        with open(themes_config_path + baksuffix, "w", encoding="utf-8") as fh:
+                            json.dump(base_themes_config, fh, indent=2, separators=(',', ': '))
+
+                        with open(themes_config_path, "w", encoding="utf-8") as fh:
+                            json.dump(themes_config, fh, indent=2, separators=(',', ': '))
+                        self.logger.info("Themes configuration has been updated.")
+                    else: self.logger.info("Themes configuration did not change.")
+                except IOError as e:
+                    msg = "Failed to backup/save themes configuration %s: %s" % (themes_config_path, e.strerror)
+                    self.logger.error(msg)
+            elif isinstance(base_themes_config, dict):
+                tenant_config_path = os.path.join(
+                    config_file_dir, 'tenantConfig.json'
+                )
+                # Read ConfigGenerator config file
+                try:
+                    with open(tenant_config_path, encoding='utf-8') as fh:
+                        tenant_config = json.load(fh, object_pairs_hook=OrderedDict)
+                except IOError as e:
+                    self.logger.error("Error reading tenantConfig.json: {}".format(
+                        e.strerror))
+                if tenant_config["themesConfig"] != themes_config:
+                    # Backup and save config file with new themes_config
+                    try:
+                        with open(tenant_config_path + baksuffix, "w", encoding="utf-8") as fh:
+                            json.dump(tenant_config, fh, indent=2, separators=(',', ': '))
+
+                        tenant_config["themesConfig"] = themes_config
+                        with open(tenant_config_path, "w", encoding="utf-8") as fh:
+                            json.dump(tenant_config, fh, indent=2, separators=(',', ': '))
+                        self.logger.info("Themes configuration has been updated.")
+                    except IOError as e:
+                        msg = "Failed to backup/save themes configuration %s: %s" % (tenant_config_path, e.strerror)
+                        self.logger.error(msg)
+                else: self.logger.info("Themes configuration did not change.")
+            else:
+                msg = "Missing or invalid themes configuration in tenantConfig.json"
+                self.logger.error(msg)
+
     def search_print_layouts(self, generator_config):
         qgis_print_layouts_dir = generator_config.get(
             'qgis_print_layouts_dir', '/layouts')