380 sync task registry update ak requirements and measures synchronization

.github/workflows/sync.yml

@@ -60,6 +60,7 @@ jobs:
           MATTERMOST_WEBHOOK_URL: ${{ secrets.MM_WEBHOOK_DEV }}
           TEXT: |
             Wijziging geconstateerd in bestanden van het Algoritmekader die invloed hebben op het Task Registry.
+            [Link naar de GitHub Action run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
             ```
             ${{ needs.sync.outputs.file_contents }}
             ```

script/conversion/add_links_to_measures

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import os
+import sys
 from pathlib import Path
 from typing import Any, TextIO
 
@@ -50,26 +51,30 @@ def link_measures_to_requirements(measure_folder: Path) -> dict[Any, Any]:
 @click.argument("requirement_folder", type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path))
 @click.argument("measure_folder", type=click.Path(exists=False, file_okay=False, dir_okay=True, path_type=Path))
 def insert_into_requirements(requirement_folder: Path, measure_folder: Path):  # noqa
-    urns_to_original = link_measures_to_requirements(measure_folder)
-
-    for filename in sorted(os.listdir(requirement_folder)):
-        if filename.endswith(".yaml"):
-            with open(os.path.join(requirement_folder, filename)) as file:
-                data: dict[Any, Any] = ordered_load(file, loader=yaml.FullLoader)  # Load while preserving order
-
-                if "links" in data and isinstance(data["links"], list):
-                    # Create reverse links based on the URN
-                    data["links"] = urns_to_original.get(data.get("urn"), [])
-
-                if "ai_act_profile" in data:
-                    for profile in data["ai_act_profile"]:
-                        for key in profile:
-                            if isinstance(profile[key], list):
-                                profile[key] = [rename_ai_act_labels_dict.get(item, item) for item in profile[key]]
-
-                # Write out the modified data to the secondary folder
-                with open(os.path.join(requirement_folder, filename), "w") as new_file:
-                    ordered_dump(data, new_file)  # Dump with order preserved
+    try:
+        urns_to_original = link_measures_to_requirements(measure_folder)
+
+        for filename in sorted(os.listdir(requirement_folder)):
+            if filename.endswith(".yaml"):
+                with open(os.path.join(requirement_folder, filename)) as file:
+                    data: dict[Any, Any] = ordered_load(file, loader=yaml.FullLoader)  # Load while preserving order
+
+                    if "links" in data and isinstance(data["links"], list):
+                        # Create reverse links based on the URN
+                        data["links"] = urns_to_original.get(data.get("urn"), [])
+
+                    if "ai_act_profile" in data:
+                        for profile in data["ai_act_profile"]:
+                            for key in profile:
+                                if isinstance(profile[key], list):
+                                    profile[key] = [rename_ai_act_labels_dict.get(item, item) for item in profile[key]]
+
+                    # Write out the modified data to the secondary folder
+                    with open(os.path.join(requirement_folder, filename), "w") as new_file:
+                        ordered_dump(data, new_file)  # Dump with order preserved
+    except Exception as e:  # Catch any exception
+        print(f"Error: {e}", file=sys.stderr)  # Print error to stderr
+        sys.exit(1)
 
 
 insert_into_requirements()

script/conversion/convert_mkdocs_to_yaml_lifecycles

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import re
+import sys
 from pathlib import Path
 
 import click
@@ -11,17 +12,21 @@ import yaml
 @click.argument("source", type=click.Path(exists=True, file_okay=True, dir_okay=False, path_type=Path))
 @click.argument("destination", type=click.Path(exists=False, file_okay=False, dir_okay=True, path_type=Path))
 def do_sync(source: Path, destination: Path) -> None:
-    destination.mkdir(parents=True, exist_ok=True)
-    with open(source, encoding="utf-8") as file:
-        document = file.read()
-    regex = r"[\s\t]?<li>\s?<a href=\".*\">(.*)</a></li>"
-    matches = re.findall(regex, document)
-    if not matches:
-        regex = r"\d\.[\s\t]*\[(.*)\]\(.*\)"
+    try:
+        destination.mkdir(parents=True, exist_ok=True)
+        with open(source, encoding="utf-8") as file:
+            document = file.read()
+        regex = r"[\s\t]?<li>\s?<a href=\".*\">(.*)</a></li>"
         matches = re.findall(regex, document)
+        if not matches:
+            regex = r"\d\.[\s\t]*\[(.*)\]\(.*\)"
+            matches = re.findall(regex, document)
 
-    with open(destination / Path("lifecycles.yaml"), "w", encoding="utf-8") as file:
-        yaml.dump({"lifecycles": matches}, file, allow_unicode=True, sort_keys=False)
+        with open(destination / Path("lifecycles.yaml"), "w", encoding="utf-8") as file:
+            yaml.dump({"lifecycles": matches}, file, allow_unicode=True, sort_keys=False)
+    except Exception as e:  # Catch any exception
+        print(f"Error: {e}", file=sys.stderr)  # Print error to stderr
+        sys.exit(1)
 
 
 do_sync()

script/conversion/convert_mkdocs_to_yaml_measure

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 import os
 import re
+import sys
 from pathlib import Path
 
 import click
@@ -81,11 +82,15 @@ def parse_markdown_to_yaml(input_file: str, destination: Path) -> None:
 @click.argument("source", type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path))
 @click.argument("destination", type=click.Path(exists=False, file_okay=False, dir_okay=True, path_type=Path))
 def read_folder(source: Path, destination: Path) -> None:
-    destination.mkdir(parents=True, exist_ok=True)
-    for filename in sorted(os.listdir(source)):
-        file_path: str = os.path.join(source, filename)
-        if os.path.isfile(file_path) and "index" not in file_path:
-            parse_markdown_to_yaml(file_path, destination)
+    try:
+        destination.mkdir(parents=True, exist_ok=True)
+        for filename in sorted(os.listdir(source)):
+            file_path: str = os.path.join(source, filename)
+            if os.path.isfile(file_path) and "index" not in file_path:
+                parse_markdown_to_yaml(file_path, destination)
+    except Exception as e:  # Catch any exception
+        print(f"Error: {e}", file=sys.stderr)  # Print error to stderr
+        sys.exit(1)
 
 
 read_folder()

script/conversion/convert_mkdocs_to_yaml_requirements

@@ -2,6 +2,7 @@
 
 import os
 import re
+import sys
 from pathlib import Path
 from typing import Any
 
@@ -96,19 +97,23 @@ def parse_markdown_to_yaml(input_file: str, destination: Path) -> None:
 @click.argument("source", type=click.Path(exists=False, file_okay=False, dir_okay=True, path_type=Path))
 @click.argument("destination", type=click.Path(exists=False, file_okay=False, dir_okay=True, path_type=Path))
 def read_folder(source: Path, destination: Path) -> None:
-    requirements_destination = destination / Path("requirements")
-    requirements_destination.mkdir(parents=True, exist_ok=True)
-    for filename in sorted(os.listdir(source)):
-        file_path: str = os.path.join(source, filename)
-        if os.path.isfile(file_path) and "index" not in file_path:
-            parse_markdown_to_yaml(file_path, requirements_destination)
-
-    labels_destination = destination / Path("labels")
-    labels_destination.mkdir(parents=True, exist_ok=True)
-    # prepare data for yaml
-    yaml_output = {"ai-act": {key: sorted(value) for key, value in ai_act_values.items()}}
-    with open(labels_destination / Path("labels.yaml"), "w", encoding="utf-8") as file:
-        yaml.dump(yaml_output, file, allow_unicode=True, sort_keys=False)
+    try:
+        requirements_destination = destination / Path("requirements")
+        requirements_destination.mkdir(parents=True, exist_ok=True)
+        for filename in sorted(os.listdir(source)):
+            file_path: str = os.path.join(source, filename)
+            if os.path.isfile(file_path) and "index" not in file_path:
+                parse_markdown_to_yaml(file_path, requirements_destination)
+
+        labels_destination = destination / Path("labels")
+        labels_destination.mkdir(parents=True, exist_ok=True)
+        # prepare data for yaml
+        yaml_output = {"ai-act": {key: sorted(value) for key, value in ai_act_values.items()}}
+        with open(labels_destination / Path("labels.yaml"), "w", encoding="utf-8") as file:
+            yaml.dump(yaml_output, file, allow_unicode=True, sort_keys=False)
+    except Exception as e:  # Catch any exception
+        print(f"Error: {e}", file=sys.stderr)  # Print error to stderr
+        sys.exit(1)
 
 
 read_folder()

script/conversion/convert_mkdocs_to_yaml_roles

@@ -2,6 +2,7 @@
 
 import os
 import re
+import sys
 from pathlib import Path
 from typing import Any
 
@@ -23,15 +24,19 @@ def extract_title_from_file(input_file: str) -> str:
 @click.argument("source", type=click.Path(exists=False, file_okay=False, dir_okay=True, path_type=Path))
 @click.argument("destination", type=click.Path(exists=False, file_okay=False, dir_okay=True, path_type=Path))
 def create_roles_file(source: Path, destination: Path) -> None:
-    destination.mkdir(parents=True, exist_ok=True)
-    roles: list[str] = []
-    for filename in sorted(os.listdir(source)):
-        file_path: str = os.path.join(source, filename)
-        if os.path.isfile(file_path) and "index" not in file_path:
-            roles.append(extract_title_from_file(file_path))
-
-    with open(destination / Path("roles.yaml"), "w", encoding="utf-8") as file:
-        yaml.dump({"roles": sorted(roles)}, file, allow_unicode=True, sort_keys=False)
+    try:
+        destination.mkdir(parents=True, exist_ok=True)
+        roles: list[str] = []
+        for filename in sorted(os.listdir(source)):
+            file_path: str = os.path.join(source, filename)
+            if os.path.isfile(file_path) and "index" not in file_path:
+                roles.append(extract_title_from_file(file_path))
+
+        with open(destination / Path("roles.yaml"), "w", encoding="utf-8") as file:
+            yaml.dump({"roles": sorted(roles)}, file, allow_unicode=True, sort_keys=False)
+    except Exception as e:  # Catch any exception
+        print(f"Error: {e}", file=sys.stderr)  # Print error to stderr
+        sys.exit(1)
 
 
 create_roles_file()

script/sync

@@ -54,11 +54,21 @@ if [ "$UPDATE" = "commit" ] && [ "$RECENT_COMMITS" = "" ]; then
 fi
 
 # Run the conversion scripts
-poetry run ../script/conversion/convert_mkdocs_to_yaml_measure algoritmekader/docs/maatregelen "${DESTINATION}"/measures
-poetry run ../script/conversion/convert_mkdocs_to_yaml_requirements algoritmekader/docs/vereisten "${DESTINATION}"
-poetry run ../script/conversion/add_links_to_measures "${DESTINATION}"/requirements "${DESTINATION}"/measures
-poetry run ../script/conversion/convert_mkdocs_to_yaml_lifecycles algoritmekader/docs/levenscyclus/index.md "${DESTINATION}"/lifecycles
-poetry run ../script/conversion/convert_mkdocs_to_yaml_roles algoritmekader/docs/rollen "${DESTINATION}"/roles
+commands=(
+  "poetry run ../script/conversion/convert_mkdocs_to_yaml_measure algoritmekader/docs/voldoen-aan-wetten-en-regels/maatregelen \"${DESTINATION}\"/measures"
+  "poetry run ../script/conversion/convert_mkdocs_to_yaml_requirements algoritmekader/docs/voldoen-aan-wetten-en-regels/vereisten \"${DESTINATION}\""
+  "poetry run ../script/conversion/add_links_to_measures \"${DESTINATION}\"/requirements \"${DESTINATION}\"/measures"
+  "poetry run ../script/conversion/convert_mkdocs_to_yaml_lifecycles algoritmekader/docs/levenscyclus/index.md \"${DESTINATION}\"/lifecycles"
+  "poetry run ../script/conversion/convert_mkdocs_to_yaml_roles algoritmekader/docs/rollen \"${DESTINATION}\"/roles"
+)
+
+for command in "${commands[@]}"; do
+  eval "$command"
+  if [[ $? -ne 0 ]]; then
+    echo "Error: $(basename "$command") failed"
+    exit 1  # Example: Exit the script on error
+  fi
+done
 
 if [ "$MODE" = "compare" ]; then
     diff -burN "${DESTINATION}"/measures ../measures; (( exit_status = exit_status || $? ))