From 5ca9970385e21ebe4809e6fce19b241c758b0377 Mon Sep 17 00:00:00 2001 From: Laurens Weijs Date: Tue, 3 Dec 2024 12:46:30 +0100 Subject: [PATCH 1/4] Add link to notify message --- .github/workflows/sync.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index bbcb3b8..b8c74da 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -60,6 +60,7 @@ jobs: MATTERMOST_WEBHOOK_URL: ${{ secrets.MM_WEBHOOK_DEV }} TEXT: | Wijziging geconstateerd in bestanden van het Algoritmekader die invloed hebben op het Task Registry. + [Link naar de GitHub Action run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) ``` ${{ needs.sync.outputs.file_contents }} ``` From 76250d01651628d552f42720caa9a156ceaad4bc Mon Sep 17 00:00:00 2001 From: Laurens Weijs Date: Tue, 3 Dec 2024 12:58:52 +0100 Subject: [PATCH 2/4] Change the sync script to fail when the conversion scripts fail --- script/sync | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/script/sync b/script/sync index 07b4e0d..1b6a992 100755 --- a/script/sync +++ b/script/sync @@ -54,11 +54,21 @@ if [ "$UPDATE" = "commit" ] && [ "$RECENT_COMMITS" = "" ]; then fi # Run the conversion scripts -poetry run ../script/conversion/convert_mkdocs_to_yaml_measure algoritmekader/docs/maatregelen "${DESTINATION}"/measures -poetry run ../script/conversion/convert_mkdocs_to_yaml_requirements algoritmekader/docs/vereisten "${DESTINATION}" -poetry run ../script/conversion/add_links_to_measures "${DESTINATION}"/requirements "${DESTINATION}"/measures -poetry run ../script/conversion/convert_mkdocs_to_yaml_lifecycles algoritmekader/docs/levenscyclus/index.md "${DESTINATION}"/lifecycles -poetry run ../script/conversion/convert_mkdocs_to_yaml_roles algoritmekader/docs/rollen "${DESTINATION}"/roles +commands=( + "poetry run ../script/conversion/convert_mkdocs_to_yaml_measure algoritmekader/docs/voldoen-aan-wetten-en-regels/maatregelen \"${DESTINATION}\"/measures" + "poetry run ../script/conversion/convert_mkdocs_to_yaml_requirements algoritmekader/docs/voldoen-aan-wetten-en-regels/vereisten \"${DESTINATION}\"" + "poetry run ../script/conversion/add_links_to_measures \"${DESTINATION}\"/requirements \"${DESTINATION}\"/measures" + "poetry run ../script/conversion/convert_mkdocs_to_yaml_lifecycles algoritmekader/docs/levenscyclus/index.md \"${DESTINATION}\"/lifecycles" + "poetry run ../script/conversion/convert_mkdocs_to_yaml_roles algoritmekader/docs/rollen \"${DESTINATION}\"/roles" +) + +for command in "${commands[@]}"; do + eval "$command" + if [[ $? -ne 0 ]]; then + echo "Error: $(basename "$command") failed" + exit 1 # Example: Exit the script on error + fi +done if [ "$MODE" = "compare" ]; then diff -burN "${DESTINATION}"/measures ../measures; (( exit_status = exit_status || $? )) From bf9bdf331db660ae48e524a8be7700b0654ec7cf Mon Sep 17 00:00:00 2001 From: Laurens Weijs Date: Tue, 3 Dec 2024 13:04:57 +0100 Subject: [PATCH 3/4] Add error handling to conversion scripts --- script/conversion/add_links_to_measures | 45 ++++++++++--------- .../convert_mkdocs_to_yaml_lifecycles | 23 ++++++---- .../conversion/convert_mkdocs_to_yaml_measure | 15 ++++--- .../convert_mkdocs_to_yaml_requirements | 31 +++++++------ .../conversion/convert_mkdocs_to_yaml_roles | 23 ++++++---- 5 files changed, 81 insertions(+), 56 deletions(-) diff --git a/script/conversion/add_links_to_measures b/script/conversion/add_links_to_measures index 46a3dbe..ebf1ba6 100755 --- a/script/conversion/add_links_to_measures +++ b/script/conversion/add_links_to_measures @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import os +import sys from pathlib import Path from typing import Any, TextIO @@ -50,26 +51,30 @@ def link_measures_to_requirements(measure_folder: Path) -> dict[Any, Any]: @click.argument("requirement_folder", type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path)) @click.argument("measure_folder", type=click.Path(exists=False, file_okay=False, dir_okay=True, path_type=Path)) def insert_into_requirements(requirement_folder: Path, measure_folder: Path): # noqa - urns_to_original = link_measures_to_requirements(measure_folder) - - for filename in sorted(os.listdir(requirement_folder)): - if filename.endswith(".yaml"): - with open(os.path.join(requirement_folder, filename)) as file: - data: dict[Any, Any] = ordered_load(file, loader=yaml.FullLoader) # Load while preserving order - - if "links" in data and isinstance(data["links"], list): - # Create reverse links based on the URN - data["links"] = urns_to_original.get(data.get("urn"), []) - - if "ai_act_profile" in data: - for profile in data["ai_act_profile"]: - for key in profile: - if isinstance(profile[key], list): - profile[key] = [rename_ai_act_labels_dict.get(item, item) for item in profile[key]] - - # Write out the modified data to the secondary folder - with open(os.path.join(requirement_folder, filename), "w") as new_file: - ordered_dump(data, new_file) # Dump with order preserved + try: + urns_to_original = link_measures_to_requirements(measure_folder) + + for filename in sorted(os.listdir(requirement_folder)): + if filename.endswith(".yaml"): + with open(os.path.join(requirement_folder, filename)) as file: + data: dict[Any, Any] = ordered_load(file, loader=yaml.FullLoader) # Load while preserving order + + if "links" in data and isinstance(data["links"], list): + # Create reverse links based on the URN + data["links"] = urns_to_original.get(data.get("urn"), []) + + if "ai_act_profile" in data: + for profile in data["ai_act_profile"]: + for key in profile: + if isinstance(profile[key], list): + profile[key] = [rename_ai_act_labels_dict.get(item, item) for item in profile[key]] + + # Write out the modified data to the secondary folder + with open(os.path.join(requirement_folder, filename), "w") as new_file: + ordered_dump(data, new_file) # Dump with order preserved + except Exception as e: # Catch any exception + print(f"Error: {e}", file=sys.stderr) # Print error to stderr + sys.exit(1) insert_into_requirements() diff --git a/script/conversion/convert_mkdocs_to_yaml_lifecycles b/script/conversion/convert_mkdocs_to_yaml_lifecycles index b2f3400..607b302 100755 --- a/script/conversion/convert_mkdocs_to_yaml_lifecycles +++ b/script/conversion/convert_mkdocs_to_yaml_lifecycles @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import re +import sys from pathlib import Path import click @@ -11,17 +12,21 @@ import yaml @click.argument("source", type=click.Path(exists=True, file_okay=True, dir_okay=False, path_type=Path)) @click.argument("destination", type=click.Path(exists=False, file_okay=False, dir_okay=True, path_type=Path)) def do_sync(source: Path, destination: Path) -> None: - destination.mkdir(parents=True, exist_ok=True) - with open(source, encoding="utf-8") as file: - document = file.read() - regex = r"[\s\t]?
  • \s?(.*)
    • " - matches = re.findall(regex, document) - if not matches: - regex = r"\d\.[\s\t]*\[(.*)\]\(.*\)" + try: + destination.mkdir(parents=True, exist_ok=True) + with open(source, encoding="utf-8") as file: + document = file.read() + regex = r"[\s\t]?
  • \s?(.*)
    • " matches = re.findall(regex, document) + if not matches: + regex = r"\d\.[\s\t]*\[(.*)\]\(.*\)" + matches = re.findall(regex, document) - with open(destination / Path("lifecycles.yaml"), "w", encoding="utf-8") as file: - yaml.dump({"lifecycles": matches}, file, allow_unicode=True, sort_keys=False) + with open(destination / Path("lifecycles.yaml"), "w", encoding="utf-8") as file: + yaml.dump({"lifecycles": matches}, file, allow_unicode=True, sort_keys=False) + except Exception as e: # Catch any exception + print(f"Error: {e}", file=sys.stderr) # Print error to stderr + sys.exit(1) do_sync() diff --git a/script/conversion/convert_mkdocs_to_yaml_measure b/script/conversion/convert_mkdocs_to_yaml_measure index 283a179..b6011ac 100755 --- a/script/conversion/convert_mkdocs_to_yaml_measure +++ b/script/conversion/convert_mkdocs_to_yaml_measure @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import os import re +import sys from pathlib import Path import click @@ -81,11 +82,15 @@ def parse_markdown_to_yaml(input_file: str, destination: Path) -> None: @click.argument("source", type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path)) @click.argument("destination", type=click.Path(exists=False, file_okay=False, dir_okay=True, path_type=Path)) def read_folder(source: Path, destination: Path) -> None: - destination.mkdir(parents=True, exist_ok=True) - for filename in sorted(os.listdir(source)): - file_path: str = os.path.join(source, filename) - if os.path.isfile(file_path) and "index" not in file_path: - parse_markdown_to_yaml(file_path, destination) + try: + destination.mkdir(parents=True, exist_ok=True) + for filename in sorted(os.listdir(source)): + file_path: str = os.path.join(source, filename) + if os.path.isfile(file_path) and "index" not in file_path: + parse_markdown_to_yaml(file_path, destination) + except Exception as e: # Catch any exception + print(f"Error: {e}", file=sys.stderr) # Print error to stderr + sys.exit(1) read_folder() diff --git a/script/conversion/convert_mkdocs_to_yaml_requirements b/script/conversion/convert_mkdocs_to_yaml_requirements index f9b3581..3610113 100755 --- a/script/conversion/convert_mkdocs_to_yaml_requirements +++ b/script/conversion/convert_mkdocs_to_yaml_requirements @@ -2,6 +2,7 @@ import os import re +import sys from pathlib import Path from typing import Any @@ -96,19 +97,23 @@ def parse_markdown_to_yaml(input_file: str, destination: Path) -> None: @click.argument("source", type=click.Path(exists=False, file_okay=False, dir_okay=True, path_type=Path)) @click.argument("destination", type=click.Path(exists=False, file_okay=False, dir_okay=True, path_type=Path)) def read_folder(source: Path, destination: Path) -> None: - requirements_destination = destination / Path("requirements") - requirements_destination.mkdir(parents=True, exist_ok=True) - for filename in sorted(os.listdir(source)): - file_path: str = os.path.join(source, filename) - if os.path.isfile(file_path) and "index" not in file_path: - parse_markdown_to_yaml(file_path, requirements_destination) - - labels_destination = destination / Path("labels") - labels_destination.mkdir(parents=True, exist_ok=True) - # prepare data for yaml - yaml_output = {"ai-act": {key: sorted(value) for key, value in ai_act_values.items()}} - with open(labels_destination / Path("labels.yaml"), "w", encoding="utf-8") as file: - yaml.dump(yaml_output, file, allow_unicode=True, sort_keys=False) + try: + requirements_destination = destination / Path("requirements") + requirements_destination.mkdir(parents=True, exist_ok=True) + for filename in sorted(os.listdir(source)): + file_path: str = os.path.join(source, filename) + if os.path.isfile(file_path) and "index" not in file_path: + parse_markdown_to_yaml(file_path, requirements_destination) + + labels_destination = destination / Path("labels") + labels_destination.mkdir(parents=True, exist_ok=True) + # prepare data for yaml + yaml_output = {"ai-act": {key: sorted(value) for key, value in ai_act_values.items()}} + with open(labels_destination / Path("labels.yaml"), "w", encoding="utf-8") as file: + yaml.dump(yaml_output, file, allow_unicode=True, sort_keys=False) + except Exception as e: # Catch any exception + print(f"Error: {e}", file=sys.stderr) # Print error to stderr + sys.exit(1) read_folder() diff --git a/script/conversion/convert_mkdocs_to_yaml_roles b/script/conversion/convert_mkdocs_to_yaml_roles index b58c0e4..cc684a1 100755 --- a/script/conversion/convert_mkdocs_to_yaml_roles +++ b/script/conversion/convert_mkdocs_to_yaml_roles @@ -2,6 +2,7 @@ import os import re +import sys from pathlib import Path from typing import Any @@ -23,15 +24,19 @@ def extract_title_from_file(input_file: str) -> str: @click.argument("source", type=click.Path(exists=False, file_okay=False, dir_okay=True, path_type=Path)) @click.argument("destination", type=click.Path(exists=False, file_okay=False, dir_okay=True, path_type=Path)) def create_roles_file(source: Path, destination: Path) -> None: - destination.mkdir(parents=True, exist_ok=True) - roles: list[str] = [] - for filename in sorted(os.listdir(source)): - file_path: str = os.path.join(source, filename) - if os.path.isfile(file_path) and "index" not in file_path: - roles.append(extract_title_from_file(file_path)) - - with open(destination / Path("roles.yaml"), "w", encoding="utf-8") as file: - yaml.dump({"roles": sorted(roles)}, file, allow_unicode=True, sort_keys=False) + try: + destination.mkdir(parents=True, exist_ok=True) + roles: list[str] = [] + for filename in sorted(os.listdir(source)): + file_path: str = os.path.join(source, filename) + if os.path.isfile(file_path) and "index" not in file_path: + roles.append(extract_title_from_file(file_path)) + + with open(destination / Path("roles.yaml"), "w", encoding="utf-8") as file: + yaml.dump({"roles": sorted(roles)}, file, allow_unicode=True, sort_keys=False) + except Exception as e: # Catch any exception + print(f"Error: {e}", file=sys.stderr) # Print error to stderr + sys.exit(1) create_roles_file() From 0f66255d59cf202450be448588a5d68d02e45db4 Mon Sep 17 00:00:00 2001 From: Laurens Weijs Date: Wed, 4 Dec 2024 14:47:44 +0100 Subject: [PATCH 4/4] Implement feedback Ravi --- script/conversion/convert_mkdocs_to_yaml_measure | 2 +- script/conversion/convert_mkdocs_to_yaml_requirements | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/script/conversion/convert_mkdocs_to_yaml_measure b/script/conversion/convert_mkdocs_to_yaml_measure index b6011ac..df2b506 100755 --- a/script/conversion/convert_mkdocs_to_yaml_measure +++ b/script/conversion/convert_mkdocs_to_yaml_measure @@ -54,7 +54,7 @@ def parse_markdown_to_yaml(input_file: str, destination: Path) -> None: "language": "nl", "owners": [{"organization": "Algoritmekader", "name": "", "email": "", "role": ""}], "date": "", - "url": f"https://minbzk.github.io/Algoritmekader/maatregelen/{url_title}/index.html", + "url": f"https://minbzk.github.io/Algoritmekader/voldoen-aan-wetten-en-regels/maatregelen/{url_title}/index.html", "subject": front_matter_data.get("onderwerp", []), "suggested_roles": front_matter_data.get("rollen", []), "lifecycle": front_matter_data.get("levenscyclus", []), diff --git a/script/conversion/convert_mkdocs_to_yaml_requirements b/script/conversion/convert_mkdocs_to_yaml_requirements index 3610113..9d3424c 100755 --- a/script/conversion/convert_mkdocs_to_yaml_requirements +++ b/script/conversion/convert_mkdocs_to_yaml_requirements @@ -60,7 +60,7 @@ def parse_markdown_to_yaml(input_file: str, destination: Path) -> None: "language": "nl", "owners": [{"organization": "Algoritmekader", "name": "", "email": "", "role": ""}], "date": "", - "url": f"https://minbzk.github.io/Algoritmekader/vereisten/{url_title}/index.html", + "url": f"https://minbzk.github.io/Algoritmekader/voldoen-aan-wetten-en-regels/vereisten/{url_title}/index.html", "subject": front_matter_data.get("onderwerp", []), "lifecycle": front_matter_data.get("levenscyclus", []), "links": [],