add better error messages and add check to ci

Makefile

@@ -60,6 +60,7 @@ format:
 	# poetry run isort ./
 
 test-and-lint-snippets:
+	cd docs/tools && poetry run python check_embedded_snippets.py
 	poetry run mypy --config-file mypy.ini docs/website docs/examples docs/tools
 	poetry run flake8 --max-line-length=200 docs/website docs/examples docs/tools
 	cd docs/website/docs && poetry run pytest --ignore=node_modules

docs/tools/check_embedded_snippets.py

@@ -1,13 +1,10 @@
 """
 Walks through all markdown files, finds all code snippets, and checks wether they are parseable.
 """
-import os
-from typing import TypedDict, List
-import ast
+from typing import TypedDict, List, Dict
+
+import os, ast, json, yaml, tomlkit  # noqa: I251
 from textwrap import dedent
-import tomlkit
-import json
-import yaml
 
 DOCS_DIR = "../website/docs"
 
@@ -24,7 +21,7 @@ class Snippet(TypedDict):
 if __name__ == "__main__":
     # discover all markdown files to be processed
     markdown_files = []
-    for path, directories, files in os.walk(DOCS_DIR):
+    for path, _, files in os.walk(DOCS_DIR):
         if "api_reference" in path:
             continue
         if "jaffle_shop" in path:
@@ -35,11 +32,12 @@ class Snippet(TypedDict):
 
     # extract snippets from markdown files
     snippets: List[Snippet] = []
-    for file in markdown_files:
-        print(f"Processing file {file}")
+    count: Dict[str, int] = {}
+    failed_count: Dict[str, int] = {}
 
+    for file in markdown_files:
         # go line by line and find all code  blocks
-        with open(file, "r") as f:
+        with open(file, "r", encoding="utf-8") as f:
             current_snippet: Snippet = None
             lint_count = 0
             for line in f.readlines():
@@ -49,6 +47,8 @@ class Snippet(TypedDict):
                         # process snippet
                         snippets.append(current_snippet)
                         current_snippet["code"] = dedent(current_snippet["code"])
+                        language = current_snippet["language"] or "unknown"
+                        count[language] = count.get(language, 0) + 1
                         current_snippet = None
                     else:
                         # start new snippet
@@ -60,17 +60,30 @@ class Snippet(TypedDict):
                         }
                 elif current_snippet:
                     current_snippet["code"] += line
-        assert not current_snippet
+        assert not current_snippet, (
+            "It seems that the last snippet in the file was not closed. Please check the file "
+            + file
+        )
+
+    print("Found", len(snippets), "snippets", "in", len(markdown_files), "markdown files: ")
+    print(count)
+    print(
+        "Snippets of the types py, yaml, toml and json will now be parsed. All other snippets will"
+        " be accepted as they are."
+    )
+    print("---")
+
+    assert len(snippets) > 100, "Found too few snippets. Something went wrong."  # sanity check
+    assert (
+        len(markdown_files) > 50
+    ), "Found too few markdown files. Something went wrong."  # sanity check
 
     # parse python snippets for now
-    count = {}
     total = 0
-    failed_count = {}
     for snippet in snippets:
         language = snippet["language"] or "unknown"
         code = snippet["code"]
         total += 1
-        count[language] = count.get(language, 0) + 1
 
         # print(
         #     "Processing snippet no",
@@ -99,25 +112,29 @@ class Snippet(TypedDict):
             elif language in ["sql"]:
                 pass
             else:
-                assert False, "Unknown language. Please choose the correct language for the snippet: py, toml, json, yaml, text, shell, bat or sql."
-
-        except Exception as e:
+                raise AssertionError("""
+Unknown language. Please choose the correct language for the snippet: py, toml, json, yaml, text, shell, bat or sql."
+All shell commands, except for windows, should be marked as shell.
+All code blocks that are not a specific (markup-) language should be marked as text.
+""")
+        except Exception:
             print(
-                "---\n"
-                "Failed parsing snippet no",
+                "---\nFailed parsing snippet no",
                 total,
                 "at line",
                 snippet["line"],
                 "in file",
                 snippet["file"],
                 "with language",
                 language,
-                "\n---"
+                "\n---",
             )
-            raise
             failed_count[language] = failed_count.get(language, 0) + 1
 
-    assert len(snippets) > 100, "Found too few snippets. Something went wrong."  # sanity check
+            raise  # you can comment this out to see all failed snippets
 
-    print(count)
-    print(failed_count)
+    if failed_count:
+        print("Failed to parse the following amount of snippets")
+        print(failed_count)
+    else:
+        print("All snippets could be parsed.")