separate examples testing into own make command

.github/workflows/test_doc_snippets.yml

@@ -60,9 +60,17 @@ jobs:
         # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
         run: poetry install --no-interaction -E duckdb -E weaviate -E parquet -E qdrant -E bigquery --with docs,sentry-sdk --without airflow
 
-      - name: create secrets.toml
+      - name: create secrets.toml for examples
+        run: pwd && echo "$DLT_SECRETS_TOML" > docs/examples/.dlt/secrets.toml
+
+      - name: create secrets.toml for snippets
         run: pwd && echo "$DLT_SECRETS_TOML" > docs/website/docs/.dlt/secrets.toml
 
-      - name: Run linter and tests
-        run: make test-and-lint-snippets
+      - name: Run linter and tests on examples
+        run: make lint-and-test-examples
+
+      - name: Run linter and tests on snippets
+        run: make lint-and-test-snippets
+
+
 

.gitignore

@@ -12,7 +12,6 @@ experiments/*
 # !experiments/pipeline/
 # !experiments/pipeline/*
 secrets.toml
-!docs/**/secrets.toml
 *.session.sql
 *.duckdb
 *.wal
@@ -141,4 +140,7 @@ tmp
 **/tmp
 
 # Qdrant embedding models cache
-local_cache/
+local_cache/
+
+# test file for examples are generated and should not be committed
+docs/examples/**/test*.py

Makefile

@@ -27,7 +27,7 @@ help:
 	@echo "			tests all components using local destinations: duckdb and postgres"
 	@echo "		test-common"
 	@echo "			tests common components"
-	@echo "		test-and-lint-snippets"
+	@echo "		lint-and-test-snippets"
 	@echo "			tests and lints snippets and examples in docs"
 	@echo "		build-library"
 	@echo "			makes dev and then builds dlt package for distribution"
@@ -60,12 +60,21 @@ format:
 	poetry run black dlt docs tests --exclude=".*syntax_error.py|\.venv.*|_storage/.*"
 	# poetry run isort ./
 
-test-and-lint-snippets:
+lint-and-test-snippets:
 	cd docs/tools && poetry run python check_embedded_snippets.py full
 	poetry run mypy --config-file mypy.ini docs/website docs/examples docs/tools --exclude docs/tools/lint_setup
 	poetry run flake8 --max-line-length=200 docs/website docs/examples docs/tools
 	cd docs/website/docs && poetry run pytest --ignore=node_modules
 
+lint-and-test-examples:
+	poetry run mypy --config-file mypy.ini docs/examples
+	poetry run flake8 --max-line-length=200 docs/examples
+	cd docs/examples && poetry run pytest
+
+
+test-examples:
+	cd docs/examples && poetry run pytest
+
 lint-security:
 	poetry run bandit -r dlt/ -n 3 -l