From 51705cf11d12926cc0c8160eafa322649d793482 Mon Sep 17 00:00:00 2001 From: Wolfgang Ulmer Date: Mon, 7 Oct 2024 14:32:01 +0000 Subject: [PATCH] chore: Updated README and Makefile --- .vscode/extensions.json | 13 + .vscode/launch.json | 24 ++ .vscode/settings.json | 76 ++++++ Makefile | 103 ++++++++ README.md | 228 +++++++++++++++++- .../tests/test_artifactory_fetcher.py | 3 +- apps/artifactory-fetcher/tests/test_cli.py | 3 +- apps/artifactory-fetcher/tests/test_main.py | 1 + cicd/check-vscode-python-paths.py | 9 + cicd/coverage-check.sh | 23 ++ 10 files changed, 480 insertions(+), 3 deletions(-) create mode 100644 .vscode/extensions.json create mode 100644 .vscode/launch.json create mode 100644 .vscode/settings.json create mode 100644 Makefile create mode 100644 cicd/check-vscode-python-paths.py create mode 100755 cicd/coverage-check.sh diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..8a79801 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,13 @@ +{ + "recommendations": [ + "ms-python.isort", + "ms-python.mypy-type-checker", + "ms-python.vscode-pylance", + "ms-python.python", + "redhat.vscode-yaml", + "vivaxy.vscode-conventional-commits", + "ryanluker.vscode-coverage-gutters", + "ms-python.debugpy", + "charliermarsh.ruff" + ] +} \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..9c84a37 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,24 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Remote Attach", + "type": "debugpy", + "request": "attach", + "connect": { + "host": "localhost", + "port": 5678 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder}", + "remoteRoot": "." + } + ], + "justMyCode": true + } + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..87a748e --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,76 @@ +{ + "python.analysis.extraPaths": [ + ".", + "apps/artifactory-fetcher/src", + "apps/excel-tools/src", + "apps/filecheck/src", + "apps/papsr/src", + "apps/pdf-signature-evaluator/src", + "apps/pex-tool/src", + "apps/security-scanner/src", + "apps/sharepoint-evaluator/src", + "apps/sharepoint-fetcher/src", + "apps/sharepoint/src", + "apps/splunk-fetcher/src", + "packages/autopilot-utils/src", + "packages/autopilot-utils/tests/app_chained_multi_evaluator/src", + "packages/autopilot-utils/tests/app_multi_command/src", + "packages/autopilot-utils/tests/app_multi_evaluator/src", + "packages/autopilot-utils/tests/app_single_command/src", + "packages/autopilot-utils/tests/app_single_evaluator/src", + "packages/autopilot-utils/tests/app_subprocess_steps/src", + "packages/autopilot-utils/tests/app_super_simple_evaluator/src", + "packages/autopilot-utils/tests/demo_app/src" + ], + "python.defaultInterpreterPath": "dist/export/python/virtualenvs/python-default/3.10.15/bin/python", + "python.testing.pytestArgs": [ + "." + ], + "python.testing.pytestEnabled": true, + "python.testing.pytestPath": "dist/export/python/virtualenvs/python-default/3.10.15/bin/pytest", + "python.testing.unittestEnabled": false, + "mypy-type-checker.interpreter": [ + "dist/export/python/virtualenvs/python-default/3.10.15/bin/python" + ], + "coverage-gutters.coverageBaseDir": "dist/coverage/**", + "coverage-gutters.coverageFileNames": [ + "coverage.xml" + ], + "editor.formatOnSave": true, + "cSpell.words": [ + "capsys", + "commitlint", + "dateutil", + "distinfo", + "distinfos", + "distutils", + "dohq", + "itoa", + "LICENCE", + "lockfiles", + "loguru", + "loguru", + "maxsplit", + "mypy", + "ntlm", + "oneq", + "papsr", + "pydantic", + "pytz", + "rdparty", + "rdparty", + "reqs", + "SCIE", + "splunklib", + "subtargets", + "testcov", + "testint", + "urllib", + "virtualenv", + "Yaku" + ], + "[python]": { + "editor.defaultFormatter": "charliermarsh.ruff", + "editor.formatOnSave": true + } +} diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..57a3c56 --- /dev/null +++ b/Makefile @@ -0,0 +1,103 @@ +SHELL=bash + +.PHONY: help +help: + @echo "Targets:" + @echo "" + @echo " install-fast-hooks Set up Git hooks (linting, formatting)" + @echo " install-slow-hooks Set up Git hooks (linting, formatting, tests)" + @echo " remove-hooks Uninstall Git hooks" + @echo "" + @echo " vscode-integration Set up VS Code so that it works nicely with Pants" + @echo " update-lockfiles Update Python lockfiles for Pants" + @echo "" + @echo " format Run formatters" + @echo " lint Run linters (e.g. in CI/CD pipeline)" + @echo " test Run tests (e.g. in CI/CD pipeline)" + @echo " testcov Run tests with coverage analysis" + @echo " testint Run integration tests only (for external systems)" + @echo " check Run checks (e.g. in CI/CD pipeline)" + @echo " package Package everything" + @echo "" + @echo " coverage-check Compare list of Python files with coverage analysis files" + @echo "" + +FOLDER ?= '.' +.PHONY: install-slow-hooks install-fast-hooks remove-hooks +install-fast-hooks: remove-hooks install-std-hooks + ln -s ../../cicd/pre-commit-hook-fast .git/hooks/pre-commit +install-slow-hooks: remove-hooks install-std-hooks + ln -s ../../cicd/pre-commit-hook-slow .git/hooks/pre-commit +install-std-hooks: remove-hooks + ln -s ../../cicd/commit-msg-hook .git/hooks/commit-msg +remove-hooks: + rm -f .git/hooks/commit-msg + rm -f .git/hooks/pre-commit + +.PHONY: vscode-integration +vscode-integration: vscode-python-paths vscode-virtualenvs + +.PHONY: vscode-virtualenvs +vscode-virtualenvs: + pants export \ + --py-resolve-format=symlinked_immutable_virtualenv \ + --resolve=python-default + export LATEST_PYTHON_VERSION=$$(ls -1 dist/export/python/virtualenvs/python-default | sort | tail -n 1) ; \ + export PYTHON_PREFIX=dist/export/python/virtualenvs/python-default/$${LATEST_PYTHON_VERSION}/ ; \ + if yq -V > /dev/null; then \ + yq -oj -i '."python.defaultInterpreterPath" = "'$${PYTHON_PREFIX}'bin/python"' .vscode/settings.json ; \ + yq -oj -i '."python.testing.pytestPath" = "'$${PYTHON_PREFIX}'bin/pytest"' .vscode/settings.json ; \ + yq -oj -i '."mypy-type-checker.interpreter" = ["'$${PYTHON_PREFIX}'bin/python"]' .vscode/settings.json ; \ + echo "Successfully updated .vscode/settings.json" ; \ + else \ + echo "No 'yq' installed. Please adapt python.defaultInterpreterPath and python.testing.pytestPath" ; \ + echo "manually in .vscode/settings.json." ; \ + fi + +.PHONY: vscode-python-paths +vscode-python-paths: + python3 cicd/check-vscode-python-paths.py + +.PHONY: _generate_lockfiles +_generate_lockfiles: + pants generate-lockfiles + +.PHONY: _convert_lockfile +_convert_lockfile: + grep -A1000000 -e '^[^/]' 3rdparty/python-lockfile.txt| jq -r '.locked_resolves[].locked_requirements[] | [ .project_name, .version ] | join("==") ' | LC_ALL=C sort -d -s > dependabot-requirements.txt + +.PHONY: update-lockfiles +update-lockfiles: _generate_lockfiles _convert_lockfile + +.PHONY: test +test: + pants --tag="-integration" test ${FOLDER}/:: + +.PHONY: testcov +testcov: + pants --tag="-integration" test --use-coverage --coverage-py-filter=${FOLDER} ${FOLDER}/:: + +.PHONY: testint +testint: + pants --tag="integration" test ${FOLDER}/:: + +.PHONY: lint +lint: + pants lint :: + +.PHONY: check +check: + pants check :: + +.PHONY: fmt format +fmt: format +format: + pants fmt :: + +.PHONY: package +package: + pants package ${FOLDER}/:: + +.PHONY: coverage-check +coverage-check: testcov + cicd/coverage-check.sh diff --git a/README.md b/README.md index d31a9cb..3db6a8a 100644 --- a/README.md +++ b/README.md @@ -1 +1,227 @@ -# yaku-apps-python \ No newline at end of file +# yaku-apps-python + +This repo contains apps for accessing and evaluating +content from different sources like SharePoint, Artifactory, Splunk, etc. + +For example: + +* `sharepoint-fetcher` fetches files from SharePoint +* `artifactory-fetcher` fetches files from Artifactory +* `splunk-fetcher` fetches data from Splunk +* `pdf-signature-evaluator` evaluates digital signatures in PDF files + +## Development + +We are using [pants](https://www.pantsbuild.org) for the monorepo setup. + +Get `pants` from . +You might need to install some +[prerequisites](https://www.pantsbuild.org/docs/prerequisites) like +`python3-dev` and `python3-distutils`. + +### Upgrading pants + +Pants itself is upgraded automatically, so that the pants version given in +[`pants.toml`](pants.toml) is used. + +But the pants launcher needs to be updated from time to time. +To [update the pants launcher](https://www.pantsbuild.org/docs/installation), execute: + +```sh +SCIE_BOOT=update pants +``` + +### Git hooks + +To install Git hooks, run `make install-slow-hooks` or `make install-fast-hooks`. +See `make help` for details. + +### Lock files + +If you encounter error messages like + +> To regenerate your lockfile, run `pants generate-lockfiles --resolve=python-default`. + +you have to recompute your Python lockfiles. Please use `make update-lockfiles` +to regenerate the lockfiles. This automatically updates the +`dependabot-requirements.txt` file. + +### Updating vulnerable packages + +As mentioned in the section above about lock files, the GitHub dependabot +will read the `dependabot-requirements.txt` file and warn in case of vulnerable +dependencies. + +In that case, simply run `make update-lockfiles` to update the lockfiles. This +will also update the `dependabot-requirements.txt` file. + +If you want to tighten our dependencies' versions, you can also start adding +more precise version requirements into the requirements files, add exact +version requirements to `3rdparty/requirements.txt`. The next time, when +there is a warning by dependabot, you can then just bump the version to the +next secure version. + +For example if there is a line for `urllib==2.0.6` and you need to update +`urllib` to 2.0.7, just change the line to `urllib==2.0.7` and then run +`make update-lockfiles` again. + +### Formatting and linting + +To lint files, run `pants lint ::`, and to apply formatting, run `pants fmt ::`. +Or just `make lint` and `make format`. + +### Running tests + +To run tests, just enter `pants test ::` on the console. You can also specify +subtargets, e.g. `pants test apps/artifactory-fetcher/::` or`pants test +apps/artifactory-fetcher/tests/` or even `pants test +apps/artifactory-fetcher/tests/test_main.py`. + +For _print-debugging_ tests, run `pants --test-debug test ...` which will not +redirect I/O and will then work with `print()` and `breakpoint()`. + +This will run both unit and integration tests. To run only the unit tests you can run: + +```sh +pants --tag="-integration" test :: +# or +make test +``` + +#### Tests with coverage + +To run the tests, and collect and print coverage information, run: + +```sh +pants test --test-use-coverage :: +# or +make testcov +``` + +#### Integration tests + +To run integration tests against real services, you need to run: + +```sh +pants --tag="integration" test +# or +make testint +``` + +In order to pass down environment variables to the integration tests you have to use the `PANTS_TEST_EXTRA_ENV_VARS` environment variable. +For example, to pass down the `SHAREPOINT_USERNAME` environment variable you have to run the following command: + +```sh +PANTS_TEST_EXTRA_ENV_VARS="['SHAREPOINT_PROJECT_SITE', 'SHAREPOINT_USERNAME', 'SHAREPOINT_PASSWORD']" +``` + +To run the integration tests, you need to export and pass down the following additional environment +variables: + +* `SHAREPOINT_PROJECT_SITE` (e.g. `https://sites.internal.company.com/sites/123456/`) +* `SHAREPOINT_USERNAME` (set to your user) +* `SHAREPOINT_PASSWORD` (set to your password) + +### Building and packaging + +To build [pex](https://pex.readthedocs.io/en/latest/index.html) files of the +autopilots, simply run + +```sh +pants package :: +# or +make package +``` + +This will build all .pex files and store them in `dist/{PEX_NAME}` where +`{PEX_NAME}` consists of the name given in the `pex_binary()` target in the +`BUILD` files of the different autopilot apps. + +You can execute them simply by calling them, e.g. + +```sh +dist/apps.splunk-fetcher/splunk-fetcher.pex +``` + +### Releases + +to be defined + +## Integration with VS Code + +### Preparation + +VS Code doesn't know of Pants' isolated build environments, so you need to +manually export a virtualenv with all the dependencies so that VS Code can find +them. + +Run `make vscode-integration`. + +This will also update `.vscode/settings.json` file with the necessary settings. + +### Debugging + +Pants supports [remote debugging for Python](https://www.pantsbuild.org/docs/python-run-goal#debugging). + +To debug tests you can use the following steps: + +1. **Set a breakpoint** in VS Code in your code, e.g. in a test file. +2. **Run the test via pants** and add the `--debug-adapter` argument, e.g.: + + ```sh + pants test --debug-adapter apps/artifactory-fetcher/tests + ``` + +3. **Connect VS Code** to the remote debugger. Press `` to launch the command + defined in [launch.json](.vscode/launch.json). + +To debug a function of an app, you can use the following steps: + +1. **Set a breakpoint** in VS Code in your code +2. **Run the script via pants** and add the `--debug-adapter` argument, e.g.: + + ```sh + pants run --debug-adapter apps/pdf-signature-evaluator/src/yaku/pdf_signature_evaluator/digital_signature_verification.py + ``` + +3. **Connect VS Code** to the remote debugger. Press `` to launch the command + defined in [launch.json](.vscode/launch.json). + +## Yaku App Concepts + +The following sections are only relevant for making the apps compatible to Yaku. +If you are just using the apps , you can skip this section. + +### Status reporting and exit codes + +For normal operations, the autopilot apps should always report their status +using JSON line data (at least with a `status` and `comment`). +Also, the exit code should be `0` for normal operations, even if the status is `FAILED`. +The exit code should be unequal to zero if something unexpected happened, +e.g. a network connection failed or a file could not be written due to IO errors. +In this case, a full stack trace should be displayed in addition to the +`FAILED` status and the `comment` about the error. + +Logging can be used to provide additional messages that are not directly +related to the autopilot output, e.g. progress messages or non-critical +warnings. The log messages will be made available to the user in case +of non-zero exit codes. + +In Python autopilots, normal errors can be raised as `AutopilotError` (defined +in `yaku.autopilot_utils.errors`) and in the CLI main function, those exceptions +can be distinguished from other non-expected exceptions, so that in one case the +error message can be printed as `comment` and in the other case, additionally +the stack trace can be printed. + +See `apps/sharepoint-fetcher/src/yaku/sharepoint_fetcher/__main__.py` for an example. + +### Logging + +We are using [loguru](https://github.com/Delgan/loguru) for logging. This means +that you must _not_ use the standard `logging` module, but _always_ use: + +```python +from loguru import logger +... +logger.info("Some log info about: {}", my_variable) # no %-formatting allowed here! +``` diff --git a/apps/artifactory-fetcher/tests/test_artifactory_fetcher.py b/apps/artifactory-fetcher/tests/test_artifactory_fetcher.py index 94561ce..de94e50 100644 --- a/apps/artifactory-fetcher/tests/test_artifactory_fetcher.py +++ b/apps/artifactory-fetcher/tests/test_artifactory_fetcher.py @@ -6,12 +6,13 @@ import pytest from artifactory import ArtifactoryException from pytest_mock import MockerFixture +from yaku.autopilot_utils.errors import AutopilotError + from yaku.artifactory_fetcher.artifactory_fetcher import ( create_artifactory_client, download_file, get_file_checksum, ) -from yaku.autopilot_utils.errors import AutopilotError class TestCreateArtifactoryClient: diff --git a/apps/artifactory-fetcher/tests/test_cli.py b/apps/artifactory-fetcher/tests/test_cli.py index 54385d9..da41361 100644 --- a/apps/artifactory-fetcher/tests/test_cli.py +++ b/apps/artifactory-fetcher/tests/test_cli.py @@ -3,9 +3,10 @@ import mock import pytest -from yaku.artifactory_fetcher.cli import CLI from yaku.autopilot_utils.errors import AutopilotFailure +from yaku.artifactory_fetcher.cli import CLI + @mock.patch.dict( os.environ, diff --git a/apps/artifactory-fetcher/tests/test_main.py b/apps/artifactory-fetcher/tests/test_main.py index 812c3aa..95ce894 100644 --- a/apps/artifactory-fetcher/tests/test_main.py +++ b/apps/artifactory-fetcher/tests/test_main.py @@ -2,6 +2,7 @@ import mock from click.testing import CliRunner + from yaku.artifactory_fetcher.cli import main diff --git a/cicd/check-vscode-python-paths.py b/cicd/check-vscode-python-paths.py new file mode 100644 index 0000000..3b1f8d8 --- /dev/null +++ b/cicd/check-vscode-python-paths.py @@ -0,0 +1,9 @@ +import json +import subprocess +from pathlib import Path + +settings_file = Path(".vscode/settings.json") +pants_roots = subprocess.check_output(["pants", "roots", "--sep=:"], encoding="utf-8") +settings = json.loads(settings_file.read_text()) +settings["python.analysis.extraPaths"] = list(filter(None, pants_roots.split(":"))) +settings_file.write_text(json.dumps(settings, indent=2)) diff --git a/cicd/coverage-check.sh b/cicd/coverage-check.sh new file mode 100755 index 0000000..6461dae --- /dev/null +++ b/cicd/coverage-check.sh @@ -0,0 +1,23 @@ +COVERAGE_XML_FILE=dist/coverage/python/coverage.xml +COVERAGE_TMP_BASE_DIR=dist/coverage +COVERAGE_TMP_DIR=$(mktemp -d -p ${COVERAGE_TMP_BASE_DIR}) + +# create temp dir +mkdir -p ${COVERAGE_TMP_BASE_DIR} +COVERAGE_TMP_DIR_NAME=$(basename ${COVERAGE_TMP_DIR}) + +# get list of files from coverage xml file +xmllint --xpath //@filename ${COVERAGE_XML_FILE} | sed -e 's/^\s*filename="\(.*\)"$/\1/' | grep -v "__global_coverage__" | sort > ${COVERAGE_TMP_DIR}/covered-files.txt + +# get list of all Python files +find apps -name "*.py" -not -path '*/tests/*' -not -path '*/tests-pex/*' -not -path '*/tests-integration/*' | sort > ${COVERAGE_TMP_DIR}/all-files.txt +find packages -name "*.py" -not -path '*/tests/*' -not -path '*/tests-pex/*' -not -path '*/tests-integration/*' | sort >> ${COVERAGE_TMP_DIR}/all-files.txt + +# compare the two lists of files and print difference +echo -n "\n\n=======================================================================\n" +echo "The following files are not contained in the coverage report:\n\n" +diff --changed-group-format='%<%>' --unchanged-group-format='' ${COVERAGE_TMP_DIR}/covered-files.txt ${COVERAGE_TMP_DIR}/all-files.txt +echo -n "=======================================================================\n" + +# clean up temporary directory +rm -rf dist/coverage/${COVERAGE_TMP_DIR_NAME}