diff --git a/.circleci/config.yml b/.circleci/config.yml index d2d873e..69b1860 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,7 +13,7 @@ filters: &semver-tagged executors: pytest: machine: - # use an older ubuntu image with python 3.11 until pytest 8.0 is out + # python 3.11, needed for ansible-test 2.15 image: ubuntu-2204:2023.10.1 resource_class: large @@ -34,6 +34,7 @@ jobs: steps: - collection-testing/pytest: pytest-args: > + --ci --ansible-version << parameters.ansible-version >> --node-python-version << parameters.node-python-version >> diff --git a/.config/molecule/config.yml b/.config/molecule/config.yml index 1ed3a67..13839ff 100644 --- a/.config/molecule/config.yml +++ b/.config/molecule/config.yml @@ -1,6 +1,8 @@ --- dependency: name: galaxy + options: + requirements-file: requirements.molecule.yml driver: name: docker @@ -8,18 +10,18 @@ driver: provisioner: name: ansible env: - ANSIBLE_PIPELINING: true + ANSIBLE_PIPELINING: false scenario: test_sequence: - - destroy - dependency - syntax + - destroy - create - prepare - converge - idempotence - - check # also run check mode in regular tests + - check # also run check mode in regular tests - side_effect - verify - destroy diff --git a/.github/renovate.json5 b/.github/renovate.json5 index 4574387..6d4db2d 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -1,5 +1,14 @@ { "extends": [ "github>maxhoesel-ansible/.github:renovate-config" - ] + ], + "pip-compile": { + "fileMatch": ["^requirements\\.txt$"] + }, + "pip_requirements": { + "enabled": false + }, + "pip_setup": { + "enabled": false + } } diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3b25ea3..a626b9d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -119,6 +119,15 @@ TBD ## Maintainer information +### Updating Dependencies + +While the *Ansible* collection itself doesn't have any dependencies outside of ansible itself, the tooling used to build and test the collection does. +We use [`pip-tools`](https://github.com/jazzband/pip-tools/) to lock these dependencies to a specific version for testing. +This prevents random CI failures because of [`requests` updates et. al.](https://github.com/docker/docker-py/pull/3257), but still gives us a simple `requirements.txt` that anyone can install. + +The direct dependencies are stored in `requirements.in`, use `scripts/udate_requirements.sh` to generate a new `requirements.txt`. +Do **not** generate `requirements.txt` in another way or remove the header, else renovate [won't be able to resolve and update dependencies in CI!](https://docs.renovatebot.com/modules/manager/pip-compile/#assumption-of-header-with-a-command) + ### Bumping supported ansible-core versions 1. Update the versions in the [CI config](./.circleci/config.yml) diff --git a/galaxy.yml b/galaxy.yml index aa4dfc3..b2ae0c9 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -1,27 +1,26 @@ authors: -- Max Hösel + - Max Hösel build_ignore: -- .circleci -- .github -- .config -- .vscode -- dist -- CODE_OF_CONDUCT.md -- CONTRIBUTING.md -- .yamllint -- '*.tar.gz' -- .venv -- .tox -- .pytest_cache -- scripts -- .pre-commit-config.yaml -- .readthedocs.yaml -- pyproject.toml -- requirements.txt -- '**/requirements.txt' + - .circleci + - .github + - .vscode + - dist + - CODE_OF_CONDUCT.md + - CONTRIBUTING.md + - .yamllint + - "*.tar.gz" + - .venv + - .tox + - .pytest_cache + - scripts + - .pre-commit-config.yaml + - .readthedocs.yaml + - pyproject.toml + - requirements.txt + - "**/requirements.txt" dependencies: - community.crypto: '>=1.0.0' - community.general: '>=1.0.0' + community.crypto: ">=1.0.0" + community.general: ">=1.0.0" description: Install and configure the Pterodactyl Gaming Server Panel and Daemon/Wings issues: https://github.com/maxhoesel-ansible/ansible-collection-pterodactyl/issues license_file: LICENSE @@ -30,8 +29,8 @@ namespace: maxhoesel readme: README.md repository: https://github.com/maxhoesel-ansible/ansible-collection-pterodactyl tags: -- pterodactyl -- application -- gaming -- linux + - pterodactyl + - application + - gaming + - linux version: 2.0.1 diff --git a/requirements.in b/requirements.in new file mode 100644 index 0000000..7a5fe6a --- /dev/null +++ b/requirements.in @@ -0,0 +1,37 @@ +# This file is only a template! +# Use requirements.txt generated by pip-compile to install dependencies + +# Requirements for developing this collection +# Includes utilities, CLI helpers and so on + +# Include a version of ansible-core for IDE hints and the default pytest version +# It is also needed for docs generation. +# +# This installed version can be overriden by running pytest with the --ansible-version command. +# Ideally pytest would install another venv just for the test, but the pytest-virtualenv package +# is ancient and incompatible with python 3.12. +ansible-core==2.16.6 + +# Linting & Formatting +ansible-lint==24.5.0 +pylint==3.2.0 +autopep8==2.1.0 +pre-commit==3.7.1 + +# Utility packages used in test fixtures and scripts +pytest==8.2.1 +docker==7.1.0 +pyyaml==6.0.1 +packaging==24.0 +# Dependencies for executing the role scenarios. +molecule==6.0.2 +molecule-plugins[docker]==23.4.1 + +# Generating requirements and syncing venv +pip-tools==7.4.1 + +# Docs +antsibull-docs==2.11.0 +ansible-pygments==0.1.1 +sphinx==7.3.7 +sphinx-ansible-theme==0.10.3 diff --git a/requirements.txt b/requirements.txt index fa4dd2f..89eb3ef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,21 +1,337 @@ -# Requirements for developing this collection -# Includes utilities, CLI helpers and so on - -# Linting & Formatting +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile requirements.in +# +aiofiles==23.2.1 + # via antsibull-core +aiohttp==3.9.5 + # via + # antsibull-core + # antsibull-docs +aiosignal==1.3.1 + # via aiohttp +alabaster==0.7.16 + # via sphinx +annotated-types==0.7.0 + # via pydantic +ansible-compat==24.6.1 + # via + # ansible-lint + # molecule +ansible-core==2.16.6 + # via + # -r requirements.in + # ansible-compat + # ansible-lint + # molecule ansible-lint==24.5.0 -pylint==3.2.1 + # via -r requirements.in +ansible-pygments==0.1.1 + # via + # -r requirements.in + # antsibull-docs + # sphinx-ansible-theme +antsibull-changelog==0.28.0 + # via antsibull-docs +antsibull-core==3.0.1 + # via antsibull-docs +antsibull-docs==2.11.0 + # via -r requirements.in +antsibull-docs-parser==1.0.1 + # via antsibull-docs +astroid==3.2.2 + # via pylint +asyncio-pool==0.6.0 + # via antsibull-docs +attrs==23.2.0 + # via + # aiohttp + # jsonschema + # referencing autopep8==2.1.0 + # via -r requirements.in +babel==2.15.0 + # via sphinx +black==24.4.2 + # via ansible-lint +bracex==2.4 + # via wcmatch +build==1.2.1 + # via + # antsibull-core + # pip-tools +certifi==2024.6.2 + # via requests +cffi==1.16.0 + # via cryptography +cfgv==3.4.0 + # via pre-commit +charset-normalizer==3.3.2 + # via requests +click==8.1.7 + # via + # black + # click-help-colors + # molecule + # pip-tools + # typer +click-help-colors==0.9.4 + # via molecule +cryptography==42.0.8 + # via ansible-core +dill==0.3.8 + # via pylint +distlib==0.3.8 + # via virtualenv +distro==1.9.0 + # via selinux +docker==7.1.0 + # via + # -r requirements.in + # molecule-plugins +docutils==0.20.1 + # via + # antsibull-changelog + # antsibull-docs + # rstcheck-core + # sphinx + # sphinx-rtd-theme +enrich==1.2.7 + # via molecule +filelock==3.14.0 + # via + # ansible-lint + # virtualenv +frozenlist==1.4.1 + # via + # aiohttp + # aiosignal +identify==2.5.36 + # via pre-commit +idna==3.7 + # via + # requests + # yarl +imagesize==1.4.1 + # via sphinx +importlib-metadata==7.1.0 + # via ansible-lint +iniconfig==2.0.0 + # via pytest +isort==5.13.2 + # via pylint +jinja2==3.1.4 + # via + # ansible-core + # antsibull-docs + # molecule + # sphinx +jsonschema==4.22.0 + # via + # ansible-compat + # ansible-lint + # molecule +jsonschema-specifications==2023.12.1 + # via jsonschema +markdown-it-py==3.0.0 + # via rich +markupsafe==2.1.5 + # via jinja2 +mccabe==0.7.0 + # via pylint +mdurl==0.1.2 + # via markdown-it-py +molecule==6.0.2 + # via + # -r requirements.in + # molecule-plugins +molecule-plugins[docker]==23.4.1 + # via -r requirements.in +multidict==6.0.5 + # via + # aiohttp + # yarl +mypy-extensions==1.0.0 + # via black +nodeenv==1.9.1 + # via pre-commit +packaging==24.0 + # via + # -r requirements.in + # ansible-compat + # ansible-core + # ansible-lint + # antsibull-changelog + # antsibull-core + # antsibull-docs + # black + # build + # molecule + # pytest + # sphinx +pathspec==0.12.1 + # via + # ansible-lint + # black + # yamllint +perky==0.9.2 + # via antsibull-core +pip-tools==7.4.1 + # via -r requirements.in +platformdirs==4.2.2 + # via + # black + # pylint + # virtualenv +pluggy==1.5.0 + # via + # molecule + # pytest pre-commit==3.7.1 - -# Testing libraries -pytest==8.2.0 -pytest-virtualenv==1.7.0 -docker==7.0.0 - -# Utility packages used in scripts + # via -r requirements.in +pycodestyle==2.11.1 + # via autopep8 +pycparser==2.22 + # via cffi +pydantic==2.7.3 + # via + # antsibull-core + # antsibull-docs + # rstcheck-core +pydantic-core==2.18.4 + # via pydantic +pygments==2.18.0 + # via + # ansible-pygments + # rich + # sphinx +pylint==3.2.0 + # via -r requirements.in +pyproject-hooks==1.1.0 + # via + # build + # pip-tools +pytest==8.2.1 + # via -r requirements.in pyyaml==6.0.1 -packaging==24.0 + # via + # -r requirements.in + # ansible-compat + # ansible-core + # ansible-lint + # antsibull-changelog + # antsibull-core + # antsibull-docs + # molecule + # pre-commit + # yamllint +referencing==0.35.1 + # via + # jsonschema + # jsonschema-specifications +requests==2.31.0 + # via + # docker + # molecule-plugins + # sphinx +resolvelib==1.0.1 + # via ansible-core +rich==13.7.1 + # via + # ansible-lint + # enrich + # molecule + # typer +rpds-py==0.18.1 + # via + # jsonschema + # referencing +rstcheck==6.2.1 + # via + # antsibull-changelog + # antsibull-docs +rstcheck-core==1.2.1 + # via rstcheck +ruamel-yaml==0.18.6 + # via ansible-lint +ruamel-yaml-clib==0.2.8 + # via ruamel-yaml +selinux==0.3.0 + # via molecule-plugins +semantic-version==2.10.0 + # via + # antsibull-changelog + # antsibull-core + # antsibull-docs +shellingham==1.5.4 + # via typer +six==1.16.0 + # via twiggy +snowballstemmer==2.2.0 + # via sphinx +sphinx==7.3.7 + # via + # -r requirements.in + # antsibull-docs + # sphinx-ansible-theme + # sphinx-rtd-theme + # sphinxcontrib-jquery +sphinx-ansible-theme==0.10.3 + # via -r requirements.in +sphinx-rtd-theme==2.0.0 + # via sphinx-ansible-theme +sphinxcontrib-applehelp==1.0.8 + # via sphinx +sphinxcontrib-devhelp==1.0.6 + # via sphinx +sphinxcontrib-htmlhelp==2.0.5 + # via sphinx +sphinxcontrib-jquery==4.1 + # via sphinx-rtd-theme +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==1.0.7 + # via sphinx +sphinxcontrib-serializinghtml==1.1.10 + # via sphinx +subprocess-tee==0.4.1 + # via + # ansible-compat + # ansible-lint +tomlkit==0.12.5 + # via pylint +twiggy==0.5.1 + # via + # antsibull-core + # antsibull-docs +typer[all]==0.12.3 + # via rstcheck +typing-extensions==4.12.1 + # via + # pydantic + # pydantic-core + # typer +urllib3==2.2.1 + # via + # docker + # requests +virtualenv==20.26.2 + # via pre-commit +wcmatch==8.5.2 + # via + # ansible-lint + # molecule +wheel==0.43.0 + # via pip-tools +yamllint==1.35.1 + # via ansible-lint +yarl==1.9.4 + # via aiohttp +zipp==3.19.2 + # via importlib-metadata -# Also include a version of ansible-core for IDE hints -# and as the default version used in tests -ansible-core==2.16.6 +# The following packages are considered to be unsafe in a requirements file: +# pip +# setuptools diff --git a/roles/pterodactyl_panel/requirements.molecule.yml b/roles/pterodactyl_panel/requirements.molecule.yml new file mode 120000 index 0000000..5ebcde0 --- /dev/null +++ b/roles/pterodactyl_panel/requirements.molecule.yml @@ -0,0 +1 @@ +../requirements.molecule.yml \ No newline at end of file diff --git a/roles/pterodactyl_wings/requirements.molecule.yml b/roles/pterodactyl_wings/requirements.molecule.yml new file mode 120000 index 0000000..5ebcde0 --- /dev/null +++ b/roles/pterodactyl_wings/requirements.molecule.yml @@ -0,0 +1 @@ +../requirements.molecule.yml \ No newline at end of file diff --git a/tests/roles/requirements.yml b/roles/requirements.molecule.yml similarity index 100% rename from tests/roles/requirements.yml rename to roles/requirements.molecule.yml diff --git a/scripts/setup.sh b/scripts/setup.sh index bc12a9d..fc425b5 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -11,8 +11,7 @@ source .venv/bin/activate printf "Installing development requirements..." python3 -m pip install --upgrade pip --quiet python3 -m pip install --quiet -r requirements.txt --upgrade -# Also install the scenario requirements so we can run them directly -python3 -m pip install --quiet -r tests/roles/requirements.txt --upgrade +python3 -m piptools sync --pip-args "--quiet" # ensure contents stay synced with lockfile printf "OK\n" printf "Installing pre-commit hook..." diff --git a/scripts/update_requirements.sh b/scripts/update_requirements.sh new file mode 100755 index 0000000..d8e7186 --- /dev/null +++ b/scripts/update_requirements.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +pip-compile -U requirements.in diff --git a/tests/conftest.py b/tests/conftest.py index 19e4710..3ad63c7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,12 +1,13 @@ from dataclasses import dataclass +import importlib +import importlib.metadata import os from pathlib import Path import subprocess +import sys from packaging import version -import pkg_resources import pytest -from pytest_virtualenv import VirtualEnv import yaml NODE_PYTHON_DEFAULT_VERSION = "3.6" @@ -15,64 +16,87 @@ GALAXY_YML = yaml.safe_load(f) -class TestEnv(): - def __init__(self, virtualenv: VirtualEnv) -> None: - self.virtualenv = virtualenv +@dataclass +class TestVersions: + node_python_version: str + ansible_version: str + + +class CollectionTestEnv(): + # pylint: disable=redefined-outer-name + def __init__(self, request, test_versions: TestVersions, tmp_path_factory: pytest.TempPathFactory) -> None: + # Sanity check against modifying user/system packages. + # We need to manually pip-install a version of ansible-core to test with, lets make sure + # we don't hose the users personal install. + in_venv = sys.prefix != sys.base_prefix + if not in_venv and not request.config.getoption("--ci"): + raise ValueError("Aborting ansible-core install for tests because we are not inside a virtualenv. " + "Please use ./scripts/setup.sh to setup a virtualenv, then activate it.") + # install the specified version of ansible-core into our venv + subprocess.run(["pip", "install", f"ansible-core~={test_versions.ansible_version}.0"], check=True) + + # Build collection into isolated install path + build_path: Path = tmp_path_factory.mktemp("build") + collection_tar = build_path / f"{GALAXY_YML['namespace']}-{GALAXY_YML['name']}-{GALAXY_YML['version']}.tar.gz" + subprocess.run( + ["ansible-galaxy", "collection", "build", "--output-path", build_path], + check=True + ) + + # Now install the collection into our temporary structure + self._collections_path = tmp_path_factory.mktemp("collections") + self._env = { + "ANSIBLE_COLLECTIONS_PATH": self._collections_path, + } + subprocess.run( + ["ansible-galaxy", "collection", "install", "-p", self._collections_path, collection_tar], + check=True, env={**os.environ.copy(), **self._env} + ) + + # in case subprocesses need to override the directory, such as for molecule tests + self.cwd: Path = (self._collections_path / "ansible_collections" / + GALAXY_YML["namespace"] / GALAXY_YML["name"]).resolve() def run(self, *args, **kwargs): - # Combine any passed in env with the virtualenv to ensure proper PATH + # merge env if "env" in kwargs: - kwargs["env"] = {**kwargs["env"], **self.virtualenv.env} - self.virtualenv.run(*args, **kwargs) + kwargs["env"] = {**kwargs["env"], **self._env} + else: + kwargs["env"] = {**os.environ.copy(), **self._env} + if "cwd" not in kwargs: + kwargs["cwd"] = self.cwd # let subclasses overwrite the path, they can get it from self.cwd + if "check" not in kwargs: + kwargs["check"] = True # check by default + # pylint: disable=subprocess-run-check + subprocess.run(*args, **kwargs) -def get_ansible_version(): - base_version = version.parse(pkg_resources.get_distribution("ansible-core").version) - return f"{base_version.major}.{base_version.minor}" - -def pytest_addoption(parser): - parser.addoption("--ansible-version", action="store", default=get_ansible_version(), - help="Version of ansible to use for tests, in the format '2.xx'. Default: see requirements.txt") - parser.addoption("--node-python-version", action="store", default=NODE_PYTHON_DEFAULT_VERSION, - help="Python version to test Ansible modules with, " - f"in the format '3.x'. Default: '{NODE_PYTHON_DEFAULT_VERSION}'") +@pytest.fixture() +# pylint: disable=redefined-outer-name +def collection_test_env(request, test_versions, tmp_path_factory) -> CollectionTestEnv: + return CollectionTestEnv(request, test_versions, tmp_path_factory) @pytest.fixture(scope="session") -def collection_path(tmp_path_factory) -> Path: - build_path: Path = tmp_path_factory.mktemp("build") - collection_file = build_path / f"{GALAXY_YML['namespace']}-{GALAXY_YML['name']}-{GALAXY_YML['version']}.tar.gz" - subprocess.run( - ["ansible-galaxy", "collection", "build", "--output-path", build_path], - check=True, - ) - - install_path: Path = tmp_path_factory.mktemp("collections") - env = os.environ.copy() - env["ANSIBLE_COLLECTIONS_PATH"] = install_path.resolve().as_posix() - subprocess.run( - ["ansible-galaxy", "collection", "install", collection_file], - env=env, check=True, +def test_versions(request) -> TestVersions: + return TestVersions( + request.config.getoption("--node-python-version"), + request.config.getoption("--ansible-version") ) - return install_path -@dataclass -class TestVersions: - ansible_version: str - node_python_version: str - - @property - def ansible_version_pip(self): - major, minor = self.ansible_version.split(".") - next_minor = int(minor) + 1 - return f"ansible-core>={self.ansible_version},<{major}.{next_minor}" +def get_ansible_version(): + base_version = version.parse(importlib.metadata.version("ansible-core")) + return f"{base_version.major}.{base_version.minor}" -@pytest.fixture(scope="session") -def test_versions(request) -> TestVersions: - return TestVersions( - request.config.getoption("--ansible-version"), - request.config.getoption("--node-python-version") - ) +def pytest_addoption(parser): + parser.addoption("--ci", action="store_true", default=False, + help="Allow unsafe actions such as installing ansible-core directly in CI environments") + parser.addoption("--ansible-version", action="store", default=get_ansible_version(), + help="Version of ansible to use for tests, in format 'x.yy', such as '2.15'." + "Default: no constraint (latest/installed)") + parser.addoption("--node-python-version", action="store", default=NODE_PYTHON_DEFAULT_VERSION, + help="Python version to test Ansible modules with, " + f"in the format '3.x'. Default: '{NODE_PYTHON_DEFAULT_VERSION}'") diff --git a/tests/roles/__init.py__ b/tests/roles/__init__.py similarity index 100% rename from tests/roles/__init.py__ rename to tests/roles/__init__.py diff --git a/tests/roles/conftest.py b/tests/roles/conftest.py index 1b9bdeb..e69de29 100644 --- a/tests/roles/conftest.py +++ b/tests/roles/conftest.py @@ -1,43 +0,0 @@ -import os -from pathlib import Path -from typing import Optional - -import pytest - -from tests.conftest import TestEnv - -MOLECULE_REQUIREMENTS_PIP = Path("tests/roles/requirements.txt").resolve() -MOLECULE_REQUIREMENTS_ANSIBLE = Path("tests/roles/requirements.yml").resolve() - - -class MoleculeTestEnv(TestEnv): - # pylint: disable=redefined-outer-name - def __init__(self, virtualenv, test_versions, collection_path) -> None: - self.env = {**os.environ.copy(), **{ - "ANSIBLE_COLLECTIONS_PATH": collection_path, - }} - super().__init__(virtualenv) - - self.run(["pip", "install", test_versions.ansible_version_pip]) - self.run(["pip", "install", "-r", MOLECULE_REQUIREMENTS_PIP]) - self.run(["ansible-galaxy", "collection", "install", "-r", MOLECULE_REQUIREMENTS_ANSIBLE]) - - def run(self, *args, **kwargs): - kwargs["env"] = self.env - return super().run(*args, **kwargs) - - -MOLECULE_ENV: Optional[MoleculeTestEnv] = None - - -@pytest.fixture() -# This fixture should be session-scoped, but cannot be since it requires the function-scoped virtualenv fixture. -# Use memoization for now. -# pylint: disable=redefined-outer-name -def molecule_env(virtualenv, test_versions, collection_path) -> MoleculeTestEnv: - global MOLECULE_ENV # pylint: disable=global-statement - if MOLECULE_ENV is not None: - return MOLECULE_ENV - - MOLECULE_ENV = MoleculeTestEnv(virtualenv, test_versions, collection_path) - return MOLECULE_ENV diff --git a/tests/roles/requirements.txt b/tests/roles/requirements.txt deleted file mode 100644 index 578fa5b..0000000 --- a/tests/roles/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -# Dependencies for executing the role scenarios. -molecule==6.0.2 -molecule-plugins[docker]==23.4.1 diff --git a/tests/roles/test_molecule.py b/tests/roles/test_molecule.py index c4d5111..75c66d1 100644 --- a/tests/roles/test_molecule.py +++ b/tests/roles/test_molecule.py @@ -16,8 +16,10 @@ def scenario_id(path: Path) -> str: @pytest.mark.parametrize("scenario", MOLECULE_SCENARIOS, ids=scenario_id) -def test_scenario(scenario: Path, molecule_env) -> None: - molecule_env.run( - ["molecule", "test", "-s", scenario.name], - cwd=scenario.parent.parent.resolve() - ) +def test_scenario(scenario: Path, collection_test_env) -> None: + role_dir = collection_test_env.cwd / scenario.parent.parent + + # Apparently molecule needs this to pick up on project-level config files in .config/molecule 🤷 + # https://github.com/ansible/molecule/blob/e6d63adea6be74a8548dab30ba00bf8474d6c088/src/molecule/util.py#L339 + collection_test_env.run(["git", "init"]) + collection_test_env.run(["molecule", "test", "-s", scenario.name], cwd=role_dir)