diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 821f2c434..c822150c3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,114 +1,97 @@ +# Copyright 2022 Canonical Ltd. +# See LICENSE file for licensing details. name: Tests + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + on: - workflow_call: pull_request: + schedule: + - cron: '53 0 * * *' # Daily at 00:53 UTC + # Triggered on push to branch "main" by .github/workflows/release.yaml + workflow_call: jobs: lint: name: Lint runs-on: ubuntu-22.04 + timeout-minutes: 5 steps: - name: Checkout uses: actions/checkout@v3 - - name: Install dependencies + - name: Install tox + # TODO: Consider replacing with custom image on self-hosted runner OR pinning version run: python3 -m pip install tox - name: Run linters - run: tox -e lint + run: tox run -e lint unit-test: name: Unit tests runs-on: ubuntu-22.04 + timeout-minutes: 5 needs: - lint steps: - name: Checkout uses: actions/checkout@v3 - - name: Install dependencies - run: python -m pip install tox + - name: Install tox + # TODO: Consider replacing with custom image on self-hosted runner OR pinning version + run: python3 -m pip install tox - name: Run tests - run: tox -e unit + run: tox run -e unit - integration-test-lxd-charm: - name: Integration tests for the charm (lxd) - runs-on: ubuntu-22.04 - needs: - - unit-test - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Setup operator environment - uses: charmed-kubernetes/actions-operator@main - with: - provider: lxd - - name: Run integration tests - run: | - # set sysctl values in case the cloudinit-userdata not applied - sudo sysctl -w vm.max_map_count=262144 - sudo sysctl -w vm.swappiness=0 - sudo sysctl -w net.ipv4.tcp_retries2=5 + build: + name: Build charms + uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@v2 - tox -e charm-integration - - integration-test-lxd-tls: - name: Integration tests for TLS (lxd) - runs-on: ubuntu-22.04 - needs: - - unit-test - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Setup operator environment - uses: charmed-kubernetes/actions-operator@main - with: - provider: lxd - - name: Run integration tests - run: | - # set sysctl values in case the cloudinit-userdata not applied - sudo sysctl -w vm.max_map_count=262144 - sudo sysctl -w vm.swappiness=0 - sudo sysctl -w net.ipv4.tcp_retries2=5 - - tox -e tls-integration - -# Commented due to failures because of HW related issues of the github runners: -# integration-test-lxd-ha: -# name: Integration tests for HA (lxd) -# runs-on: ubuntu-22.04 -# needs: -# - unit-test -# steps: -# - name: Checkout -# uses: actions/checkout@v3 -# - name: Setup operator environment -# uses: charmed-kubernetes/actions-operator@main -# with: -# provider: lxd -# - name: Run integration tests -# run: | -# # set sysctl values in case the cloudinit-userdata not applied -# sudo sysctl -w vm.max_map_count=262144 -# sudo sysctl -w vm.swappiness=0 -# sudo sysctl -w net.ipv4.tcp_retries2=5 -# -# tox -e ha-integration - - integration-test-client-relation: - name: Integration tests for client relation - runs-on: ubuntu-22.04 + integration-test: + strategy: + fail-fast: false + matrix: + tox-environments: + - charm-integration + - tls-integration +# - ha-integration + - client-integration + name: ${{ matrix.tox-environments }} needs: - lint - unit-test + - build + runs-on: ubuntu-latest + timeout-minutes: 120 steps: - name: Checkout uses: actions/checkout@v3 - name: Setup operator environment + # TODO: Replace with custom image on self-hosted runner uses: charmed-kubernetes/actions-operator@main with: provider: lxd + - name: Download packed charm(s) + uses: actions/download-artifact@v3 + with: + name: ${{ needs.build.outputs.artifact-name }} + - name: Select tests + id: select-tests + run: | + if [ "${{ github.event_name }}" == "schedule" ] + then + echo Running unstable and stable tests + echo "mark_expression=" >> $GITHUB_OUTPUT + else + echo Skipping unstable tests + echo "mark_expression=not unstable" >> $GITHUB_OUTPUT + fi - name: Run integration tests run: | # set sysctl values in case the cloudinit-userdata not applied sudo sysctl -w vm.max_map_count=262144 sudo sysctl -w vm.swappiness=0 sudo sysctl -w net.ipv4.tcp_retries2=5 - tox -e client-integration + + tox run -e ${{ matrix.tox-environments }} -- -m '${{ steps.select-tests.outputs.mark_expression }}' + env: + CI_PACKED_CHARMS: ${{ needs.build.outputs.charms }} diff --git a/.github/workflows/on_push.yaml b/.github/workflows/on_push.yaml deleted file mode 100644 index 346065cdc..000000000 --- a/.github/workflows/on_push.yaml +++ /dev/null @@ -1,12 +0,0 @@ -name: Tests -on: - push: - branches: - - main - - track/** - - feature/** - -jobs: - run-tests: - name: Run Tests - uses: ./.github/workflows/ci.yaml diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 240b39c42..9f5eaa792 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -9,6 +9,7 @@ jobs: lib-check: name: Check libraries runs-on: ubuntu-22.04 + timeout-minutes: 5 steps: - name: Checkout uses: actions/checkout@v3 @@ -29,6 +30,7 @@ jobs: - lib-check - ci-tests runs-on: ubuntu-22.04 + timeout-minutes: 60 steps: - name: Checkout uses: actions/checkout@v3 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 70c9c1b43..c1cfa3ab5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,21 +23,21 @@ this operator. ## Developing -You can use the environments created by `tox` for development: +You can create an environment for development with `tox`: ```shell -tox --notest -e unit -source .tox/unit/bin/activate +tox devenv -e integration +source venv/bin/activate ``` ### Testing ```shell -tox -e fmt # update your code according to linting rules -tox -e lint # code style -tox -e unit # unit tests -tox -e integration # integration tests -tox # runs 'lint' and 'unit' environments +tox run -e format # update your code according to linting rules +tox run -e lint # code style +tox run -e unit # unit tests +tox run -e integration # integration tests +tox # runs 'format', 'lint', and 'unit' environments ``` ## Build charm diff --git a/pyproject.toml b/pyproject.toml index f4ed7e804..44aa33b16 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,12 +12,7 @@ show_missing = true minversion = "6.0" log_cli_level = "INFO" asyncio_mode = "auto" -markers = [ - "charm_tests", - "tls_tests", - "ha_tests", - "client_relation: integration tests that test modern client relations", -] +markers = ["unstable"] # Formatting tools configuration [tool.black] diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py new file mode 100644 index 000000000..da132958d --- /dev/null +++ b/tests/integration/conftest.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +# Copyright 2022 Canonical Ltd. +# See LICENSE file for licensing details. + +import json +import os +from pathlib import Path + +import pytest +from pytest_operator.plugin import OpsTest + + +@pytest.fixture(scope="module") +def ops_test(ops_test: OpsTest) -> OpsTest: + if os.environ.get("CI") == "true": + # Running in GitHub Actions; skip build step + # (GitHub Actions uses a separate, cached build step. See .github/workflows/ci.yaml) + packed_charms = json.loads(os.environ["CI_PACKED_CHARMS"]) + + async def build_charm(charm_path, bases_index: int = None) -> Path: + for charm in packed_charms: + if Path(charm_path) == Path(charm["directory_path"]): + if bases_index is None or bases_index == charm["bases_index"]: + return charm["file_path"] + raise ValueError(f"Unable to find .charm file for {bases_index=} at {charm_path=}") + + ops_test.build_charm = build_charm + return ops_test diff --git a/tests/integration/ha/test_horizontal_scaling.py b/tests/integration/ha/test_horizontal_scaling.py index b3c0840db..ced0e0e4d 100644 --- a/tests/integration/ha/test_horizontal_scaling.py +++ b/tests/integration/ha/test_horizontal_scaling.py @@ -33,7 +33,6 @@ logger = logging.getLogger(__name__) -@pytest.mark.ha_tests @pytest.mark.abort_on_fail @pytest.mark.skip_if_deployed async def test_build_and_deploy(ops_test: OpsTest) -> None: @@ -65,7 +64,6 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None: ) -@pytest.mark.ha_tests @pytest.mark.abort_on_fail async def test_horizontal_scale_up(ops_test: OpsTest) -> None: """Tests that new added units to the cluster are discoverable.""" @@ -92,7 +90,6 @@ async def test_horizontal_scale_up(ops_test: OpsTest) -> None: assert ClusterTopology.nodes_count_by_role(nodes)["cluster_manager"] == 3 -@pytest.mark.ha_tests @pytest.mark.abort_on_fail async def test_safe_scale_down_shards_realloc(ops_test: OpsTest) -> None: """Tests the shutdown of a node, and re-allocation of shards to a newly joined unit. @@ -180,7 +177,6 @@ async def test_safe_scale_down_shards_realloc(ops_test: OpsTest) -> None: assert new_shards_per_node.get(-1, 0) == 0 -@pytest.mark.ha_tests @pytest.mark.abort_on_fail async def test_safe_scale_down_roles_reassigning(ops_test: OpsTest) -> None: """Tests the shutdown of a node with a role requiring the re-balance of the cluster roles. diff --git a/tests/integration/relations/opensearch_provider/conftest.py b/tests/integration/relations/opensearch_provider/conftest.py index 98aaba9f3..522cd3af7 100644 --- a/tests/integration/relations/opensearch_provider/conftest.py +++ b/tests/integration/relations/opensearch_provider/conftest.py @@ -6,7 +6,6 @@ from pytest_operator.plugin import OpsTest -@pytest.mark.abort_on_fail @pytest.fixture(scope="module") async def application_charm(ops_test: OpsTest): """Build the application charm.""" @@ -14,7 +13,6 @@ async def application_charm(ops_test: OpsTest): return await ops_test.build_charm(test_charm_path) -@pytest.mark.abort_on_fail @pytest.fixture(scope="module") async def opensearch_charm(ops_test: OpsTest): """Build the opensearch charm.""" diff --git a/tests/integration/relations/opensearch_provider/test_opensearch_provider.py b/tests/integration/relations/opensearch_provider/test_opensearch_provider.py index 00262b875..9fbdd442e 100644 --- a/tests/integration/relations/opensearch_provider/test_opensearch_provider.py +++ b/tests/integration/relations/opensearch_provider/test_opensearch_provider.py @@ -37,7 +37,6 @@ @pytest.mark.abort_on_fail -@pytest.mark.client_relation async def test_database_relation_with_charm_libraries( ops_test: OpsTest, application_charm, opensearch_charm ): @@ -77,7 +76,6 @@ async def test_database_relation_with_charm_libraries( # ) -@pytest.mark.client_relation async def test_database_usage(ops_test: OpsTest): """Check we can update and delete things.""" await run_request( @@ -108,7 +106,6 @@ async def test_database_usage(ops_test: OpsTest): ) -@pytest.mark.client_relation async def test_database_bulk_usage(ops_test: OpsTest): """Check we can update and delete things using bulk api.""" bulk_payload = """{ "index" : { "_index": "albums", "_id" : "2" } } @@ -149,7 +146,6 @@ async def test_database_bulk_usage(ops_test: OpsTest): assert set(artists) == {"Herbie Hancock", "Lydian Collective", "Vulfpeck"} -@pytest.mark.client_relation async def test_database_version(ops_test: OpsTest): """Check version is accurate.""" run_version_request = await run_request( @@ -170,7 +166,6 @@ async def test_database_version(ops_test: OpsTest): assert version == results.get("version", {}).get("number") -@pytest.mark.client_relation async def test_multiple_relations(ops_test: OpsTest, application_charm): """Test that two different applications can connect to the database.""" # Deploy secondary application. @@ -198,7 +193,6 @@ async def test_multiple_relations(ops_test: OpsTest, application_charm): ) -@pytest.mark.client_relation async def test_relation_broken(ops_test: OpsTest): """Test that the user is removed when the relation is broken.""" # Retrieve the relation user. diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 93bed015d..9536413ce 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -26,7 +26,6 @@ DEFAULT_NUM_UNITS = 2 -@pytest.mark.charm_tests @pytest.mark.abort_on_fail @pytest.mark.skip_if_deployed async def test_build_and_deploy(ops_test: OpsTest) -> None: @@ -42,7 +41,6 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None: await ops_test.model.wait_for_idle(wait_for_exact_units=DEFAULT_NUM_UNITS) -@pytest.mark.charm_tests @pytest.mark.abort_on_fail async def test_status(ops_test: OpsTest) -> None: """Verifies that the application and unit are active.""" @@ -52,7 +50,6 @@ async def test_status(ops_test: OpsTest) -> None: assert len(ops_test.model.applications[APP_NAME].units) == DEFAULT_NUM_UNITS -@pytest.mark.charm_tests @pytest.mark.abort_on_fail async def test_actions_get_admin_password(ops_test: OpsTest) -> None: """Test the retrieval of admin secrets.""" @@ -91,7 +88,6 @@ async def test_actions_get_admin_password(ops_test: OpsTest) -> None: assert result.status == "failed" -@pytest.mark.charm_tests @pytest.mark.abort_on_fail async def test_actions_rotate_admin_password(ops_test: OpsTest) -> None: """Test the rotation and change of admin password.""" diff --git a/tests/integration/tls/test_tls.py b/tests/integration/tls/test_tls.py index 3a1e963af..267f9bed9 100644 --- a/tests/integration/tls/test_tls.py +++ b/tests/integration/tls/test_tls.py @@ -27,7 +27,6 @@ TLS_CERTIFICATES_APP_NAME = "tls-certificates-operator" -@pytest.mark.tls_tests @pytest.mark.abort_on_fail @pytest.mark.skip_if_deployed async def test_build_and_deploy_active(ops_test: OpsTest) -> None: @@ -57,7 +56,6 @@ async def test_build_and_deploy_active(ops_test: OpsTest) -> None: ) -@pytest.mark.tls_tests @pytest.mark.abort_on_fail async def test_security_index_initialised(ops_test: OpsTest) -> None: """Test that the security index is well initialised.""" @@ -66,7 +64,6 @@ async def test_security_index_initialised(ops_test: OpsTest) -> None: assert await check_security_index_initialised(ops_test, leader_unit_ip) -@pytest.mark.tls_tests @pytest.mark.abort_on_fail async def test_tls_configured(ops_test: OpsTest) -> None: """Test that TLS is enabled when relating to the TLS Certificates Operator.""" @@ -74,7 +71,6 @@ async def test_tls_configured(ops_test: OpsTest) -> None: assert await check_unit_tls_configured(ops_test, unit_ip, unit_name) -@pytest.mark.tls_tests @pytest.mark.abort_on_fail async def test_cluster_formation_after_tls(ops_test: OpsTest) -> None: """Test that the cluster formation is successful after TLS setup.""" diff --git a/tox.ini b/tox.ini index 36826d917..cfa193307 100644 --- a/tox.ini +++ b/tox.ini @@ -2,27 +2,27 @@ # See LICENSE file for licensing details. [tox] -skipsdist=True +no_package = True skip_missing_interpreters = True -envlist = fmt, lint, unit +env_list = format, lint, unit [vars] -src_path = {toxinidir}/src/ -tst_path = {toxinidir}/tests/ -lib_path = {toxinidir}/lib/charms/opensearch/v0/ -all_path = {[vars]src_path} {[vars]lib_path} {[vars]tst_path} +src_path = {tox_root}/src +tests_path = {tox_root}/tests +lib_path = {tox_root}/lib/charms/opensearch/v0 +all_path = {[vars]src_path} {[vars]lib_path} {[vars]tests_path} [testenv] -setenv = - PYTHONPATH = {toxinidir}:{toxinidir}/lib:{[vars]src_path} - PYTHONBREAKPOINT=ipdb.set_trace - PY_COLORS=1 -passenv = - PYTHONPATH - CHARM_BUILD_DIR - MODEL_SETTINGS +set_env = + PYTHONPATH = {tox_root}:{tox_root}/lib:{[vars]src_path} + PYTHONBREAKPOINT=ipdb.set_trace + PY_COLORS=1 +pass_env = + PYTHONPATH + CHARM_BUILD_DIR + MODEL_SETTINGS -[testenv:fmt] +[testenv:format] description = Apply coding style standards to code deps = black @@ -45,9 +45,9 @@ deps = codespell commands = codespell {[vars]lib_path} - codespell {toxinidir} --skip {toxinidir}/.git --skip {toxinidir}/.tox \ - --skip {toxinidir}/build --skip {toxinidir}/lib --skip {toxinidir}/venv \ - --skip {toxinidir}/.mypy_cache --skip {toxinidir}/icon.svg + codespell {tox_root} --skip {tox_root}/.git --skip {tox_root}/.tox \ + --skip {tox_root}/build --skip {tox_root}/lib --skip {tox_root}/venv \ + --skip {tox_root}/.mypy_cache --skip {tox_root}/icon.svg # pflake8 wrapper supports config from pyproject.toml pflake8 {[vars]all_path} isort --check-only --diff {[vars]all_path} @@ -59,63 +59,83 @@ deps = pytest pytest-asyncio coverage[toml] - -r{toxinidir}/requirements.txt + -r {tox_root}/requirements.txt commands = coverage run --source={[vars]src_path} --source={[vars]lib_path} \ - -m pytest --ignore={[vars]tst_path}integration -v --tb native -s {posargs} + -m pytest -v --tb native -s {posargs} {[vars]tests_path}/unit coverage report [testenv:charm-integration] description = Run charm integration tests +pass_env = + {[testenv]pass_env} + CI + CI_PACKED_CHARMS deps = pytest pytest-asyncio juju==2.9.38.1 # juju 3.0.0 has issues with retrieving action results pytest-operator - -r{toxinidir}/requirements.txt + -r {tox_root}/requirements.txt commands = - pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} -m charm_tests + pytest -v --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/test_charm.py [testenv:tls-integration] description = Run TLS integration tests +pass_env = + {[testenv]pass_env} + CI + CI_PACKED_CHARMS deps = pytest pytest-asyncio juju==2.9.38.1 # juju 3.0.0 has issues with retrieving action results pytest-operator - -r{toxinidir}/requirements.txt + -r {tox_root}/requirements.txt commands = - pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} -m tls_tests + pytest -v --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/tls/test_tls.py [testenv:ha-integration] description = Run HA integration tests +pass_env = + {[testenv]pass_env} + CI + CI_PACKED_CHARMS deps = pytest pytest-asyncio juju==2.9.38.1 # juju 3.0.0 has issues with retrieving action results pytest-operator - -r{toxinidir}/requirements.txt + -r {tox_root}/requirements.txt commands = - pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} -m ha_tests + pytest -v --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/ha/test_horizontal_scaling.py [testenv:client-integration] description = Run client relation integration tests +pass_env = + {[testenv]pass_env} + CI + CI_PACKED_CHARMS deps = pytest pytest-asyncio juju==2.9.38.1 # juju 3.0.0 has issues with retrieving action results pytest-operator - -r{toxinidir}/requirements.txt + -r {tox_root}/requirements.txt commands = - pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} -m client_relation + pytest -v --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/relations/opensearch_provider/test_opensearch_provider.py [testenv:integration] description = Run all integration tests +pass_env = + {[testenv]pass_env} + CI + CI_PACKED_CHARMS deps = pytest pytest-asyncio juju==2.9.38.1 # juju 3.0.0 has issues with retrieving action results pytest-operator - -r{toxinidir}/requirements.txt + -r {tox_root}/requirements.txt commands = - pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} + pytest -v --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/