diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 000000000..b0b310bd4 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,106 @@ +# Copyright 2022 Canonical Ltd. +# See LICENSE file for licensing details. +name: Tests + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + pull_request: + schedule: + - cron: '53 0 * * *' # Daily at 00:53 UTC + # Triggered on push to branch "main" by .github/workflows/release.yaml + workflow_call: + +jobs: + lint: + name: Lint + uses: canonical/data-platform-workflows/.github/workflows/lint.yaml@main + + unit-test: + name: Unit test charm + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Install tox & poetry + run: | + pipx install tox + pipx install poetry + - name: Run tests + run: tox run -e unit + + + lib-check: + name: Check libraries + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - run: | + # Workaround for https://github.com/canonical/charmcraft/issues/1389#issuecomment-1880921728 + touch requirements.txt + - name: Check libs + uses: canonical/charming-actions/check-libraries@2.4.0 + with: + credentials: ${{ secrets.CHARMHUB_TOKEN }} + github-token: ${{ secrets.GITHUB_TOKEN }} + use-labels: false + fail-build: ${{ github.event_name == 'pull_request' }} + + build: + name: Build charm + strategy: + fail-fast: false + matrix: + paths: + - to-charm-directory: "." + - to-charm-directory: "./tests/integration/relations/opensearch_provider/application-charm/" + uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@main + with: + path-to-charm-directory: ${{ matrix.paths.to-charm-directory }} + charmcraft-snap-channel: beta + cache: true + + integration-test: + strategy: + fail-fast: false + matrix: + juju: + - agent: 3.3.1 + libjuju: 3.3.0.0 + juju-snap: 3.3/stable + - agent: 3.2.4 + libjuju: ^3 + juju-snap: 3.2/stable + - agent: 3.1.7 + libjuju: ^3 + juju-snap: 3.1/stable + - agent: 3.1.6 + libjuju: ^3 + juju-snap: 3.1/stable + name: Integration test charm | ${{ matrix.juju.agent }} + needs: + - lint + - unit-test + - build + uses: canonical/data-platform-workflows/.github/workflows/integration_test_charm.yaml@main + with: + artifact-prefix: ${{ needs.build.outputs.artifact-prefix }} + cloud: lxd + juju-agent-version: ${{ matrix.juju.agent }} + libjuju-version-constraint: ${{ matrix.juju.libjuju }} + secrets: + integration-test: | + { + "AWS_ACCESS_KEY": "${{ secrets.AWS_ACCESS_KEY }}", + "AWS_SECRET_KEY": "${{ secrets.AWS_SECRET_KEY }}", + "GCP_ACCESS_KEY": "${{ secrets.GCP_ACCESS_KEY }}", + "GCP_SECRET_KEY": "${{ secrets.GCP_SECRET_KEY }}", + "GCP_SERVICE_ACCOUNT": "${{ secrets.GCP_SERVICE_ACCOUNT }}", + } diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml deleted file mode 100644 index c29be9e4d..000000000 --- a/.github/workflows/integration.yaml +++ /dev/null @@ -1,202 +0,0 @@ -# Copyright 2023 Canonical Ltd. -# See LICENSE file for licensing details. -on: - workflow_call: - inputs: - juju-snap-channel: - description: Snap channel for Juju CLI - default: 3.1/stable - type: string - libjuju-version-specifier: - description: | - python-libjuju version specifier (e.g. ">=1.3") - https://packaging.python.org/en/latest/glossary/#term-Version-Specifier - required: false - type: string -# secrets: -# integration-test: -# description: | -# Secrets needed in integration tests -# -# Passed to tests with `SECRETS_FROM_GITHUB` environment variable -# -# Use a string representation of a Python dict[str, str] built from multiple GitHub secrets -# Do NOT put the string into a single GitHub secret—build the string from multiple GitHub secrets so that GitHub is more likely to redact the secrets in GitHub Actions logs. -# -# Python code to verify the string format: -# ``` -# import ast -# secrets = ast.literal_eval("") -# assert isinstance(secrets, dict) -# for key, value in secrets.items(): -# assert isinstance(key, str) and isinstance(value, str) -# ``` -# required: false - -jobs: - build: - name: Build charms - uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@v2 - - integration-test: - needs: - - build - strategy: - fail-fast: false - matrix: - tox-environments: - - charm-integration - - tls-integration - - client-integration - - ha-base-integration - - ha-networking-integration - - ha-multi-clusters-integration - - large-deployments-integration - - plugins-integration -# - ha-backup-integration - runner: ["ubuntu-22.04"] - include: - - tox-environments: h-scaling-integration - runner: "Ubuntu_4C_16G" - name: ${{ matrix.tox-environments }} - runs-on: ${{ matrix.runner }} - timeout-minutes: 360 - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Upgrade LXD - run: | - sudo snap refresh lxd --channel=latest/stable - - - name: Setup Juju environment - # Runs on juju 3 by default - # TODO: Replace with custom image on self-hosted runner - uses: charmed-kubernetes/actions-operator@main - with: - provider: lxd - juju-channel: ${{ inputs.juju-snap-channel }} - - - name: Download packed charm(s) - uses: actions/download-artifact@v3 - with: - name: ${{ needs.build.outputs.artifact-name }} - -# - name: Install CLI -# run: | -# sudo apt update -# sudo apt install -y pipx -# pipx install git+https://github.com/canonical/data-platform-workflows#subdirectory=python/cli -# - name: Redact secrets from log -# run: redact-secrets -# env: -# SECRETS: ${{ secrets.integration-test }} - - - name: Select tests - id: select-tests - run: | - if [ "${{ github.event_name }}" == "schedule" ] - then - echo Running unstable and stable tests - echo "mark_expression=" >> $GITHUB_OUTPUT - else - echo Skipping unstable tests - echo "mark_expression=not unstable" >> $GITHUB_OUTPUT - fi - - - name: Run integration tests - run: | - # free space in the runner - sudo rm -rf /usr/share/dotnet - sudo rm -rf /opt/ghc - sudo rm -rf /usr/local/share/boost - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - - # Set kernel params for OpenSearch - sudo sysctl -w vm.max_map_count=262144 vm.swappiness=0 net.ipv4.tcp_retries2=5 - # Set kernel params for testing - sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1 net.ipv6.conf.default.disable_ipv6=1 net.ipv6.conf.all.autoconf=0 - tox run -e ${{ matrix.tox-environments }} -- -m '${{ steps.select-tests.outputs.mark_expression }}' - env: - CI_PACKED_CHARMS: ${{ needs.build.outputs.charms }} - LIBJUJU_VERSION_SPECIFIER: ${{ inputs.libjuju-version-specifier }} - SECRETS_FROM_GITHUB: "" -# SECRETS_FROM_GITHUB: ${{ secrets.integration-test }} - - - - backup-microceph-integration-test: - needs: - - build - name: backup-microceph-integration-test - runs-on: "ubuntu-22.04" - timeout-minutes: 360 - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Upgrade LXD - run: | - sudo snap refresh lxd --channel=latest/stable - - - name: Setup Juju environment - # Runs on juju 3 by default - # TODO: Replace with custom image on self-hosted runner - uses: charmed-kubernetes/actions-operator@main - with: - provider: lxd - juju-channel: ${{ inputs.juju-snap-channel }} - - - name: Download packed charm(s) - uses: actions/download-artifact@v3 - with: - name: ${{ needs.build.outputs.artifact-name }} - - - name: Free space in the runner - id: free-space-runner - run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf /opt/ghc - sudo rm -rf /usr/local/share/boost - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - - - name: Select tests - id: select-tests - run: | - if [ "${{ github.event_name }}" == "schedule" ] - then - echo Running unstable and stable tests - echo "mark_expression=" >> $GITHUB_OUTPUT - else - echo Skipping unstable tests - echo "mark_expression=not unstable" >> $GITHUB_OUTPUT - fi - - - name: Run backup integration - run: | - sudo sysctl -w vm.max_map_count=262144 vm.swappiness=0 net.ipv4.tcp_retries2=5 - tox run -e ha-backup-integration -- -m '${{ steps.select-tests.outputs.mark_expression }}' - env: - CI_PACKED_CHARMS: ${{ needs.build.outputs.charms }} - LIBJUJU_VERSION_SPECIFIER: ${{ inputs.libjuju-version-specifier }} - -# - name: Run backup test with AWS -# run: tox run -e ha-backup-integration -- -m '${{ steps.select-tests.outputs.mark_expression }}' -# env: -# CI_PACKED_CHARMS: ${{ needs.build.outputs.charms }} -# LIBJUJU_VERSION_SPECIFIER: ${{ matrix.libjuju-version }} -# SECRETS_FROM_GITHUB: | -# { -# "AWS_ACCESS_KEY": "${{ secrets.AWS_ACCESS_KEY }}", -# "AWS_SECRET_KEY": "${{ secrets.AWS_SECRET_KEY }}", -# } -# - name: Run backup test with GCP -# run: tox run -e ha-backup-integration -- -m '${{ steps.select-tests.outputs.mark_expression }}' -# env: -# CI_PACKED_CHARMS: ${{ needs.build.outputs.charms }} -# LIBJUJU_VERSION_SPECIFIER: ${{ matrix.libjuju-version }} -# SECRETS_FROM_GITHUB: | -# { -# "GCP_ACCESS_KEY": "${{ secrets.GCP_ACCESS_KEY }}", -# "GCP_SECRET_KEY": "${{ secrets.GCP_SECRET_KEY }}", -# } diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml deleted file mode 100644 index 012b8db0d..000000000 --- a/.github/workflows/pr.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2023 Canonical Ltd. -# See LICENSE file for licensing details. -name: PR CI - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -on: - pull_request: - -jobs: - pre-integration-checks: - secrets: inherit - uses: ./.github/workflows/pre_integration_checks.yaml - with: - libjuju-version-specifier: ==3.3.0.0 - - integration: - needs: - - pre-integration-checks - uses: ./.github/workflows/integration.yaml - with: - libjuju-version-specifier: ==3.3.0.0 diff --git a/.github/workflows/pre_integration_checks.yaml b/.github/workflows/pre_integration_checks.yaml deleted file mode 100644 index 373e2c370..000000000 --- a/.github/workflows/pre_integration_checks.yaml +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2023 Canonical Ltd. -# See LICENSE file for licensing details. -name: Pre-integration checks - -on: - workflow_call: - inputs: - libjuju-version-specifier: - description: | - python-libjuju version specifier (e.g. ">=1.3") - https://packaging.python.org/en/latest/glossary/#term-Version-Specifier - required: false - type: string - -jobs: - lint: - name: Lint - runs-on: ubuntu-22.04 - timeout-minutes: 5 - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Install tox - # TODO: Consider replacing with custom image on self-hosted runner OR pinning version - run: python3 -m pip install tox - - name: Run linters - run: tox run -e lint - - unit-test: - name: Unit tests - runs-on: ubuntu-22.04 - timeout-minutes: 5 - needs: - - lint - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Install tox - # TODO: Consider replacing with custom image on self-hosted runner OR pinning version - run: python3 -m pip install tox - - name: Run tests - run: tox run -e unit - env: - LIBJUJU_VERSION_SPECIFIER: ${{ inputs.libjuju-version-specifier }} - - lib-check: - name: Check libraries - runs-on: ubuntu-22.04 - timeout-minutes: 5 - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Check libs - uses: canonical/charming-actions/check-libraries@2.1.1 - with: - credentials: "${{ secrets.CHARMHUB_TOKEN }}" # FIXME: current token will expire in 2023-07-04 - github-token: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 9ca3bc83d..58c0288df 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -8,48 +8,46 @@ on: - main jobs: - pre-integration-checks: + ci-tests: + name: Tests + uses: ./.github/workflows/ci.yaml secrets: inherit - strategy: - fail-fast: false - matrix: - include: - - libjuju: ==3.3.0.0 - uses: ./.github/workflows/pre_integration_checks.yaml - with: - libjuju-version-specifier: ${{ matrix.libjuju }} + permissions: + actions: write # Needed to manage GitHub Actions cache - integration: - needs: - - pre-integration-checks - strategy: - fail-fast: false - matrix: - include: - - snap: 3.1/stable - libjuju: ==3.3.0.0 - uses: ./.github/workflows/integration.yaml + # release-libraries: + # name: Release libraries + # needs: + # - ci-tests + # runs-on: ubuntu-latest + # timeout-minutes: 60 + # steps: + # - name: Checkout + # uses: actions/checkout@v3 + # with: + # fetch-depth: 0 + # - name: Release charm libraries + # uses: canonical/charming-actions/release-libraries@2.3.0 + # with: + # credentials: ${{ secrets.CHARMHUB_TOKEN }} + # github-token: ${{ secrets.GITHUB_TOKEN }} + + build: + name: Build charm + uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@main with: - juju-snap-channel: ${{ matrix.snap }} - libjuju-version-specifier: ${{ matrix.libjuju }} + charmcraft-snap-channel: beta - release-to-charmhub: - name: Release to CharmHub + release: + name: Release charm needs: - - pre-integration-checks - - integration - runs-on: ubuntu-22.04 - timeout-minutes: 60 - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Select charmhub channel - uses: canonical/charming-actions/channel@2.1.1 - id: channel - - name: Upload charm to charmhub - uses: canonical/charming-actions/upload-charm@2.1.1 - with: - credentials: "${{ secrets.CHARMHUB_TOKEN }}" - github-token: "${{ secrets.GITHUB_TOKEN }}" - channel: "2/edge" + - build + uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v0.0.0 + with: + channel: latest/edge + artifact-prefix: ${{ needs.build.outputs.artifact-prefix }} + secrets: + charmhub-token: ${{ secrets.CHARMHUB_TOKEN }} + permissions: + contents: write # Needed to create GitHub release diff --git a/.github/workflows/scheduled_tests.yaml b/.github/workflows/scheduled_tests.yaml deleted file mode 100644 index 10507a6b3..000000000 --- a/.github/workflows/scheduled_tests.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2023 Canonical Ltd. -# See LICENSE file for licensing details. -name: Scheduled CI - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -on: - schedule: - - cron: '53 0 * * *' # Daily at 00:53 UTC - -jobs: - pre-integration-checks: - secrets: inherit - uses: ./.github/workflows/pre_integration_checks.yaml - - integration: - needs: - - pre-integration-checks - strategy: - fail-fast: false - matrix: - include: - - snap: 3.1/stable - libjuju: ==3.3.0.0 - - snap: 2.9/stable - libjuju: ==2.9.44.0 - uses: ./.github/workflows/integration.yaml - with: - juju-snap-channel: ${{ matrix.snap }} - libjuju-version-specifier: ${{ matrix.libjuju }} diff --git a/.gitignore b/.gitignore index f1b819d2b..51ccad860 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,7 @@ __pycache__/ *.tar.xz cloudinit-userdata.yaml /.pytest_cache/ + +# Moving to Poetry, we do not need this file to be pushed any longer +/requirements.txt +/requirements-last-build.txt diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 72e5cde06..7d16019f4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,6 +21,41 @@ this operator. - Please help us out in ensuring easy to review branches by rebasing your pull request branch onto the `main` branch. This also avoids merge commits and creates a linear Git commit history. + +## Build charm + +Build the charm in this git repository using tox. + +There are two alternatives to build the charm: using the charm cache or not. +Cache will speed the build by downloading all dependencies from charmcraftcache-hub. + +### Build Without Cache + +To run the traditional build only using `charmcraft`, run the following command: + +```shell +tox -e build-production +``` + +### Build With Cache + +First, ensure you have the right dependencies: +* charmcraft v2.5.4+ +* charmcraftcache + +By running the following commands: + +```shell +pip install charmcraftcache +sudo snap refresh charmcraft --channel=latest/beta +``` + +Then, start the build: + +```shell +tox -e build-dev +``` + ## Developing You can create an environment for development with `tox`: @@ -32,41 +67,67 @@ source venv/bin/activate ### Testing -To run tests, run the following +To run tests, first build the charm as described above, then run the following ```shell tox -e format # update your code according to linting rules tox -e lint # code style tox -e unit # unit tests -tox -m integration # integration tests, running on juju 2. +tox -m integration # integration tests, running on juju 3. tox # runs 'format', 'lint', and 'unit' environments ``` -Integration tests can be run for separate areas of functionality: +Integration tests can be run for separate files: ```shell -tox -e charm-integration # basic charm integration tests -tox -e tls-integration # TLS-specific integration tests -tox -e client-integration # Validates the `opensearch-client` integration -tox -e ha-integration # HA tests -tox -e h-scaling-integration # HA tests specific to horizontal scaling +tox -e integration -- tests/integration/ha/test_storage.py ``` -If you're running tests on juju 3, run the following command to change libjuju to the correct version: +#### Running different versions of libjuju + +To try different versions of libjuju, run: ```shell -export LIBJUJU_VERSION_SPECIFIER="==3.3.0.0" +poetry add --lock --group integration,unit juju@ ``` -## Build charm +#### Testing Backups -Build the charm in this git repository using: +Backup testing installs microceph and, optionally, can run on AWS and GCP object stores. +For that, pass the access / secret / service account information as env. variables. + +To run the test only against microceph: ```shell -charmcraft pack +tox -e integration -- tests/integration/ha/test_backups.py --group='microceph' # test backup service +``` + +And against public clouds + microceph: + +```shell +SECRETS_FROM_GITHUB=$(cat /credentials.json) tox -e integration -- tests/integration/ha/test_backups.py +``` + +Where, for AWS only, `credentials.json` should look like (the `IGNORE` is necessary to remove GCP): +```json +{ "AWS_ACCESS_KEY": ..., "AWS_SECRET_KEY": ..., "GCP_SECRET_KEY": "IGNORE" } ``` -### Deploy +For GCP only, some additional information is needed. The pytest script runs with `boto3` and needs `GCP_{ACCESS,SECRET}_KEY` to clean up the object store; whereas OpenSearch uses the `service account` json file. + +```json +{ "AWS_SECRET_KEY": "IGNORE", "GCP_ACCESS_KEY": ..., "GCP_SECRET_KEY": ..., "GCP_SERVICE_ACCOUNT": ... } +``` + +Combine both if AWS and GCP are to be used. + + +#### Cleaning Test Environment + +Remove the following files / folders: `poetry.lock`, `.tox`, `requirements*`. + + +## Deploy OpenSearch has a set of system requirements to correctly function, you can find the list [here](https://opensearch.org/docs/latest/install-and-configure/install-opensearch/index/). Some of those settings must be set using cloudinit-userdata on the model, while others must be set on the host machine: diff --git a/charmcraft.yaml b/charmcraft.yaml index bd045e1ed..e55daa571 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -3,15 +3,26 @@ type: charm bases: + # Whenever "bases" is changed: + # - Update tests/integration/conftest.py::pytest_configure() + # - Update .github/workflow/ci.yaml integration-test matrix - build-on: - name: "ubuntu" channel: "22.04" run-on: - name: "ubuntu" channel: "22.04" - parts: charm: + override-pull: | + craftctl default + if [[ ! -f requirements.txt ]] + then + echo 'ERROR: Use "tox run -e build-dev" instead of calling "charmcraft pack" directly' >&2 + exit 1 + fi + charm-strict-dependencies: true + charm-entrypoint: src/charm.py build-packages: - cargo - libffi-dev diff --git a/pyproject.toml b/pyproject.toml index 44aa33b16..eec6c956b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,92 @@ # Copyright 2023 Canonical Ltd. # See LICENSE file for licensing details. -# Testing tools configuration +[tool.poetry] +# Charm is not packed as a standard Python package; this information is not used +name = "charm" +version = "0.1.0" +description = "" +authors = [] + +[tool.poetry.dependencies] +python = "^3.10" +ops = "^2.5.0" +tenacity = "^8.2.2" +boto3 = "^1.28.22" +jinja2 = "^3.1.2" +overrides = "7.4.0" +requests = "2.31.0" +# Official name: ruamel.yaml, but due to Poetry GH#109 - replace dots with dashs +ruamel-yaml = "0.17.35" +shortuuid = "1.0.11" +jproperties = "2.1.1" +pydantic = "^1.10, <2" +cryptography = "^42.0.2" +jsonschema = "^4.21.1" + + +[tool.poetry.group.charm-libs.dependencies] +# data_platform_libs/v0/data_interfaces.py +ops = ">=2.0.0" +# data_platform_libs/v0/upgrade.py +# grafana_agent/v0/cos_agent.py requires pydantic <2 +poetry-core = "^1.8.1" +pydantic = "^1.10, <2" +# tls_certificates_interface/v1/tls_certificates.py +cryptography = "^42.0.2" +jsonschema = "^4.21.1" +# grafana_agent/v0/cos_agent.py +cosl = ">=0.0.7" +bcrypt = "4.0.1" + +[tool.poetry.group.format] +optional = true + +[tool.poetry.group.format.dependencies] +black = "^23.7.0" +isort = "^5.12.0" + +[tool.poetry.group.lint] +optional = true + +[tool.poetry.group.lint.dependencies] +black = "^23.7.0" +isort = "^5.12.0" +flake8 = "^6.0.0" +flake8-docstrings = "^1.7.0" +flake8-copyright = "^0.2.4" +flake8-builtins = "^2.1.0" +pyproject-flake8 = "^6.0.0.post1" +pep8-naming = "^0.13.3" +codespell = "^2.2.5" +shellcheck-py = "^0.9.0.5" + +[tool.poetry.group.unit.dependencies] +juju = "^3.2.2" +pytest = "^7.4.0" +pytest-asyncio = "<0.23" +pytest-mock = "^3.11.1" +coverage = {extras = ["toml"], version = "^7.4.1"} +parameterized = "^0.9.0" + +[tool.poetry.group.integration.dependencies] +boto3 = "^1.28.23" +pytest = "^7.4.0" +pytest-github-secrets = {git = "https://github.com/canonical/data-platform-workflows", branch = "main", subdirectory = "python/pytest_plugins/github_secrets"} +pytest-asyncio = "<0.23" +pytest-operator = "^0.32.0" +pytest-operator-cache = {git = "https://github.com/canonical/data-platform-workflows", branch = "main", subdirectory = "python/pytest_plugins/pytest_operator_cache"} +pytest-operator-groups = {git = "https://github.com/canonical/data-platform-workflows", branch = "main", subdirectory = "python/pytest_plugins/pytest_operator_groups"} +pytest-microceph = {git = "https://github.com/canonical/data-platform-workflows", branch = "main", subdirectory = "python/pytest_plugins/microceph"} +juju = "^3.2.2" +ops = "^2.5.0" +pytest-mock = "^3.11.1" +tenacity = "^8.2.2" +pyyaml = "^6.0.1" +urllib3 = "^1.26.16" +protobuf = "3.20.0" +opensearch-py = "^2.4.2" + [tool.coverage.run] branch = true @@ -11,8 +96,8 @@ show_missing = true [tool.pytest.ini_options] minversion = "6.0" log_cli_level = "INFO" -asyncio_mode = "auto" markers = ["unstable"] +asyncio_mode = "auto" # Formatting tools configuration [tool.black] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index c1befdf4b..000000000 --- a/requirements.txt +++ /dev/null @@ -1,15 +0,0 @@ -bcrypt==4.1.2 -cosl==0.0.7 -cryptography==42.0.1 -jproperties==2.1.1 -jsonschema==4.21.1 -ops==2.9.0 -overrides==7.6.0 -parameterized==0.9.0 -pydantic==1.10.14 -pytest-mock==3.12.0 -requests==2.31.0 -ruamel.yaml==0.18.5 -shortuuid==1.0.11 -tenacity==8.2.3 -urllib3==2.1.0 diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index 60fdaf947..000000000 --- a/tests/conftest.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2023 Canonical Ltd. -# See LICENSE file for licensing details. -import os -from unittest.mock import PropertyMock - -import pytest -from ops import JujuVersion -from pytest_mock import MockerFixture - - -@pytest.fixture(autouse=True) -def juju_has_secrets(mocker: MockerFixture): - """This fixture will force the usage of secrets whenever run on Juju 3.x. - - NOTE: This is needed, as normally JujuVersion is set to 0.0.0 in tests - (i.e. not the real juju version) - """ - juju_version = os.environ["LIBJUJU_VERSION_SPECIFIER"] - if juju_version.startswith("=="): - juju_version = juju_version[2:] # Removing == symbol - - if juju_version < "3": - mocker.patch.object(JujuVersion, "has_secrets", new_callable=PropertyMock).return_value = ( - False - ) - return False - - mocker.patch.object(JujuVersion, "has_secrets", new_callable=PropertyMock).return_value = True - return True - - -@pytest.fixture -def only_with_juju_secrets(juju_has_secrets): - """Pretty way to skip Juju 3 tests.""" - if not juju_has_secrets: - pytest.skip("Secrets test only applies on Juju 3.x") - - -@pytest.fixture -def only_without_juju_secrets(juju_has_secrets): - """Pretty way to skip Juju 2-specific tests. - - Typically: to save CI time, when the same check were executed in a Juju 3-specific way already - """ - if juju_has_secrets: - pytest.skip("Skipping legacy secrets tests") diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 000000000..db3bfe1a6 --- /dev/null +++ b/tests/integration/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py deleted file mode 100644 index da132958d..000000000 --- a/tests/integration/conftest.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2022 Canonical Ltd. -# See LICENSE file for licensing details. - -import json -import os -from pathlib import Path - -import pytest -from pytest_operator.plugin import OpsTest - - -@pytest.fixture(scope="module") -def ops_test(ops_test: OpsTest) -> OpsTest: - if os.environ.get("CI") == "true": - # Running in GitHub Actions; skip build step - # (GitHub Actions uses a separate, cached build step. See .github/workflows/ci.yaml) - packed_charms = json.loads(os.environ["CI_PACKED_CHARMS"]) - - async def build_charm(charm_path, bases_index: int = None) -> Path: - for charm in packed_charms: - if Path(charm_path) == Path(charm["directory_path"]): - if bases_index is None or bases_index == charm["bases_index"]: - return charm["file_path"] - raise ValueError(f"Unable to find .charm file for {bases_index=} at {charm_path=}") - - ops_test.build_charm = build_charm - return ops_test diff --git a/tests/integration/ha/__init__.py b/tests/integration/ha/__init__.py new file mode 100644 index 000000000..db3bfe1a6 --- /dev/null +++ b/tests/integration/ha/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. diff --git a/tests/integration/ha/conftest.py b/tests/integration/ha/conftest.py new file mode 100644 index 000000000..6448f997d --- /dev/null +++ b/tests/integration/ha/conftest.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. + +import logging + +import pytest +from pytest_operator.plugin import OpsTest + +from ..helpers import APP_NAME, get_application_unit_ids +from .continuous_writes import ContinuousWrites +from .helpers import ORIGINAL_RESTART_DELAY, app_name, update_restart_delay + +logger = logging.getLogger(__name__) + + +SECOND_APP_NAME = "second-opensearch" +RESTART_DELAY = 360 + + +@pytest.fixture(scope="function") +async def reset_restart_delay(ops_test: OpsTest): + """Resets service file delay on all units.""" + yield + app = (await app_name(ops_test)) or APP_NAME + for unit_id in get_application_unit_ids(ops_test, app): + await update_restart_delay(ops_test, app, unit_id, ORIGINAL_RESTART_DELAY) + + +@pytest.fixture(scope="function") +async def c_writes(ops_test: OpsTest): + """Creates instance of the ContinuousWrites.""" + app = (await app_name(ops_test)) or APP_NAME + return ContinuousWrites(ops_test, app) + + +@pytest.fixture(scope="function") +async def c_writes_runner(ops_test: OpsTest, c_writes: ContinuousWrites): + """Starts continuous write operations and clears writes at the end of the test.""" + await c_writes.start() + yield + await c_writes.clear() + logger.info("\n\n\n\nThe writes have been cleared.\n\n\n\n") + + +@pytest.fixture(scope="function") +async def c_balanced_writes_runner(ops_test: OpsTest, c_writes: ContinuousWrites): + """Same as previous runner, but starts continuous writes on cluster wide replicated index.""" + await c_writes.start(repl_on_all_nodes=True) + yield + await c_writes.clear() + logger.info("\n\n\n\nThe writes have been cleared.\n\n\n\n") diff --git a/tests/integration/ha/continuous_writes.py b/tests/integration/ha/continuous_writes.py index 7481a2135..610e300eb 100644 --- a/tests/integration/ha/continuous_writes.py +++ b/tests/integration/ha/continuous_writes.py @@ -22,11 +22,7 @@ wait_random, ) -from tests.integration.helpers import ( - get_admin_secrets, - get_application_unit_ips, - opensearch_client, -) +from ..helpers import get_admin_secrets, get_application_unit_ips, opensearch_client logging.getLogger("opensearch").setLevel(logging.ERROR) logging.getLogger("opensearchpy.helpers").setLevel(logging.ERROR) diff --git a/tests/integration/ha/helpers.py b/tests/integration/ha/helpers.py index 4c73a8ea7..1e528d2aa 100644 --- a/tests/integration/ha/helpers.py +++ b/tests/integration/ha/helpers.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 # Copyright 2023 Canonical Ltd. # See LICENSE file for licensing details. + import json import logging import subprocess @@ -18,9 +19,7 @@ wait_random, ) -from tests.integration.ha.continuous_writes import ContinuousWrites -from tests.integration.ha.helpers_data import index_docs_count -from tests.integration.helpers import ( +from ..helpers import ( get_application_unit_ids, get_application_unit_ids_hostnames, get_application_unit_ids_ips, @@ -28,10 +27,17 @@ juju_version_major, run_action, ) +from .continuous_writes import ContinuousWrites +from .helpers_data import index_docs_count OPENSEARCH_SERVICE_PATH = "/etc/systemd/system/snap.opensearch.daemon.service" +SECOND_APP_NAME = "second-opensearch" +ORIGINAL_RESTART_DELAY = 20 +RESTART_DELAY = 360 + + logger = logging.getLogger(__name__) @@ -207,6 +213,16 @@ async def all_nodes(ops_test: OpsTest, unit_ip: str) -> List[Node]: return result +async def assert_continuous_writes_increasing( + c_writes: ContinuousWrites, +) -> None: + """Asserts that the continuous writes are increasing.""" + writes_count = await c_writes.count() + time.sleep(5) + more_writes = await c_writes.count() + assert more_writes > writes_count, "Writes not continuing to DB" + + async def assert_continuous_writes_consistency( ops_test: OpsTest, c_writes: ContinuousWrites, app: str ) -> None: @@ -476,8 +492,8 @@ async def wait_restore_finish(ops_test, unit_ip): raise Exception() -async def continuous_writes_increases(ops_test: OpsTest, unit_ip: str, app: str) -> bool: - """Asserts that TEST_BACKUP_INDEX is writable while under continuous writes. +async def start_and_check_continuous_writes(ops_test: OpsTest, unit_ip: str, app: str) -> bool: + """Asserts that ContinuousWrites.INDEX_NAME is writable. Given we are restoring an index, we need to make sure ContinuousWrites restart at the tip of that index instead of doc_id = 0. @@ -490,18 +506,19 @@ async def continuous_writes_increases(ops_test: OpsTest, unit_ip: str, app: str) ) writer = ContinuousWrites(ops_test, app, initial_count=initial_count) await writer.start() - time.sleep(5) + time.sleep(10) result = await writer.stop() return result.count > initial_count -async def backup_cluster(ops_test: OpsTest, leader_id: int) -> bool: +async def backup_cluster(ops_test: OpsTest, leader_id: int) -> int: """Runs the backup of the cluster.""" action = await run_action(ops_test, leader_id, "create-backup") logger.debug(f"create-backup output: {action}") await wait_backup_finish(ops_test, leader_id) - return action.status == "completed" + assert action.status == "completed" + return int(action.response["backup-id"]) async def restore_cluster(ops_test: OpsTest, backup_id: int, unit_ip: str, leader_id: int) -> bool: diff --git a/tests/integration/ha/helpers_data.py b/tests/integration/ha/helpers_data.py index 5b194e4a9..3e98fcc17 100644 --- a/tests/integration/ha/helpers_data.py +++ b/tests/integration/ha/helpers_data.py @@ -10,7 +10,7 @@ from pytest_operator.plugin import OpsTest from tenacity import Retrying, retry, stop_after_attempt, wait_fixed, wait_random -from tests.integration.helpers import http_request +from ..helpers import http_request logger = logging.getLogger(__name__) diff --git a/tests/integration/ha/test_backups.py b/tests/integration/ha/test_backups.py index e9bd9ec83..dcf0878a5 100644 --- a/tests/integration/ha/test_backups.py +++ b/tests/integration/ha/test_backups.py @@ -4,215 +4,163 @@ import asyncio import logging -import os -import random import subprocess +import time +import uuid +from pathlib import Path -# from pathlib import Path -# -# import boto3 +import boto3 import pytest -import requests from pytest_operator.plugin import OpsTest -from tests.integration.ha.continuous_writes import ContinuousWrites -from tests.integration.ha.helpers import ( - app_name, - assert_continuous_writes_consistency, - backup_cluster, - continuous_writes_increases, - restore_cluster, -) -from tests.integration.ha.helpers_data import index_docs_count -from tests.integration.ha.test_horizontal_scaling import IDLE_PERIOD -from tests.integration.helpers import ( +from ..ha.continuous_writes import ContinuousWrites +from ..ha.test_horizontal_scaling import IDLE_PERIOD +from ..helpers import ( APP_NAME, MODEL_CONFIG, SERIES, get_leader_unit_id, get_leader_unit_ip, - get_reachable_unit_ips, - http_request, run_action, ) -from tests.integration.tls.test_tls import TLS_CERTIFICATES_APP_NAME +from ..tls.test_tls import TLS_CERTIFICATES_APP_NAME +from .helpers import ( + app_name, + assert_continuous_writes_consistency, + backup_cluster, + restore_cluster, + start_and_check_continuous_writes, +) +from .helpers_data import index_docs_count logger = logging.getLogger(__name__) -S3_INTEGRATOR_NAME = "s3-integrator" -TEST_BACKUP_DOC_ID = 10 -CLOUD_CONFIGS = { - "aws": { - "endpoint": "https://s3.amazonaws.com", - "bucket": "data-charms-testing", - "path": "opensearch", - "region": "us-east-1", - }, - "gcp": { - "endpoint": "https://storage.googleapis.com", - "bucket": "data-charms-testing", - "path": "opensearch", - "region": "", - }, -} - backups_by_cloud = {} value_before_backup, value_after_backup = None, None -# @pytest.fixture(scope="session") -# def cloud_credentials(github_secrets) -> dict[str, dict[str, str]]: -# """Read cloud credentials.""" -# return { -# "aws": { -# "access-key": github_secrets["AWS_ACCESS_KEY"], -# "secret-key": github_secrets["AWS_SECRET_KEY"], -# }, -# "gcp": { -# "access-key": github_secrets["GCP_ACCESS_KEY"], -# "secret-key": github_secrets["GCP_SECRET_KEY"], -# }, -# } - - -# @pytest.fixture(scope="session", autouse=True) -# def clean_backups_from_buckets(cloud_credentials) -> None: -# """Teardown to clean up created backups from clouds.""" -# yield -# -# logger.info("Cleaning backups from cloud buckets") -# for cloud_name, config in CLOUD_CONFIGS.items(): -# backup = backups_by_cloud.get(cloud_name) -# -# if not backup: -# continue -# -# session = boto3.session.Session( -# aws_access_key_id=cloud_credentials[cloud_name]["access-key"], -# aws_secret_access_key=cloud_credentials[cloud_name]["secret-key"], -# region_name=config["region"], -# ) -# s3 = session.resource("s3", endpoint_url=config["endpoint"]) -# bucket = s3.Bucket(config["bucket"]) -# -# # GCS doesn't support batch delete operation, so delete the objects one by one -# backup_path = str(Path(config["path"]) / backups_by_cloud[cloud_name]) -# for bucket_object in bucket.objects.filter(Prefix=backup_path): -# bucket_object.delete() - - -@pytest.fixture() -async def c_writes(ops_test: OpsTest): - """Creates instance of the ContinuousWrites.""" - app = (await app_name(ops_test)) or APP_NAME - return ContinuousWrites(ops_test, app) +@pytest.fixture(scope="session") +def cloud_configs(github_secrets, microceph): + # Add UUID to path to avoid conflict with tests running in parallel (e.g. multiple Juju + # versions on a PR, multiple PRs) + path = f"opensearch/{uuid.uuid4()}" + + ip = subprocess.check_output(["hostname", "-I"]).decode().split()[0] + results = { + "microceph": { + "endpoint": f"http://{ip}", + "bucket": microceph.bucket, + "path": path, + "region": "default", + }, + } + if "AWS_ACCESS_KEY" in github_secrets: + results["aws"] = { + "endpoint": "https://s3.amazonaws.com", + "bucket": "data-charms-testing", + "path": path, + "region": "us-east-1", + } + return results -@pytest.fixture() -async def c_writes_runner(ops_test: OpsTest, c_writes: ContinuousWrites): - """Starts continuous write operations and clears writes at the end of the test.""" - await c_writes.start() - yield +@pytest.fixture(scope="session") +def cloud_credentials(github_secrets, microceph) -> dict[str, dict[str, str]]: + """Read cloud credentials.""" + results = { + "microceph": { + "access-key": microceph.access_key_id, + "secret-key": microceph.secret_access_key, + }, + } + if "AWS_ACCESS_KEY" in github_secrets: + results["aws"] = { + "access-key": github_secrets["AWS_ACCESS_KEY"], + "secret-key": github_secrets["AWS_SECRET_KEY"], + } + return results - reachable_ip = random.choice(await get_reachable_unit_ips(ops_test)) - await http_request(ops_test, "GET", f"https://{reachable_ip}:9200/_cat/nodes", json_resp=False) - await http_request( - ops_test, "GET", f"https://{reachable_ip}:9200/_cat/shards", json_resp=False - ) - await c_writes.clear() - logger.info("\n\n\n\nThe writes have been cleared.\n\n\n\n") +@pytest.fixture(scope="session", autouse=True) +def clean_backups_from_buckets(github_secrets, cloud_configs, cloud_credentials) -> None: + """Teardown to clean up created backups from clouds.""" + yield + creds = cloud_credentials.copy() + logger.info("Cleaning backups from cloud buckets") + for cloud_name, config in cloud_configs.items(): + backup = backups_by_cloud.get(cloud_name) -# TODO: Remove this method as soon as poetry gets merged. -@pytest.fixture(scope="session") -def microceph(): - """Starts microceph radosgw.""" - if "microceph" not in subprocess.check_output(["sudo", "snap", "list"]).decode(): - uceph = "/tmp/microceph.sh" - - with open(uceph, "w") as f: - # TODO: if this code stays, then the script below should be added as a file - # in the charm. - resp = requests.get( - "https://raw.githubusercontent.com/canonical/microceph-action/main/microceph.sh" - ) - f.write(resp.content.decode()) - - os.chmod(uceph, 0o755) - subprocess.check_output( - [ - "sudo", - uceph, - "-c", - "latest/edge", - "-d", - "/dev/sdc", - "-a", - "accesskey", - "-s", - "secretkey", - "-b", - "data-charms-testing", - "-z", - "5G", - ] + if not backup: + continue + + session = boto3.session.Session( + aws_access_key_id=creds[cloud_name]["access-key"], + aws_secret_access_key=creds[cloud_name]["secret-key"], + region_name=config["region"], ) - ip = subprocess.check_output(["hostname", "-I"]).decode().split()[0] - # TODO: if this code stays, then we should generate random keys for the test. - return {"url": f"http://{ip}", "access-key": "accesskey", "secret-key": "secretkey"} + s3 = session.resource("s3", endpoint_url=config["endpoint"]) + bucket = s3.Bucket(config["bucket"]) + + for f in backups_by_cloud[cloud_name]: + backup_path = str(Path(config["path"]) / Path(str(f))) + for bucket_object in bucket.objects.filter(Prefix=backup_path): + bucket_object.delete() + + +async def _configure_s3(ops_test, config, credentials, app_name): + await ops_test.model.applications[S3_INTEGRATOR].set_config(config) + await run_action( + ops_test, + 0, + "sync-s3-credentials", + params=credentials, + app=S3_INTEGRATOR, + ) + await ops_test.model.wait_for_idle( + apps=[app_name, S3_INTEGRATOR], + status="active", + timeout=TIMEOUT, + ) +S3_INTEGRATOR = "s3-integrator" +S3_INTEGRATOR_CHANNEL = "latest/edge" +TIMEOUT = 10 * 60 + + +@pytest.mark.parametrize( + "cloud_name", + [ + (pytest.param("microceph", marks=pytest.mark.group("microceph"))), + (pytest.param("aws", marks=pytest.mark.group("aws"))), + ], +) @pytest.mark.abort_on_fail @pytest.mark.skip_if_deployed -async def test_build_and_deploy( - ops_test: OpsTest, microceph -) -> None: # , cloud_credentials) -> None: +async def test_build_and_deploy(ops_test: OpsTest, cloud_name) -> None: """Build and deploy an HA cluster of OpenSearch and corresponding S3 integration.""" - # it is possible for users to provide their own cluster for HA testing. - # Hence, check if there is a pre-existing cluster. - if await app_name(ops_test): return - s3_config = { - "bucket": "data-charms-testing", - "path": "/", - "endpoint": microceph["url"], - "region": "default", - } - my_charm = await ops_test.build_charm(".") await ops_test.model.set_config(MODEL_CONFIG) - # Deploy TLS Certificates operator. - tls_config = {"ca-common-name": "CN_CA"} + config = {"ca-common-name": "CN_CA"} + await ops_test.model.deploy(TLS_CERTIFICATES_APP_NAME, channel="stable", config=config), + s3_charm = S3_INTEGRATOR # Convert to integer as environ always returns string app_num_units = 3 - await asyncio.gather( - ops_test.model.deploy(TLS_CERTIFICATES_APP_NAME, channel="stable", config=tls_config), - ops_test.model.deploy(S3_INTEGRATOR_NAME, channel="stable", config=s3_config), + ops_test.model.deploy(s3_charm, channel=S3_INTEGRATOR_CHANNEL), ops_test.model.deploy(my_charm, num_units=app_num_units, series=SERIES), ) - s3_creds = { - "access-key": microceph["access-key"], - "secret-key": microceph["secret-key"], - } - - await run_action( - ops_test, - 0, - "sync-s3-credentials", - params=s3_creds, - app=S3_INTEGRATOR_NAME, - ) # Relate it to OpenSearch to set up TLS. await ops_test.model.integrate(APP_NAME, TLS_CERTIFICATES_APP_NAME) - await ops_test.model.integrate(APP_NAME, S3_INTEGRATOR_NAME) + await ops_test.model.integrate(APP_NAME, S3_INTEGRATOR) await ops_test.model.wait_for_idle( apps=[TLS_CERTIFICATES_APP_NAME, APP_NAME], status="active", @@ -221,42 +169,92 @@ async def test_build_and_deploy( ) +@pytest.mark.parametrize( + "cloud_name", + [ + (pytest.param("microceph", marks=pytest.mark.group("microceph"))), + (pytest.param("aws", marks=pytest.mark.group("aws"))), + ], +) @pytest.mark.abort_on_fail async def test_backup_cluster( - ops_test: OpsTest, c_writes: ContinuousWrites, c_writes_runner + ops_test: OpsTest, + c_writes: ContinuousWrites, + c_writes_runner, + cloud_configs, + cloud_credentials, + cloud_name, ) -> None: """Runs the backup process whilst writing to the cluster into 'noisy-index'.""" app = (await app_name(ops_test)) or APP_NAME leader_id = await get_leader_unit_id(ops_test) + unit_ip = await get_leader_unit_ip(ops_test) + config = cloud_configs[cloud_name] + + logger.info(f"Syncing credentials for {cloud_name}") + await _configure_s3(ops_test, config, cloud_credentials[cloud_name], app) - assert await backup_cluster( + logger.info("Creating backup") + backup_id = await backup_cluster( ops_test, leader_id, ) + assert backup_id > 0 + if cloud_name not in backups_by_cloud: + backups_by_cloud[cloud_name] = [] + backups_by_cloud[cloud_name].append(backup_id) + + # Comparing the number of docs without stopping c_writes + initial_count = await index_docs_count(ops_test, app, unit_ip, ContinuousWrites.INDEX_NAME) + time.sleep(5) + count = await index_docs_count(ops_test, app, unit_ip, ContinuousWrites.INDEX_NAME) + assert count > initial_count + # continuous writes checks await assert_continuous_writes_consistency(ops_test, c_writes, app) +@pytest.mark.parametrize( + "cloud_name", + [ + (pytest.param("microceph", marks=pytest.mark.group("microceph"))), + (pytest.param("aws", marks=pytest.mark.group("aws"))), + ], +) @pytest.mark.abort_on_fail -async def test_restore_cluster(ops_test: OpsTest) -> None: - """Deletes the TEST_BACKUP_INDEX, restores the cluster and tries to search for index.""" +async def test_restore_cluster( + ops_test: OpsTest, cloud_configs, cloud_credentials, cloud_name +) -> None: + """Restores the cluster and tries to search for index.""" unit_ip = await get_leader_unit_ip(ops_test) app = (await app_name(ops_test)) or APP_NAME leader_id = await get_leader_unit_id(ops_test) + config = cloud_configs[cloud_name] + + logger.info(f"Syncing credentials for {cloud_name}") + await _configure_s3(ops_test, config, cloud_credentials[cloud_name], app) + logger.info("Restoring backup") assert await restore_cluster( ops_test, 1, # backup_id unit_ip, leader_id, ) - count = await index_docs_count(ops_test, app, unit_ip, ContinuousWrites.INDEX_NAME) - assert count > 0 - await continuous_writes_increases(ops_test, unit_ip, app) + assert await start_and_check_continuous_writes(ops_test, unit_ip, app) +@pytest.mark.parametrize( + "cloud_name", + [ + (pytest.param("microceph", marks=pytest.mark.group("microceph"))), + (pytest.param("aws", marks=pytest.mark.group("aws"))), + ], +) @pytest.mark.abort_on_fail -async def test_restore_cluster_after_app_destroyed(ops_test: OpsTest) -> None: +async def test_restore_cluster_after_app_destroyed( + ops_test: OpsTest, cloud_configs, cloud_credentials, cloud_name +) -> None: """Deletes the entire OpenSearch cluster and redeploys from scratch. Restores the backup and then checks if the same TEST_BACKUP_INDEX is there. @@ -267,13 +265,15 @@ async def test_restore_cluster_after_app_destroyed(ops_test: OpsTest) -> None: await ops_test.model.remove_application(app, block_until_done=True) app_num_units = 3 my_charm = await ops_test.build_charm(".") + config = cloud_configs[cloud_name] + # Redeploy await asyncio.gather( ops_test.model.deploy(my_charm, num_units=app_num_units, series=SERIES), ) # Relate it to OpenSearch to set up TLS. await ops_test.model.integrate(APP_NAME, TLS_CERTIFICATES_APP_NAME) - await ops_test.model.integrate(APP_NAME, S3_INTEGRATOR_NAME) + await ops_test.model.integrate(APP_NAME, S3_INTEGRATOR) await ops_test.model.wait_for_idle( apps=[APP_NAME], status="active", @@ -283,29 +283,51 @@ async def test_restore_cluster_after_app_destroyed(ops_test: OpsTest) -> None: leader_id = await get_leader_unit_id(ops_test) leader_unit_ip = await get_leader_unit_ip(ops_test, app=app) + + logger.info(f"Syncing credentials for {cloud_name}") + await _configure_s3(ops_test, config, cloud_credentials[cloud_name], app) + + logger.info("Restoring backup") assert await restore_cluster( ops_test, 1, # backup_id leader_unit_ip, leader_id, ) - # Count the number of docs in the index - count = await index_docs_count(ops_test, app, leader_unit_ip, ContinuousWrites.INDEX_NAME) - assert count > 0 - await continuous_writes_increases(ops_test, leader_unit_ip, app) - + logger.info("Creating backup") + backup_id = await backup_cluster( + ops_test, + leader_id, + ) + assert backup_id > 0 + if cloud_name not in backups_by_cloud: + backups_by_cloud[cloud_name] = [] + backups_by_cloud[cloud_name].append(backup_id) + assert await start_and_check_continuous_writes(ops_test, leader_unit_ip, app) + + +@pytest.mark.parametrize( + "cloud_name", + [ + (pytest.param("microceph", marks=pytest.mark.group("microceph"))), + (pytest.param("aws", marks=pytest.mark.group("aws"))), + ], +) @pytest.mark.abort_on_fail -async def test_remove_and_readd_s3_relation(ops_test: OpsTest) -> None: +async def test_remove_and_readd_s3_relation( + ops_test: OpsTest, cloud_configs, cloud_credentials, cloud_name +) -> None: """Removes and re-adds the s3-credentials relation to test backup and restore.""" app = (await app_name(ops_test)) or APP_NAME leader_id = await get_leader_unit_id(ops_test) unit_ip = await get_leader_unit_ip(ops_test) + config = cloud_configs[cloud_name] logger.info("Remove s3-credentials relation") # Remove relation await ops_test.model.applications[app].destroy_relation( - "s3-credentials", f"{S3_INTEGRATOR_NAME}:s3-credentials" + "s3-credentials", f"{S3_INTEGRATOR}:s3-credentials" ) await ops_test.model.wait_for_idle( apps=[app], @@ -315,7 +337,7 @@ async def test_remove_and_readd_s3_relation(ops_test: OpsTest) -> None: ) logger.info("Re-add s3-credentials relation") - await ops_test.model.integrate(APP_NAME, S3_INTEGRATOR_NAME) + await ops_test.model.integrate(APP_NAME, S3_INTEGRATOR) await ops_test.model.wait_for_idle( apps=[app], status="active", @@ -323,16 +345,25 @@ async def test_remove_and_readd_s3_relation(ops_test: OpsTest) -> None: idle_period=IDLE_PERIOD, ) - assert await backup_cluster( + logger.info(f"Syncing credentials for {cloud_name}") + await _configure_s3(ops_test, config, cloud_credentials[cloud_name], app) + + logger.info("Creating backup") + backup_id = await backup_cluster( ops_test, leader_id, ) - assert await restore_cluster( - ops_test, - 1, # backup_id - unit_ip, - leader_id, - ) - count = await index_docs_count(ops_test, app, unit_ip, ContinuousWrites.INDEX_NAME) - assert count > 0 - await continuous_writes_increases(ops_test, unit_ip, app) + assert backup_id > 0 + if cloud_name not in backups_by_cloud: + backups_by_cloud[cloud_name] = [] + backups_by_cloud[cloud_name].append(backup_id) + + for id in [1, backup_id]: + logger.info(f"Restoring backup-id: {id}") + assert await restore_cluster( + ops_test, + id, # backup_id of the 1st backup and then the latest backup + unit_ip, + leader_id, + ) + assert await start_and_check_continuous_writes(ops_test, unit_ip, app) diff --git a/tests/integration/ha/test_ha.py b/tests/integration/ha/test_ha.py index ab5beb1b1..2fcb9277e 100644 --- a/tests/integration/ha/test_ha.py +++ b/tests/integration/ha/test_ha.py @@ -9,25 +9,7 @@ import pytest from pytest_operator.plugin import OpsTest -from tests.integration.ha.continuous_writes import ContinuousWrites -from tests.integration.ha.helpers import ( - all_processes_down, - app_name, - assert_continuous_writes_consistency, - get_elected_cm_unit_id, - get_shards_by_index, - send_kill_signal_to_process, - update_restart_delay, -) -from tests.integration.ha.helpers_data import ( - create_index, - default_doc, - delete_index, - index_doc, - search, -) -from tests.integration.ha.test_horizontal_scaling import IDLE_PERIOD -from tests.integration.helpers import ( +from ..helpers import ( APP_NAME, MODEL_CONFIG, SERIES, @@ -40,51 +22,28 @@ get_reachable_unit_ips, is_up, ) -from tests.integration.helpers_deployments import wait_until -from tests.integration.tls.test_tls import TLS_CERTIFICATES_APP_NAME +from ..helpers_deployments import wait_until +from ..tls.test_tls import TLS_CERTIFICATES_APP_NAME +from .continuous_writes import ContinuousWrites +from .helpers import ( + ORIGINAL_RESTART_DELAY, + RESTART_DELAY, + all_processes_down, + app_name, + assert_continuous_writes_consistency, + assert_continuous_writes_increasing, + get_elected_cm_unit_id, + get_shards_by_index, + send_kill_signal_to_process, + update_restart_delay, +) +from .helpers_data import create_index, default_doc, delete_index, index_doc, search +from .test_horizontal_scaling import IDLE_PERIOD logger = logging.getLogger(__name__) -SECOND_APP_NAME = "second-opensearch" -ORIGINAL_RESTART_DELAY = 20 -RESTART_DELAY = 360 - - -@pytest.fixture() -async def reset_restart_delay(ops_test: OpsTest): - """Resets service file delay on all units.""" - yield - app = (await app_name(ops_test)) or APP_NAME - for unit_id in get_application_unit_ids(ops_test, app): - await update_restart_delay(ops_test, app, unit_id, ORIGINAL_RESTART_DELAY) - - -@pytest.fixture() -async def c_writes(ops_test: OpsTest): - """Creates instance of the ContinuousWrites.""" - app = (await app_name(ops_test)) or APP_NAME - return ContinuousWrites(ops_test, app) - - -@pytest.fixture() -async def c_writes_runner(ops_test: OpsTest, c_writes: ContinuousWrites): - """Starts continuous write operations and clears writes at the end of the test.""" - await c_writes.start() - yield - await c_writes.clear() - logger.info("\n\n\n\nThe writes have been cleared.\n\n\n\n") - - -@pytest.fixture() -async def c_balanced_writes_runner(ops_test: OpsTest, c_writes: ContinuousWrites): - """Same as previous runner, but starts continuous writes on cluster wide replicated index.""" - await c_writes.start(repl_on_all_nodes=True) - yield - await c_writes.clear() - logger.info("\n\n\n\nThe writes have been cleared.\n\n\n\n") - - +@pytest.mark.group(1) @pytest.mark.abort_on_fail @pytest.mark.skip_if_deployed async def test_build_and_deploy(ops_test: OpsTest) -> None: @@ -96,7 +55,6 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None: my_charm = await ops_test.build_charm(".") await ops_test.model.set_config(MODEL_CONFIG) - # Deploy TLS Certificates operator. config = {"ca-common-name": "CN_CA"} await asyncio.gather( @@ -115,6 +73,7 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None: assert len(ops_test.model.applications[APP_NAME].units) == 3 +@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_replication_across_members( ops_test: OpsTest, c_writes: ContinuousWrites, c_writes_runner @@ -139,7 +98,7 @@ async def test_replication_across_members( await index_doc(ops_test, app, leader_unit_ip, index_name, doc_id) # check that the doc can be retrieved from any node - for u_id, u_ip in units.items(): + for u_ip in units.values(): docs = await search( ops_test, app, @@ -157,6 +116,7 @@ async def test_replication_across_members( await assert_continuous_writes_consistency(ops_test, c_writes, app) +@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_kill_db_process_node_with_primary_shard( ops_test: OpsTest, c_writes: ContinuousWrites, c_balanced_writes_runner @@ -189,12 +149,7 @@ async def test_kill_db_process_node_with_primary_shard( ops_test, app, first_unit_with_primary_shard, signal="SIGKILL" ) - # verify new writes are continuing by counting the number of writes before and after 5 seconds - # should also be plenty for the shard primary reelection to happen - writes = await c_writes.count() - time.sleep(5) - more_writes = await c_writes.count() - assert more_writes > writes, "Writes not continuing to DB" + await assert_continuous_writes_increasing(c_writes) # verify that the opensearch service is back running on the old primary unit assert await is_up( @@ -223,6 +178,7 @@ async def test_kill_db_process_node_with_primary_shard( await assert_continuous_writes_consistency(ops_test, c_writes, app) +@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_kill_db_process_node_with_elected_cm( ops_test: OpsTest, c_writes: ContinuousWrites, c_balanced_writes_runner @@ -252,12 +208,7 @@ async def test_kill_db_process_node_with_elected_cm( # Kill the opensearch process await send_kill_signal_to_process(ops_test, app, first_elected_cm_unit_id, signal="SIGKILL") - # verify new writes are continuing by counting the number of writes before and after 5 seconds - # should also be plenty for the cluster manager reelection to happen - writes = await c_writes.count() - time.sleep(5) - more_writes = await c_writes.count() - assert more_writes > writes, "Writes not continuing to DB" + await assert_continuous_writes_increasing(c_writes) # verify that the opensearch service is back running on the old elected cm unit assert await is_up( @@ -279,6 +230,7 @@ async def test_kill_db_process_node_with_elected_cm( await assert_continuous_writes_consistency(ops_test, c_writes, app) +@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_freeze_db_process_node_with_primary_shard( ops_test: OpsTest, c_writes: ContinuousWrites, c_balanced_writes_runner @@ -318,12 +270,7 @@ async def test_freeze_db_process_node_with_primary_shard( is_node_up = await is_up(ops_test, units_ips[first_unit_with_primary_shard], retries=3) assert not is_node_up - # verify new writes are continuing by counting the number of writes before and after 5 seconds - # should also be plenty for the shard primary reelection to happen - writes = await c_writes.count() - time.sleep(5) - more_writes = await c_writes.count() - assert more_writes > writes, "writes not continuing to DB" + await assert_continuous_writes_increasing(c_writes) # get reachable unit to perform requests against, in case the previously stopped unit # is leader unit, so its address is not reachable @@ -368,6 +315,7 @@ async def test_freeze_db_process_node_with_primary_shard( await assert_continuous_writes_consistency(ops_test, c_writes, app) +@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_freeze_db_process_node_with_elected_cm( ops_test: OpsTest, c_writes: ContinuousWrites, c_balanced_writes_runner @@ -406,12 +354,7 @@ async def test_freeze_db_process_node_with_elected_cm( is_node_up = await is_up(ops_test, units_ips[first_elected_cm_unit_id], retries=3) assert not is_node_up - # verify new writes are continuing by counting the number of writes before and after 5 seconds - # should also be plenty for the cluster manager reelection to happen - writes = await c_writes.count() - time.sleep(5) - more_writes = await c_writes.count() - assert more_writes > writes, "writes not continuing to DB" + await assert_continuous_writes_increasing(c_writes) # get reachable unit to perform requests against, in case the previously stopped unit # is leader unit, so its address is not reachable @@ -446,6 +389,7 @@ async def test_freeze_db_process_node_with_elected_cm( await assert_continuous_writes_consistency(ops_test, c_writes, app) +@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_restart_db_process_node_with_elected_cm( ops_test: OpsTest, c_writes: ContinuousWrites, c_balanced_writes_runner @@ -475,12 +419,7 @@ async def test_restart_db_process_node_with_elected_cm( # restart the opensearch process await send_kill_signal_to_process(ops_test, app, first_elected_cm_unit_id, signal="SIGTERM") - # verify new writes are continuing by counting the number of writes before and after 5 seconds - # should also be plenty for the cluster manager reelection to happen - writes = await c_writes.count() - time.sleep(5) - more_writes = await c_writes.count() - assert more_writes > writes, "writes not continuing to DB" + await assert_continuous_writes_increasing(c_writes) # verify that the opensearch service is back running on the unit previously elected CM unit assert await is_up( @@ -501,6 +440,7 @@ async def test_restart_db_process_node_with_elected_cm( await assert_continuous_writes_consistency(ops_test, c_writes, app) +@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_restart_db_process_node_with_primary_shard( ops_test: OpsTest, c_writes: ContinuousWrites, c_balanced_writes_runner @@ -533,12 +473,7 @@ async def test_restart_db_process_node_with_primary_shard( ops_test, app, first_unit_with_primary_shard, signal="SIGTERM" ) - # verify new writes are continuing by counting the number of writes before and after 5 seconds - # should also be plenty for the cluster manager reelection to happen - writes = await c_writes.count() - time.sleep(5) - more_writes = await c_writes.count() - assert more_writes > writes, "writes not continuing to DB" + await assert_continuous_writes_increasing(c_writes) # verify that the opensearch service is back running on the previous primary shard unit assert await is_up( @@ -566,6 +501,7 @@ async def test_restart_db_process_node_with_primary_shard( await assert_continuous_writes_consistency(ops_test, c_writes, app) +@pytest.mark.group(1) async def test_full_cluster_crash( ops_test: OpsTest, c_writes: ContinuousWrites, c_balanced_writes_runner, reset_restart_delay ) -> None: @@ -607,11 +543,7 @@ async def test_full_cluster_crash( ops_test, leader_ip, get_application_unit_names(ops_test, app=app) ) - # verify new writes are continuing by counting the number of writes before and after 5 seconds - writes = await c_writes.count() - time.sleep(5) - more_writes = await c_writes.count() - assert more_writes > writes, "Writes not continuing to DB" + await assert_continuous_writes_increasing(c_writes) # check that cluster health is green (all primary and replica shards allocated) health_resp = await cluster_health(ops_test, leader_ip) @@ -621,6 +553,7 @@ async def test_full_cluster_crash( await assert_continuous_writes_consistency(ops_test, c_writes, app) +@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_full_cluster_restart( ops_test: OpsTest, c_writes: ContinuousWrites, c_balanced_writes_runner, reset_restart_delay @@ -663,11 +596,7 @@ async def test_full_cluster_restart( ops_test, leader_ip, get_application_unit_names(ops_test, app=app) ) - # verify new writes are continuing by counting the number of writes before and after 5 seconds - writes = await c_writes.count() - time.sleep(5) - more_writes = await c_writes.count() - assert more_writes > writes, "Writes not continuing to DB" + await assert_continuous_writes_increasing(c_writes) # check that cluster health is green (all primary and replica shards allocated) health_resp = await cluster_health(ops_test, leader_ip) diff --git a/tests/integration/ha/test_ha_multi_clusters.py b/tests/integration/ha/test_ha_multi_clusters.py index fd1524bd2..67c3616d7 100644 --- a/tests/integration/ha/test_ha_multi_clusters.py +++ b/tests/integration/ha/test_ha_multi_clusters.py @@ -8,66 +8,25 @@ import pytest from pytest_operator.plugin import OpsTest -from tests.integration.ha.continuous_writes import ContinuousWrites -from tests.integration.ha.helpers import ( - app_name, - assert_continuous_writes_consistency, - update_restart_delay, -) -from tests.integration.ha.helpers_data import delete_index, index_doc, search -from tests.integration.ha.test_horizontal_scaling import IDLE_PERIOD -from tests.integration.helpers import ( +from ..helpers import ( APP_NAME, MODEL_CONFIG, SERIES, + app_name, get_application_unit_ids, get_leader_unit_ip, ) -from tests.integration.helpers_deployments import wait_until -from tests.integration.tls.test_tls import TLS_CERTIFICATES_APP_NAME +from ..helpers_deployments import wait_until +from ..tls.test_tls import TLS_CERTIFICATES_APP_NAME +from .continuous_writes import ContinuousWrites +from .helpers import SECOND_APP_NAME, assert_continuous_writes_consistency +from .helpers_data import delete_index, index_doc, search +from .test_horizontal_scaling import IDLE_PERIOD logger = logging.getLogger(__name__) -SECOND_APP_NAME = "second-opensearch" -ORIGINAL_RESTART_DELAY = 20 -RESTART_DELAY = 360 - - -@pytest.fixture() -async def reset_restart_delay(ops_test: OpsTest): - """Resets service file delay on all units.""" - yield - app = (await app_name(ops_test)) or APP_NAME - for unit_id in get_application_unit_ids(ops_test, app): - await update_restart_delay(ops_test, app, unit_id, ORIGINAL_RESTART_DELAY) - - -@pytest.fixture() -async def c_writes(ops_test: OpsTest): - """Creates instance of the ContinuousWrites.""" - app = (await app_name(ops_test)) or APP_NAME - return ContinuousWrites(ops_test, app) - - -@pytest.fixture() -async def c_writes_runner(ops_test: OpsTest, c_writes: ContinuousWrites): - """Starts continuous write operations and clears writes at the end of the test.""" - await c_writes.start() - yield - await c_writes.clear() - logger.info("\n\n\n\nThe writes have been cleared.\n\n\n\n") - - -@pytest.fixture() -async def c_balanced_writes_runner(ops_test: OpsTest, c_writes: ContinuousWrites): - """Same as previous runner, but starts continuous writes on cluster wide replicated index.""" - await c_writes.start(repl_on_all_nodes=True) - yield - await c_writes.clear() - logger.info("\n\n\n\nThe writes have been cleared.\n\n\n\n") - - +@pytest.mark.group(1) @pytest.mark.abort_on_fail @pytest.mark.skip_if_deployed async def test_build_and_deploy(ops_test: OpsTest) -> None: @@ -79,7 +38,6 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None: my_charm = await ops_test.build_charm(".") await ops_test.model.set_config(MODEL_CONFIG) - # Deploy TLS Certificates operator. config = {"ca-common-name": "CN_CA"} await asyncio.gather( @@ -101,6 +59,7 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None: # put this test at the end of the list of tests, as we delete an app during cleanup # and the safeguards we have on the charm prevent us from doing so, so we'll keep # using a unit without need - when other tests may need the unit on the CI +@pytest.mark.group(1) async def test_multi_clusters_db_isolation( ops_test: OpsTest, c_writes: ContinuousWrites, c_writes_runner ) -> None: diff --git a/tests/integration/ha/test_ha_networking.py b/tests/integration/ha/test_ha_networking.py index eabe08682..d3d431667 100644 --- a/tests/integration/ha/test_ha_networking.py +++ b/tests/integration/ha/test_ha_networking.py @@ -4,15 +4,13 @@ import asyncio import logging -import time import pytest from pytest_operator.plugin import OpsTest -from tests.integration.ha.continuous_writes import ContinuousWrites -from tests.integration.ha.helpers import ( - app_name, +from ..ha.helpers import ( assert_continuous_writes_consistency, + assert_continuous_writes_increasing, cut_network_from_unit_with_ip_change, cut_network_from_unit_without_ip_change, get_elected_cm_unit_id, @@ -22,11 +20,11 @@ restore_network_for_unit_with_ip_change, restore_network_for_unit_without_ip_change, ) -from tests.integration.ha.test_horizontal_scaling import IDLE_PERIOD -from tests.integration.helpers import ( +from ..helpers import ( APP_NAME, MODEL_CONFIG, SERIES, + app_name, check_cluster_formation_successful, get_application_unit_ids_hostnames, get_application_unit_ids_ips, @@ -35,40 +33,21 @@ get_leader_unit_ip, is_up, ) -from tests.integration.helpers_deployments import wait_until -from tests.integration.tls.test_tls import TLS_CERTIFICATES_APP_NAME +from ..helpers_deployments import wait_until +from ..tls.test_tls import TLS_CERTIFICATES_APP_NAME +from .continuous_writes import ContinuousWrites +from .test_horizontal_scaling import IDLE_PERIOD logger = logging.getLogger(__name__) -@pytest.fixture() -async def c_writes(ops_test: OpsTest): - """Creates instance of the ContinuousWrites.""" - app = (await app_name(ops_test)) or APP_NAME - return ContinuousWrites(ops_test, app) - - -@pytest.fixture() -async def c_balanced_writes_runner(ops_test: OpsTest, c_writes: ContinuousWrites): - """Same as previous runner, but starts continuous writes on cluster wide replicated index.""" - await c_writes.start(repl_on_all_nodes=True) - yield - await c_writes.clear() - logger.info("\n\n\n\nThe writes have been cleared.\n\n\n\n") - - +@pytest.mark.group(1) @pytest.mark.abort_on_fail @pytest.mark.skip_if_deployed async def test_build_and_deploy(ops_test: OpsTest) -> None: """Build and deploy one unit of OpenSearch.""" - # it is possible for users to provide their own cluster for HA testing. - # Hence, check if there is a pre-existing cluster. - if await app_name(ops_test): - return - my_charm = await ops_test.build_charm(".") await ops_test.model.set_config(MODEL_CONFIG) - # Deploy TLS Certificates operator. config = {"ca-common-name": "CN_CA"} await asyncio.gather( @@ -87,6 +66,7 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None: assert len(ops_test.model.applications[APP_NAME].units) == 3 +@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_full_network_cut_with_ip_change_node_with_elected_cm( ops_test: OpsTest, c_writes: ContinuousWrites, c_balanced_writes_runner @@ -146,11 +126,7 @@ async def test_full_network_cut_with_ip_change_node_with_elected_cm( ops_test, first_elected_cm_unit_ip, retries=3 ), "Connection still possible to the first CM node where the network was cut." - # verify new writes are continuing by counting the number of writes before and after 5 seconds - writes = await c_writes.count() - time.sleep(5) - more_writes = await c_writes.count() - assert more_writes > writes, "Writes not continuing to DB" + assert_continuous_writes_increasing(c_writes) # check new CM got elected leader_unit_ip = await get_leader_unit_ip(ops_test, app=app) @@ -192,6 +168,7 @@ async def test_full_network_cut_with_ip_change_node_with_elected_cm( await assert_continuous_writes_consistency(ops_test, c_writes, app) +@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_full_network_cut_with_ip_change_node_with_primary_shard( ops_test: OpsTest, c_writes: ContinuousWrites, c_balanced_writes_runner @@ -252,11 +229,7 @@ async def test_full_network_cut_with_ip_change_node_with_primary_shard( ops_test, first_unit_with_primary_shard_ip, retries=3 ), "Connection still possible to the first unit with primary shard where the network was cut." - # verify new writes are continuing by counting the number of writes before and after 5 seconds - writes = await c_writes.count() - time.sleep(5) - more_writes = await c_writes.count() - assert more_writes > writes, "Writes not continuing to DB" + assert_continuous_writes_increasing(c_writes) # check new primary shard got elected leader_unit_ip = await get_leader_unit_ip(ops_test, app=app) @@ -313,6 +286,7 @@ async def test_full_network_cut_with_ip_change_node_with_primary_shard( await assert_continuous_writes_consistency(ops_test, c_writes, app) +@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_full_network_cut_without_ip_change_node_with_elected_cm( ops_test: OpsTest, c_writes: ContinuousWrites, c_balanced_writes_runner @@ -367,11 +341,7 @@ async def test_full_network_cut_without_ip_change_node_with_elected_cm( ops_test, first_elected_cm_unit_ip, retries=3 ), "Connection still possible to the first CM node where the network was cut." - # verify new writes are continuing by counting the number of writes before and after 5 seconds - writes = await c_writes.count() - time.sleep(5) - more_writes = await c_writes.count() - assert more_writes > writes, "Writes not continuing to DB" + assert_continuous_writes_increasing(c_writes) # check new CM got elected leader_unit_ip = await get_leader_unit_ip(ops_test, app=app) @@ -404,6 +374,7 @@ async def test_full_network_cut_without_ip_change_node_with_elected_cm( await assert_continuous_writes_consistency(ops_test, c_writes, app) +@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_full_network_cut_without_ip_change_node_with_primary_shard( ops_test: OpsTest, c_writes: ContinuousWrites, c_balanced_writes_runner @@ -461,11 +432,7 @@ async def test_full_network_cut_without_ip_change_node_with_primary_shard( ops_test, first_unit_with_primary_shard_ip, retries=3 ), "Connection still possible to the first unit with primary shard where the network was cut." - # verify new writes are continuing by counting the number of writes before and after 5 seconds - writes = await c_writes.count() - time.sleep(5) - more_writes = await c_writes.count() - assert more_writes > writes, "Writes not continuing to DB" + assert_continuous_writes_increasing(c_writes) # check new primary shard got elected leader_unit_ip = await get_leader_unit_ip(ops_test, app=app) diff --git a/tests/integration/ha/test_horizontal_scaling.py b/tests/integration/ha/test_horizontal_scaling.py index 7df21c784..2f4897643 100644 --- a/tests/integration/ha/test_horizontal_scaling.py +++ b/tests/integration/ha/test_horizontal_scaling.py @@ -11,8 +11,7 @@ from charms.opensearch.v0.helper_cluster import ClusterTopology from pytest_operator.plugin import OpsTest -from tests.integration.ha.continuous_writes import ContinuousWrites -from tests.integration.ha.helpers import ( +from ..ha.helpers import ( all_nodes, assert_continuous_writes_consistency, get_elected_cm_unit_id, @@ -20,12 +19,7 @@ get_shards_by_index, get_shards_by_state, ) -from tests.integration.ha.helpers_data import ( - create_dummy_docs, - create_dummy_indexes, - delete_dummy_indexes, -) -from tests.integration.helpers import ( +from ..helpers import ( APP_NAME, IDLE_PERIOD, MODEL_CONFIG, @@ -38,28 +32,15 @@ get_leader_unit_id, get_leader_unit_ip, ) -from tests.integration.helpers_deployments import wait_until -from tests.integration.tls.test_tls import TLS_CERTIFICATES_APP_NAME +from ..helpers_deployments import wait_until +from ..tls.test_tls import TLS_CERTIFICATES_APP_NAME +from .continuous_writes import ContinuousWrites +from .helpers_data import create_dummy_docs, create_dummy_indexes, delete_dummy_indexes logger = logging.getLogger(__name__) -@pytest.fixture() -async def c_writes(ops_test: OpsTest): - """Creates instance of the ContinuousWrites.""" - app = (await app_name(ops_test)) or APP_NAME - return ContinuousWrites(ops_test, app) - - -@pytest.fixture() -async def c_writes_runner(ops_test: OpsTest, c_writes: ContinuousWrites): - """Starts continuous write operations and clears writes at the end of the test.""" - await c_writes.start() - yield - await c_writes.clear() - logger.info("\n\n\n\nThe writes have been cleared.\n\n\n\n") - - +@pytest.mark.group(1) @pytest.mark.abort_on_fail @pytest.mark.skip_if_deployed async def test_build_and_deploy(ops_test: OpsTest) -> None: @@ -71,7 +52,6 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None: my_charm = await ops_test.build_charm(".") await ops_test.model.set_config(MODEL_CONFIG) - # Deploy TLS Certificates operator. config = {"ca-common-name": "CN_CA"} await asyncio.gather( @@ -87,6 +67,7 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None: assert len(ops_test.model.applications[APP_NAME].units) == 1 +@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_horizontal_scale_up( ops_test: OpsTest, c_writes: ContinuousWrites, c_writes_runner @@ -134,6 +115,7 @@ async def test_horizontal_scale_up( await assert_continuous_writes_consistency(ops_test, c_writes, app) +@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_safe_scale_down_shards_realloc( ops_test: OpsTest, c_writes: ContinuousWrites, c_writes_runner @@ -243,6 +225,7 @@ async def test_safe_scale_down_shards_realloc( await assert_continuous_writes_consistency(ops_test, c_writes, app) +@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_safe_scale_down_roles_reassigning( ops_test: OpsTest, c_writes: ContinuousWrites, c_writes_runner @@ -337,6 +320,7 @@ async def test_safe_scale_down_roles_reassigning( await assert_continuous_writes_consistency(ops_test, c_writes, app) +@pytest.mark.group(1) async def test_safe_scale_down_remove_leaders( ops_test: OpsTest, c_writes: ContinuousWrites, c_writes_runner ) -> None: @@ -352,6 +336,8 @@ async def test_safe_scale_down_remove_leaders( app = (await app_name(ops_test)) or APP_NAME init_units_count = len(ops_test.model.applications[app].units) + c_writes_obj = c_writes + # scale up by 2 units await ops_test.model.applications[app].add_unit(count=3) await wait_until( @@ -422,7 +408,7 @@ async def test_safe_scale_down_remove_leaders( # sleep for a couple of minutes for the model to stabilise time.sleep(IDLE_PERIOD + 60) - writes = await c_writes.count() + writes = await c_writes_obj.count() # check that the primary shard reelection happened leader_unit_ip = await get_leader_unit_ip(ops_test, app=app) @@ -437,8 +423,8 @@ async def test_safe_scale_down_remove_leaders( # check that writes are still going after the removal / p_shard reelection time.sleep(3) - new_writes = await c_writes.count() + new_writes = await c_writes_obj.count() assert new_writes > writes # continuous writes checks - await assert_continuous_writes_consistency(ops_test, c_writes, app) + await assert_continuous_writes_consistency(ops_test, c_writes_obj, app) diff --git a/tests/integration/ha/test_large_deployments.py b/tests/integration/ha/test_large_deployments.py index 0eec7a8dc..8e5a72fa1 100644 --- a/tests/integration/ha/test_large_deployments.py +++ b/tests/integration/ha/test_large_deployments.py @@ -9,63 +9,26 @@ from charms.opensearch.v0.constants_charm import PClusterWrongNodesCountForQuorum from pytest_operator.plugin import OpsTest -from tests.integration.ha.continuous_writes import ContinuousWrites -from tests.integration.ha.helpers import all_nodes, app_name, update_restart_delay -from tests.integration.ha.test_horizontal_scaling import IDLE_PERIOD -from tests.integration.helpers import ( +from ..helpers import ( APP_NAME, MODEL_CONFIG, SERIES, check_cluster_formation_successful, cluster_health, - get_application_unit_ids, get_application_unit_names, get_application_unit_status, get_leader_unit_ip, ) -from tests.integration.helpers_deployments import wait_until -from tests.integration.tls.test_tls import TLS_CERTIFICATES_APP_NAME +from ..helpers_deployments import wait_until +from ..tls.test_tls import TLS_CERTIFICATES_APP_NAME +from .continuous_writes import ContinuousWrites +from .helpers import all_nodes, app_name +from .test_horizontal_scaling import IDLE_PERIOD logger = logging.getLogger(__name__) -ORIGINAL_RESTART_DELAY = 20 - - -@pytest.fixture() -async def reset_restart_delay(ops_test: OpsTest): - """Resets service file delay on all units.""" - yield - app = (await app_name(ops_test)) or APP_NAME - for unit_id in get_application_unit_ids(ops_test, app): - await update_restart_delay(ops_test, app, unit_id, ORIGINAL_RESTART_DELAY) - - -@pytest.fixture() -async def c_writes(ops_test: OpsTest): - """Creates instance of the ContinuousWrites.""" - app = (await app_name(ops_test)) or APP_NAME - return ContinuousWrites(ops_test, app) - - -@pytest.fixture() -async def c_writes_runner(ops_test: OpsTest, c_writes: ContinuousWrites): - """Starts continuous write operations and clears writes at the end of the test.""" - await c_writes.start() - yield - await c_writes.clear() - logger.info("\n\n\n\nThe writes have been cleared.\n\n\n\n") - - -@pytest.fixture() -async def c_balanced_writes_runner(ops_test: OpsTest, c_writes: ContinuousWrites): - """Same as previous runner, but starts continuous writes on cluster wide replicated index.""" - await c_writes.start(repl_on_all_nodes=True) - yield - await c_writes.clear() - logger.info("\n\n\n\nThe writes have been cleared.\n\n\n\n") - - +@pytest.mark.group(1) @pytest.mark.abort_on_fail @pytest.mark.skip_if_deployed async def test_build_and_deploy(ops_test: OpsTest) -> None: @@ -77,7 +40,6 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None: my_charm = await ops_test.build_charm(".") await ops_test.model.set_config(MODEL_CONFIG) - # Deploy TLS Certificates operator. config = {"ca-common-name": "CN_CA"} await asyncio.gather( @@ -98,6 +60,7 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None: assert len(ops_test.model.applications[APP_NAME].units) == 3 +@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_set_roles_manually( ops_test: OpsTest, c_writes: ContinuousWrites, c_writes_runner diff --git a/tests/integration/ha/test_storage.py b/tests/integration/ha/test_storage.py index 32f7e51c5..e36f96388 100644 --- a/tests/integration/ha/test_storage.py +++ b/tests/integration/ha/test_storage.py @@ -4,98 +4,27 @@ import asyncio import logging -import random import time import pytest from pytest_operator.plugin import OpsTest -from tests.integration.ha.continuous_writes import ContinuousWrites -from tests.integration.ha.helpers import ( - app_name, - storage_id, - storage_type, - update_restart_delay, -) -from tests.integration.ha.test_horizontal_scaling import IDLE_PERIOD -from tests.integration.helpers import ( - APP_NAME, - MODEL_CONFIG, - SERIES, - get_application_unit_ids, - get_reachable_unit_ips, - http_request, -) -from tests.integration.tls.test_tls import TLS_CERTIFICATES_APP_NAME +from ..ha.helpers import app_name, storage_id, storage_type +from ..ha.test_horizontal_scaling import IDLE_PERIOD +from ..helpers import APP_NAME, MODEL_CONFIG, SERIES, get_application_unit_ids +from ..tls.test_tls import TLS_CERTIFICATES_APP_NAME +from .continuous_writes import ContinuousWrites logger = logging.getLogger(__name__) -SECOND_APP_NAME = "second-opensearch" -ORIGINAL_RESTART_DELAY = 20 -RESTART_DELAY = 360 - - -@pytest.fixture() -async def reset_restart_delay(ops_test: OpsTest): - """Resets service file delay on all units.""" - yield - app = (await app_name(ops_test)) or APP_NAME - for unit_id in get_application_unit_ids(ops_test, app): - await update_restart_delay(ops_test, app, unit_id, ORIGINAL_RESTART_DELAY) - - -@pytest.fixture() -async def c_writes(ops_test: OpsTest): - """Creates instance of the ContinuousWrites.""" - app = (await app_name(ops_test)) or APP_NAME - return ContinuousWrites(ops_test, app) - - -@pytest.fixture() -async def c_writes_runner(ops_test: OpsTest, c_writes: ContinuousWrites): - """Starts continuous write operations and clears writes at the end of the test.""" - await c_writes.start() - yield - - reachable_ip = random.choice(await get_reachable_unit_ips(ops_test)) - await http_request(ops_test, "GET", f"https://{reachable_ip}:9200/_cat/nodes", json_resp=False) - await http_request( - ops_test, "GET", f"https://{reachable_ip}:9200/_cat/shards", json_resp=False - ) - - await c_writes.clear() - logger.info("\n\n\n\nThe writes have been cleared.\n\n\n\n") - - -@pytest.fixture() -async def c_balanced_writes_runner(ops_test: OpsTest, c_writes: ContinuousWrites): - """Same as previous runner, but starts continuous writes on cluster wide replicated index.""" - await c_writes.start(repl_on_all_nodes=True) - yield - - reachable_ip = random.choice(await get_reachable_unit_ips(ops_test)) - await http_request(ops_test, "GET", f"https://{reachable_ip}:9200/_cat/nodes", json_resp=False) - await http_request( - ops_test, "GET", f"https://{reachable_ip}:9200/_cat/shards", json_resp=False - ) - - await c_writes.clear() - logger.info("\n\n\n\nThe writes have been cleared.\n\n\n\n") - - +@pytest.mark.group(1) @pytest.mark.abort_on_fail @pytest.mark.skip_if_deployed async def test_build_and_deploy(ops_test: OpsTest) -> None: """Build and deploy one unit of OpenSearch.""" - # it is possible for users to provide their own cluster for HA testing. - # Hence, check if there is a pre-existing cluster. - if await app_name(ops_test): - return - my_charm = await ops_test.build_charm(".") await ops_test.model.set_config(MODEL_CONFIG) - # Deploy TLS Certificates operator. config = {"ca-common-name": "CN_CA"} await asyncio.gather( @@ -114,6 +43,7 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None: assert len(ops_test.model.applications[APP_NAME].units) == 1 +@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_storage_reuse_after_scale_down( ops_test: OpsTest, c_writes: ContinuousWrites, c_writes_runner @@ -176,9 +106,10 @@ async def test_storage_reuse_after_scale_down( assert writes_result.max_stored_id == (await c_writes.max_stored_id()) +@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_storage_reuse_in_new_cluster_after_app_removal( - ops_test: OpsTest, c_writes: ContinuousWrites, c_balanced_writes_runner + ops_test: OpsTest, c_writes: ContinuousWrites, c_writes_runner ): """Check storage is reused and data accessible after removing app and deploying new cluster.""" app = (await app_name(ops_test)) or APP_NAME diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index 339647f00..aa6aeee72 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -13,11 +13,6 @@ import requests import yaml from charms.opensearch.v0.helper_networking import is_reachable -from integration.helpers_deployments import ( - Status, - get_application_units, - get_unit_hostname, -) from opensearchpy import OpenSearch from pytest_operator.plugin import OpsTest from tenacity import ( @@ -29,6 +24,8 @@ wait_random, ) +from .helpers_deployments import Status, get_application_units, get_unit_hostname + METADATA = yaml.safe_load(Path("./metadata.yaml").read_text()) APP_NAME = METADATA["name"] diff --git a/tests/integration/plugins/__init__.py b/tests/integration/plugins/__init__.py new file mode 100644 index 000000000..db3bfe1a6 --- /dev/null +++ b/tests/integration/plugins/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. diff --git a/tests/integration/plugins/helpers.py b/tests/integration/plugins/helpers.py index fc033079f..d6ab3bb15 100644 --- a/tests/integration/plugins/helpers.py +++ b/tests/integration/plugins/helpers.py @@ -18,8 +18,8 @@ wait_random, ) -from tests.integration.ha.helpers_data import bulk_insert, create_index -from tests.integration.helpers import get_application_unit_ids, http_request +from ..ha.helpers_data import bulk_insert, create_index +from ..helpers import get_application_unit_ids, http_request logger = logging.getLogger(__name__) diff --git a/tests/integration/plugins/test_plugins.py b/tests/integration/plugins/test_plugins.py index 1c9ec5aa2..45827271a 100644 --- a/tests/integration/plugins/test_plugins.py +++ b/tests/integration/plugins/test_plugins.py @@ -9,10 +9,10 @@ from pytest_operator.plugin import OpsTest from tenacity import RetryError -from tests.integration.ha.helpers import app_name -from tests.integration.ha.helpers_data import bulk_insert, create_index, search -from tests.integration.ha.test_horizontal_scaling import IDLE_PERIOD -from tests.integration.helpers import ( +from ..ha.helpers import app_name +from ..ha.helpers_data import bulk_insert, create_index, search +from ..ha.test_horizontal_scaling import IDLE_PERIOD +from ..helpers import ( APP_NAME, MODEL_CONFIG, SERIES, @@ -25,7 +25,7 @@ http_request, run_action, ) -from tests.integration.plugins.helpers import ( +from ..plugins.helpers import ( create_index_and_bulk_insert, generate_bulk_training_data, get_application_unit_ids_start_time, @@ -33,15 +33,14 @@ is_knn_training_complete, run_knn_training, ) -from tests.integration.relations.opensearch_provider.helpers import ( - get_unit_relation_data, -) -from tests.integration.tls.test_tls import TLS_CERTIFICATES_APP_NAME +from ..relations.helpers import get_unit_relation_data +from ..tls.test_tls import TLS_CERTIFICATES_APP_NAME COS_APP_NAME = "grafana-agent" COS_RELATION_NAME = "cos-agent" +@pytest.mark.group(1) @pytest.mark.abort_on_fail @pytest.mark.skip_if_deployed async def test_build_and_deploy(ops_test: OpsTest) -> None: @@ -62,10 +61,10 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None: # Deploy TLS Certificates operator. config = {"ca-common-name": "CN_CA"} await asyncio.gather( - ops_test.model.deploy(TLS_CERTIFICATES_APP_NAME, channel="stable", config=config), ops_test.model.deploy( my_charm, num_units=3, series=SERIES, config={"plugin_opensearch_knn": True} ), + ops_test.model.deploy(TLS_CERTIFICATES_APP_NAME, channel="stable", config=config), ) # Relate it to OpenSearch to set up TLS. @@ -79,6 +78,7 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None: assert len(ops_test.model.applications[APP_NAME].units) == 3 +@pytest.mark.group(1) async def test_prometheus_exporter_enabled_by_default(ops_test): """Test that Prometheus Exporter is running before the relation is there.""" leader_unit_ip = await get_leader_unit_ip(ops_test, app=APP_NAME) @@ -90,6 +90,7 @@ async def test_prometheus_exporter_enabled_by_default(ops_test): assert len(response_str.split("\n")) > 500 +@pytest.mark.group(1) async def test_prometheus_exporter_cos_relation(ops_test): await ops_test.model.deploy(COS_APP_NAME, channel="edge"), await ops_test.model.integrate(APP_NAME, COS_APP_NAME) @@ -119,6 +120,7 @@ async def test_prometheus_exporter_cos_relation(ops_test): assert relation_data["scheme"] == "https" +@pytest.mark.group(1) async def test_monitoring_user_fetch_prometheus_data(ops_test): leader_unit_ip = await get_leader_unit_ip(ops_test, app=APP_NAME) endpoint = f"https://{leader_unit_ip}:9200/_prometheus/metrics" @@ -139,6 +141,7 @@ async def test_monitoring_user_fetch_prometheus_data(ops_test): assert len(response_str.split("\n")) > 500 +@pytest.mark.group(1) async def test_prometheus_monitor_user_password_change(ops_test): # Password change applied as expected leader_id = await get_leader_unit_id(ops_test, APP_NAME) @@ -161,6 +164,7 @@ async def test_prometheus_monitor_user_password_change(ops_test): assert relation_data["password"] == new_password +@pytest.mark.group(1) async def test_knn_enabled_disabled(ops_test): config = await ops_test.model.applications[APP_NAME].get_config() assert config["plugin_opensearch_knn"]["default"] is True @@ -179,6 +183,7 @@ async def test_knn_enabled_disabled(ops_test): assert config["plugin_opensearch_knn"]["value"] is True +@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_knn_search_with_hnsw_faiss(ops_test: OpsTest) -> None: """Uploads data and runs a query search against the FAISS KNNEngine.""" @@ -222,6 +227,7 @@ async def test_knn_search_with_hnsw_faiss(ops_test: OpsTest) -> None: assert len(docs) == 2 +@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_knn_search_with_hnsw_nmslib(ops_test: OpsTest) -> None: """Uploads data and runs a query search against the NMSLIB KNNEngine.""" @@ -265,6 +271,7 @@ async def test_knn_search_with_hnsw_nmslib(ops_test: OpsTest) -> None: assert len(docs) == 2 +@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_knn_training_search(ops_test: OpsTest) -> None: """Tests the entire cycle of KNN plugin. diff --git a/tests/integration/relations/__init__.py b/tests/integration/relations/__init__.py new file mode 100644 index 000000000..db3bfe1a6 --- /dev/null +++ b/tests/integration/relations/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. diff --git a/tests/integration/relations/conftest.py b/tests/integration/relations/conftest.py new file mode 100644 index 000000000..aa54a92e6 --- /dev/null +++ b/tests/integration/relations/conftest.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. + +import glob +import os +import pathlib +import shutil +from importlib import reload + +import pytest +import pytest_operator +from pytest_operator.plugin import OpsTest + + +def should_rebuild_charm(target: pathlib.Path) -> bool: + """Returns the latest change in the charm path. + + Compare it with the *.charm file, if source files were changed more recently, + then rebuild the charm. + """ + ignored_prefixes = [".", "_"] + target_path = os.path.dirname(target) + for root, dirs, files in os.walk(target_path): + if any([p for p in ignored_prefixes if root.startswith(p)]): + continue + for f in files: + if f.endswith(".charm") or any([p for p in ignored_prefixes if f.startswith(p)]): + continue + if os.path.getctime(target) < os.path.getctime(f): + return True + return False + + +async def _build(ops_test, charmpath): + """Decides if we should rebuild the charm or not. + + If the charm is not built yet or the source files were changed more recently than .charm, + then rebuild the charm. + + Besides that, the current DP workflow only checks for the actual charm. It disconsiders + any other testing charms existing in the tests/ folder. + """ + if ( + # This is a build process that should be only valid whenever we are not in CI + ("CI" not in os.environ or os.environ["CI"] != "true") + and ( + # Now, check for the .charm file and if we had any recent updates in the sources + not glob.glob(f"{charmpath}/*.charm") + or should_rebuild_charm(glob.glob(f"{charmpath}/*.charm")[0]) + ) + ): + name = await reload(pytest_operator.plugin).OpsTest.build_charm(ops_test, charmpath) + return name + return await ops_test.build_charm(charmpath) + + +@pytest.fixture(scope="module") +async def application_charm(ops_test: OpsTest): + """Build the application charm.""" + shutil.copyfile( + "./lib/charms/data_platform_libs/v0/data_interfaces.py", + "./tests/integration/relations/opensearch_provider/application-charm/lib/charms/data_platform_libs/v0/data_interfaces.py", + ) + test_charm_path = "./tests/integration/relations/opensearch_provider/application-charm" + return await _build(ops_test, test_charm_path) + + +@pytest.fixture(scope="module") +async def opensearch_charm(ops_test: OpsTest): + """Build the opensearch charm as well.""" + return await _build(ops_test, ".") diff --git a/tests/integration/relations/opensearch_provider/helpers.py b/tests/integration/relations/helpers.py similarity index 100% rename from tests/integration/relations/opensearch_provider/helpers.py rename to tests/integration/relations/helpers.py diff --git a/tests/integration/relations/opensearch_provider/conftest.py b/tests/integration/relations/opensearch_provider/conftest.py deleted file mode 100644 index 6d6bbf6bf..000000000 --- a/tests/integration/relations/opensearch_provider/conftest.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2023 Canonical Ltd. -# See LICENSE file for licensing details. - -import shutil - -import pytest -from pytest_operator.plugin import OpsTest - - -@pytest.fixture(scope="module") -async def application_charm(ops_test: OpsTest): - """Build the application charm.""" - shutil.copyfile( - "./lib/charms/data_platform_libs/v0/data_interfaces.py", - "./tests/integration/relations/opensearch_provider/application-charm/lib/charms/data_platform_libs/v0/data_interfaces.py", - ) - test_charm_path = "./tests/integration/relations/opensearch_provider/application-charm" - return await ops_test.build_charm(test_charm_path) - - -@pytest.fixture(scope="module") -async def opensearch_charm(ops_test: OpsTest): - """Build the opensearch charm.""" - return await ops_test.build_charm(".") diff --git a/tests/integration/relations/opensearch_provider/test_opensearch_provider.py b/tests/integration/relations/test_opensearch_provider.py similarity index 77% rename from tests/integration/relations/opensearch_provider/test_opensearch_provider.py rename to tests/integration/relations/test_opensearch_provider.py index 5f7b63b70..459b072b9 100644 --- a/tests/integration/relations/opensearch_provider/test_opensearch_provider.py +++ b/tests/integration/relations/test_opensearch_provider.py @@ -11,8 +11,8 @@ from charms.opensearch.v0.constants_charm import ClientRelationName from pytest_operator.plugin import OpsTest -from tests.integration.helpers import APP_NAME as OPENSEARCH_APP_NAME -from tests.integration.helpers import ( +from ..helpers import APP_NAME as OPENSEARCH_APP_NAME +from ..helpers import ( MODEL_CONFIG, SERIES, get_application_unit_ids, @@ -20,8 +20,9 @@ http_request, scale_application, ) -from tests.integration.helpers_deployments import wait_until -from tests.integration.relations.opensearch_provider.helpers import ( +from ..helpers_deployments import wait_until +from ..tls.test_tls import TLS_CERTIFICATES_APP_NAME +from .helpers import ( get_application_relation_data, ip_to_url, run_request, @@ -32,7 +33,6 @@ CLIENT_APP_NAME = "application" SECONDARY_CLIENT_APP_NAME = "secondary-application" -TLS_CERTIFICATES_APP_NAME = "self-signed-certificates" ALL_APPS = [OPENSEARCH_APP_NAME, TLS_CERTIFICATES_APP_NAME, CLIENT_APP_NAME] NUM_UNITS = 3 @@ -51,6 +51,7 @@ ] +@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_create_relation(ops_test: OpsTest, application_charm, opensearch_charm): """Test basic functionality of relation interface.""" @@ -59,8 +60,10 @@ async def test_create_relation(ops_test: OpsTest, application_charm, opensearch_ new_model_conf = MODEL_CONFIG.copy() new_model_conf["update-status-hook-interval"] = "1m" + config = {"ca-common-name": "CN_CA"} + await ops_test.model.deploy(TLS_CERTIFICATES_APP_NAME, channel="stable", config=config) + await ops_test.model.set_config(new_model_conf) - tls_config = {"ca-common-name": "CN_CA"} await asyncio.gather( ops_test.model.deploy( application_charm, @@ -72,7 +75,6 @@ async def test_create_relation(ops_test: OpsTest, application_charm, opensearch_ num_units=NUM_UNITS, series=SERIES, ), - ops_test.model.deploy(TLS_CERTIFICATES_APP_NAME, channel="stable", config=tls_config), ) await ops_test.model.integrate(OPENSEARCH_APP_NAME, TLS_CERTIFICATES_APP_NAME) wait_for_relation_joined_between(ops_test, OPENSEARCH_APP_NAME, TLS_CERTIFICATES_APP_NAME) @@ -89,8 +91,11 @@ async def test_create_relation(ops_test: OpsTest, application_charm, opensearch_ timeout=1600, status="active", ) + # Return all update-status-hook-interval to 5m, as all idle_period values are set to 70s + await ops_test.model.set_config({"update-status-hook-interval": "5m"}) +@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_index_usage(ops_test: OpsTest): """Check we can update and delete things. @@ -128,6 +133,7 @@ async def test_index_usage(ops_test: OpsTest): ) +@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_bulk_index_usage(ops_test: OpsTest): """Check we can update and delete things using bulk api.""" @@ -168,6 +174,7 @@ async def test_bulk_index_usage(ops_test: OpsTest): assert set(artists) == {"Herbie Hancock", "Lydian Collective", "Vulfpeck"} +@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_version(ops_test: OpsTest): """Check version reported in the databag is consistent with the version on the charm.""" @@ -188,8 +195,8 @@ async def test_version(ops_test: OpsTest): assert version == results.get("version", {}).get("number"), results +@pytest.mark.group(1) @pytest.mark.abort_on_fail -@pytest.mark.usefixtures("only_without_juju_secrets") async def test_scaling(ops_test: OpsTest): """Test that scaling correctly updates endpoints in databag. @@ -253,8 +260,8 @@ async def get_secret_data(ops_test, secret_uri): return json.loads(stdout)[secret_unique_id]["content"]["Data"] +@pytest.mark.group(1) @pytest.mark.abort_on_fail -@pytest.mark.usefixtures("only_with_juju_secrets") async def test_scaling_secrets(ops_test: OpsTest): """Test that scaling correctly updates endpoints in databag. @@ -317,6 +324,7 @@ async def _is_number_of_endpoints_valid(client_app: str, rel: str) -> bool: ), await rel_endpoints(CLIENT_APP_NAME, FIRST_RELATION_NAME) +@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_multiple_relations(ops_test: OpsTest, application_charm): """Test that two different applications can connect to the database.""" @@ -381,6 +389,7 @@ async def test_multiple_relations(ops_test: OpsTest, application_charm): assert "403 Client Error: Forbidden for url:" in results[0], results +@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_multiple_relations_accessing_same_index(ops_test: OpsTest): """Test that two different applications can connect to the database.""" @@ -417,6 +426,7 @@ async def test_multiple_relations_accessing_same_index(ops_test: OpsTest): assert set(artists) == {"Herbie Hancock", "Lydian Collective", "Vulfpeck"} +@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_admin_relation(ops_test: OpsTest): """Test we can create relations with admin permissions.""" @@ -452,72 +462,8 @@ async def test_admin_relation(ops_test: OpsTest): assert set(artists) == {"Herbie Hancock", "Lydian Collective", "Vulfpeck"} +@pytest.mark.group(1) @pytest.mark.abort_on_fail -@pytest.mark.usefixtures("only_without_juju_secrets") -async def test_admin_permissions(ops_test: OpsTest): - """Test admin permissions behave the way we want. - - admin-only actions include: - - creating multiple indices - - removing indices they've created - - set cluster roles. - - verify that: - - we can't remove .opendistro_security index - - otherwise create client-admin-role - - verify neither admin nor default users can access user api - - otherwise create client-default-role - """ - test_unit = ops_test.model.applications[CLIENT_APP_NAME].units[0] - # Verify admin can't access security API - security_api_endpoint = "/_plugins/_security/api/internalusers" - run_dump_users = await run_request( - ops_test, - unit_name=test_unit.name, - endpoint=security_api_endpoint, - method="GET", - relation_id=admin_relation.id, - relation_name=ADMIN_RELATION_NAME, - ) - results = json.loads(run_dump_users["results"]) - logging.info(results) - assert "403 Client Error: Forbidden for url:" in results[0], results - - # verify admin can't delete users - first_relation_user = await get_application_relation_data( - ops_test, f"{CLIENT_APP_NAME}/0", FIRST_RELATION_NAME, "username" - ) - first_relation_user_endpoint = f"/_plugins/_security/api/internalusers/{first_relation_user}" - run_delete_users = await run_request( - ops_test, - unit_name=test_unit.name, - endpoint=first_relation_user_endpoint, - method="DELETE", - relation_id=admin_relation.id, - relation_name=ADMIN_RELATION_NAME, - ) - results = json.loads(run_delete_users["results"]) - logging.info(results) - assert "403 Client Error: Forbidden for url:" in results[0], results - - # verify admin can't modify protected indices - for protected_index in PROTECTED_INDICES: - protected_index_endpoint = f"/{protected_index}" - run_remove_distro = await run_request( - ops_test, - unit_name=test_unit.name, - endpoint=protected_index_endpoint, - method="DELETE", - relation_id=admin_relation.id, - relation_name=ADMIN_RELATION_NAME, - ) - results = json.loads(run_remove_distro["results"]) - logging.info(results) - assert "Error:" in results[0], results - - -@pytest.mark.abort_on_fail -@pytest.mark.usefixtures("only_with_juju_secrets") async def test_admin_permissions_secrets(ops_test: OpsTest): """Test admin permissions behave the way we want. @@ -584,66 +530,8 @@ async def test_admin_permissions_secrets(ops_test: OpsTest): assert "Error:" in results[0], results +@pytest.mark.group(1) @pytest.mark.abort_on_fail -@pytest.mark.usefixtures("only_without_juju_secrets") -async def test_normal_user_permissions(ops_test: OpsTest): - """Test normal user permissions behave the way we want. - - verify that: - - we can't remove .opendistro_security index - - verify neither admin nor default users can access user api - """ - test_unit = ops_test.model.applications[CLIENT_APP_NAME].units[0] - - # Verify normal users can't access security API - security_api_endpoint = "/_plugins/_security/api/internalusers" - run_dump_users = await run_request( - ops_test, - unit_name=test_unit.name, - endpoint=security_api_endpoint, - method="GET", - relation_id=client_relation.id, - relation_name=FIRST_RELATION_NAME, - ) - results = json.loads(run_dump_users["results"]) - logging.info(results) - assert "403 Client Error: Forbidden for url:" in results[0], results - - # verify normal users can't delete users - first_relation_user = await get_application_relation_data( - ops_test, f"{CLIENT_APP_NAME}/0", FIRST_RELATION_NAME, "username" - ) - first_relation_user_endpoint = f"/_plugins/_security/api/internalusers/{first_relation_user}" - run_delete_users = await run_request( - ops_test, - unit_name=test_unit.name, - endpoint=first_relation_user_endpoint, - method="DELETE", - relation_id=client_relation.id, - relation_name=FIRST_RELATION_NAME, - ) - results = json.loads(run_delete_users["results"]) - logging.info(results) - assert "403 Client Error: Forbidden for url:" in results[0], results - - # verify user can't modify protected indices - for protected_index in PROTECTED_INDICES: - protected_index_endpoint = f"/{protected_index}" - run_remove_index = await run_request( - ops_test, - unit_name=test_unit.name, - endpoint=protected_index_endpoint, - method="DELETE", - relation_id=client_relation.id, - relation_name=FIRST_RELATION_NAME, - ) - results = json.loads(run_remove_index["results"]) - logging.info(results) - assert "Error:" in results[0], results - - -@pytest.mark.abort_on_fail -@pytest.mark.usefixtures("only_with_juju_secrets") async def test_normal_user_permissions_secrets(ops_test: OpsTest): """Test normal user permissions behave the way we want. @@ -703,57 +591,8 @@ async def test_normal_user_permissions_secrets(ops_test: OpsTest): assert "Error:" in results[0], results +@pytest.mark.group(1) @pytest.mark.abort_on_fail -@pytest.mark.usefixtures("only_without_juju_secrets") -async def test_relation_broken(ops_test: OpsTest): - """Test that the user is removed when the relation is broken.""" - # Retrieve the relation user. - relation_user = await get_application_relation_data( - ops_test, f"{CLIENT_APP_NAME}/0", FIRST_RELATION_NAME, "username" - ) - await ops_test.model.wait_for_idle( - status="active", - apps=[SECONDARY_CLIENT_APP_NAME] + ALL_APPS, - idle_period=70, - timeout=1600, - ) - - # Break the relation. - await asyncio.gather( - ops_test.model.applications[OPENSEARCH_APP_NAME].remove_relation( - f"{OPENSEARCH_APP_NAME}:{ClientRelationName}", - f"{CLIENT_APP_NAME}:{FIRST_RELATION_NAME}", - ), - ops_test.model.applications[OPENSEARCH_APP_NAME].remove_relation( - f"{OPENSEARCH_APP_NAME}:{ClientRelationName}", - f"{CLIENT_APP_NAME}:{ADMIN_RELATION_NAME}", - ), - ) - - await asyncio.gather( - ops_test.model.wait_for_idle(apps=[CLIENT_APP_NAME], status="blocked", idle_period=70), - ops_test.model.wait_for_idle( - apps=[OPENSEARCH_APP_NAME, TLS_CERTIFICATES_APP_NAME, SECONDARY_CLIENT_APP_NAME], - status="active", - idle_period=70, - timeout=1600, - ), - ) - - leader_ip = await get_leader_unit_ip(ops_test) - users = await http_request( - ops_test, - "GET", - f"https://{ip_to_url(leader_ip)}:9200/_plugins/_security/api/internalusers/", - verify=False, - ) - logger.info(relation_user) - logger.info(users) - assert relation_user not in users.keys() - - -@pytest.mark.abort_on_fail -@pytest.mark.usefixtures("only_with_juju_secrets") async def test_relation_broken_secrets(ops_test: OpsTest): """Test that the user is removed when the relation is broken.""" # Retrieve the relation user. @@ -807,6 +646,7 @@ async def test_relation_broken_secrets(ops_test: OpsTest): assert relation_user not in users.keys() +@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_data_persists_on_relation_rejoin(ops_test: OpsTest): """Verify that if we recreate a relation, we can access the same index.""" diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 401eb13d4..ee28eacc7 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -7,7 +7,7 @@ import pytest from pytest_operator.plugin import OpsTest -from tests.integration.helpers import ( +from .helpers import ( APP_NAME, MODEL_CONFIG, SERIES, @@ -18,8 +18,8 @@ http_request, run_action, ) -from tests.integration.helpers_deployments import wait_until -from tests.integration.tls.test_tls import TLS_CERTIFICATES_APP_NAME +from .helpers_deployments import wait_until +from .tls.test_tls import TLS_CERTIFICATES_APP_NAME logger = logging.getLogger(__name__) @@ -27,6 +27,7 @@ DEFAULT_NUM_UNITS = 2 +@pytest.mark.group(1) @pytest.mark.abort_on_fail @pytest.mark.skip_if_deployed async def test_build_and_deploy(ops_test: OpsTest) -> None: @@ -42,6 +43,7 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None: await ops_test.model.wait_for_idle(wait_for_exact_units=DEFAULT_NUM_UNITS, timeout=1800) +@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_status(ops_test: OpsTest) -> None: """Verifies that the application and unit are active.""" @@ -54,6 +56,7 @@ async def test_status(ops_test: OpsTest) -> None: assert len(ops_test.model.applications[APP_NAME].units) == DEFAULT_NUM_UNITS +@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_actions_get_admin_password(ops_test: OpsTest) -> None: """Test the retrieval of admin secrets.""" @@ -64,14 +67,13 @@ async def test_actions_get_admin_password(ops_test: OpsTest) -> None: # Deploy TLS Certificates operator. config = {"ca-common-name": "CN_CA"} await ops_test.model.deploy(TLS_CERTIFICATES_APP_NAME, channel="stable", config=config) - await ops_test.model.wait_for_idle( - apps=[TLS_CERTIFICATES_APP_NAME], status="active", timeout=1000 - ) - # Relate it to OpenSearch to set up TLS. await ops_test.model.integrate(APP_NAME, TLS_CERTIFICATES_APP_NAME) await ops_test.model.wait_for_idle( - apps=[APP_NAME], status="active", timeout=1200, wait_for_exact_units=DEFAULT_NUM_UNITS + apps=[APP_NAME], + status="active", + timeout=1200, + wait_for_exact_units=DEFAULT_NUM_UNITS, ) leader_ip = await get_leader_unit_ip(ops_test) @@ -92,6 +94,7 @@ async def test_actions_get_admin_password(ops_test: OpsTest) -> None: assert result.status == "failed" +@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_actions_rotate_admin_password(ops_test: OpsTest) -> None: """Test the rotation and change of admin password.""" diff --git a/tests/integration/tls/__init__.py b/tests/integration/tls/__init__.py new file mode 100644 index 000000000..db3bfe1a6 --- /dev/null +++ b/tests/integration/tls/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. diff --git a/tests/integration/tls/helpers.py b/tests/integration/tls/helpers.py index 3c3542988..ce8c18478 100644 --- a/tests/integration/tls/helpers.py +++ b/tests/integration/tls/helpers.py @@ -5,7 +5,7 @@ from pytest_operator.plugin import OpsTest from tenacity import retry, stop_after_attempt, wait_fixed, wait_random -from tests.integration.helpers import http_request +from ..helpers import http_request @retry( diff --git a/tests/integration/tls/test_tls.py b/tests/integration/tls/test_tls.py index fb12c4c12..f7d19fcda 100644 --- a/tests/integration/tls/test_tls.py +++ b/tests/integration/tls/test_tls.py @@ -7,40 +7,32 @@ import pytest from pytest_operator.plugin import OpsTest -from tests.integration.helpers import ( +from ..helpers import ( APP_NAME, MODEL_CONFIG, - SERIES, UNIT_IDS, check_cluster_formation_successful, get_application_unit_ips_names, get_application_unit_names, get_leader_unit_ip, ) -from tests.integration.helpers_deployments import wait_until -from tests.integration.tls.helpers import ( - check_security_index_initialised, - check_unit_tls_configured, -) +from ..helpers_deployments import wait_until +from ..tls.helpers import check_security_index_initialised, check_unit_tls_configured logger = logging.getLogger(__name__) + TLS_CERTIFICATES_APP_NAME = "self-signed-certificates" +@pytest.mark.group(1) @pytest.mark.abort_on_fail @pytest.mark.skip_if_deployed async def test_build_and_deploy_active(ops_test: OpsTest) -> None: """Build and deploy one unit of OpenSearch.""" - my_charm = await ops_test.build_charm(".") + await ops_test.build_charm(".") await ops_test.model.set_config(MODEL_CONFIG) - await ops_test.model.deploy( - my_charm, - num_units=len(UNIT_IDS), - series=SERIES, - ) - # Deploy TLS Certificates operator. config = {"ca-common-name": "CN_CA"} await ops_test.model.deploy(TLS_CERTIFICATES_APP_NAME, channel="stable", config=config) @@ -58,6 +50,7 @@ async def test_build_and_deploy_active(ops_test: OpsTest) -> None: assert len(ops_test.model.applications[APP_NAME].units) == len(UNIT_IDS) +@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_security_index_initialised(ops_test: OpsTest) -> None: """Test that the security index is well initialised.""" @@ -66,6 +59,7 @@ async def test_security_index_initialised(ops_test: OpsTest) -> None: assert await check_security_index_initialised(ops_test, leader_unit_ip) +@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_tls_configured(ops_test: OpsTest) -> None: """Test that TLS is enabled when relating to the TLS Certificates Operator.""" @@ -73,6 +67,7 @@ async def test_tls_configured(ops_test: OpsTest) -> None: assert await check_unit_tls_configured(ops_test, unit_ip, unit_name) +@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_cluster_formation_after_tls(ops_test: OpsTest) -> None: """Test that the cluster formation is successful after TLS setup.""" diff --git a/tests/unit/lib/test_opensearch_secrets.py b/tests/unit/lib/test_opensearch_secrets.py index 60fa01103..77f6cfda8 100644 --- a/tests/unit/lib/test_opensearch_secrets.py +++ b/tests/unit/lib/test_opensearch_secrets.py @@ -3,7 +3,6 @@ from unittest.mock import MagicMock, patch -import pytest from charms.opensearch.v0.constants_charm import ClientRelationName, PeerRelationName from charms.opensearch.v0.constants_tls import CertType from charms.opensearch.v0.opensearch_base_charm import SERVICE_MANAGER @@ -17,7 +16,11 @@ from charm import OpenSearchOperatorCharm -@pytest.mark.usefixtures("only_with_juju_secrets") +class JujuVersionMock: + def has_secrets(self): + return True + + class TestOpenSearchSecrets(TestOpenSearchInternalData): """Ensuring that secrets interfaces and expected behavior are preserved. @@ -37,6 +40,8 @@ def setUp(self): self.secrets = self.charm.secrets self.store = self.charm.secrets + JujuVersion.from_environ = MagicMock(return_value=JujuVersionMock()) + self.peers_rel_id = self.harness.add_relation(PeerRelationName, self.charm.app.name) self.service_rel_id = self.harness.add_relation(SERVICE_MANAGER, self.charm.app.name) self.client_rel_id = self.harness.add_relation(ClientRelationName, "application") diff --git a/tox.ini b/tox.ini index 9b1827461..200a27b26 100644 --- a/tox.ini +++ b/tox.ini @@ -1,147 +1,108 @@ + # Copyright 2023 Canonical Ltd. # See LICENSE file for licensing details. [tox] no_package = True -skip_missing_interpreters = True -env_list = format, lint, unit -labels = - # Don't run this group in parallel, or with --keep-models because it creates a lot of local VMs. - integration = {charm, tls, ha, ha-networking, ha-multi-clusters, large-deployments, client, h-scaling, ha-storage}-integration +env_list = lint, unit [vars] src_path = {tox_root}/src tests_path = {tox_root}/tests -lib_path = {tox_root}/lib/charms/opensearch/v0 -all_path = {[vars]src_path} {[vars]lib_path} {[vars]tests_path} +lib_path = {tox_root}/lib/charms/opensearch +all_path = {[vars]src_path} {[vars]tests_path} {[vars]lib_path} [testenv] set_env = - PYTHONPATH = {tox_root}:{tox_root}/lib:{[vars]src_path} - PYTHONBREAKPOINT=ipdb.set_trace + PYTHONPATH = {[vars]src_path}:{tox_root}/lib PY_COLORS=1 - LIBJUJU_VERSION_SPECIFIER={env:LIBJUJU_VERSION_SPECIFIER:==3.3.0.0} +allowlist_externals = + poetry +[testenv:build-{production,dev,wrapper}] +# Wrap `charmcraft pack` pass_env = - PYTHONPATH - CHARM_BUILD_DIR - MODEL_SETTINGS - LIBJUJU_VERSION_SPECIFIER + CI +allowlist_externals = + {[testenv]allowlist_externals} + charmcraft + charmcraftcache + mv +commands_pre = + poetry export --only main,charm-libs --output requirements.txt +commands = + build-production: charmcraft pack {posargs} + build-dev: charmcraftcache pack {posargs} +commands_post = + mv requirements.txt requirements-last-build.txt [testenv:format] description = Apply coding style standards to code -deps = - black - isort +commands_pre = + poetry install --only format commands = - isort {[vars]all_path} - black {[vars]all_path} + poetry lock --no-update + poetry run isort {[vars]all_path} + poetry run black {[vars]all_path} [testenv:lint] description = Check code against coding style standards -pass_env = - {[testenv]pass_env} -deps = - black - flake8 - flake8-docstrings - flake8-copyright - flake8-builtins - pyproject-flake8 - pep8-naming - isort - codespell +allowlist_externals = + {[testenv]allowlist_externals} + find +commands_pre = + poetry install --only lint commands = - codespell {[vars]lib_path} - codespell {tox_root} --skip {tox_root}/.git --skip {tox_root}/.tox \ - --skip {tox_root}/build --skip {tox_root}/lib --skip {tox_root}/venv \ - --skip {tox_root}/.mypy_cache --skip {tox_root}/icon.svg + poetry check --lock + poetry run codespell {[vars]all_path} # pflake8 wrapper supports config from pyproject.toml - pflake8 {[vars]all_path} - isort --check-only --diff {[vars]all_path} - black --check --diff {[vars]all_path} + # exclude the lib folder, that is copied from lib/charms/data_platform_libs/ + poetry run pflake8 --exclude '.git,__pycache__,.tox,build,dist,*.egg_info,venv,tests/integration/relations/opensearch_provider/application-charm/lib/charms/data_platform_libs/' {[vars]all_path} + poetry run isort --check-only --diff {[vars]all_path} + poetry run black --check --diff {[vars]all_path} + find {[vars]all_path} -type f \( -name "*.sh" -o -name "*.bash" \) -exec poetry run shellcheck --color=always \{\} + [testenv:unit] description = Run unit tests -deps = - pytest - pytest-asyncio - coverage[toml] - -r {tox_root}/requirements.txt +set_env = + {[testenv]set_env} + LIBJUJU_VERSION_SPECIFIER = {env:LIBJUJU_VERSION_SPECIFIER:3.2.3.0} +commands_pre = + poetry install --only main,charm-libs,unit commands = - coverage run --source={[vars]src_path} --source={[vars]lib_path} \ + poetry run coverage run --source={[vars]src_path},{[vars]lib_path} \ -m pytest -v --tb native -s {posargs} {[vars]tests_path}/unit - coverage report + poetry run coverage report + poetry run coverage xml -[testenv:{charm, tls, client, ha-base, h-scaling, ha-storage, ha-networking, ha-multi-clusters, large-deployments, plugins}-integration] +[testenv:integration] description = Run integration tests -pass_env = - {[testenv]pass_env} - CI - CI_PACKED_CHARMS - S3_ACCESS_KEY - S3_SECRET_KEY - S3_BUCKET - S3_REGION - S3_SERVER_URL - S3_CA_BUNDLE_PATH - # For AWS testing - AWS_ACCESS_KEY - AWS_SECRET_KEY - # For GCP testing - GCP_ACCESS_KEY - GCP_SECRET_KEY - # Generic secrets from CI: - SECRETS_FROM_GITHUB - TEST_NUM_APP_UNITS -deps = - # This applies to libjuju, not Juju. - juju{env:LIBJUJU_VERSION_SPECIFIER} - opensearch-py - pytest - pytest-asyncio - pytest-operator - -r {tox_root}/requirements.txt -commands = - charm: pytest -v --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/test_charm.py - tls: pytest -v --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/tls/test_tls.py - client: pytest -v --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/relations/opensearch_provider/test_opensearch_provider.py - ha-base: pytest -v --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/ha/test_ha.py - # h-scaling must run on a machine with more than 2 cores - h-scaling: pytest -v --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/ha/test_horizontal_scaling.py - ha-storage: pytest -v --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/ha/test_storage.py - ha-networking: pytest -v --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/ha/test_ha_networking.py - ha-multi-clusters: pytest -v --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/ha/test_ha_multi_clusters.py - large-deployments: pytest -v --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/ha/test_large_deployments.py - plugins: pytest -v --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/plugins/test_plugins.py +set_env = + {[testenv]set_env} + # Workaround for https://github.com/python-poetry/poetry/issues/6958 + POETRY_INSTALLER_PARALLEL = false + LIBJUJU_VERSION_SPECIFIER = {env:LIBJUJU_VERSION_SPECIFIER:3.2.3.0} + CI={env:CI:true} + GITHUB_OUTPUT={env:GITHUB_OUTPUT:} + S3_INTEGRATOR_CHARMPATH={env:S3_INTEGRATOR_CHARMPATH:} + # If not specified, it passes a dummy value that will ignore. + # To not be ignored, the AWS|GCP cloud MUST specify the *_ACCESS_KEY + SECRETS_FROM_GITHUB={env:SECRETS_FROM_GITHUB:{"AWS_SECRET_KEY": "IGNORE", "GCP_SECRET_KEY": "IGNORE"}} +allowlist_externals = + {[testenv:build-wrapper]allowlist_externals} -[testenv:ha-backup-integration] -description = Run integration tests -pass_env = - {[testenv]pass_env} - CI - CI_PACKED_CHARMS - # For microceph testing - S3_ACCESS_KEY - S3_SECRET_KEY - S3_BUCKET - S3_REGION - S3_SERVER_URL - S3_CA_BUNDLE_PATH - # For AWS testing - AWS_ACCESS_KEY - AWS_SECRET_KEY - # For GCP testing - GCP_ACCESS_KEY - GCP_SECRET_KEY - TEST_NUM_APP_UNITS -deps = - # This applies to libjuju, not Juju. - juju{env:LIBJUJU_VERSION_SPECIFIER} - opensearch-py - pytest - pytest-asyncio - pytest-operator - -r {tox_root}/requirements.txt + # Set the testing host before starting the lxd cloud + sudo + sysctl + +commands_pre = + poetry install --only main,charm-libs,integration + + # Set the testing host before starting the lxd cloud + sudo sysctl -w vm.max_map_count=262144 vm.swappiness=0 net.ipv4.tcp_retries2=5 + + {[testenv:build-wrapper]commands_pre} commands = - ha-backup: pytest -v --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/ha/test_backups.py + poetry run pytest -v --tb native --log-cli-level=INFO -s --ignore={[vars]tests_path}/unit/ {posargs} +commands_post = + {[testenv:build-wrapper]commands_post}