diff --git a/.github/workflows/requirements-molecule.txt b/.github/workflows/requirements-molecule.txt new file mode 100644 index 0000000..e558845 --- /dev/null +++ b/.github/workflows/requirements-molecule.txt @@ -0,0 +1,51 @@ +ansible-lint==24.5.0 +attrs==23.2.0 +black==24.4.2 +bracex==2.4 +certifi==2024.2.2 +cffi==1.16.0 +charset-normalizer==3.3.2 +click==8.1.7 +click-help-colors==0.9.4 +cryptography==42.0.7 +distro==1.9.0 +enrich==1.2.7 +filelock==3.14.0 +idna==3.7 +importlib_metadata==7.1.0 +Jinja2==3.1.4 +jmespath==1.0.1 +jsonschema==4.22.0 +jsonschema-specifications==2023.12.1 +markdown-it-py==3.0.0 +MarkupSafe==2.1.5 +mdurl==0.1.2 +molecule==24.2.1 +molecule-plugins==23.5.3 +mypy-extensions==1.0.0 +packaging==24.0 +pathspec==0.12.1 +platformdirs==4.2.2 +pluggy==1.5.0 +pycparser==2.22 +Pygments==2.18.0 +PyYAML==6.0.1 +referencing==0.35.1 +resolvelib==1.0.1 +rich==13.7.1 +rpds-py==0.18.1 +ruamel.yaml==0.18.6 +ruamel.yaml.clib==0.2.8 +selinux==0.3.0 +subprocess-tee==0.4.1 +urllib3==2.2.1 +wcmatch==8.5.2 +yamllint==1.35.1 +zipp==3.18.2 +# requests > 2.31.0 brakes Docker Python library +# The fix https://github.com/docker/docker-py/pull/3257/files +# has not been in a release of the Docker Python library +# they should be updated together +# when new versions of the Docker Python library > 7.1.0 are released +docker==7.1.0 +requests==2.31.0 diff --git a/.github/workflows/reusable-molecule.yml b/.github/workflows/reusable-molecule.yml index f12ef89..78d85a5 100644 --- a/.github/workflows/reusable-molecule.yml +++ b/.github/workflows/reusable-molecule.yml @@ -24,30 +24,26 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.x' + python-version: '3.12' - name: Check molecule run: | if [ -d "molecule" ]; then echo "MOLECULE_IS_PRESENT=PRESENT" >> "${GITHUB_ENV}" fi working-directory: "${{ github.repository }}/roles/${{ inputs.role-name }}" - - name: Setup molecule + - name: Install Python modules run: | - pip3 install --no-cache-dir yamllint ansible==${{ inputs.ansible-version }} ansible-lint \ - molecule molecule-plugins[docker] docker \ - jmespath - - name: Print Ansible version - run: ansible --version - - name: Setup LXD - if: ${{ env.MOLECULE_IS_PRESENT && inputs.molecule-driver == 'lxd' }} - # https://github.com/canonical/setup-lxd - uses: canonical/setup-lxd@v0.1.1 - with: - channel: latest/stable - - name: Setup LXD molecule module - if: ${{ env.MOLECULE_IS_PRESENT && inputs.molecule-driver == 'lxd' }} + set -e + echo "ansible==${{ inputs.ansible-version }}" >> requirements-molecule.txt + pip3 install --no-cache-dir -r requirements-molecule.txt + working-directory: "${{ github.repository }}/.github/workflows" + - name: Print versions run: | - pip3 install --no-cache-dir molecule-lxd + set -e + ansible --version + molecule --version + yamllint --version + ansible-lint --version - name: Run lint run: | set -e diff --git a/galaxy.yml b/galaxy.yml index 5f8d2c7..6a0e9b1 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -8,7 +8,7 @@ namespace: paritytech name: chain # The version of the collection. Must be compatible with semantic versioning -version: 1.9.0 +version: 1.9.1 # The path to the Markdown (.md) readme file. This path is relative to the root of the collection readme: README.md diff --git a/roles/nginx/molecule/default/README.md b/roles/nginx/molecule/default/README.md index 8ae68a1..3daa9a6 100644 --- a/roles/nginx/molecule/default/README.md +++ b/roles/nginx/molecule/default/README.md @@ -8,11 +8,3 @@ molecule verify molecule destroy ``` -#### LXD -Test role with LXD driver -```shell -DRIVER=lxd molecule create -DRIVER=lxd molecule converge -DRIVER=lxd molecule verify -DRIVER=lxd molecule destroy -``` diff --git a/roles/nginx/molecule/default/molecule.yml b/roles/nginx/molecule/default/molecule.yml index 0fa28b6..7f9dc45 100644 --- a/roles/nginx/molecule/default/molecule.yml +++ b/roles/nginx/molecule/default/molecule.yml @@ -5,9 +5,6 @@ driver: name: ${DRIVER:-docker} platforms: - name: molecule-instance-nginx - # LXD - source: - alias: debian/bullseye/amd64 # DOCKER image: paritytech/debian11:latest command: ${MOLECULE_DOCKER_COMMAND:-""} diff --git a/roles/node/molecule/default/group_vars/all.yml b/roles/node/molecule/default/group_vars/all.yml index 1654cf1..7768945 100644 --- a/roles/node/molecule/default/group_vars/all.yml +++ b/roles/node/molecule/default/group_vars/all.yml @@ -5,7 +5,7 @@ ansible_user: root ## Node node_chain: polkadot node_app_name: "{{ node_chain }}" -node_binary_version: v1.3.0 +node_binary_version: v1.12.0 node_legacy_rpc_flags: false node_rpc_port: 9944 node_binary: https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-{{ node_binary_version }}/polkadot diff --git a/roles/node/molecule/default/molecule.yml b/roles/node/molecule/default/molecule.yml index 305a0ca..c6f04e4 100644 --- a/roles/node/molecule/default/molecule.yml +++ b/roles/node/molecule/default/molecule.yml @@ -5,9 +5,6 @@ driver: name: ${DRIVER:-docker} platforms: - name: molecule-instance-node - # LXD - source: - alias: debian/bullseye/amd64 # DOCKER image: paritytech/debian11:latest command: ${MOLECULE_DOCKER_COMMAND:-""} diff --git a/roles/node/molecule/parachain/molecule.yml b/roles/node/molecule/parachain/molecule.yml index b63f72a..4bb12e9 100644 --- a/roles/node/molecule/parachain/molecule.yml +++ b/roles/node/molecule/parachain/molecule.yml @@ -5,9 +5,6 @@ driver: name: ${DRIVER:-docker} platforms: - name: molecule-instance-node-parachain - # LXD - source: - alias: debian/bullseye/amd64 # DOCKER image: paritytech/debian11:latest command: ${MOLECULE_DOCKER_COMMAND:-""} diff --git a/roles/node/molecule/parachain_remote_rc/molecule.yml b/roles/node/molecule/parachain_remote_rc/molecule.yml index 04eecfd..61b7653 100644 --- a/roles/node/molecule/parachain_remote_rc/molecule.yml +++ b/roles/node/molecule/parachain_remote_rc/molecule.yml @@ -5,8 +5,6 @@ driver: name: ${DRIVER:-docker} platforms: - name: molecule-instance-node-parachain-remote-rc - source: - alias: debian/bullseye/amd64 # DOCKER image: paritytech/debian11:latest command: ${MOLECULE_DOCKER_COMMAND:-""} diff --git a/roles/node_backup/molecule/default/README.md b/roles/node_backup/molecule/default/README.md index cde3444..8321361 100644 --- a/roles/node_backup/molecule/default/README.md +++ b/roles/node_backup/molecule/default/README.md @@ -15,11 +15,4 @@ molecule converge molecule destroy ``` -#### LXD -Test role with LXD driver -```shell -DRIVER=lxd molecule create -DRIVER=lxd molecule converge -DRIVER=lxd molecule destroy -``` diff --git a/roles/node_backup/molecule/default/group_vars/all.yml b/roles/node_backup/molecule/default/group_vars/all.yml index 016c908..538548a 100644 --- a/roles/node_backup/molecule/default/group_vars/all.yml +++ b/roles/node_backup/molecule/default/group_vars/all.yml @@ -2,34 +2,35 @@ ## Molecule ansible_user: root -## prepare.yml -#node_legacy_rpc_flags: false -node_binary: https://github.com/paritytech/polkadot/releases/download/v0.9.42/polkadot -node_chain: rococo-local -node_data_root_path: /opt/{{ node_app_name }} +## node +node_role: "validator" +node_user: polkadot +node_chain: westend-local +node_chain_backup_chain_path: "westend_local_testnet" node_chain_backup_restoring_type: none -node_pruning: 256 -# node_binary_deployment: false +node_parachain_chain_backup_restoring_type: none +node_p2p_bind_addr: "127.0.0.1" +node_ansible_annotation_path: /tmp/substrate.prom +node_binary_version: v1.12.0 +node_binary: "https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-{{ node_binary_version }}/polkadot" +node_prepare_worker_binary: "https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-{{ node_binary_version }}/polkadot-prepare-worker" +node_execute_worker_binary: "https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-{{ node_binary_version }}/polkadot-execute-worker" + # node_backup _gcp_bucket: test-blockstore-backups -node_backup_user: parity +node_backup_user: polkadot node_backup_r2_access_key_id: abc node_backup_r2_secret_access_key: cba node_backup_r2_api_url: https://a.b node_backup_targets: - - service_name: rococo-alice-rocksdb-prune - local_path: /opt/rococo-alice-rocksdb-prune/chains/rococo_local_testnet/db + - service_name: alice-rocksdb-prune + local_path: /opt/alice-rocksdb-prune/chains/{{ node_chain_backup_chain_path }}/db rpc_port: 9933 bucket_name: "{{ _gcp_bucket }}" - type: gcp-native - - service_name: rococo-bob-paritydb-prune - local_path: /opt/rococo-bob-paritydb-prune/chains/rococo_local_testnet/paritydb - rpc_port: 9934 - bucket_name: "{{ _gcp_bucket }}" type: gcp-rclone - - service_name: rococo-bob-paritydb-prune - local_path: /opt/rococo-bob-paritydb-prune/chains/rococo_local_testnet/paritydb + - service_name: bob-paritydb-prune + local_path: /opt/bob-paritydb-prune/chains/{{ node_chain_backup_chain_path }}/paritydb rpc_port: 9934 bucket_name: "{{ _gcp_bucket }}" type: r2-rclone diff --git a/roles/node_backup/molecule/default/molecule.yml b/roles/node_backup/molecule/default/molecule.yml index 2dcf72f..6ad8784 100644 --- a/roles/node_backup/molecule/default/molecule.yml +++ b/roles/node_backup/molecule/default/molecule.yml @@ -5,9 +5,6 @@ driver: name: ${DRIVER:-docker} platforms: - name: molecule-instance-node-backup - # LXD - source: - alias: debian/bullseye/amd64 # DOCKER image: paritytech/debian11:latest command: ${MOLECULE_DOCKER_COMMAND:-""} diff --git a/roles/node_backup/molecule/default/prepare.yml b/roles/node_backup/molecule/default/prepare.yml index f8c8ac2..aabc8c5 100644 --- a/roles/node_backup/molecule/default/prepare.yml +++ b/roles/node_backup/molecule/default/prepare.yml @@ -3,42 +3,53 @@ hosts: all gather_facts: false pre_tasks: - - name: Install Python3 + - name: Prepare | Install Python3 ansible.builtin.raw: apt -y update && apt install -y python3 changed_when: false + - name: Prepare | Install required packages + ansible.builtin.apt: + name: + - gpg + - cron + update_cache: false + changed_when: false - name: Prepare | create user parity ansible.builtin.user: - name: parity + name: polkadot tasks: - - name: rococo-alice local + - name: Include node alice + # use include role to skip ansible-lint ansible.builtin.include_role: - name: parity.chain.node + name: node vars: - node_rpc_port: 9933 - node_app_name: rococo-alice-rocksdb-prune + node_app_name: "alice-rocksdb-prune" + node_handler_id: "{{ node_app_name }}" + node_data_root_path: "/opt/{{ node_app_name }}" + # 12D3KooWHhB5LqXji1moEvbzCEq7HzJuMvqi5E3BkoceUcQFPX2f + node_p2p_private_key: "4f1ae54a051e08161456b74a70b85e45e161fd4f614637f50a2a5f09ba7afb2e" node_custom_options: - - --alice - - name: rococo-bob local + - "--alice" + - "--bootnodes /ip4/127.0.0.1/tcp/30334/p2p/12D3KooWKvNLq5fFMcQvdZHejhUvRQcSxDpbWFBo1mXh4kGR949r" + - "--no-hardware-benchmarks" + - "--rpc-cors '*'" + node_p2p_port: "30333" + node_prometheus_port: "9615" + node_rpc_port: "9933" + - name: Include node bob ansible.builtin.include_role: - name: parity.chain.node + name: node vars: - node_rpc_port: 9934 - node_paritydb_enable: true - node_app_name: rococo-bob-paritydb-prune + node_app_name: "bob-paritydb-prune" + node_handler_id: "{{ node_app_name }}" + node_data_root_path: /opt/{{ node_app_name }} + # 12D3KooWKvNLq5fFMcQvdZHejhUvRQcSxDpbWFBo1mXh4kGR949r + node_p2p_private_key: "042cd72c647f27c6da663f15665b59f707bce7de4b771b098361ff756cde168e" node_custom_options: - - --bob - - name: Pretend we are in gcp | Install cron, gnupg - ansible.builtin.package: - name: - - cron - - gnupg - state: present - update_cache: true - - name: Pretend we are in gcp | Add an Apt signing key - ansible.builtin.apt_key: - url: https://packages.cloud.google.com/apt/doc/apt-key.gpg - state: present - - name: Pretend we are in gcp | Add apt repository into sources list - ansible.builtin.apt_repository: - repo: deb https://packages.cloud.google.com/apt cloud-sdk main - state: present + - "--bob" + - "--bootnodes /ip4/127.0.0.1/tcp/30333/p2p/12D3KooWHhB5LqXji1moEvbzCEq7HzJuMvqi5E3BkoceUcQFPX2f" + - "--no-hardware-benchmarks" + - "--rpc-cors '*'" + node_p2p_port: "30334" + node_prometheus_port: "9616" + node_rpc_port: "9934" + node_paritydb_enable: true diff --git a/roles/node_backup/molecule/default/verify.yml b/roles/node_backup/molecule/default/verify.yml index 10846bb..10b8fb8 100644 --- a/roles/node_backup/molecule/default/verify.yml +++ b/roles/node_backup/molecule/default/verify.yml @@ -22,26 +22,10 @@ - name: Print current block ansible.builtin.debug: var: _node_backup_register_header.json.result.number | int(base=16) -# # todo add tests -# -## a) upload to gcp -# GCP storage emulator is not available yet (https://github.com/googleapis/google-cloud-python/issues/10300), -# there are third party emulator, but support of gsutils is broken (https://github.com/oittaa/gcp-storage-emulator/issues/186) -# when emulator will be available: -# 1. run and configure emulator -# 2. run script: -# - name: run backup script -# ansible.builtin.command: /home/parity/bin/node_backup.sh -# 3. -# - name: "rococo-bob local" -# ansible.builtin.include_role: -# name: parity.chain.node -# vars: -# node_rpc_port: 9935 -# node_paritydb_enable: true -# node_app_name: "rococo-local-rpc" -# -## b) Test backup-exporter: -# We can push fake data to backup-exporter (like run bash script). + +# TODO - add tests +# Test backup-exporter: +# 1. We can push fake data to backup-exporter (like run bash script). # Then we can check the Prometheus endpoint to check and match the results. # This will allow checking the code of the exporter. +# 2. We can upload data to local MinIO \ No newline at end of file diff --git a/roles/ws_health_exporter/molecule/default/collections.yml b/roles/ws_health_exporter/molecule/default/collections.yml new file mode 100644 index 0000000..7a0c167 --- /dev/null +++ b/roles/ws_health_exporter/molecule/default/collections.yml @@ -0,0 +1,4 @@ +collections: + - name: https://github.com/paritytech/ansible-galaxy.git + type: git + version: main diff --git a/roles/ws_health_exporter/molecule/default/collections.yml.bak b/roles/ws_health_exporter/molecule/default/collections.yml.bak deleted file mode 100644 index 7650095..0000000 --- a/roles/ws_health_exporter/molecule/default/collections.yml.bak +++ /dev/null @@ -1,4 +0,0 @@ -#collections: -# - name: https://github.com/paritytech/ansible-galaxy.git -# type: git -# version: main diff --git a/roles/ws_health_exporter/molecule/default/group_vars/all.yml b/roles/ws_health_exporter/molecule/default/group_vars/all.yml index d2ea45c..9d98804 100644 --- a/roles/ws_health_exporter/molecule/default/group_vars/all.yml +++ b/roles/ws_health_exporter/molecule/default/group_vars/all.yml @@ -3,13 +3,18 @@ ansible_user: root ## node -node_binary_version: v0.9.42 +node_role: "validator" node_user: polkadot -node_binary: https://github.com/paritytech/polkadot/releases/download/{{ node_binary_version }}/polkadot -node_chain: rococo-local +node_chain: westend-local +node_chain_backup_chain_path: "westend_local_testnet" node_chain_backup_restoring_type: none node_parachain_chain_backup_restoring_type: none +node_p2p_bind_addr: "127.0.0.1" node_ansible_annotation_path: /tmp/substrate.prom +node_binary_version: v1.12.0 +node_binary: "https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-{{ node_binary_version }}/polkadot" +node_prepare_worker_binary: "https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-{{ node_binary_version }}/polkadot-prepare-worker" +node_execute_worker_binary: "https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-{{ node_binary_version }}/polkadot-execute-worker" ## ws_health_exporter ws_health_exporter_user: "{{ node_user }}" @@ -19,5 +24,5 @@ ws_health_exporter_node_max_unsynchronized_block_drift: 2 ws_health_exporter_node_min_peers: 1 ws_health_exporter_ws_urls: - - ws://127.0.0.1:9944 - - ws://127.0.0.1:9945 + - ws://127.0.0.1:9933 + - ws://127.0.0.1:9934 diff --git a/roles/ws_health_exporter/molecule/default/molecule.yml b/roles/ws_health_exporter/molecule/default/molecule.yml index adc3f6c..72d9d2e 100644 --- a/roles/ws_health_exporter/molecule/default/molecule.yml +++ b/roles/ws_health_exporter/molecule/default/molecule.yml @@ -5,9 +5,6 @@ driver: name: ${DRIVER:-docker} platforms: - name: molecule-instance-ws-health-exporter - # LXD - source: - alias: debian/bullseye/amd64 # DOCKER image: paritytech/debian11:latest command: ${MOLECULE_DOCKER_COMMAND:-""} diff --git a/roles/ws_health_exporter/molecule/default/prepare.yml b/roles/ws_health_exporter/molecule/default/prepare.yml index f879773..79f6624 100644 --- a/roles/ws_health_exporter/molecule/default/prepare.yml +++ b/roles/ws_health_exporter/molecule/default/prepare.yml @@ -6,29 +6,45 @@ - name: Install Python3 ansible.builtin.raw: apt -y update && apt install -y python3 changed_when: false + - name: Install required packages + ansible.builtin.apt: + name: + - gpg + update_cache: false + changed_when: false tasks: - name: Include node alice # use include role to skip ansible-lint ansible.builtin.include_role: name: node vars: - node_app_name: rococo-alice - node_data_root_path: /opt/{{ node_app_name }} + node_app_name: "alice" + node_handler_id: "{{ node_app_name }}" + node_data_root_path: "/opt/{{ node_app_name }}" + # 12D3KooWHhB5LqXji1moEvbzCEq7HzJuMvqi5E3BkoceUcQFPX2f + node_p2p_private_key: "4f1ae54a051e08161456b74a70b85e45e161fd4f614637f50a2a5f09ba7afb2e" node_custom_options: - - --alice + - "--alice" + - "--bootnodes /ip4/127.0.0.1/tcp/30334/p2p/12D3KooWKvNLq5fFMcQvdZHejhUvRQcSxDpbWFBo1mXh4kGR949r" + - "--no-hardware-benchmarks" + - "--rpc-cors '*'" node_p2p_port: "30333" node_prometheus_port: "9615" node_rpc_port: "9933" - node_rpc_ws_port: "9944" - name: Include node bob ansible.builtin.include_role: name: node vars: - node_app_name: rococo-bob + node_app_name: bob + node_handler_id: "{{ node_app_name }}" node_data_root_path: /opt/{{ node_app_name }} + # 12D3KooWKvNLq5fFMcQvdZHejhUvRQcSxDpbWFBo1mXh4kGR949r + node_p2p_private_key: "042cd72c647f27c6da663f15665b59f707bce7de4b771b098361ff756cde168e" node_custom_options: - - --bob + - "--bob" + - "--bootnodes /ip4/127.0.0.1/tcp/30333/p2p/12D3KooWHhB5LqXji1moEvbzCEq7HzJuMvqi5E3BkoceUcQFPX2f" + - "--no-hardware-benchmarks" + - "--rpc-cors '*'" node_p2p_port: "30334" node_prometheus_port: "9616" node_rpc_port: "9934" - node_rpc_ws_port: "9945" diff --git a/roles/ws_health_exporter/molecule/default/verify.yml b/roles/ws_health_exporter/molecule/default/verify.yml index d01c460..fd4718b 100644 --- a/roles/ws_health_exporter/molecule/default/verify.yml +++ b/roles/ws_health_exporter/molecule/default/verify.yml @@ -10,15 +10,15 @@ ansible.builtin.debug: var: ansible_facts.services[item+'.service'] loop: - - rococo-alice - - rococo-bob + - alice + - bob - name: check service ansible.builtin.assert: that: ansible_facts.services[item+'.service'].state == 'running' loop: - - rococo-alice - - rococo-bob + - alice + - bob - name: check ws health exporter ansible.builtin.uri: