fix ci (#68)

* fix ci --------- Signed-off-by: kogeler <[email protected]>
paritytech · May 27, 2024 · 628c78d · 628c78d
1 parent de2adfc
commit 628c78d
Show file tree

Hide file tree

Showing 20 changed files with 168 additions and 136 deletions.
diff --git a/.github/workflows/requirements-molecule.txt b/.github/workflows/requirements-molecule.txt
@@ -0,0 +1,51 @@
+ansible-lint==24.5.0
+attrs==23.2.0
+black==24.4.2
+bracex==2.4
+certifi==2024.2.2
+cffi==1.16.0
+charset-normalizer==3.3.2
+click==8.1.7
+click-help-colors==0.9.4
+cryptography==42.0.7
+distro==1.9.0
+enrich==1.2.7
+filelock==3.14.0
+idna==3.7
+importlib_metadata==7.1.0
+Jinja2==3.1.4
+jmespath==1.0.1
+jsonschema==4.22.0
+jsonschema-specifications==2023.12.1
+markdown-it-py==3.0.0
+MarkupSafe==2.1.5
+mdurl==0.1.2
+molecule==24.2.1
+molecule-plugins==23.5.3
+mypy-extensions==1.0.0
+packaging==24.0
+pathspec==0.12.1
+platformdirs==4.2.2
+pluggy==1.5.0
+pycparser==2.22
+Pygments==2.18.0
+PyYAML==6.0.1
+referencing==0.35.1
+resolvelib==1.0.1
+rich==13.7.1
+rpds-py==0.18.1
+ruamel.yaml==0.18.6
+ruamel.yaml.clib==0.2.8
+selinux==0.3.0
+subprocess-tee==0.4.1
+urllib3==2.2.1
+wcmatch==8.5.2
+yamllint==1.35.1
+zipp==3.18.2
+# requests > 2.31.0 brakes Docker Python library
+# The fix https://github.com/docker/docker-py/pull/3257/files
+# has not been in a release of the Docker Python library
+# they should be updated together
+# when new versions of the Docker Python library > 7.1.0 are released
+docker==7.1.0
+requests==2.31.0
diff --git a/.github/workflows/reusable-molecule.yml b/.github/workflows/reusable-molecule.yml
@@ -24,30 +24,26 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.x'
+          python-version: '3.12'
       - name: Check molecule
         run: |
           if [ -d "molecule" ]; then
             echo "MOLECULE_IS_PRESENT=PRESENT" >> "${GITHUB_ENV}"
           fi
         working-directory: "${{ github.repository }}/roles/${{ inputs.role-name }}"
-      - name: Setup molecule
+      - name: Install Python modules
         run: |
-          pip3 install --no-cache-dir yamllint ansible==${{ inputs.ansible-version }} ansible-lint \
-          molecule molecule-plugins[docker] docker \
-          jmespath
-      - name: Print Ansible version
-        run: ansible --version
-      - name: Setup LXD
-        if: ${{ env.MOLECULE_IS_PRESENT && inputs.molecule-driver == 'lxd' }}
-        # https://github.com/canonical/setup-lxd
-        uses: canonical/[email protected]
-        with:
-          channel: latest/stable
-      - name: Setup LXD molecule module
-        if: ${{ env.MOLECULE_IS_PRESENT && inputs.molecule-driver == 'lxd' }}
+          set -e
+          echo "ansible==${{ inputs.ansible-version }}" >> requirements-molecule.txt
+          pip3 install --no-cache-dir -r requirements-molecule.txt
+        working-directory: "${{ github.repository }}/.github/workflows"
+      - name: Print versions
         run: |
-          pip3 install --no-cache-dir molecule-lxd
+          set -e
+          ansible --version
+          molecule --version
+          yamllint --version
+          ansible-lint --version
       - name: Run lint
         run: |
           set -e

diff --git a/galaxy.yml b/galaxy.yml
@@ -8,7 +8,7 @@ namespace: paritytech
 name: chain
 
 # The version of the collection. Must be compatible with semantic versioning
-version: 1.9.0
+version: 1.9.1
 
 # The path to the Markdown (.md) readme file. This path is relative to the root of the collection
 readme: README.md

diff --git a/roles/nginx/molecule/default/README.md b/roles/nginx/molecule/default/README.md
@@ -8,11 +8,3 @@ molecule verify
 molecule destroy
 ```
 
-#### LXD 
-Test role with LXD driver 
-```shell
-DRIVER=lxd molecule create
-DRIVER=lxd molecule converge
-DRIVER=lxd molecule verify
-DRIVER=lxd molecule destroy
-```
diff --git a/roles/nginx/molecule/default/molecule.yml b/roles/nginx/molecule/default/molecule.yml
@@ -5,9 +5,6 @@ driver:
   name: ${DRIVER:-docker}
 platforms:
   - name: molecule-instance-nginx
-    # LXD
-    source:
-      alias: debian/bullseye/amd64
     # DOCKER
     image: paritytech/debian11:latest
     command: ${MOLECULE_DOCKER_COMMAND:-""}

diff --git a/roles/node/molecule/default/group_vars/all.yml b/roles/node/molecule/default/group_vars/all.yml
@@ -5,7 +5,7 @@ ansible_user: root
 ## Node
 node_chain: polkadot
 node_app_name: "{{ node_chain }}"
-node_binary_version: v1.3.0
+node_binary_version: v1.12.0
 node_legacy_rpc_flags: false
 node_rpc_port: 9944
 node_binary: https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-{{ node_binary_version }}/polkadot

diff --git a/roles/node/molecule/default/molecule.yml b/roles/node/molecule/default/molecule.yml
@@ -5,9 +5,6 @@ driver:
   name: ${DRIVER:-docker}
 platforms:
   - name: molecule-instance-node
-    # LXD
-    source:
-      alias: debian/bullseye/amd64
     # DOCKER
     image: paritytech/debian11:latest
     command: ${MOLECULE_DOCKER_COMMAND:-""}

diff --git a/roles/node/molecule/parachain/molecule.yml b/roles/node/molecule/parachain/molecule.yml
@@ -5,9 +5,6 @@ driver:
   name: ${DRIVER:-docker}
 platforms:
   - name: molecule-instance-node-parachain
-    # LXD
-    source:
-      alias: debian/bullseye/amd64
     # DOCKER
     image: paritytech/debian11:latest
     command: ${MOLECULE_DOCKER_COMMAND:-""}

diff --git a/roles/node/molecule/parachain_remote_rc/molecule.yml b/roles/node/molecule/parachain_remote_rc/molecule.yml
@@ -5,8 +5,6 @@ driver:
   name: ${DRIVER:-docker}
 platforms:
   - name: molecule-instance-node-parachain-remote-rc
-    source:
-      alias: debian/bullseye/amd64
     # DOCKER
     image: paritytech/debian11:latest
     command: ${MOLECULE_DOCKER_COMMAND:-""}

diff --git a/roles/node_backup/molecule/default/README.md b/roles/node_backup/molecule/default/README.md
@@ -15,11 +15,4 @@ molecule converge
 molecule destroy
 ```
 
-#### LXD 
-Test role with LXD driver 
-```shell
-DRIVER=lxd molecule create
-DRIVER=lxd molecule converge
-DRIVER=lxd molecule destroy
-```
 
diff --git a/roles/node_backup/molecule/default/group_vars/all.yml b/roles/node_backup/molecule/default/group_vars/all.yml
@@ -2,34 +2,35 @@
 ## Molecule
 ansible_user: root
 
-## prepare.yml
-#node_legacy_rpc_flags: false
-node_binary: https://github.com/paritytech/polkadot/releases/download/v0.9.42/polkadot
-node_chain: rococo-local
-node_data_root_path: /opt/{{ node_app_name }}
+## node
+node_role: "validator"
+node_user: polkadot
+node_chain: westend-local
+node_chain_backup_chain_path: "westend_local_testnet"
 node_chain_backup_restoring_type: none
-node_pruning: 256
-# node_binary_deployment: false
+node_parachain_chain_backup_restoring_type: none
+node_p2p_bind_addr: "127.0.0.1"
+node_ansible_annotation_path: /tmp/substrate.prom
+node_binary_version: v1.12.0
+node_binary: "https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-{{ node_binary_version }}/polkadot"
+node_prepare_worker_binary: "https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-{{ node_binary_version }}/polkadot-prepare-worker"
+node_execute_worker_binary: "https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-{{ node_binary_version }}/polkadot-execute-worker"
+
 
 # node_backup
 _gcp_bucket: test-blockstore-backups
-node_backup_user: parity
+node_backup_user: polkadot
 node_backup_r2_access_key_id: abc
 node_backup_r2_secret_access_key: cba
 node_backup_r2_api_url: https://a.b
 node_backup_targets:
-  - service_name: rococo-alice-rocksdb-prune
-    local_path: /opt/rococo-alice-rocksdb-prune/chains/rococo_local_testnet/db
+  - service_name: alice-rocksdb-prune
+    local_path: /opt/alice-rocksdb-prune/chains/{{ node_chain_backup_chain_path }}/db
     rpc_port: 9933
     bucket_name: "{{ _gcp_bucket }}"
-    type: gcp-native
-  - service_name: rococo-bob-paritydb-prune
-    local_path: /opt/rococo-bob-paritydb-prune/chains/rococo_local_testnet/paritydb
-    rpc_port: 9934
-    bucket_name: "{{ _gcp_bucket }}"
     type: gcp-rclone
-  - service_name: rococo-bob-paritydb-prune
-    local_path: /opt/rococo-bob-paritydb-prune/chains/rococo_local_testnet/paritydb
+  - service_name: bob-paritydb-prune
+    local_path: /opt/bob-paritydb-prune/chains/{{ node_chain_backup_chain_path }}/paritydb
     rpc_port: 9934
     bucket_name: "{{ _gcp_bucket }}"
     type: r2-rclone

diff --git a/roles/node_backup/molecule/default/molecule.yml b/roles/node_backup/molecule/default/molecule.yml
@@ -5,9 +5,6 @@ driver:
   name: ${DRIVER:-docker}
 platforms:
   - name: molecule-instance-node-backup
-    # LXD
-    source:
-      alias: debian/bullseye/amd64
     # DOCKER
     image: paritytech/debian11:latest
     command: ${MOLECULE_DOCKER_COMMAND:-""}

diff --git a/roles/node_backup/molecule/default/prepare.yml b/roles/node_backup/molecule/default/prepare.yml
@@ -3,42 +3,53 @@
   hosts: all
   gather_facts: false
   pre_tasks:
-    - name: Install Python3
+    - name: Prepare | Install Python3
       ansible.builtin.raw: apt -y update && apt install -y python3
       changed_when: false
+    - name: Prepare | Install required packages
+      ansible.builtin.apt:
+        name:
+          - gpg
+          - cron
+        update_cache: false
+      changed_when: false
     - name: Prepare | create user parity
       ansible.builtin.user:
-        name: parity
+        name: polkadot
   tasks:
-    - name: rococo-alice local
+    - name: Include node alice
+      # use include role to skip ansible-lint
       ansible.builtin.include_role:
-        name: parity.chain.node
+        name: node
       vars:
-        node_rpc_port: 9933
-        node_app_name: rococo-alice-rocksdb-prune
+        node_app_name: "alice-rocksdb-prune"
+        node_handler_id: "{{ node_app_name }}"
+        node_data_root_path: "/opt/{{ node_app_name }}"
+        # 12D3KooWHhB5LqXji1moEvbzCEq7HzJuMvqi5E3BkoceUcQFPX2f
+        node_p2p_private_key: "4f1ae54a051e08161456b74a70b85e45e161fd4f614637f50a2a5f09ba7afb2e"
         node_custom_options:
-          - --alice
-    - name: rococo-bob local
+          - "--alice"
+          - "--bootnodes /ip4/127.0.0.1/tcp/30334/p2p/12D3KooWKvNLq5fFMcQvdZHejhUvRQcSxDpbWFBo1mXh4kGR949r"
+          - "--no-hardware-benchmarks"
+          - "--rpc-cors '*'"
+        node_p2p_port: "30333"
+        node_prometheus_port: "9615"
+        node_rpc_port: "9933"
+    - name: Include node bob
       ansible.builtin.include_role:
-        name: parity.chain.node
+        name: node
       vars:
-        node_rpc_port: 9934
-        node_paritydb_enable: true
-        node_app_name: rococo-bob-paritydb-prune
+        node_app_name: "bob-paritydb-prune"
+        node_handler_id: "{{ node_app_name }}"
+        node_data_root_path: /opt/{{ node_app_name }}
+        # 12D3KooWKvNLq5fFMcQvdZHejhUvRQcSxDpbWFBo1mXh4kGR949r
+        node_p2p_private_key: "042cd72c647f27c6da663f15665b59f707bce7de4b771b098361ff756cde168e"
         node_custom_options:
-          - --bob
-    - name: Pretend we are in gcp | Install cron, gnupg
-      ansible.builtin.package:
-        name:
-          - cron
-          - gnupg
-        state: present
-        update_cache: true
-    - name: Pretend we are in gcp | Add an Apt signing key
-      ansible.builtin.apt_key:
-        url: https://packages.cloud.google.com/apt/doc/apt-key.gpg
-        state: present
-    - name: Pretend we are in gcp | Add apt repository into sources list
-      ansible.builtin.apt_repository:
-        repo: deb https://packages.cloud.google.com/apt cloud-sdk main
-        state: present
+          - "--bob"
+          - "--bootnodes /ip4/127.0.0.1/tcp/30333/p2p/12D3KooWHhB5LqXji1moEvbzCEq7HzJuMvqi5E3BkoceUcQFPX2f"
+          - "--no-hardware-benchmarks"
+          - "--rpc-cors '*'"
+        node_p2p_port: "30334"
+        node_prometheus_port: "9616"
+        node_rpc_port: "9934"
+        node_paritydb_enable: true
diff --git a/roles/node_backup/molecule/default/verify.yml b/roles/node_backup/molecule/default/verify.yml
@@ -22,26 +22,10 @@
     - name: Print current block
       ansible.builtin.debug:
         var: _node_backup_register_header.json.result.number |  int(base=16)
-#   # todo add tests
-#
-## a) upload to gcp
-# GCP storage emulator is not available yet (https://github.com/googleapis/google-cloud-python/issues/10300),
-# there are third party emulator, but support of gsutils is broken (https://github.com/oittaa/gcp-storage-emulator/issues/186)
-# when emulator will be available:
-#   1. run and configure emulator
-#   2. run script:
-#  - name: run backup script
-#    ansible.builtin.command: /home/parity/bin/node_backup.sh
-#   3.
-#  - name: "rococo-bob local"
-#    ansible.builtin.include_role:
-#      name: parity.chain.node
-#    vars:
-#      node_rpc_port: 9935
-#      node_paritydb_enable: true
-#      node_app_name: "rococo-local-rpc"
-#
-## b) Test backup-exporter:
-# We can push fake data to backup-exporter (like run bash script).
+
+# TODO - add tests
+# Test backup-exporter:
+# 1. We can push fake data to backup-exporter (like run bash script).
 # Then we can check the Prometheus endpoint to check and match the results.
 # This will allow checking the code of the exporter.
+# 2. We can upload data to local MinIO
diff --git a/roles/ws_health_exporter/molecule/default/collections.yml b/roles/ws_health_exporter/molecule/default/collections.yml
@@ -0,0 +1,4 @@
+collections:
+  - name: https://github.com/paritytech/ansible-galaxy.git
+    type: git
+    version: main
diff --git a/roles/ws_health_exporter/molecule/default/collections.yml.bak b/roles/ws_health_exporter/molecule/default/collections.yml.bak
diff --git a/roles/ws_health_exporter/molecule/default/group_vars/all.yml b/roles/ws_health_exporter/molecule/default/group_vars/all.yml
@@ -3,13 +3,18 @@
 ansible_user: root
 
 ## node
-node_binary_version: v0.9.42
+node_role: "validator"
 node_user: polkadot
-node_binary: https://github.com/paritytech/polkadot/releases/download/{{ node_binary_version }}/polkadot
-node_chain: rococo-local
+node_chain: westend-local
+node_chain_backup_chain_path: "westend_local_testnet"
 node_chain_backup_restoring_type: none
 node_parachain_chain_backup_restoring_type: none
+node_p2p_bind_addr: "127.0.0.1"
 node_ansible_annotation_path: /tmp/substrate.prom
+node_binary_version: v1.12.0
+node_binary: "https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-{{ node_binary_version }}/polkadot"
+node_prepare_worker_binary: "https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-{{ node_binary_version }}/polkadot-prepare-worker"
+node_execute_worker_binary: "https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-{{ node_binary_version }}/polkadot-execute-worker"
 
 ## ws_health_exporter
 ws_health_exporter_user: "{{ node_user }}"
@@ -19,5 +24,5 @@ ws_health_exporter_node_max_unsynchronized_block_drift: 2
 ws_health_exporter_node_min_peers: 1
 
 ws_health_exporter_ws_urls:
-  - ws://127.0.0.1:9944
-  - ws://127.0.0.1:9945
+  - ws://127.0.0.1:9933
+  - ws://127.0.0.1:9934