From a2c729e2a8c720c28cb1ca4cf690037bed86858d Mon Sep 17 00:00:00 2001 From: kogeler <25884155+kogeler@users.noreply.github.com> Date: Thu, 23 Nov 2023 11:29:08 +0200 Subject: [PATCH] GCP backups were removed from the node role (#63) * gcp backups were removed from the node role Signed-off-by: kogeler --- README.md | 6 +- galaxy.yml | 2 +- roles/node/defaults/main.yml | 18 +--- roles/node/tasks/100-tests.yml | 22 +---- roles/node/tasks/800-restore-chain.yml | 7 -- roles/node/tasks/802-restore-chain-gcp.yml | 101 --------------------- roles/node/vars/main.yml | 10 -- 7 files changed, 11 insertions(+), 155 deletions(-) delete mode 100644 roles/node/tasks/802-restore-chain-gcp.yml diff --git a/README.md b/README.md index 8f58d5f..278ecf4 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Create `requirements.yml` file in your playbook repository (or add to the existi collections: - name: https://github.com/paritytech/ansible-galaxy.git type: git - version: 1.6.2 + version: 1.8.0 ``` or @@ -15,7 +15,7 @@ or ```yaml collections: - name: paritytech.chain - version: 1.6.2 + version: 1.8.0 ``` If you want to install collections in the project space, you have to run: @@ -38,3 +38,5 @@ ansible-galaxy collection install -f -r requirements.yml * secure_apt - [README](./roles/secure_apt/README.md) * state_exporter - [README](./roles/state_exporter/README.md) * ws_health_exporter - [README](./roles/ws_health_exporter/README.md) +* nginx - [README](./roles/nginx/README.md) +* nginx_exporter - [README](./roles/nginx_exporter/README.md) diff --git a/galaxy.yml b/galaxy.yml index 9dd2553..f271479 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -8,7 +8,7 @@ namespace: paritytech name: chain # The version of the collection. Must be compatible with semantic versioning -version: 1.7.0 +version: 1.8.0 # The path to the Markdown (.md) readme file. This path is relative to the root of the collection readme: README.md diff --git a/roles/node/defaults/main.yml b/roles/node/defaults/main.yml index 4cfda32..3db6a39 100644 --- a/roles/node/defaults/main.yml +++ b/roles/node/defaults/main.yml @@ -110,7 +110,7 @@ node_custom_options: [] node_enable_detailed_log_output: true ### Backup restoring -# Can be "http", "gcp", "tar" or "none". If it's "none", the role will not restore the blockstore of relaychain. +# Can be "http", "tar" or "none". If it's "none", the role will not restore the blockstore of relaychain. # It will also not perform a restore if the db folder exists and is not empty node_chain_backup_restoring_type: none # Name of a folder that is used by different networks to store chain. @@ -135,14 +135,6 @@ node_chain_backup_http_install_rclone: false # A tar archive must contain database on the root level node_chain_backup_url: "" -## GCP backups -# You can find the structure of a bucket in README.md -node_chain_backup_gcp_bucket: "" -# full link to a GCP backup, you have to specify a full path including a bucket name. -node_chain_backup_gcp_path: "" -# If you don't have pre-installed google SDK the role can install it -node_chain_backup_gcp_install_sdk: false - ### Loging and telemetry node_telemetry_enable: true # If you set an empty value, it will use the default telemetry server @@ -219,7 +211,7 @@ node_parachain_custom_options: [] # - "--rpc-methods Unsafe" ### Backup restoring -# Can be "http", "gcp", "tar" or "none". If it's "none", the role will not restore the blockstore of parachain. +# Can be "http", "tar" or "none". If it's "none", the role will not restore the blockstore of parachain. # It will also not perform a restore if the db folder exists and is not empty node_parachain_chain_backup_restoring_type: none # Name of a folder that is used by different parachain networks to store chain. @@ -238,12 +230,6 @@ node_parachain_chain_backup_http_url: "" # A tar archive must contain database on the root level node_parachain_chain_backup_url: "" -## GCP backups -# You can find the structure of a bucket in README.md -node_parachain_chain_backup_gcp_bucket: "" -# Link to a gcp backup, you have to specify a full path including a bucket name. -node_parachain_chain_backup_gcp_path: "" - ### Loging and telemetry node_parachain_telemetry_enable: true # If you set an empty value, it will use the default telemetry server diff --git a/roles/node/tasks/100-tests.yml b/roles/node/tasks/100-tests.yml index 3487e46..7413f81 100644 --- a/roles/node/tasks/100-tests.yml +++ b/roles/node/tasks/100-tests.yml @@ -52,27 +52,13 @@ - name: Test | Check node_chain_backup_restoring_type ansible.builtin.fail: - msg: "The 'node_chain_backup_restoring_type' variable can contain only 'gcp', 'tar' or 'none' values!" - when: node_chain_backup_restoring_type not in ["http", "gcp", "tar", "none"] + msg: "The 'node_chain_backup_restoring_type' variable can contain only 'http', 'tar' or 'none' values!" + when: node_chain_backup_restoring_type not in ["http", "tar", "none"] - name: Test | Check node_parachain_chain_backup_restoring_type ansible.builtin.fail: - msg: "The 'node_parachain_chain_backup_restoring_type' variable can contain only 'gcp', 'tar' or 'none' values!" - when: node_parachain_chain_backup_restoring_type not in ["http", "gcp", "tar", "none"] - -- name: Test | Check gcp backup - ansible.builtin.fail: - msg: "If you use GCP backups for the node that uses chain: {{ node_chain }}, - pruning state: {{ (node_pruning > 0) }}, - you have to set the node_chain_backup_gcp_bucket variable or the node_chain_backup_gcp_path" - when: node_chain_backup_restoring_type == 'gcp' and node_chain_backup_gcp_bucket == '' and node_chain_backup_gcp_path == '' - -- name: Test | Check parachain gcp backup - ansible.builtin.fail: - msg: "If you use GCP backups for the node that uses parachain: {{ node_parachain_chain }}, - pruning state: {{ (node_parachain_pruning > 0) }}, - you have to set the node_parachain_chain_backup_gcp_bucket variable or the node_parachain_chain_backup_gcp_path variable" - when: node_parachain_role != '' and node_parachain_chain_backup_restoring_type == 'gcp' and node_parachain_chain_backup_gcp_bucket == '' and node_parachain_chain_backup_gcp_path == '' + msg: "The 'node_parachain_chain_backup_restoring_type' variable can contain only 'http', 'tar' or 'none' values!" + when: node_parachain_chain_backup_restoring_type not in ["http", "tar", "none"] - name: Test | Check node_chain_backup_url ansible.builtin.fail: diff --git a/roles/node/tasks/800-restore-chain.yml b/roles/node/tasks/800-restore-chain.yml index 750d459..4aad4d8 100644 --- a/roles/node/tasks/800-restore-chain.yml +++ b/roles/node/tasks/800-restore-chain.yml @@ -81,13 +81,6 @@ tags: [ 'node', 'node-restore-chain' ] when: item.restoring_type == 'tar' - - name: Restore {{ item.part }} | GCP restoring - ansible.builtin.include_tasks: - file: 802-restore-chain-gcp.yml - apply: - tags: [ 'node', 'node-restore-chain' ] - when: item.restoring_type == 'gcp' - - name: Restore {{ item.part }} | HTTP restoring ansible.builtin.include_tasks: file: 803-restore-chain-http.yml diff --git a/roles/node/tasks/802-restore-chain-gcp.yml b/roles/node/tasks/802-restore-chain-gcp.yml deleted file mode 100644 index 5c1c4a5..0000000 --- a/roles/node/tasks/802-restore-chain-gcp.yml +++ /dev/null @@ -1,101 +0,0 @@ ---- - -- block: - - - name: Restore {{ item.part }} | GCP restoring | Install packages - ansible.builtin.package: - name: gpg - update_cache: yes - - - name: Restore {{ item.part }} | GCP restoring | Add Google cloud apt GPG key - ansible.builtin.apt_key: - url: https://packages.cloud.google.com/apt/doc/apt-key.gpg - - - name: Restore {{ item.part }} | GCP restoring | Add gce-sdk repository - ansible.builtin.apt_repository: - repo: deb http://packages.cloud.google.com/apt cloud-sdk-{{ ansible_distribution_release }} main - filename: /etc/apt/sources.list.d/gce_sdk.list - update_cache: yes - - - name: Restore {{ item.part }} | GCP restoring | Install packages - ansible.builtin.package: - name: google-cloud-sdk - update_cache: no - - when: node_chain_backup_gcp_install_sdk | bool - -- name: Restore {{ item.part }} | GCP restoring | Check last version - ansible.builtin.command: gsutil cat {{ item.gcp_url }}/latest_version.meta.txt - register: _node_last_backup_version - check_mode: false - changed_when: false - when: item.custom_gcp_url == '' - -- name: Restore {{ item.part }} | GCP restoring | Setup _node_chain_backup_gcp_full_path - ansible.builtin.set_fact: - _node_chain_backup_gcp_full_path: "{% if item.custom_gcp_url == '' %} - {{ item.gcp_url }}/{{ _node_last_backup_version.stdout }} - {% else %}{{ item.custom_gcp_url }}{% endif %}" - -- name: Restore {{ item.part }} | GCP restoring | Check the size of the backup - ansible.builtin.shell: gsutil du -s {{ _node_chain_backup_gcp_full_path | trim | quote }} | awk '{print $1}' - check_mode: false - changed_when: false - register: _node_restore_backup_size - when: _node_run_check_size - -- name: Restore {{ item.part }} | GCP restoring | Print backup size - ansible.builtin.debug: - msg: Backup size = {{ _node_restore_backup_size.stdout | int | filesizeformat(true) }} - when: _node_run_check_size - -- name: Restore {{ item.part }} | GCP restoring | Fail if free space <500MB - ansible.builtin.fail: - msg: | - Not enough free space to perform the restore, you should set the node_data_root_path variable - to a path on a different drive with enough free space - when: - - _node_run_check_size - - _node_restore_free_space | int - _node_restore_backup_size.stdout | int < 500 * 1024 * 1024 - # Skipped during check mode because it doesn't actually delete the db, making the result of this task wrong - - not ansible_check_mode - -- name: Restore {{ item.part }} | GCP restoring | Print backup path - ansible.builtin.debug: - msg: "{{ _node_chain_backup_gcp_full_path }}" - -- name: Restore {{ item.part }} | GCP restoring | Stop service - ansible.builtin.systemd: - name: "{{ node_app_name }}" - state: stopped - notify: restart service {{ node_handler_id }} - ignore_errors: "{{ not _node_systemd_unit_file_stat.stat.exists }}" - when: not _node_use_tmp_restore_path - -- name: Restore {{ item.part }} | GCP restoring | Download chain backup - ansible.builtin.command: | - gsutil -m -o 'GSUtil:parallel_process_count={{ ansible_processor_vcpus }}' -o 'GSUtil:parallel_thread_count=12' - rsync -d -r {{ _node_chain_backup_gcp_full_path }} {{ _node_backup_dl_path | quote }} - changed_when: true - notify: restart service {{ node_handler_id }} - -- name: Restore {{ item.part }} | GCP restoring | Manage node_chain_backup_tmp_restore_path - block: - - - name: Restore {{ item.part }} | GCP restoring | Stop service and cleanup DB - ansible.builtin.include_tasks: includes/_delete_db_folder.yml - - - name: Restore {{ item.part }} | GCP restoring | Copy backup from temporary folder - ansible.builtin.copy: - src: "{{ node_chain_backup_tmp_restore_path }}/" - dest: "{{ item.chain_path }}/{{ item.db_folder }}" - owner: "{{ node_user }}" - group: "{{ node_user }}" - remote_src: true - - - name: Restore {{ item.part }} | GCP restoring | Delete temporary folder - ansible.builtin.file: - path: "{{ node_chain_backup_tmp_restore_path }}" - state: absent - - when: _node_use_tmp_restore_path diff --git a/roles/node/vars/main.yml b/roles/node/vars/main.yml index 2bd8691..e832993 100644 --- a/roles/node/vars/main.yml +++ b/roles/node/vars/main.yml @@ -73,7 +73,6 @@ _node_chainspec_file: "{{ _node_user_home_path }}/chainspecs/{{ node_app_name }} _node_restore_relaychain: "{{ (node_chain_backup_restoring_type == 'http' and ( node_chain_backup_http_base_url != '' or node_chain_backup_http_url != '')) or - (node_chain_backup_restoring_type == 'gcp' and ( node_chain_backup_gcp_bucket != '' or node_chain_backup_gcp_path != '')) or (node_chain_backup_restoring_type == 'tar' and node_chain_backup_url != '') }}" _node_chain_backup_data: @@ -82,10 +81,6 @@ _node_chain_backup_data: chain_path: "{{ _node_data_chain_path }}" db_folder: "{{ 'paritydb' if node_paritydb_enable else 'db' }}" tar_url: "{{ node_chain_backup_url }}" - gcp_url: "{{ node_chain_backup_gcp_bucket + - node_chain + - ('-paritydb' if node_paritydb_enable else '-rocksdb') + ('-prune' if node_pruning > 0 else '-archive') }}" - custom_gcp_url: "{{ node_chain_backup_gcp_path }}" http_url: "{{ node_chain_backup_http_base_url + '/' + node_chain + ('-paritydb' if node_paritydb_enable else '-rocksdb') + ('-prune' if node_pruning > 0 else '-archive') }}" @@ -103,7 +98,6 @@ _node_parachain_chainspec_file: "{{ _node_user_home_path }}/chainspecs/{{ node_a _node_restore_parachain: "{{ node_parachain_role != '' and ((node_parachain_chain_backup_restoring_type == 'http' and (node_parachain_chain_backup_http_base_url != '' or node_parachain_chain_backup_http_url != '' )) or - (node_parachain_chain_backup_restoring_type == 'gcp' and (node_parachain_chain_backup_gcp_bucket != '' or node_parachain_chain_backup_gcp_path != '' )) or (node_parachain_chain_backup_restoring_type == 'tar' and node_parachain_chain_backup_url != '')) }}" _node_parachain_chain_backup_data: @@ -112,10 +106,6 @@ _node_parachain_chain_backup_data: chain_path: "{{ _node_parachain_data_chain_path }}" db_folder: "{{ 'paritydb' if node_parachain_paritydb_enable else 'db' }}" tar_url: "{{ node_parachain_chain_backup_url }}" - gcp_url: "{{ node_parachain_chain_backup_gcp_bucket + - node_parachain_chain + - ('-paritydb' if node_parachain_paritydb_enable else '-rocksdb') + ('-prune' if node_parachain_pruning > 0 else '-archive') }}" - custom_gcp_url: "{{ node_parachain_chain_backup_gcp_path }}" http_url: "{{ node_parachain_chain_backup_http_base_url + '/' + node_parachain_chain + ('-paritydb' if node_parachain_paritydb_enable else '-rocksdb') + ('-prune' if node_parachain_pruning > 0 else '-archive') }}"