From cc8bb33c4f25aeb0a2e6e021e619f5374fb20203 Mon Sep 17 00:00:00 2001 From: Keshav Prasad Date: Tue, 6 Sep 2022 14:08:01 +0530 Subject: [PATCH] feat: adding gcp es snapshot module --- ansible/es.yml | 10 ++--- .../roles/es-azure-snapshot/tasks/main.yml | 2 +- .../roles/es-gcs-snapshot/defaults/main.yml | 12 ++++++ .../tasks/main.yml | 15 +------ .../defaults/main.yml | 1 - ansible/roles/es-s3-snapshot/tasks/main.yml | 42 +++++++++++++++++++ ansible/roles/es6/tasks/main.yml | 6 +-- .../es6/tasks/plugins/repository-gcs.yml | 19 +++++++-- .../ansible/inventory/dev/Core/common.yml | 5 +++ .../ansible/inventory/dev/Core/secrets.yml | 6 +++ 10 files changed, 91 insertions(+), 27 deletions(-) create mode 100644 ansible/roles/es-gcs-snapshot/defaults/main.yml rename ansible/roles/{es-aws-snapshot => es-gcs-snapshot}/tasks/main.yml (65%) rename ansible/roles/{es-aws-snapshot => es-s3-snapshot}/defaults/main.yml (99%) create mode 100644 ansible/roles/es-s3-snapshot/tasks/main.yml diff --git a/ansible/es.yml b/ansible/es.yml index 17aef43a63..0cd3ceb02d 100644 --- a/ansible/es.yml +++ b/ansible/es.yml @@ -7,11 +7,11 @@ - snapshot_base_path: "{{app_snapshot_base_path}}" roles: - role: es-azure-snapshot - when: cloud_provider == "azure" - - role: es-aws-snapshot - when: cloud_provider == "aws" -# - role: es-gcloud-snapshot -# when: cloud_provider == "gcloud" + when: cloud_service_provider == "azure" + - role: es-s3-snapshot + when: cloud_service_provider == "aws" + - role: es-gcs-snapshot + when: cloud_service_provider == "gcloud" - role: es5-snapshot-purge tags: - es_backup diff --git a/ansible/roles/es-azure-snapshot/tasks/main.yml b/ansible/roles/es-azure-snapshot/tasks/main.yml index 965cc96eda..def329a545 100644 --- a/ansible/roles/es-azure-snapshot/tasks/main.yml +++ b/ansible/roles/es-azure-snapshot/tasks/main.yml @@ -17,7 +17,7 @@ AZURE_SECRET: "{{ azure_management_client_secret }}" delegate_to: localhost -- name: Create azure snapshot +- name: Create Azure Repository uri: url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}" method: PUT diff --git a/ansible/roles/es-gcs-snapshot/defaults/main.yml b/ansible/roles/es-gcs-snapshot/defaults/main.yml new file mode 100644 index 0000000000..3c2efa9a7f --- /dev/null +++ b/ansible/roles/es-gcs-snapshot/defaults/main.yml @@ -0,0 +1,12 @@ +snapshot_create_request_body: { + type: gcs, + settings: { + bucket: "{{ gcs_management_bucket_name }}", + base_path: "{{ es_gcs_backup_folder_name }}/{{ snapshot_base_path }}_{{ base_path_date }}" + } +} + +# Override these values +es_snapshot_host: "localhost" +snapshot_base_path: "default" +es_gcs_backup_folder_name: "elasticsearch-snapshots" \ No newline at end of file diff --git a/ansible/roles/es-aws-snapshot/tasks/main.yml b/ansible/roles/es-gcs-snapshot/tasks/main.yml similarity index 65% rename from ansible/roles/es-aws-snapshot/tasks/main.yml rename to ansible/roles/es-gcs-snapshot/tasks/main.yml index d428f337a9..55f50b17ad 100644 --- a/ansible/roles/es-aws-snapshot/tasks/main.yml +++ b/ansible/roles/es-gcs-snapshot/tasks/main.yml @@ -4,20 +4,7 @@ - set_fact: snapshot_number="snapshot_{{ lookup('pipe','date +%s') }}" -# - name: Ensure backup folder exists in aws bucket -# azure_rm_storageblob: -# resource_group: "{{ azure_management_resource_group_name }}" -# storage_account_name: "{{ azure_management_storage_account_name }}" -# container: "{{ es_azure_backup_container_name }}" -# state: present -# environment: -# AZURE_TENANT: "{{ azure_management_tenant_id }}" -# AZURE_SUBSCRIPTION_ID: "{{ azure_management_subscription_id }}" -# AZURE_CLIENT_ID: "{{ azure_management_client_id }}" -# AZURE_SECRET: "{{ azure_management_client_secret }}" -# delegate_to: localhost - -- name: Create aws snapshot +- name: Create GCS Repository uri: url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}" method: PUT diff --git a/ansible/roles/es-aws-snapshot/defaults/main.yml b/ansible/roles/es-s3-snapshot/defaults/main.yml similarity index 99% rename from ansible/roles/es-aws-snapshot/defaults/main.yml rename to ansible/roles/es-s3-snapshot/defaults/main.yml index 4d9ff91c84..2ca18929b4 100644 --- a/ansible/roles/es-aws-snapshot/defaults/main.yml +++ b/ansible/roles/es-s3-snapshot/defaults/main.yml @@ -9,5 +9,4 @@ snapshot_create_request_body: { # Override these values es_snapshot_host: "localhost" snapshot_base_path: "default" - es_s3_backup_folder_name: "elasticsearch-snapshots" \ No newline at end of file diff --git a/ansible/roles/es-s3-snapshot/tasks/main.yml b/ansible/roles/es-s3-snapshot/tasks/main.yml new file mode 100644 index 0000000000..aee768626c --- /dev/null +++ b/ansible/roles/es-s3-snapshot/tasks/main.yml @@ -0,0 +1,42 @@ +--- + +- set_fact: base_path_date="{{ lookup('pipe','date +%Y-%m') }}" + +- set_fact: snapshot_number="snapshot_{{ lookup('pipe','date +%s') }}" + +- name: Create S3 Repository + uri: + url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}" + method: PUT + body: "{{ snapshot_create_request_body | to_json }}" + headers: + Content-Type: "application/json" + +- name: Take new snapshot + uri: + url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}" + method: PUT + headers: + Content-Type: "application/json" + +- name: Print all snapshots + uri: + url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/_all" + method: GET + +- name: Print status of current snapshot + uri: + url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}" + method: GET + +- name: "Wait for backup to be completed" + uri: + url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}" + method: GET + return_content: yes + status_code: 200 + body_format: json + register: result + until: result.json.snapshots[0].state == 'SUCCESS' + retries: 120 + delay: 10 diff --git a/ansible/roles/es6/tasks/main.yml b/ansible/roles/es6/tasks/main.yml index 0c1ed0e3a7..1de1eeff8e 100644 --- a/ansible/roles/es6/tasks/main.yml +++ b/ansible/roles/es6/tasks/main.yml @@ -42,15 +42,15 @@ - name: include plugins/repository-azure.yml include: plugins/repository-azure.yml - when: cloud_provider == "azure" + when: cloud_service_provider == "azure" - name: include plugins/repository-s3.yml include: plugins/repository-s3.yml - when: cloud_provider == "aws" + when: cloud_service_provider == "aws" - name: include plugins/repository-gcs.yml include: plugins/repository-gcs.yml - when: cloud_provider == "gcloud" + when: cloud_service_provider == "gcloud" - name: flush handlers meta: flush_handlers diff --git a/ansible/roles/es6/tasks/plugins/repository-gcs.yml b/ansible/roles/es6/tasks/plugins/repository-gcs.yml index 59da8461a3..7d5c32e52e 100644 --- a/ansible/roles/es6/tasks/plugins/repository-gcs.yml +++ b/ansible/roles/es6/tasks/plugins/repository-gcs.yml @@ -1,5 +1,18 @@ --- -- name: Add gcs credentials file to keystore +- name: Create the gcs service account file from variable become: yes - shell: echo "{{ gcs_management_bucket_service_account }}" | {{ es_home }}/bin/elasticsearch-keystore add-file -f gcs.client.default.credentials_file google_cloud_store_management_service_account.json - no_log: True \ No newline at end of file + copy: + dest: "{{ conf_dir }}/gcs_management_bucket_service_account.json" + content: "{{ gcs_management_bucket_service_account }}" + +- name: Add gcs service account file to keystore + become: yes + shell: "{{ es_home }}/bin/elasticsearch-keystore add-file -f gcs.client.default.credentials_file {{ conf_dir }}/gcs_management_bucket_service_account.json" + no_log: True + environment: + ES_PATH_CONF: "{{ conf_dir }}" + +- name: Remove the service account file + file: + path: "{{ conf_dir }}/gcs_management_bucket_service_account.json" + state: absent \ No newline at end of file diff --git a/private_repo/ansible/inventory/dev/Core/common.yml b/private_repo/ansible/inventory/dev/Core/common.yml index 3150b983d1..6264c92555 100644 --- a/private_repo/ansible/inventory/dev/Core/common.yml +++ b/private_repo/ansible/inventory/dev/Core/common.yml @@ -38,6 +38,11 @@ azure_management_storage_account_name: "" # Management Storage Bucket aws_management_bucket_name: "" +# Define the below if you are using Google Cloud +# Ansible GCP Module Variables +# Management Storage Bucket +gcs_management_bucket_name: "" + # ------------------------------------------------------------------------------------------------------------ # # Cloud / Infra Specific values - Check these and update accordingly diff --git a/private_repo/ansible/inventory/dev/Core/secrets.yml b/private_repo/ansible/inventory/dev/Core/secrets.yml index 36a4d5aa40..f951d8ffc9 100644 --- a/private_repo/ansible/inventory/dev/Core/secrets.yml +++ b/private_repo/ansible/inventory/dev/Core/secrets.yml @@ -30,6 +30,12 @@ aws_management_bucket_user_access_key: "" aws_management_bucket_user_secret_key: "" +# Define the below if you are using Google Cloud +# Ansible GCP Module Variables +# Management Storage Bucket +gcs_management_bucket_service_account: | + + # The proxy key and crt values should be padded to the right by a couple of spaces # Example: # core_vault_proxy_site_key: |+