feat: adding gcp es snapshot module

ansible/es.yml

@@ -7,11 +7,11 @@
     - snapshot_base_path: "{{app_snapshot_base_path}}"
   roles:
     - role: es-azure-snapshot
-      when: cloud_provider == "azure"
-    - role: es-aws-snapshot
-      when: cloud_provider == "aws"
-#    - role: es-gcloud-snapshot
-#      when: cloud_provider == "gcloud"
+      when: cloud_service_provider == "azure"
+    - role: es-s3-snapshot
+      when: cloud_service_provider == "aws"
+    - role: es-gcs-snapshot
+      when: cloud_service_provider == "gcloud"
     - role: es5-snapshot-purge
   tags:
     - es_backup

ansible/roles/es-azure-snapshot/tasks/main.yml

@@ -17,7 +17,7 @@
         AZURE_SECRET: "{{ azure_management_client_secret }}"
   delegate_to: localhost
 
-- name: Create azure snapshot
+- name: Create Azure Repository
   uri:
     url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}"
     method: PUT

ansible/roles/es-gcs-snapshot/defaults/main.yml

@@ -0,0 +1,12 @@
+snapshot_create_request_body: {
+    type: gcs,
+    settings: {
+      bucket: "{{ gcs_management_bucket_name }}",
+      base_path: "{{ es_gcs_backup_folder_name }}/{{ snapshot_base_path }}_{{ base_path_date }}"
+    }
+}
+
+# Override these values
+es_snapshot_host: "localhost"
+snapshot_base_path: "default"
+es_gcs_backup_folder_name: "elasticsearch-snapshots"

ansible/roles/es-aws-snapshot/tasks/main.yml → ansible/roles/es-gcs-snapshot/tasks/main.yml

@@ -4,20 +4,7 @@
 
 - set_fact: snapshot_number="snapshot_{{ lookup('pipe','date +%s') }}"
 
-# - name: Ensure backup folder exists in aws bucket
-#   azure_rm_storageblob:
-#     resource_group: "{{ azure_management_resource_group_name }}"
-#     storage_account_name: "{{ azure_management_storage_account_name }}"
-#     container: "{{ es_azure_backup_container_name }}"
-#     state: present
-#   environment:
-#         AZURE_TENANT: "{{ azure_management_tenant_id }}"
-#         AZURE_SUBSCRIPTION_ID: "{{ azure_management_subscription_id }}"
-#         AZURE_CLIENT_ID: "{{ azure_management_client_id }}"
-#         AZURE_SECRET: "{{ azure_management_client_secret }}"
-#   delegate_to: localhost
-
-- name: Create aws snapshot
+- name: Create GCS Repository
   uri:
     url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}"
     method: PUT

ansible/roles/es-aws-snapshot/defaults/main.yml → ansible/roles/es-s3-snapshot/defaults/main.yml

@@ -9,5 +9,4 @@ snapshot_create_request_body: {
 # Override these values
 es_snapshot_host: "localhost"
 snapshot_base_path: "default"
-
 es_s3_backup_folder_name: "elasticsearch-snapshots"

ansible/roles/es-s3-snapshot/tasks/main.yml

@@ -0,0 +1,42 @@
+---
+
+- set_fact: base_path_date="{{ lookup('pipe','date +%Y-%m') }}"
+
+- set_fact: snapshot_number="snapshot_{{ lookup('pipe','date +%s') }}"
+
+- name: Create S3 Repository
+  uri:
+    url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}"
+    method: PUT
+    body: "{{ snapshot_create_request_body | to_json }}"
+    headers:
+      Content-Type: "application/json"
+
+- name: Take new snapshot
+  uri:
+    url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}"
+    method: PUT
+    headers:
+      Content-Type: "application/json"
+
+- name: Print all snapshots
+  uri:
+    url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/_all"
+    method: GET
+
+- name: Print status of current snapshot
+  uri:
+    url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}"
+    method: GET
+
+- name: "Wait for backup to be completed"
+  uri:
+    url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}"
+    method: GET
+    return_content: yes
+    status_code: 200
+    body_format: json
+  register: result
+  until: result.json.snapshots[0].state == 'SUCCESS'
+  retries: 120
+  delay: 10

ansible/roles/es6/tasks/main.yml

@@ -42,15 +42,15 @@
 
 - name: include plugins/repository-azure.yml
   include: plugins/repository-azure.yml
-  when: cloud_provider == "azure"
+  when: cloud_service_provider == "azure"
 
 - name: include plugins/repository-s3.yml
   include: plugins/repository-s3.yml
-  when: cloud_provider == "aws"
+  when: cloud_service_provider == "aws"
 
 - name: include plugins/repository-gcs.yml
   include: plugins/repository-gcs.yml
-  when: cloud_provider == "gcloud"
+  when: cloud_service_provider == "gcloud"
 
 - name: flush handlers
   meta: flush_handlers

ansible/roles/es6/tasks/plugins/repository-gcs.yml

@@ -1,5 +1,18 @@
 ---
-- name: Add gcs credentials file to keystore
+- name: Create the gcs service account file from variable
   become: yes
-  shell: echo "{{ gcs_management_bucket_service_account }}" | {{ es_home }}/bin/elasticsearch-keystore add-file -f gcs.client.default.credentials_file google_cloud_store_management_service_account.json
-  no_log: True
+  copy:
+    dest: "{{ conf_dir }}/gcs_management_bucket_service_account.json"
+    content: "{{ gcs_management_bucket_service_account }}"
+
+- name: Add gcs service account file to keystore
+  become: yes
+  shell: "{{ es_home }}/bin/elasticsearch-keystore add-file -f gcs.client.default.credentials_file {{ conf_dir }}/gcs_management_bucket_service_account.json"
+  no_log: True
+  environment:
+    ES_PATH_CONF: "{{ conf_dir }}"
+
+- name: Remove the service account file
+  file:
+    path: "{{ conf_dir }}/gcs_management_bucket_service_account.json"
+    state: absent

private_repo/ansible/inventory/dev/Core/common.yml

@@ -38,6 +38,11 @@ azure_management_storage_account_name: ""
 # Management Storage Bucket
 aws_management_bucket_name: ""
 
+# Define the below if you are using Google Cloud
+# Ansible GCP Module Variables
+# Management Storage Bucket
+gcs_management_bucket_name: ""
+
 # ------------------------------------------------------------------------------------------------------------ #
 # Cloud / Infra Specific values - Check these and update accordingly
 

private_repo/ansible/inventory/dev/Core/secrets.yml

@@ -30,6 +30,12 @@ aws_management_bucket_user_access_key: ""
 aws_management_bucket_user_secret_key: ""
 
 
+# Define the below if you are using Google Cloud
+# Ansible GCP Module Variables
+# Management Storage Bucket
+gcs_management_bucket_service_account: |
+
+
 # The proxy key and crt values should be padded to the right by a couple of spaces 
 # Example:
 # core_vault_proxy_site_key: |+