feat: updated log-es role to support multiple clouds

ansible/es.yml

@@ -25,7 +25,12 @@
     - es_snapshot_host: "{{log_es_snapshot_host}}"
     - snapshot_base_path: "{{log_snapshot_base_path}}"
   roles:
-    - es-azure-snapshot
+    - role: es-azure-snapshot
+      when: cloud_service_provider == "azure"
+    - role: es-s3-snapshot
+      when: cloud_service_provider == "aws"
+    - role: es-gcs-snapshot
+      when: cloud_service_provider == "gcloud"
     - es5-snapshot-purge
   tags:
     - log_es_backup

ansible/provision-log-es.yml

@@ -1,13 +1,10 @@
 - hosts: log-es
   become: yes
   vars:
-    es_version: "6.8.10"
+    es_version: "6.8.22"
     es_etc_discovery_zen_ping_unicast_hosts: "{{ groups['log-es'] }}"
     es_etc_cluster_name: "{{ node_name }}"
     es_heap_size: "{{ log_es_heap_size | default('1g') }}"
-    es_plugins_reinstall: true
-    es_plugins:
-      - plugin: "repository-azure"
     es_config:
       cluster.name: "{{ log_es_etc_cluster_name }}"
       discovery.zen.ping.unicast.hosts: "{{ groups['log-es'] }}"

ansible/roles/log-es6/defaults/main.yml

@@ -15,7 +15,6 @@ es_start_service: true
 es_java_install: "{{ false if (es_version is version('7.0.0', '>=')) else true }}"
 update_java: false
 es_restart_on_change: true
-es_plugins_reinstall: false
 es_templates: false
 es_user: elasticsearch
 es_group: elasticsearch
@@ -69,3 +68,8 @@ es_ssl_verification_mode: "certificate"
 es_validate_certs: "yes"
 es_delete_unmanaged_file: true
 es_delete_unmanaged_native: true
+es_plugins_reinstall: true
+es_plugins:
+  - plugin: "repository-azure"
+  - plugin: "repository-s3"
+  - plugin: "repository-gcs"

ansible/roles/log-es6/tasks/main.yml

@@ -59,6 +59,21 @@
   tags:
       - xpack
 
+- name: include plugins/create-keystore.yml
+  include: plugins/create-keystore.yml
+
+- name: include plugins/repository-azure.yml
+  include: plugins/repository-azure.yml
+  when: cloud_service_provider == "azure"
+
+- name: include plugins/repository-s3.yml
+  include: plugins/repository-s3.yml
+  when: cloud_service_provider == "aws"
+
+- name: include plugins/repository-gcs.yml
+  include: plugins/repository-gcs.yml
+  when: cloud_service_provider == "gcloud"
+
 - name: include elasticsearch-ssl.yml
   include: elasticsearch-ssl.yml
   when: es_enable_http_ssl or es_enable_transport_ssl

ansible/roles/log-es6/tasks/plugins/create-keystore.yml

@@ -0,0 +1,13 @@
+---
+- name: Check if elasticsearch keystore exists or not
+  become: yes
+  stat:
+    path: "{{ conf_dir }}/elasticsearch.keystore"
+  register: elasticsearch_keystore_file
+
+- name: Create the elasticsearch keystore if not exists
+  become: yes
+  command: "{{es_home}}/bin/elasticsearch-keystore create"
+  environment:
+    ES_PATH_CONF: "{{ conf_dir }}"
+  when: elasticsearch_keystore_file.stat.exists == false

ansible/roles/log-es6/tasks/plugins/repository-azure.yml

@@ -0,0 +1,15 @@
+---
+- name: Add default azure account name for backups
+  become: yes
+  shell: echo "{{ azure_management_storage_account_name }}" | {{ es_home }}/bin/elasticsearch-keystore add -f azure.client.default.account
+  no_log: True
+  environment:
+    ES_PATH_CONF: "{{ conf_dir }}"
+
+
+- name: Add default azure account key for backups
+  become: yes
+  shell: echo "{{ azure_management_storage_account_key }}" | {{ es_home }}/bin/elasticsearch-keystore add -f azure.client.default.key
+  no_log: True
+  environment:
+    ES_PATH_CONF: "{{ conf_dir }}"

ansible/roles/log-es6/tasks/plugins/repository-gcs.yml

@@ -0,0 +1,18 @@
+---
+- name: Create the gcs service account file from variable
+  become: yes
+  copy:
+    dest: "{{ conf_dir }}/gcs_management_bucket_service_account.json"
+    content: "{{ gcs_management_bucket_service_account }}"
+
+- name: Add gcs service account file to keystore
+  become: yes
+  shell: "{{ es_home }}/bin/elasticsearch-keystore add-file -f gcs.client.default.credentials_file {{ conf_dir }}/gcs_management_bucket_service_account.json"
+  no_log: True
+  environment:
+    ES_PATH_CONF: "{{ conf_dir }}"
+
+- name: Remove the service account file
+  file:
+    path: "{{ conf_dir }}/gcs_management_bucket_service_account.json"
+    state: absent

ansible/roles/log-es6/tasks/plugins/repository-s3.yml

@@ -0,0 +1,14 @@
+---
+- name: Add default aws account name for backups
+  become: yes
+  shell: echo "{{ aws_management_bucket_user_access_key }}" | {{ es_home }}/bin/elasticsearch-keystore add -f s3.client.default.access_key
+  no_log: True
+  environment:
+    ES_PATH_CONF: "{{ conf_dir }}"
+
+- name: Add default aws account key for backups
+  become: yes
+  shell: echo "{{ aws_management_bucket_user_secret_key }}" | {{ es_home }}/bin/elasticsearch-keystore add -f s3.client.default.secret_key
+  no_log: True
+  environment:
+    ES_PATH_CONF: "{{ conf_dir }}"