fix: updated roles and plays for azure provision and backup

ansible/es.yml

@@ -10,6 +10,7 @@
     - es5-snapshot-purge
   tags:
     - es_backup
+  run_once: true
 
 - hosts: log-es-backup
   become: yes
@@ -23,12 +24,3 @@
     - es5-snapshot-purge
   tags:
     - log_es_backup
-
-# - hosts: log-es
-#   become: yes
-#   vars_files:
-#     - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
-#   roles:
-#     - es2-data-purge
-#   tags:
-#     - purge_old_logs

ansible/roles/es-azure-snapshot/tasks/main.yml

@@ -11,7 +11,7 @@
       Content-Type: "application/json"
 
 - debug:
-    var: "{{ snapshot_base_path }}"
+    msg: "snapshot_base_path: {{ snapshot_base_path }}"
 
 - set_fact: snapshot_number="snapshot_{{ansible_date_time.epoch}}"
 

ansible/roles/es-curator/defaults/main.yml

@@ -1,6 +1,2 @@
-# es_curator_major_version: 3
-# es_curator_version: 3.4.1-1
-
 es_curator_major_version: 5
-es_curator_version: 5.7.6
-python_es_curator_version: 3.4.1-1
+es_curator_version: 5.8.4

ansible/roles/es-curator/tasks/main.yml

@@ -7,11 +7,6 @@
 - debug:
     msg: "{{ es_curator_version }}"
 
-- name: Install python elasticsearch curator
-  apt:
-    name: python-elasticsearch-curator={{ python_es_curator_version }}
-    force: yes
-
 - name: Install elasticsearch curator
   apt:
     name: elasticsearch-curator={{ es_curator_version }}

ansible/roles/es5-snapshot-purge/meta/main.yml

@@ -1,3 +1,3 @@
 ---
 dependencies:
-  - { role: es-curator, es_curator_major_version: 5, es_curator_version: 5.8.1, python_es_curator_version: 3.4.1-1 }
+  - { role: es-curator, es_curator_major_version: 5, es_curator_version: 5.8.4 }

ansible/roles/es6/defaults/main.yml

@@ -45,4 +45,8 @@ es_plugins_reinstall: true
 es_plugins:
 - plugin: "repository-azure"
 - plugin: "repository-s3"
-- plugin: "repository-gcs"
+- plugin: "repository-gcs"
+
+### Re-use exisiting variables of azure storage account
+azure_management_storage_account_name: "{{ sunbird_management_storage_account_name }}"
+azure_management_storage_account_key: "{{ sunbird_management_storage_account_key }}"

ansible/roles/es6/tasks/main.yml

@@ -37,20 +37,20 @@
   tags:
       - xpack
 
+- name: include plugins/create-keystore.yml
+  include: plugins/create-keystore.yml
+
 - name: include plugins/repository-azure.yml
-  include: xpack/repository-azure.yml
-  tags:
-      - azure
+  include: plugins/repository-azure.yml
+  when: cloud_provider == "azure"
 
 - name: include plugins/repository-aws.yml
-  include: xpack/repository-aws.yml
-  tags:
-      - aws
+  include: plugins/repository-aws.yml
+  when: cloud_provider == "aws"
 
 - name: include plugins/repository-gcs.yml
-  include: xpack/repository-gcs.yml
-  tags:
-      - gcs
+  include: plugins/repository-gcs.yml
+  when: cloud_provider == "gcloud"
 
 - name: flush handlers
   meta: flush_handlers

ansible/roles/es6/tasks/plugins/create-keystore.yml

@@ -0,0 +1,13 @@
+---
+- name: Check if elasticsearch keystore exists or not
+  become: yes
+  stat:
+    path: "{{ conf_dir }}/elasticsearch.keystore"
+  register: elasticsearch_keystore_file
+
+- name: Create the elasticsearch keystore if not exists
+  become: yes
+  command: "{{es_home}}/bin/elasticsearch-keystore create"
+  environment:
+    ES_PATH_CONF: "{{ conf_dir }}"
+  when: elasticsearch_keystore_file.stat.exists is false

ansible/roles/es6/tasks/plugins/repository-azure.yml

@@ -1,10 +1,15 @@
-----
+---
 - name: Add default azure account name for backups
   become: yes
-  shell: echo "{{ azure_management_storage_account_name }}" | {{ es_home }}/bin/elasticsearch-keystore add azure.client.default.account
+  shell: echo "{{ azure_management_storage_account_name }}" | {{ es_home }}/bin/elasticsearch-keystore add -f azure.client.default.account
   no_log: True
+  environment:
+    ES_PATH_CONF: "{{ conf_dir }}"
+
 
 - name: Add default azure account key for backups
   become: yes
-  shell: echo "{{ azure_management_storage_account_key }}" | {{ es_home }}/bin/elasticsearch-keystore add azure.client.default.key
-  no_log: True
+  shell: echo "{{ azure_management_storage_account_key }}" | {{ es_home }}/bin/elasticsearch-keystore add -f azure.client.default.key
+  no_log: True
+  environment:
+    ES_PATH_CONF: "{{ conf_dir }}"

ansible/roles/es6/tasks/plugins/repository-gcs.yaml → ansible/roles/es6/tasks/plugins/repository-gcs.yml

@@ -1,5 +1,5 @@
-----
+---
 - name: Add gcs credentials file to keystore
   become: yes
-  shell: echo "{{ s3_management_bucket_access_key }}" | {{ es_home }}/bin/elasticsearch-keystore add-file gcs.client.default.credentials_file google_cloud_store_management_service_account.json
+  shell: echo "{{ s3_management_bucket_access_key }}" | {{ es_home }}/bin/elasticsearch-keystore add-file -f gcs.client.default.credentials_file google_cloud_store_management_service_account.json
   no_log: True

ansible/roles/es6/tasks/plugins/repository-s3.yaml → ansible/roles/es6/tasks/plugins/repository-s3.yml

@@ -1,10 +1,10 @@
-----
+---
 - name: Add default aws account name for backups
   become: yes
-  shell: echo "{{ s3_management_bucket_access_key }}" | {{ es_home }}/bin/elasticsearch-keystore add s3.client.default.access_key
+  shell: echo "{{ s3_management_bucket_access_key }}" | {{ es_home }}/bin/elasticsearch-keystore add -f s3.client.default.access_key
   no_log: True
 
 - name: Add default aws account key for backups
   become: yes
-  shell: echo "{{ s3_management_bucket_secret_key }}" | {{ es_home }}/bin/elasticsearch-keystore add s3.client.default.secret_key
+  shell: echo "{{ s3_management_bucket_secret_key }}" | {{ es_home }}/bin/elasticsearch-keystore add -f s3.client.default.secret_key
   no_log: True

kubernetes/ansible/roles/logging/templates/elasticsearch-curator.yaml

@@ -8,7 +8,7 @@ cronjob:
 
 image:
   repository: untergeek/curator
-  tag: 5.8.1
+  tag: 5.8.4
   pullPolicy: IfNotPresent
 
 configMaps: