From 43d0c95ff5a58a8a8baea2207d3b66e1e84a64da Mon Sep 17 00:00:00 2001 From: Keshav Prasad Date: Mon, 29 Aug 2022 20:40:24 +0530 Subject: [PATCH] fix: updated roles and plays for azure provision and backup --- ansible/es.yml | 10 +--------- ansible/roles/es-azure-snapshot/tasks/main.yml | 2 +- ansible/roles/es-curator/defaults/main.yml | 6 +----- ansible/roles/es-curator/tasks/main.yml | 5 ----- ansible/roles/es5-snapshot-purge/meta/main.yml | 2 +- ansible/roles/es6/defaults/main.yml | 6 +++++- ansible/roles/es6/tasks/main.yml | 18 +++++++++--------- .../es6/tasks/plugins/create-keystore.yml | 13 +++++++++++++ .../es6/tasks/plugins/repository-azure.yml | 13 +++++++++---- ...{repository-gcs.yaml => repository-gcs.yml} | 4 ++-- .../{repository-s3.yaml => repository-s3.yml} | 6 +++--- .../templates/elasticsearch-curator.yaml | 2 +- 12 files changed, 46 insertions(+), 41 deletions(-) create mode 100644 ansible/roles/es6/tasks/plugins/create-keystore.yml rename ansible/roles/es6/tasks/plugins/{repository-gcs.yaml => repository-gcs.yml} (50%) rename ansible/roles/es6/tasks/plugins/{repository-s3.yaml => repository-s3.yml} (66%) diff --git a/ansible/es.yml b/ansible/es.yml index 8042e5dbfa..b6073c5a08 100644 --- a/ansible/es.yml +++ b/ansible/es.yml @@ -10,6 +10,7 @@ - es5-snapshot-purge tags: - es_backup + run_once: true - hosts: log-es-backup become: yes @@ -23,12 +24,3 @@ - es5-snapshot-purge tags: - log_es_backup - -# - hosts: log-es -# become: yes -# vars_files: -# - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml'] -# roles: -# - es2-data-purge -# tags: -# - purge_old_logs diff --git a/ansible/roles/es-azure-snapshot/tasks/main.yml b/ansible/roles/es-azure-snapshot/tasks/main.yml index d9e01c5529..50530ce197 100644 --- a/ansible/roles/es-azure-snapshot/tasks/main.yml +++ b/ansible/roles/es-azure-snapshot/tasks/main.yml @@ -11,7 +11,7 @@ Content-Type: "application/json" - debug: - var: "{{ snapshot_base_path }}" + msg: "snapshot_base_path: {{ snapshot_base_path }}" - set_fact: snapshot_number="snapshot_{{ansible_date_time.epoch}}" diff --git a/ansible/roles/es-curator/defaults/main.yml b/ansible/roles/es-curator/defaults/main.yml index a4728a5b39..9fd4efe2c8 100644 --- a/ansible/roles/es-curator/defaults/main.yml +++ b/ansible/roles/es-curator/defaults/main.yml @@ -1,6 +1,2 @@ -# es_curator_major_version: 3 -# es_curator_version: 3.4.1-1 - es_curator_major_version: 5 -es_curator_version: 5.7.6 -python_es_curator_version: 3.4.1-1 +es_curator_version: 5.8.4 \ No newline at end of file diff --git a/ansible/roles/es-curator/tasks/main.yml b/ansible/roles/es-curator/tasks/main.yml index 689b87be46..c4a8bacee7 100644 --- a/ansible/roles/es-curator/tasks/main.yml +++ b/ansible/roles/es-curator/tasks/main.yml @@ -7,11 +7,6 @@ - debug: msg: "{{ es_curator_version }}" -- name: Install python elasticsearch curator - apt: - name: python-elasticsearch-curator={{ python_es_curator_version }} - force: yes - - name: Install elasticsearch curator apt: name: elasticsearch-curator={{ es_curator_version }} diff --git a/ansible/roles/es5-snapshot-purge/meta/main.yml b/ansible/roles/es5-snapshot-purge/meta/main.yml index e26db1f145..8b4e268b5d 100644 --- a/ansible/roles/es5-snapshot-purge/meta/main.yml +++ b/ansible/roles/es5-snapshot-purge/meta/main.yml @@ -1,3 +1,3 @@ --- dependencies: - - { role: es-curator, es_curator_major_version: 5, es_curator_version: 5.8.1, python_es_curator_version: 3.4.1-1 } + - { role: es-curator, es_curator_major_version: 5, es_curator_version: 5.8.4 } diff --git a/ansible/roles/es6/defaults/main.yml b/ansible/roles/es6/defaults/main.yml index 89a784a7de..39717196ce 100644 --- a/ansible/roles/es6/defaults/main.yml +++ b/ansible/roles/es6/defaults/main.yml @@ -45,4 +45,8 @@ es_plugins_reinstall: true es_plugins: - plugin: "repository-azure" - plugin: "repository-s3" -- plugin: "repository-gcs" \ No newline at end of file +- plugin: "repository-gcs" + +### Re-use exisiting variables of azure storage account +azure_management_storage_account_name: "{{ sunbird_management_storage_account_name }}" +azure_management_storage_account_key: "{{ sunbird_management_storage_account_key }}" \ No newline at end of file diff --git a/ansible/roles/es6/tasks/main.yml b/ansible/roles/es6/tasks/main.yml index 71400a911c..a0d877d716 100644 --- a/ansible/roles/es6/tasks/main.yml +++ b/ansible/roles/es6/tasks/main.yml @@ -37,20 +37,20 @@ tags: - xpack +- name: include plugins/create-keystore.yml + include: plugins/create-keystore.yml + - name: include plugins/repository-azure.yml - include: xpack/repository-azure.yml - tags: - - azure + include: plugins/repository-azure.yml + when: cloud_provider == "azure" - name: include plugins/repository-aws.yml - include: xpack/repository-aws.yml - tags: - - aws + include: plugins/repository-aws.yml + when: cloud_provider == "aws" - name: include plugins/repository-gcs.yml - include: xpack/repository-gcs.yml - tags: - - gcs + include: plugins/repository-gcs.yml + when: cloud_provider == "gcloud" - name: flush handlers meta: flush_handlers diff --git a/ansible/roles/es6/tasks/plugins/create-keystore.yml b/ansible/roles/es6/tasks/plugins/create-keystore.yml new file mode 100644 index 0000000000..8d1c17a209 --- /dev/null +++ b/ansible/roles/es6/tasks/plugins/create-keystore.yml @@ -0,0 +1,13 @@ +--- +- name: Check if elasticsearch keystore exists or not + become: yes + stat: + path: "{{ conf_dir }}/elasticsearch.keystore" + register: elasticsearch_keystore_file + +- name: Create the elasticsearch keystore if not exists + become: yes + command: "{{es_home}}/bin/elasticsearch-keystore create" + environment: + ES_PATH_CONF: "{{ conf_dir }}" + when: elasticsearch_keystore_file.stat.exists is false diff --git a/ansible/roles/es6/tasks/plugins/repository-azure.yml b/ansible/roles/es6/tasks/plugins/repository-azure.yml index 180a548403..170a84000e 100644 --- a/ansible/roles/es6/tasks/plugins/repository-azure.yml +++ b/ansible/roles/es6/tasks/plugins/repository-azure.yml @@ -1,10 +1,15 @@ ----- +--- - name: Add default azure account name for backups become: yes - shell: echo "{{ azure_management_storage_account_name }}" | {{ es_home }}/bin/elasticsearch-keystore add azure.client.default.account + shell: echo "{{ azure_management_storage_account_name }}" | {{ es_home }}/bin/elasticsearch-keystore add -f azure.client.default.account no_log: True + environment: + ES_PATH_CONF: "{{ conf_dir }}" + - name: Add default azure account key for backups become: yes - shell: echo "{{ azure_management_storage_account_key }}" | {{ es_home }}/bin/elasticsearch-keystore add azure.client.default.key - no_log: True \ No newline at end of file + shell: echo "{{ azure_management_storage_account_key }}" | {{ es_home }}/bin/elasticsearch-keystore add -f azure.client.default.key + no_log: True + environment: + ES_PATH_CONF: "{{ conf_dir }}" \ No newline at end of file diff --git a/ansible/roles/es6/tasks/plugins/repository-gcs.yaml b/ansible/roles/es6/tasks/plugins/repository-gcs.yml similarity index 50% rename from ansible/roles/es6/tasks/plugins/repository-gcs.yaml rename to ansible/roles/es6/tasks/plugins/repository-gcs.yml index 1aad10b617..a39e8c16a2 100644 --- a/ansible/roles/es6/tasks/plugins/repository-gcs.yaml +++ b/ansible/roles/es6/tasks/plugins/repository-gcs.yml @@ -1,5 +1,5 @@ ----- +--- - name: Add gcs credentials file to keystore become: yes - shell: echo "{{ s3_management_bucket_access_key }}" | {{ es_home }}/bin/elasticsearch-keystore add-file gcs.client.default.credentials_file google_cloud_store_management_service_account.json + shell: echo "{{ s3_management_bucket_access_key }}" | {{ es_home }}/bin/elasticsearch-keystore add-file -f gcs.client.default.credentials_file google_cloud_store_management_service_account.json no_log: True \ No newline at end of file diff --git a/ansible/roles/es6/tasks/plugins/repository-s3.yaml b/ansible/roles/es6/tasks/plugins/repository-s3.yml similarity index 66% rename from ansible/roles/es6/tasks/plugins/repository-s3.yaml rename to ansible/roles/es6/tasks/plugins/repository-s3.yml index 41e6f1d5db..73cacabf07 100644 --- a/ansible/roles/es6/tasks/plugins/repository-s3.yaml +++ b/ansible/roles/es6/tasks/plugins/repository-s3.yml @@ -1,10 +1,10 @@ ----- +--- - name: Add default aws account name for backups become: yes - shell: echo "{{ s3_management_bucket_access_key }}" | {{ es_home }}/bin/elasticsearch-keystore add s3.client.default.access_key + shell: echo "{{ s3_management_bucket_access_key }}" | {{ es_home }}/bin/elasticsearch-keystore add -f s3.client.default.access_key no_log: True - name: Add default aws account key for backups become: yes - shell: echo "{{ s3_management_bucket_secret_key }}" | {{ es_home }}/bin/elasticsearch-keystore add s3.client.default.secret_key + shell: echo "{{ s3_management_bucket_secret_key }}" | {{ es_home }}/bin/elasticsearch-keystore add -f s3.client.default.secret_key no_log: True \ No newline at end of file diff --git a/kubernetes/ansible/roles/logging/templates/elasticsearch-curator.yaml b/kubernetes/ansible/roles/logging/templates/elasticsearch-curator.yaml index b411d48adb..6253cd44e1 100644 --- a/kubernetes/ansible/roles/logging/templates/elasticsearch-curator.yaml +++ b/kubernetes/ansible/roles/logging/templates/elasticsearch-curator.yaml @@ -8,7 +8,7 @@ cronjob: image: repository: untergeek/curator - tag: 5.8.1 + tag: 5.8.4 pullPolicy: IfNotPresent configMaps: