Changes required to install Elasticsearch 6 on staging

environments/staging/public.yml

@@ -3,8 +3,8 @@ monitor_celery_heartbeat: False
 
 elasticsearch_endpoint: '{{ groups.elasticsearch.0 }}'
 elasticsearch_cluster_name: 'staginges'
-elasticsearch_version: 5.6.16
-elasticsearch_download_sha256: 6b035a59337d571ab70cea72cc55225c027ad142fbb07fd8984e54261657c77f.
+elasticsearch_version: 6.8.23
+elasticsearch_download_sha256: 73258b400cd194768058105a74ca0e811962ccf81bed8c4d3a824d4e5b97a8178a31150e60602fa1e04c6764b139980eb45ae6f246864702120653b45120b597.
 elasticsearch_delete_old_logs: True
 
 kafka_version: 3.2.3
@@ -135,7 +135,7 @@ localsettings:
   COUCH_PASSWORD: "{{ COUCH_PASSWORD }}"
   COUCH_USERNAME: "{{ COUCH_USERNAME }}"
   DEPLOY_MACHINE_NAME: "{{ ansible_hostname }}"
-  ELASTICSEARCH_MAJOR_VERSION: 5
+  ELASTICSEARCH_MAJOR_VERSION: 6
   EMAIL_SMTP_HOST: email-smtp.us-east-1.amazonaws.com
   EMAIL_SMTP_PORT: 587
   EMAIL_USE_TLS: yes

src/commcare_cloud/ansible/roles/elasticsearch/tasks/main.yml

@@ -32,6 +32,15 @@
     dest: "/home/{{ cchq_user }}/downloads/elasticsearch-{{ elasticsearch_version }}.tar.gz"
   when: elasticsearch_version == "5.6.16"
 
+- name: Download Elasticsearch 6
+  become: yes
+  get_url:
+    url: https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{ elasticsearch_version }}.tar.gz
+    dest: "/home/{{ cchq_user }}/downloads/elasticsearch-{{ elasticsearch_version }}.tar.gz"
+    sha256sum: "{{ elasticsearch_download_sha256 }}"
+  when: elasticsearch_version is version('6.0.0', '>=') and elasticsearch_version is version('7.0.0', '<')
+
+
 - name: Download Elasticsearch 7
   become: yes
   get_url:
@@ -78,6 +87,10 @@
 - include_tasks: misc_v5.yml
   when: elasticsearch_version == '5.6.16'
 
+- include_tasks: misc_v6.yml
+  when: elasticsearch_version is version('6.0.0', '>=') and elasticsearch_version is version('7.0.0', '<')
+
+
 - include_tasks: misc_v7.yml
   when: elasticsearch_version is version('7.0.0', '>=')
 

src/commcare_cloud/ansible/roles/elasticsearch/tasks/misc_v6.yml

@@ -0,0 +1,35 @@
+- name: Copy Elasticsearch logging config
+  become: yes
+  copy:
+    src: log4j2.properties
+    dest: "{{ elasticsearch_conf_dir }}/log4j2.properties"
+    owner: elasticsearch
+    group: elasticsearch
+  register: copy_elasticsearch_logging_result
+  tags:
+    - es_conf
+
+- name: Copy Elasticsearch JVM Options
+  become: yes
+  template:
+    src: config/jvm.options.j2
+    dest: "{{ elasticsearch_conf_dir }}/jvm.options"
+    owner: elasticsearch
+    group: elasticsearch
+  register: copy_elasticsearch_logging_result
+  tags:
+    - es_conf
+
+- name: Install elasticsearch python client
+  become: yes
+  pip:
+    name: elasticsearch>=6.0.0,<7.0.0
+
+- name: Check for installed plugins
+  shell: "{{ elasticsearch_home }}/bin/elasticsearch-plugin list"
+  register: installed_plugins
+
+- name: Install elasticsearch phonetic plugin
+  become: yes
+  shell: "{{ elasticsearch_home }}/bin/elasticsearch-plugin install analysis-phonetic"
+  when: (installed_plugins.stdout is defined) and ('analysis-phonetic' not in installed_plugins.stdout) and es_plugins_phonetic_enabled

src/commcare_cloud/ansible/roles/elasticsearch/templates/config/elasticsearch.yml.j2

@@ -26,22 +26,25 @@ http.max_content_length: 200mb
 path.data: {{ elasticsearch_data_dir }}/data
 path.logs: {{ elasticsearch_data_dir }}/logs
 
-{% if elasticsearch_version is version('5.6.16', '<=') %}
+{% if elasticsearch_version is version('7.0.0', '<') %}
 # discovery.zen.minimum_master_nodes is the minimum number of
 # MASTER ELIGIBLE nodes that must be reachable before a master may be elected
 # Best practice is to set it to M/2 + 1 where M is the number of master-eligible nodes,
 # but (todo) we DO NOT follow best practice.
 discovery.zen.minimum_master_nodes: {{ [2, groups.elasticsearch|map('extract', hostvars, 'elasticsearch_master_value')|select()|list | length] | min }}
+
+{% if elasticsearch_version is version('5.6.16', '<') %}
 discovery.zen.fd.ping_timeout: 90s
 discovery.zen.fd.ping_interval: 10s
 discovery.zen.fd.ping_retries: 20
+{% endif %}
 
-{% if elasticsearch_version == '5.6.16' %}
+{% if elasticsearch_version is version('6.0.0', '>=') %}
 discovery.zen.ping_timeout: 90s
 {% endif %}
 {% endif %}
 
-{% if elasticsearch_version is version('5.6.16', '<=') %}
+{% if elasticsearch_version is version('7.0.0', '<') %}
 discovery.zen.ping.unicast.hosts: [
 {% endif %}
 {% if elasticsearch_version is version('7.0.0', '>=') %}
@@ -87,8 +90,9 @@ cloud:
 {% endif %}
 
 # Only auto create elastic search internal indices
+{% if elasticsearch_version is version('5.6.16', '<=') %}
 action.auto_create_index: .watches,.triggered_watches,.watcher-history-*
-
+{% endif %}
 
 {% if es_local_repo and not backup_es_s3%}
 path.repo: ["{{ elasticsearch_data_dir }}/snapshot"]
@@ -110,7 +114,7 @@ cluster.initial_master_nodes:
 {% endif %}
 
 
-{% if elasticsearch_version is version('5.6.16', '<=') %}
+{% if elasticsearch_version is version('7.0.0', '<') %}
 # see also discovery.zen.minimum_master_nodes above for how a master gets elected
 {# only include master eligibility if it is set on at least one node in the cluster #}
 {% if groups.elasticsearch|map('extract', hostvars, 'elasticsearch_master_value')|select()|list %}

src/commcare_cloud/ansible/roles/elasticsearch/templates/systemd/elasticsearch.service.j2

@@ -7,8 +7,10 @@ After=network-online.target
 [Service]
 Type=simple
 Environment=ES_HOME={{ elasticsearch_home }}
-Environment=CONF_DIR={{ elasticsearch_conf_dir}}
 Environment=ES_PATH_CONF={{ elasticsearch_conf_dir}}
+{% if elasticsearch_version <= '5.6.16'%}
+Environment=CONF_DIR={{ elasticsearch_conf_dir}}
+{% endif %}
 {% if elasticsearch_version == '5.6.16'%}
 Environment=ES_JVM_OPTIONS={{ elasticsearch_conf_dir}}/jvm.options
 {% endif %}