Changes required to install Elasticsearch 6 on staging

environments/staging/public.yml

@@ -3,8 +3,8 @@ monitor_celery_heartbeat: False
 
 elasticsearch_endpoint: '{{ groups.elasticsearch.0 }}'
 elasticsearch_cluster_name: 'staginges'
-elasticsearch_version: 5.6.16
-elasticsearch_download_sha256: 6b035a59337d571ab70cea72cc55225c027ad142fbb07fd8984e54261657c77f.
+elasticsearch_version: 6.8.23
+elasticsearch_download_sha256: 424af91f838f9e5f13e0292f97cbd6333535450291a621d761bd479dfc2dff78.
 elasticsearch_delete_old_logs: True
 
 kafka_version: 3.2.3
@@ -135,7 +135,7 @@ localsettings:
   COUCH_PASSWORD: "{{ COUCH_PASSWORD }}"
   COUCH_USERNAME: "{{ COUCH_USERNAME }}"
   DEPLOY_MACHINE_NAME: "{{ ansible_hostname }}"
-  ELASTICSEARCH_MAJOR_VERSION: 5
+  ELASTICSEARCH_MAJOR_VERSION: 6
   EMAIL_SMTP_HOST: email-smtp.us-east-1.amazonaws.com
   EMAIL_SMTP_PORT: 587
   EMAIL_USE_TLS: yes

src/commcare_cloud/ansible/roles/elasticsearch/defaults/main.yml

@@ -12,7 +12,6 @@ elasticsearch_http_port: 9200
 #es_snapshot_bucket: "dimagi-{{ deploy_env }}-es-snapshots"
 es_repository_name: "{{ deploy_env }}_es_snapshot"
 backup_es_swift: no
-elasticsearch_enable_inline_groovy_scripts: True
 elasticsearch_service_name: elasticsearch
 elasticsearch_node_name: "{{ hostname }}"
 elasticsearch_delete_old_logs: False

src/commcare_cloud/ansible/roles/elasticsearch/tasks/main.yml

@@ -32,12 +32,21 @@
     dest: "/home/{{ cchq_user }}/downloads/elasticsearch-{{ elasticsearch_version }}.tar.gz"
   when: elasticsearch_version == "5.6.16"
 
+- name: Download Elasticsearch 6
+  become: yes
+  get_url:
+    url: https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{ elasticsearch_version }}.tar.gz
+    dest: "/home/{{ cchq_user }}/downloads/elasticsearch-{{ elasticsearch_version }}.tar.gz"
+    checksum: "sha256:{{ elasticsearch_download_sha256 }}"
+  when: elasticsearch_version is version('6.0.0', '>=') and elasticsearch_version is version('7.0.0', '<')
+
+
 - name: Download Elasticsearch 7
   become: yes
   get_url:
     url: "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{ elasticsearch_version }}-linux-x86_64.tar.gz"
     dest: "/home/{{ cchq_user }}/downloads/elasticsearch-{{ elasticsearch_version }}.tar.gz"
-    sha256sum: "{{ elasticsearch_download_sha256 }}"
+    checksum: "sha256:{{ elasticsearch_download_sha256 }}"
   when: elasticsearch_version is version('7.0.0', '>=')
 
 - name: Unarchive Elasticsearch
@@ -75,11 +84,41 @@
   tags:
     - es_conf
 
-- include_tasks: misc_v5.yml
-  when: elasticsearch_version == '5.6.16'
+- name: Copy Elasticsearch logging config
+  become: yes
+  copy:
+    src: log4j2.properties
+    dest: "{{ elasticsearch_conf_dir }}/log4j2.properties"
+    owner: elasticsearch
+    group: elasticsearch
+  register: copy_elasticsearch_logging_result
+  tags:
+    - es_conf
 
-- include_tasks: misc_v7.yml
-  when: elasticsearch_version is version('7.0.0', '>=')
+- name: Copy Elasticsearch JVM Options
+  become: yes
+  template:
+    src: config/jvm.options.j2
+    dest: "{{ elasticsearch_conf_dir }}/jvm.options"
+    owner: elasticsearch
+    group: elasticsearch
+  register: copy_elasticsearch_logging_result
+  tags:
+    - es_conf
+
+- name: Install elasticsearch python client
+  become: yes
+  pip:
+    name: "elasticsearch>={{ elasticsearch_version.split('.')[0] }}.0.0,<{{ elasticsearch_version.split('.')[0] | int + 1 }}.0.0"
+
+- name: Check for installed plugins
+  shell: "{{ elasticsearch_home }}/bin/elasticsearch-plugin list"
+  register: installed_plugins
+
+- name: Install elasticsearch phonetic plugin
+  become: yes
+  shell: "{{ elasticsearch_home }}/bin/elasticsearch-plugin install analysis-phonetic"
+  when: (installed_plugins.stdout is defined) and ('analysis-phonetic' not in installed_plugins.stdout) and es_plugins_phonetic_enabled
 
 - name: Copy Elasticsearch Systemd Conf
   become: yes
@@ -103,18 +142,26 @@
 - debug: msg="If you're just updating the ES conf then you sill need to restart the cluster. Use  the 'es_rolling_restart' playbook."
   when: copy_elasticsearch_conf_result.changed
 
-- name: Create Elasticsearch data directories
+- name: Check if Elasticsearch data directories exist
+  stat:
+    path: "{{ item }}"
+  register: dir_stat
+  with_items:
+    - "{{ elasticsearch_data_dir }}"
+    - "{{ elasticsearch_data_dir }}/data"
+    - "{{ elasticsearch_data_dir }}/logs"
+    - "{{ elasticsearch_data_dir }}/tmp"
+
+- name: Create Elasticsearch data directories if they do not exist
   become: yes
   file:
-    path: "{{ item }}"
+    path: "{{ item.item }}"
     state: directory
     owner: elasticsearch
     group: elasticsearch
     mode: 0755
-  with_items:
-    - "{{ elasticsearch_data_dir }}"
-    - "{{ elasticsearch_data_dir }}/data"
-    - "{{ elasticsearch_data_dir }}/logs"
+  when: not item.stat.exists
+  with_items: "{{ dir_stat.results }}"
 
 - name: Start Elasticsearch
   become: yes

src/commcare_cloud/ansible/roles/elasticsearch/templates/config/elasticsearch.yml.j2

@@ -26,22 +26,19 @@ http.max_content_length: 200mb
 path.data: {{ elasticsearch_data_dir }}/data
 path.logs: {{ elasticsearch_data_dir }}/logs
 
-{% if elasticsearch_version is version('5.6.16', '<=') %}
+{% if elasticsearch_version is version('7.0.0', '<') %}
 # discovery.zen.minimum_master_nodes is the minimum number of
 # MASTER ELIGIBLE nodes that must be reachable before a master may be elected
 # Best practice is to set it to M/2 + 1 where M is the number of master-eligible nodes,
 # but (todo) we DO NOT follow best practice.
 discovery.zen.minimum_master_nodes: {{ [2, groups.elasticsearch|map('extract', hostvars, 'elasticsearch_master_value')|select()|list | length] | min }}
-discovery.zen.fd.ping_timeout: 90s
-discovery.zen.fd.ping_interval: 10s
-discovery.zen.fd.ping_retries: 20
 
-{% if elasticsearch_version == '5.6.16' %}
+{% if elasticsearch_version is version('5.0.0', '>=') %}
 discovery.zen.ping_timeout: 90s
 {% endif %}
 {% endif %}
 
-{% if elasticsearch_version is version('5.6.16', '<=') %}
+{% if elasticsearch_version is version('7.0.0', '<') %}
 discovery.zen.ping.unicast.hosts: [
 {% endif %}
 {% if elasticsearch_version is version('7.0.0', '>=') %}
@@ -66,13 +63,6 @@ network.host: "{{ lookup('dig', inventory_hostname, wantlist=True)[0] }}"
 indices.fielddata.cache.size:  {{ elasticsearch_fielddata_cache_size }}
 {% endif %}
 
-{% if elasticsearch_enable_inline_groovy_scripts %}
-{% if elasticsearch_version is version('5.6.16', '<=') %}
-# somewhat of a security risk but required by pact custom reports
-script.engine.groovy.inline.aggs: true
-script.engine.groovy.inline.search: true
-{% endif %}
-{% endif %}
 
 {% if backup_es_s3 and AMAZON_S3_ACCESS_KEY %}
 cloud:
@@ -87,8 +77,9 @@ cloud:
 {% endif %}
 
 # Only auto create elastic search internal indices
+{% if elasticsearch_version is version('5.6.16', '<=') %}
 action.auto_create_index: .watches,.triggered_watches,.watcher-history-*
-
+{% endif %}
 
 {% if es_local_repo and not backup_es_s3%}
 path.repo: ["{{ elasticsearch_data_dir }}/snapshot"]
@@ -110,7 +101,7 @@ cluster.initial_master_nodes:
 {% endif %}
 
 
-{% if elasticsearch_version is version('5.6.16', '<=') %}
+{% if elasticsearch_version is version('7.0.0', '<') %}
 # see also discovery.zen.minimum_master_nodes above for how a master gets elected
 {# only include master eligibility if it is set on at least one node in the cluster #}
 {% if groups.elasticsearch|map('extract', hostvars, 'elasticsearch_master_value')|select()|list %}

src/commcare_cloud/ansible/roles/elasticsearch/templates/config/jvm.options.j2

@@ -54,7 +54,7 @@
 14-:-XX:InitiatingHeapOccupancyPercent=30
 
 ## JVM temporary directory
--Djava.io.tmpdir=${ES_TMPDIR}
+-Djava.io.tmpdir={{ elasticsearch_data_dir }}/tmp
 
 ## heap dumps
 
@@ -67,7 +67,7 @@
 -XX:HeapDumpPath={{ elasticsearch_data_dir }}/logs/heapdump.hprof
 
 # specify an alternative path for JVM fatal error logs
--XX:ErrorFile={{ elasticsearch_data_dir }}logs/hs_err_pid%p.log
+-XX:ErrorFile={{ elasticsearch_data_dir }}/logs/hs_err_pid%p.log
 
 ## JDK 8 GC logging
 8:-XX:+PrintGCDetails

src/commcare_cloud/ansible/roles/elasticsearch/templates/systemd/elasticsearch.service.j2

@@ -7,8 +7,10 @@ After=network-online.target
 [Service]
 Type=simple
 Environment=ES_HOME={{ elasticsearch_home }}
-Environment=CONF_DIR={{ elasticsearch_conf_dir}}
 Environment=ES_PATH_CONF={{ elasticsearch_conf_dir}}
+{% if elasticsearch_version <= '5.6.16'%}
+Environment=CONF_DIR={{ elasticsearch_conf_dir}}
+{% endif %}
 {% if elasticsearch_version == '5.6.16'%}
 Environment=ES_JVM_OPTIONS={{ elasticsearch_conf_dir}}/jvm.options
 {% endif %}
@@ -17,7 +19,7 @@ WorkingDirectory={{ elasticsearch_home }}
 User=elasticsearch
 Group=elasticsearch
 
-ExecStart={{ elasticsearch_home }}/bin/elasticsearch -Epath.conf=${CONF_DIR}
+ExecStart={{ elasticsearch_home }}/bin/elasticsearch  {% if elasticsearch_version == '5.6.16'%} -Epath.conf=${CONF_DIR} {% endif %}
 
 Restart=always
 RestartSec=10