Refactor (#12)

* Test mitogen * Fix config for ES7 support * Add separate es6&7 tests Signed-off-by: Timur Gadiev <[email protected]>
lean-delivery · Jun 13, 2019 · 4f8abf1 · 4f8abf1
1 parent c2971bc
commit 4f8abf1
Show file tree

Hide file tree

Showing 26 changed files with 338 additions and 171 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -5,10 +5,13 @@ stages:
   - deployment test
 
 before_script:
+  - export PY_COLORS=1
   - rm -rf molecule/resources/provisioning
+  - rm -rf /tmp/molecule/plugins
   - ansible --version
   - molecule --version
   - git clone https://github.com/lean-delivery/ansible-molecule-drivers.git molecule/resources/provisioning
+  - mkdir -p /tmp/molecule/plugins && git clone https://github.com/dw/mitogen.git /tmp/molecule/plugins/mitogen
 
 variables:
   GET_SOURCES_ATTEMPTS: "5"
@@ -19,6 +22,7 @@ Ansible lint:
     - git clone https://github.com/lean-delivery/ansible-lint-rules.git ~/ansible-lint-rules
   script:
     - ansible-lint --version
+    - yamllint . -c .yamllint
     - ansible-lint . -c .ansible-lint
   after_script:
     - rm -rf ~/ansible-lint-rules
@@ -32,12 +36,21 @@ Docker ansible-role-elasticsearch:
   tags:
     - aws
 
-AWS ansible-role-elasticsearch:
+AWS ansible-role-elasticsearch-7:
   variables:
     AWS_REGION: us-east-1
   stage: deployment test
   script:
-    - molecule test -s cloud-aws-direct
+    - molecule test -s cloud-aws-direct-7
+  tags:
+    - aws
+
+AWS ansible-role-elasticsearch-6:
+  variables:
+    AWS_REGION: us-east-1
+  stage: deployment test
+  script:
+    - molecule test -s cloud-aws-direct-6
   tags:
     - aws
 

diff --git a/.travis.yml b/.travis.yml
@@ -13,12 +13,13 @@ before_install:
   - git clone https://github.com/lean-delivery/ansible-lint-rules.git ~/ansible-lint-rules
 
 install:
-  - pip install --upgrade ansible==2.5.* docker-py molecule==2.17 pyOpenSSL PyYAML pytest==3.7.4
+  - pip install --upgrade ansible==2.7.* docker-py molecule==2.20 pyOpenSSL PyYAML pytest
   - pip install git+https://github.com/ansible/ansible-lint.git
   - ansible --version
   - ansible-lint --version
 
 script:
+  - yamllint . -c .yamllint
   - ansible-lint . -c .ansible-lint
   - molecule test -s default
 

diff --git a/README.md b/README.md
@@ -39,7 +39,7 @@ No problems in role with active SELinux were encountered. In a case of any issue
 All Elasticsearch configuration parameters are supported. This is achieved using a configuration map parameter `es_config` which is serialized into the elasticsearch.yml file.
 The use of a map ensures the Ansible playbook does not need to be updated to reflect new/deprecated/plugin configuration parameters.
 - `elastic_branch`
-Is used to select main Elasticsearch branch to be installed (5.x or 6.x current stable versions). By default this variable is set to `6`. So, 6.x version is installed by default. You can override this by setting this variable in playbook.
+Is used to select main Elasticsearch branch to be installed (6.x or 7.x current stable versions). By default this variable is set to `6`. So, 6.x version is installed by default. You can override this by setting this variable in playbook.
 - `es_version`
 Used to define full Elasticsearch version (e.g. 6.6.0). Depends on `elastic_branch` by default. Default value is `6.x`
 - `elasticsearch_host`
@@ -132,40 +132,80 @@ Example Playbook
 ```
 
 ### Installing multi node solution with elasticsearch 6.x version:
+
+Playbook structure:
+
+```yaml
+.
+├── elastic_cluster_inventory
+├── group_vars
+│   ├── all.yml
+│   ├── controller.yml
+│   └── dm.yml
+└── elastic_cluster.yml
+```
+
+elastic_cluster_inventory:
+
+```yaml
+[controller]
+node1
+
+[dm]
+node2
+node3
+node4
+
+[cluster:children]
+controller
+dm
+```
+
+group_vars/controller.yml:
+
+```yaml
+kibana_host: '{{ ansible_host }}'
+es_config: 
+  node.name: '{{ ansible_host }}'
+  cluster.name: my_cluster
+  network.host: [_local_,_site_]
+  node.master: false
+  node.data: false
+  node.ingest: false
+  discovery.zen.minimum_master_nodes: 2
+  discovery.zen.ping.unicast.hosts: '{{ cluster_list }}'
+```
+
+group_vars/dm.yml:
+
+```yaml
+es_config:
+  node.name: '{{ ansible_host }}'
+  cluster.name: my_cluster
+  network.host: [_local_,_site_]
+  node.master: true
+  node.data: true
+  discovery.zen.minimum_master_nodes: 2
+  discovery.zen.ping.unicast.hosts: '{{ cluster_list }}'
+```
+
+elastic_cluster.yml:
+
 ```yaml
-- name: Install Elasticsearch 6.x and Kibana. Configure node as controller
+- name: Install elasticsearch and configure cluster
+  hosts: all
+  vars:
+    cluster_list: "{{ groups['cluster'] | map ('extract', hostvars, ['ansible_hostname']) |  join (',') }}"
+
+  roles:
+   - role: lean_delivery.java
+   - role: lean_delivery.elasticsearch
+
+- name: Install kibana on controller node
   hosts: controller
+
   roles:
-    - role: lean_delivery.java
-    - role: lean_delivery.elasticsearch
     - role: lean_delivery.kibana
-  vars:
-    elastic_branch: 6
-    es_config:
-      node.master: false
-      node.data: false
-      node.ingest: false
-      node.name: "{{ ansible_host }}"
-      cluster.name: Cluster_Name
-      network.host: [_local_,_site_]
-      discovery.zen.ping.unicast.hosts: ["node1.example.com","node2.example.com","node3.example.com"]
-      discovery.zen.minimum_master_nodes: 2
-
-- name: Install Elasticsearch 6.x and configure nodes as data & master
-  hosts: dm
-  roles:
-    - role: lean_delivery.java
-    - role: lean_delivery.elasticsearch
-  vars:
-    elastic_branch: 6
-    es_config:
-      node.name: "{{ ansible_host }}"
-      cluster.name: Cluster_Name
-      network.host: [_local_,_site_]
-      node.master: true
-      node.data: true
-      discovery.zen.ping.unicast.hosts: ["node1.example.com","node2.example.com","node3.example.com"]
-      discovery.zen.minimum_master_nodes: 2
 ```
 
 License

diff --git a/defaults/main.yml b/defaults/main.yml
@@ -6,12 +6,12 @@ elasticsearch_port: 9200
 elasticsearch_xmx: 512m
 elasticsearch_xms: 512m
 es_jvm_custom_parameters: ''
-elasticsearch_scripts_install: False
+elasticsearch_scripts_install: false
 elasticsearch_scripts_path: /opt/es_scripts
-elasticsearch_set_settings: True
-es_restart_on_change: True
-es_start_service: True
-m_lock_enabled: False
+elasticsearch_set_settings: true
+es_restart_on_change: true
+es_start_service: true
+m_lock_enabled: false
 es_home: '{{ default["es_home"] }}'
 es_pid_dir: '{{ default["es_pid_dir"] }}'
 es_log_dir: '{{ default["es_log_dir"] }}'
@@ -24,6 +24,6 @@ es_apt_url: 'deb https://artifacts.elastic.co/packages/{{ es_repo_name }}/apt st
 es_yum_url: 'https://artifacts.elastic.co/packages/{{ es_repo_name }}/yum'
 es_repo_file: 'elastic-{{ es_major_version }}'
 elastic_gpg_key: https://artifacts.elastic.co/GPG-KEY-elasticsearch
-es_use_oss_version: False
+es_use_oss_version: false
 es_config:
   node.name: node1
diff --git a/handlers/main.yml b/handlers/main.yml
@@ -1,13 +1,13 @@
 ---
 - name: Reload systemd configuration
   systemd:
-    daemon_reload: True
-  become: True
+    daemon_reload: true
+  become: true
 
 - name: Restart elasticsearch
   service:
     name: elasticsearch
     state: restarted
   when:
-    - es_restart_on_change
-  become: True
+    - es_restart_on_change | bool
+  become: true
diff --git a/molecule/cloud-aws-direct/molecule.yml → molecule/cloud-aws-direct-6/molecule.yml b/molecule/cloud-aws-direct/molecule.yml → molecule/cloud-aws-direct-6/molecule.yml
@@ -10,12 +10,13 @@ lint:
   options:
     config-file: .yamllint
 platforms:
-  - name: test-aws-centos7-ansible-role-elasticsearch
+  - name: test-aws-centos7-elasticsearch-6
     image: ami-9887c6e7
-    instance_type: m5.large
+    platform: centos7
+    instance_type: t3.medium
     region: us-east-1
-    vpc_subnet_id: subnet-05a2ef2b767afec50
-    assign_public_ip: False
+    vpc_subnet_id: subnet-0f2b9cd66faea38af
+    assign_public_ip: false
     spot_price: 0.04
     security_group_name:
       - default
@@ -24,40 +25,29 @@ platforms:
     groups:
       - rhel_family
 
-  - name: test-aws-centos6-ansible-role-elasticsearch
+  - name: test-aws-centos6-elasticsearch-6
     image: ami-1585c46a
+    platform: centos6
     instance_type: m4.large
     region: us-east-1
-    vpc_subnet_id: subnet-05a2ef2b767afec50
-    assign_public_ip: False
+    vpc_subnet_id: subnet-0f2b9cd66faea38af
+    assign_public_ip: false
     spot_price: 0.04
     security_group_name:
       - default
     wait_timeout: 1800
     ssh_user: centos
     groups:
-     - rhel_family
-
-  - name: test-aws-ubuntu16-ansible-role-elasticsearch
-    image: ami-09677e0a6b14905b0
-    instance_type: m5.large
-    region: us-east-1
-    vpc_subnet_id: subnet-05a2ef2b767afec50
-    assign_public_ip: False
-    security_group_name:
-      - default
-    spot_price: 0.04
-    wait_timeout: 1800
-    ssh_user: ubuntu
-    groups:
-      - debian_family
+      - rhel_family
+      - yml_config
 
-  - name: test-aws-ubuntu18-ansible-role-elasticsearch
+  - name: test-aws-ubuntu18-elasticsearch-6
     image: ami-012fd5eb46f56731f
-    instance_type: m5.large
+    platform: ubuntu18
+    instance_type: t3.medium
     region: us-east-1
-    vpc_subnet_id: subnet-05a2ef2b767afec50
-    assign_public_ip: False
+    vpc_subnet_id: subnet-0f2b9cd66faea38af
+    assign_public_ip: false
     security_group_name:
       - default
     spot_price: 0.04
@@ -66,12 +56,13 @@ platforms:
     groups:
       - debian_family
 
-  - name: test-aws-Debian9-ansible-role-elasticsearch
+  - name: test-aws-Debian9-elasticsearch-6
     image: ami-003f19e0e687de1cd
-    instance_type: m5.large
+    platform: debian9
+    instance_type: t3.medium
     region: us-east-1
-    vpc_subnet_id: subnet-05a2ef2b767afec50
-    assign_public_ip: False
+    vpc_subnet_id: subnet-0f2b9cd66faea38af
+    assign_public_ip: false
     spot_price: 0.04
     security_group_name:
       - default
@@ -82,17 +73,26 @@ platforms:
 
 provisioner:
   name: ansible
-  log: False
+  log: false
+  config_options:
+    defaults:
+      callback_whitelist: profile_tasks,timer
+      strategy_plugins: /tmp/molecule/plugins/mitogen/ansible_mitogen/plugins/strategy
+  inventory:
+    group_vars:
+      yml_config:
+        es_config:
+          node.name: '{{ ansible_host }}'
+          http.port: 9200
+          bootstrap.system_call_filter: false
   playbooks:
     create: ../resources/provisioning/AWS/create.yml
-    prepare: ../resources/provisioning/AWS/prepare.yml
+    prepare: ../resources/prepare.yml
     destroy: ../resources/provisioning/AWS/destroy.yml
   lint:
     name: ansible-lint
-  env:
-    ANSIBLE_LIBRARY: ${ANSIBLE_LIBRARY}
 scenario:
-  name: cloud-aws-direct
+  name: cloud-aws-direct-6
 verifier:
   name: testinfra
   options:

diff --git a/molecule/cloud-aws-direct-6/playbook.yml b/molecule/cloud-aws-direct-6/playbook.yml
@@ -0,0 +1,8 @@
+---
+- name: Converge
+  hosts: all
+  strategy: mitogen_linear
+  roles:
+    - role: ansible-role-elasticsearch
+      elasticsearch_host: '{{ ansible_host }}'
+      elastic_branch: 6