add linux dependencies for rabbitmq server

ansible/helm_external.yml

@@ -35,3 +35,13 @@
         external_dir_name: cassandra-external
         server_type: cassandra
         network_interface: "{{ cassandra_network_interface }}"
+
+- hosts: "rmq-cluster"
+  become: false
+  tasks:
+    - name: Generate rabbitmq IPs for helm
+      include_tasks: tasks/helm_external.yml
+      vars:
+        external_dir_name: rabbitmq-external
+        server_type: rabbitmq
+        network_interface: "{{ rabbitmq_network_interface }}"

ansible/inventory/offline/99-static

@@ -30,6 +30,10 @@
 # minio2 ansible_host=XXXX
 # minio3 ansible_host=XXXX
 #
+# rabbitmq1 ansible_host=XXXX
+# rabbitmq2 ansible_host=XXXX
+# rabbitmq3 ansible_host=XXXX
+#
 # If you are in an offline environment, add an assethost here, from which
 # artifacts are served
 # assethost ansible_host=100.89.14.74 ip=10.114.0.9
@@ -57,7 +61,7 @@
 # execution is wrapped through a container, as the ssh-agent socket isn't
 # accessible there.
 
-## In the next three sections, Cassandra, elasticsearch, and MinIO need
+## In the next four sections, Cassandra, elasticsearch, rabbitmq, and MinIO need
 #  the name of the interface you want database services to run on
 #  specified. While this can be used to specify a private network just
 #  for databases, you must specify the interface name you want to run
@@ -82,6 +86,9 @@
 #domain = "example.com"
 #deeplink_title = "example.com environment"
 
+[rmq-cluster:vars]
+# rabbitmq_network_interface = enp1s0
+
 [restund:vars]
 # Uncomment if your public IP is not on the default gateway
 # restund_network_interface = enp1s0
@@ -172,3 +179,8 @@ elasticsearch
 # minio1
 # minio2
 # minio3
+
+[rmq-cluster]
+# rabbitmq1
+# rabbitmq2
+# rabbitmq3

ansible/rabbitmq.yml

@@ -0,0 +1,5 @@
+---
+- hosts: rmq-cluster
+  become: yes
+  roles:
+    - rabbitmq-cluster

ansible/roles/rabbitmq-cluster/defaults/main.yml

@@ -0,0 +1,85 @@
+---
+# Whether you need to update hosts file or not, default false.
+# This is useful when you are using AWS EC2 instance, whose default hostname is too long and doesn't have a meaning,
+# like "ip-10-101-50-12.eu-central-1.compute.internal", but you want to use something shorter and meaningful as hostname.
+# In this case you need to set this variable to true in order to update the hosts file, and you need to define a variable named "rabbitmq_hosts",
+# with the following format:
+#
+# rabbitmq_hosts: |
+#   node-1-ip node-1-FQDN
+#   node-2-ip node-2-FQDN
+#
+# example:
+#
+# rabbitmq_hosts: |
+#   10.0.0.10 eu-central-1-mq-master   (whatever the command `hostname -f` outputs on this host)
+#   10.0.0.11 eu-central-1-mq-slave-01 (whatever the command `hostname -f` outputs on this host)
+#
+update_hosts: false
+
+rabbitmq_cluster_master: ansnode1
+rabbitmq_hosts: |
+  172.16.0.132 ansnode1
+  172.16.0.133 ansnode2
+  172.16.0.134 ansnode3
+
+# erlang
+# erlang_version: "1:20.2.2"
+# erlang_download_url: "http://packages.erlang-solutions.com/site/esl/esl-erlang/FLAVOUR_1_general/esl-erlang_20.2.2-1~ubuntu~xenial_amd64.deb"
+# erlang_pkg_name: "esl-erlang_20.2.2-1~ubuntu~xenial_amd64.deb"
+
+# cluster
+rabbitmq_create_cluster: yes
+rabbitmq_erlang_cookie: WKRBTTEQRYPTQOPUKSVF
+# https://www.rabbitmq.com/configure.html#define-environment-variables
+# When set to true this will cause RabbitMQ to use fully qualified names to identify nodes.
+# This may prove useful on EC2.
+# Note that it is not possible to switch between using short and long names without resetting the node.
+rabbitmq_use_longname: 'false'
+
+# log rotate
+rabbitmq_logrotate_period: weekly
+rabbitmq_logrotate_amount: 20
+
+# https://www.rabbitmq.com/install-debian.html
+# The main setting that needs adjustment is the max number of open files, also known as ulimit -n.
+# The default value on many operating systems is too low for a messaging broker (eg. 1024 on several Linux distributions).
+# We recommend allowing for at least 65536 file descriptors for user rabbitmq in production environments.
+# 4096 should be sufficient for most development workloads
+rabbitmq_ulimit_open_files: 65536
+
+# default ports
+rabbitmq_tls_port: 5671
+rabbitmq_amqp_port: 5672
+rabbitmq_epmd_port: 4369
+rabbitmq_node_port: 25672
+
+# plugins for HTTP API monitor
+rabbitmq_plugins:
+  - rabbitmq_management
+  # - rabbitmq_management_agent
+  # - rabbitmq_shovel
+  # - rabbitmq_shovel_management
+
+# TLS/SSL support
+enable_tls: false
+# if true, only tls is supported which means amqp default port 5672 is not open anymore
+tls_only: false
+
+tls_verify: "verify_none"
+tls_fail_if_no_peer_cert: false
+
+cacertfile: ""
+cacertfile_dest: "/etc/rabbitmq/cacert.pem"
+
+certfile: ""
+certfile_dest: "/etc/rabbitmq/cert.pem"
+
+keyfile: ""
+keyfile_dest: "/etc/rabbitmq/key.pem"
+
+# By default, queues within a RabbitMQ cluster are located on a single node (the node on which they were first declared).
+# Queues can optionally be made mirrored across all nodes, or exactly N number of nodes
+# By enabling this variable to true, there will be 1 queue master and 1 queue mirror.
+# If the node running the queue master becomes unavailable, the queue mirror will be automatically promoted to master.
+backup_queues_in_two_nodes: true

ansible/roles/rabbitmq-cluster/handlers/main.yml

@@ -0,0 +1,10 @@
+---
+# Handlers for RabbitMQ
+- name: restart rabbitmq-server
+  service: name=rabbitmq-server state=restarted
+
+- name: start rabbitmq-server
+  service: name=rabbitmq-server state=started
+
+- name: stop rabbitmq-server
+  service: name=rabbitmq-server state=stopped

ansible/roles/rabbitmq-cluster/tasks/cluster.yml

@@ -0,0 +1,16 @@
+---
+- name: make sure rabbitmq server is up
+  service:
+    name: rabbitmq-server
+    state: started
+
+- name: make sure rabbitmq app is up
+  command: rabbitmqctl start_app
+
+- name: check if already in cluster
+  command: rabbitmqctl cluster_status
+  register: cluster_status
+  changed_when: false
+
+- include_tasks: join_cluster.yml
+  when: cluster_status.stdout.find("rabbit@{{ rabbitmq_cluster_master }}") == -1 and (ansible_fqdn != rabbitmq_cluster_master)

ansible/roles/rabbitmq-cluster/tasks/config.yml

@@ -0,0 +1,34 @@
+---
+- name: rabbitmq default file
+  template:
+    src: "{{ item.src }}"
+    dest: "{{ item.dest }}"
+    owner: root
+    group: root
+    mode: 0644
+  with_items:
+    - { src: etc/default/rabbitmq-server.j2 , dest: /etc/default/rabbitmq-server }
+    - { src: etc/rabbitmq/rabbitmq.config.j2, dest: /etc/rabbitmq/rabbitmq.config }
+    # - { src: etc/rabbitmq/rabbitmq-env.conf.j2, dest: /etc/rabbitmq/rabbitmq-env.conf }
+  notify:
+    restart rabbitmq-server
+
+- name: restart rabbitmq-server
+  service:
+    name: rabbitmq-server
+    state: restarted
+
+# - name: Enable the plugins is installed
+#   rabbitmq_plugin:
+#     names: "{{ item }}"
+#     prefix: /usr/lib/rabbitmq
+#     state: enabled
+#     new_only: yes
+#   with_items: "{{ rabbitmq_plugins }}"
+#   notify:
+#     restart rabbitmq-server
+
+- name: restart rabbitmq-server
+  service:
+    name: rabbitmq-server
+    state: restarted

ansible/roles/rabbitmq-cluster/tasks/configure_dns.yml

@@ -0,0 +1,14 @@
+---
+- name: Update /etc/hosts on rmq-cluster nodes
+  hosts: rmq-cluster
+  become: yes  # This allows the playbook to run with elevated privileges
+
+  tasks:
+    - name: Add entries to /etc/hosts
+      lineinfile:
+        path: /etc/hosts
+        regexp: "^{{ hostvars[item].ansible_default_ipv4.address }}\\s+{{ item }}\\s+rabbit@{{ item }}$"
+        line: "{{ hostvars[item].ansible_default_ipv4.address }} {{ item }} rabbit@{{ item }}"
+        insertbefore: EOF
+      with_items: "{{ groups['rmq-cluster'] }}"
+      when: item != inventory_hostname

ansible/roles/rabbitmq-cluster/tasks/create_users.yml

@@ -0,0 +1,16 @@
+---
+# rc 70: user already exists
+- name: create test user
+  shell: rabbitmqctl add_user test test
+  register: res
+  failed_when: res.rc != 70 and res.rc != 0
+  changed_when: res.rc != 70
+
+- name: list permissions for test user
+  shell: rabbitmqctl list_permissions
+  register: list_permissions
+  changed_when: false
+
+- name: set permissions on / vhost
+  shell: rabbitmqctl set_permissions test ".*" ".*" ".*"
+  when: list_permissions.stdout.find("test") == -1

ansible/roles/rabbitmq-cluster/tasks/enable_ha_queues.yml

@@ -0,0 +1,11 @@
+---
+- name: check if ha-mode is already enabled
+  shell: rabbitmqctl list_policies
+  register: list_policies
+  changed_when: false
+
+- name: set ha-mode to exactly two nodes for all queues for backup
+  shell: rabbitmqctl set_policy ha-exactly-two ".*" '{"ha-mode":"exactly","ha-params":2,"ha-sync-mode":"automatic"}'
+  register: res
+  failed_when: res.rc != 0
+  when: list_policies.stdout.find("ha-exactly-two") == -1

ansible/roles/rabbitmq-cluster/tasks/erlang_cookie.yml

@@ -0,0 +1,23 @@
+---
+# Configure RabbitMQ for cluster
+- name: backup old erlang cookie
+  shell: cp -a /var/lib/rabbitmq/.erlang.cookie /var/lib/rabbitmq/.erlang.cookie.old
+  changed_when: false
+
+- name: updating rabbitmq erlang cookie
+  template:
+    src: erlang.cookie.j2
+    dest: /var/lib/rabbitmq/.erlang.cookie
+    owner: rabbitmq
+    group: rabbitmq
+    mode: 0400
+  notify:
+    stop rabbitmq-server
+
+- meta: flush_handlers
+
+- name: remove old erlang cookie
+  file:
+    path: /var/lib/rabbitmq/.erlang.cookie.old
+    state: absent
+  changed_when: false

ansible/roles/rabbitmq-cluster/tasks/hosts.yml

@@ -0,0 +1,6 @@
+- blockinfile:
+    path: /etc/hosts
+    block: "{{ rabbitmq_hosts }}"
+    owner: root
+    group: root
+    mode: 0644

ansible/roles/rabbitmq-cluster/tasks/install.yml

@@ -0,0 +1,8 @@
+- name: install rabbitmq-server
+  apt:
+    update_cache: yes
+    force: yes
+    pkg: "{{ item }}"
+    state: present
+  with_items:
+    - rabbitmq-server

ansible/roles/rabbitmq-cluster/tasks/join_cluster.yml

@@ -0,0 +1,9 @@
+---
+- name: stop rabbitmq app
+  command: rabbitmqctl stop_app
+
+- name: add this node to cluster
+  command: rabbitmqctl join_cluster rabbit@{{ rabbitmq_cluster_master }}
+
+- name: start rabbitmq app
+  command: rabbitmqctl start_app

ansible/roles/rabbitmq-cluster/tasks/main.yml

@@ -0,0 +1,23 @@
+---
+- include_tasks: hosts.yml
+  when: update_hosts
+
+- include_tasks: install.yml
+
+- include_tasks: erlang_cookie.yml
+  when: rabbitmq_create_cluster
+
+- include_tasks: tls.yml
+  when: enable_tls
+
+- include_tasks: config.yml
+
+#- include_tasks: service.yml
+
+- include_tasks: cluster.yml
+  when: rabbitmq_create_cluster == true
+
+- include_tasks: create_users.yml
+
+- include_tasks: enable_ha_queues.yml
+  when: backup_queues_in_two_nodes

ansible/roles/rabbitmq-cluster/tasks/service.yaml

@@ -0,0 +1,6 @@
+---
+- name: rabbitmq service started and enabled
+  service:
+    name: rabbitmq-server
+    enabled: yes
+    state: started

ansible/roles/rabbitmq-cluster/tasks/tls.yml

@@ -0,0 +1,13 @@
+---
+# Copy cacert, cert and key file for TLS/SSL
+- name: make sure TLS/SSL certificates exists
+  template:
+    src: "{{ item.src }}"
+    dest: "{{ item.dest }}"
+    owner: root
+    group: rabbitmq
+    mode: 0644
+  with_items:
+    - { src: "{{ cacertfile }}", dest: "{{ cacertfile_dest }}" }
+    - { src: "{{ certfile }}", dest: "{{ certfile_dest }}" }
+    - { src: "{{ keyfile }}", dest: "{{ keyfile_dest }}" }

ansible/roles/rabbitmq-cluster/templates/erlang.cookie.j2

@@ -0,0 +1 @@
+{{ rabbitmq_erlang_cookie }}

ansible/roles/rabbitmq-cluster/templates/etc/default/rabbitmq-server.j2

@@ -0,0 +1,13 @@
+# This file is sourced by /etc/init.d/rabbitmq-server. Its primary
+# reason for existing is to allow adjustment of system limits for the
+# rabbitmq-server process.
+#
+# Maximum number of open file handles. This will need to be increased
+# to handle many simultaneous connections. Refer to the system
+# documentation for ulimit (in man bash) for more information.
+#
+
+ulimit -n {{ rabbitmq_ulimit_open_files }}
+
+ERL_EPMD_PORT={{ rabbitmq_epmd_port }}
+RABBITMQ_NODE_PORT= {{ rabbitmq_node_port }}

ansible/roles/rabbitmq-cluster/templates/etc/rabbitmq/rabbitmq.config.j2

@@ -0,0 +1,16 @@
+[
+  {rabbit, [
+{% if tls_only %}
+    {tcp_listeners, []},
+{% endif %}
+{% if enable_tls %}
+    {ssl_listeners, [{{ rabbitmq_tls_port }}]},
+    {ssl_options, [{cacertfile,"{{ cacertfile_dest }}"},
+                   {certfile,"{{ certfile_dest }}"},
+                   {keyfile,"{{ keyfile_dest }}"},
+                   {verify,{{ tls_verify }}},
+                   {fail_if_no_peer_cert,{{tls_fail_if_no_peer_cert|lower}}}]},
+{% endif %}
+    {loopback_users, []}
+  ]}
+].

ansible/setup-offline-sources.yml

@@ -47,7 +47,7 @@
         daemon-reload: yes
 
 - name: Set up offline repositories and remove online ones
-  hosts: k8s-cluster:etcd:restund:cassandra:elasticsearch:minio
+  hosts: k8s-cluster:etcd:restund:cassandra:elasticsearch:minio:rmq-cluster
   tasks:
     - name: Bail if GPG is not installed or installable.
       apt:

bin/offline-cluster.sh

@@ -37,4 +37,5 @@ ansible-playbook -i $ANSIBLE_DIR/inventory/offline/hosts.ini $ANSIBLE_DIR/kubern
 ansible-playbook -i $ANSIBLE_DIR/inventory/offline/hosts.ini $ANSIBLE_DIR/cassandra.yml
 ansible-playbook -i $ANSIBLE_DIR/inventory/offline/hosts.ini $ANSIBLE_DIR/elasticsearch.yml
 ansible-playbook -i $ANSIBLE_DIR/inventory/offline/hosts.ini $ANSIBLE_DIR/restund.yml
-ansible-playbook -i $ANSIBLE_DIR/inventory/offline/hosts.ini $ANSIBLE_DIR/minio.yml
+ansible-playbook -i $ANSIBLE_DIR/inventory/offline/hosts.ini $ANSIBLE_DIR/minio.yml
+ansible-playbook -i $ANSIBLE_DIR/inventory/offline/hosts.ini $ANSIBLE_DIR/rabbitmq.yml