Update docs and CI to support rabbitmq-external chart

ansible/helm_external.yml

@@ -36,12 +36,12 @@
         server_type: cassandra
         network_interface: "{{ cassandra_network_interface }}"
 
-- hosts: "rmq-cluster"
-  become: false
-  tasks:
-    - name: Generate rabbitmq IPs for helm
-      include_tasks: tasks/helm_external.yml
-      vars:
-        external_dir_name: rabbitmq-external
-        server_type: rmq-cluster
-        network_interface: "{{ rabbitmq_network_interface }}"
+# - hosts: "rmq-cluster"
+#   become: false
+#   tasks:
+#     - name: Generate rabbitmq IPs for helm
+#       include_tasks: tasks/helm_external.yml
+#       vars:
+#         external_dir_name: rabbitmq-external
+#         server_type: rmq-cluster
+#         network_interface: "{{ rabbitmq_network_interface }}"

ansible/inventory/offline/99-static

@@ -99,8 +99,8 @@
 # restund_allowed_private_network_cidrs = a.b.c.d/24
 # If you install restund together with other services on the same machine
 # you need to restund_allowed_private_network_cidrs to allow these services
-# to communicate on the private network. E.g. If your private network is 172.16.0.1/24
-# restund_allowed_private_network_cidrs = 172.16.0/24
+# to communicate on the private network. E.g. If your private network is 172.16.0.0/24
+# restund_allowed_private_network_cidrs = '["172.16.0.0/24"]'
 
 # Explicitely specify the restund user id to be "root" to override the default of "997"
 restund_uid = root

ansible/seed-offline-containerd.yml

@@ -1,5 +1,5 @@
 - name: Seed system containers
-  hosts: k8s-cluster:etcd
+  hosts: k8s-cluster
   tags: system-containers
   tasks:
     - name: load containers

bin/offline-cluster.sh

@@ -9,36 +9,60 @@ set -x
 
 ls $ANSIBLE_DIR/inventory/offline
 
+if [ -f "$ANSIBLE_DIR/inventory/offline/hosts.ini" ]
+then
+        INVENTORY_FILE="$ANSIBLE_DIR/inventory/offline/hosts.ini"
+else
+        if [ -f "$ANSIBLE_DIR/inventory/offline/inventory.yml" ]
+        then
+                INVENTORY_FILE="$ANSIBLE_DIR/inventory/offline/inventory.yml"
+        else
+                {
+                        echo "no inventory file in ansible/inventory/offline/. please supply an inventory.yml or hosts.ini."
+                        exit -1
+                }
+        fi
+fi
+
+if [ -f "$ANSIBLE_DIR/inventory/offline/hosts.ini" ]  && [ -f "$ANSIBLE_DIR/inventory/offline/inventory.ymp" ]
+then
+        {
+                echo "both hosts.ini and inventory.yml provided in ansible/inventory/offline! pick only one."
+                exit -1
+        }
+fi
+
+echo "using ansible inventory: $INVENTORY_FILE"
+cat $INVENTORY_FILE
+
 # Populate the assethost, and prepare to install images from it.
 #
 # Copy over binaries and debs, serves assets from the asset host, and configure
 # other hosts to fetch debs from it.
 #
 # If this step fails partway, and you know that parts of it completed, the `--skip-tags debs,binaries,containers,containers-helm,containers-other` tags may come in handy.
-ansible-playbook -i $ANSIBLE_DIR/inventory/offline $ANSIBLE_DIR/setup-offline-sources.yml
+ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/setup-offline-sources.yml
 
 # Run kubespray until docker is installed and runs. This allows us to preseed the docker containers that
 # are part of the offline bundle
-ansible-playbook -i $ANSIBLE_DIR/inventory/offline $ANSIBLE_DIR/kubernetes.yml --tags bastion,bootstrap-os,preinstall,container-engine
+ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/kubernetes.yml --tags bastion,bootstrap-os,preinstall,container-engine
 
 # Install docker on the restund nodes
-ansible-playbook -i $ANSIBLE_DIR/inventory/offline $ANSIBLE_DIR/restund.yml --tags docker
+ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/restund.yml --tags docker
 
 # With ctr being installed on all nodes that need it, seed all container images:
-ansible-playbook -i $ANSIBLE_DIR/inventory/offline $ANSIBLE_DIR/seed-offline-containerd.yml
+ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/seed-offline-containerd.yml
 
 # Install NTP
-ansible-playbook -i $ANSIBLE_DIR/inventory/offline $ANSIBLE_DIR/sync_time.yml -v
+ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/sync_time.yml -v
 
 # Run the rest of kubespray. This should bootstrap a kubernetes cluster successfully:
-ansible-playbook -i $ANSIBLE_DIR/inventory/offline $ANSIBLE_DIR/kubernetes.yml --skip-tags bootstrap-os,preinstall,container-engine
+ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/kubernetes.yml --skip-tags bootstrap-os,preinstall,container-engine
 
 ./bin/fix_default_router.sh
 
 # Deploy all other services which don't run in kubernetes.
-ansible-playbook -i $ANSIBLE_DIR/inventory/offline $ANSIBLE_DIR/cassandra.yml
-ansible-playbook -i $ANSIBLE_DIR/inventory/offline $ANSIBLE_DIR/elasticsearch.yml
-ansible-playbook -i $ANSIBLE_DIR/inventory/offline $ANSIBLE_DIR/restund.yml
-ansible-playbook -i $ANSIBLE_DIR/inventory/offline $ANSIBLE_DIR/minio.yml
-ansible-playbook -i $ANSIBLE_DIR/inventory/offline $ANSIBLE_DIR/rabbitmq.yml
-ansible-playbook -i $ANSIBLE_DIR/inventory/offline $ANSIBLE_DIR/helm_external.yml
+ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/cassandra.yml
+ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/elasticsearch.yml
+ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/minio.yml
+ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/restund.yml

bin/offline-deploy.sh

@@ -18,6 +18,5 @@ WSD_CONTAINER=$(sudo docker load -i $SCRIPT_DIR/../containers-adminhost/containe
 
 ./bin/offline-secrets.sh
 
-
 sudo docker run --network=host -v $SSH_AUTH_SOCK:/ssh-agent -e SSH_AUTH_SOCK=/ssh-agent -v $PWD:/wire-server-deploy $WSD_CONTAINER ./bin/offline-cluster.sh
-sudo docker run --network=host -v $PWD:/wire-server-deploy $WSD_CONTAINER ./bin/offline-helm.sh
+sudo docker run --network=host -v $SSH_AUTH_SOCK:/ssh-agent -e SSH_AUTH_SOCK=/ssh-agent -v $PWD:/wire-server-deploy $WSD_CONTAINER ./bin/offline-helm.sh

bin/offline-helm.sh

@@ -1,6 +1,11 @@
 #!/usr/bin/env bash
 
 set -euo pipefail
+set -x
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+ANSIBLE_DIR="$( cd "$SCRIPT_DIR/../ansible" && pwd )"
+ansible-playbook -i "$ANSIBLE_DIR"/inventory/offline "$ANSIBLE_DIR"/helm_external.yml -vv
 
 helm upgrade --install --wait cassandra-external ./charts/cassandra-external --values ./values/cassandra-external/values.yaml
 helm upgrade --install --wait elasticsearch-external ./charts/elasticsearch-external --values ./values/elasticsearch-external/values.yaml

bin/offline-vm-setup.sh

@@ -67,7 +67,7 @@ create_node () {
     sudo virt-install \
         --name "$name" \
         --ram 8192 \
-        --disk path=/var/kvm/images/"$name".img,size=80 \
+        --disk path=/var/kvm/images/"$name".img,size=100 \
         --vcpus 6 \
         --network bridge=br0 \
         --graphics none \

offline/ci.sh

@@ -92,6 +92,7 @@ charts=(
   wire/sftd
   wire/restund
   wire/rabbitmq
+  wire/rabbitmq-external
   # Has a weird dependency on curl:latest. out of scope
   # wire-server-metrics
   # fluent-bit

offline/docs_ubuntu_22.04.md

@@ -280,19 +280,6 @@ the `wire.com/external-ip` annotation to the public IP of the node.
 
 In order to automatically generate deeplinks, Edit the minio variables in `[minio:vars]` (`prefix`, `domain` and `deeplink_title`) by replacing `example.com` with your own domain.
 
-### Configuring rabbitmq
-
-Add the nodes in which you want to run rabbitmq to the `[rmq-cluster]` group. Also, update the `ansible/roles/rabbimq-cluster/defaults/main.yml` file with the correct configurations for your environment.
-
-Important: RabbitMQ nodes address each other using a node name, for e.g rabbitmq@ansnode1
-Please refer to official doc and configure your DNS based on the setup - https://www.rabbitmq.com/clustering.html#cluster-formation-requirements
-
-For adding entries to local host file(/etc/hosts), run
-```
-d ansible-playbook -i ansible/inventory/offline/hosts.ini ansible/roles/rabbitmq-cluster/tasks/configure_dns.yml
-```
-
-
 
 ### Example hosts.ini
 
@@ -327,7 +314,10 @@ deeplink_title = "wire demo environment, example.com"
 
 [restund:vars]
 restund_uid = root
-restund_allowed_private_network_cidrs=172.16.0.1/24
+restund_allowed_private_network_cidrs='["172.16.0.0/24"]'
+
+[rmq-cluster:vars]
+rabbitmq_network_interface = enp1s0
 
 [kube-master]
 kubenode1
@@ -394,14 +384,6 @@ Minio and restund services have shared secrets with the `wire-server` helm chart
 
 This should generate two files. `./ansible/inventory/group_vars/all/secrets.yaml` and `values/wire-server/secrets.yaml`.
 
-## Deploying Kubernetes, Restund and stateful services
-
-In order to deploy all the services run:
-```
-d ./bin/offline-cluster.sh
-```
-In case any of the steps in this script fail, see the notes in the comments that accompany each step.
-Comment out steps that have already completed when re-running the scripts.
 
 #### Ensuring kubernetes is healthy.
 
@@ -460,6 +442,25 @@ Hash: SHA1, RIPEMD160, SHA256, SHA384, SHA512, SHA224
 Compression: Uncompressed, ZIP, ZLIB, BZIP2
 ```
 
+## Deploying Kubernetes, Restund and stateful services
+
+In order to deploy all mentioned services, run:
+```
+d ./bin/offline-cluster.sh
+```
+In case any of the steps in this script fail, see the notes in the comments that accompany each step.
+Comment out steps that have already completed when re-running the scripts.
+
+#### Ensuring Kubernetes is healthy.
+
+Ensure the cluster comes up healthy. The container also contains `kubectl`, so check the node status:
+
+```
+d kubectl get nodes -owide
+```
+They should all report ready.
+
+
 #### Troubleshooting restund
 
 In case the restund firewall fails to start. Fix
@@ -507,41 +508,32 @@ ufw allow 25672/tcp;
 '
 ```
 
+### Preparation for Federation
+For enabling Federation, we need to have RabbitMQ in place. Please follow the instructions in [offline/federation_preparation.md](./federation_preparation.md) for setting up RabbitMQ.
+
+After that continue to the next steps below.
+
+
+### Preparing helm values for external services
 Afterwards, run the following playbook to create helm values that tell our helm charts
 what the IP addresses of cassandra, elasticsearch, minio and rabbitmq are.
 
 ```
 d ansible-playbook -i ./ansible/inventory/offline/hosts.ini ansible/helm_external.yml
 ```
 
-#### Installing Rabbitmq
-
-To install the rabbitmq,
-First copy the value and secret file:
-```
-cp ./values/rabbitmq/prod-values.example.yaml ./values/rabbitmq/values.yaml
-cp ./values/rabbitmq/prod-secrets.example.yaml ./values/rabbitmq/secrets.yaml
-```
-
-Now, update the `./values/rabbitmq/values.yaml` and `./values/rabbitmq/secrets.yaml` with correct values as per needed.
-
-Deploy the rabbitmq helm chart -
-```
-d helm upgrade --install rabbitmq ./charts/rabbitmq --values ./values/rabbitmq/values.yaml --values ./values/rabbitmq/secrets.yaml
-```
 
 ### Deploying Wire
 
 It's now time to deploy the helm charts on top of kubernetes, installing the Wire platform.
 
 #### Finding the stateful services
-First.  Make kubernetes aware of where alll the external stateful services are by running:
+First, setup interfaces from Kubernetes to external services by running:
 
 ```
 d helm install cassandra-external ./charts/cassandra-external --values ./values/cassandra-external/values.yaml
 d helm install elasticsearch-external ./charts/elasticsearch-external --values ./values/elasticsearch-external/values.yaml
 d helm install minio-external ./charts/minio-external --values ./values/minio-external/values.yaml
-d helm install rabbitmq-external ./charts/rabbitmq-external --values ./values/rabbitmq-external/values.yaml
 ```
 
 #### Deploying stateless dependencies

offline/federation_preparation.md

@@ -0,0 +1,85 @@
+## RabbitMQ
+
+There are two methods to deploy the RabbitMQ cluster:
+
+### Method 1: Install RabbitMQ inside kubernetes cluster with the help of helm chart
+
+To install the RabbitMQ service, first copy the value and secret files:
+```
+cp ./values/rabbitmq/prod-values.example.yaml ./values/rabbitmq/values.yaml
+cp ./values/rabbitmq/prod-secrets.example.yaml ./values/rabbitmq/secrets.yaml
+```
+By default this will create a RabbitMQ deployment with ephemeral storage. To use the local persistence storage of Kubernetes nodes, please refer to the related documentation in [offline/local_persistent_storage_k8s.md](./local_persistent_storage_k8s.md).
+
+Now, update the `./values/rabbitmq/values.yaml` and `./values/rabbitmq/secrets.yaml` with correct values as needed.
+
+Deploy the `rabbitmq` helm chart:
+```
+d helm upgrade --install rabbitmq ./charts/rabbitmq --values ./values/rabbitmq/values.yaml --values ./values/rabbitmq/secrets.yaml
+```
+
+### Method 2: Install RabbitMQ outside of the Kubernetes cluster with an Ansible playbook
+
+Add the nodes on which you want to run rabbitmq to the `[rmq-cluster]` group in the `ansible/inventory/offline/hosts.ini` file. Also, update the `ansible/roles/rabbitmq-cluster/defaults/main.yml` file with the correct configurations for your environment.
+
+If you need RabbitMQ to listen on a different interface than the default gateway, set `rabbitmq_network_interface`
+
+You should have following entries in the `/ansible/inventory/offline/hosts.ini` file. For example:
+```
+[rmq-cluster:vars]
+rabbitmq_network_interface = enp1s0
+
+[rmq-cluster]
+ansnode1
+ansnode2
+ansnode3
+```
+
+**Important:** RabbitMQ nodes address each other using a node name, for e.g rabbitmq@ansnode1
+Please refer to the official documentation and configure your DNS based on the setup - https://www.rabbitmq.com/clustering.html#cluster-formation-requirements
+
+
+For adding entries to local host file(`/etc/hosts`), run
+```
+d ansible-playbook -i ansible/inventory/offline/hosts.ini ansible/roles/rabbitmq-cluster/tasks/configure_dns.yml
+```
+
+Create the rabbitmq cluster:
+
+``` 
+d ansible-playbook -i ansible/inventory/offline/hosts.ini ansible/rabbitmq.yml
+```
+
+Uncomment the following section, in the `ansible/helm_external.yml` file:
+```
+# - hosts: "rmq-cluster"
+#   become: false
+#   tasks:
+#     - name: Generate rabbitmq IPs for helm
+#       include_tasks: tasks/helm_external.yml
+#       vars:
+#         external_dir_name: rabbitmq-external
+#         server_type: rmq-cluster
+#         network_interface: "{{ rabbitmq_network_interface }}"
+
+```
+
+and run the following playbook to create values file for helm charts to look for RabbitMQ IP addresses -
+
+```
+d ansible-playbook -i ./ansible/inventory/offline/hosts.ini ansible/helm_external.yml
+```
+
+Make Kubernetes aware of where RabbitMQ external stateful service is running:
+```
+d helm install rabbitmq-external ./charts/rabbitmq-external --values ./values/rabbitmq-external/values.yaml
+```
+
+Configure wire-server to use the external RabbitMQ service:
+
+Edit the `/values/wire-server/prod-values.yaml` file to update the RabbitMQ host
+Under `brig` and `galley` section, you will find the `rabbitmq` config, update the host to `rabbitmq-external`, it should look like this:
+```
+rabbitmq:
+  host: rabbitmq-external
+``` 

values/wire-server/prod-values.example.yaml

@@ -28,7 +28,7 @@ brig:
     elasticsearch:
       host: elasticsearch-external
     rabbitmq:
-      host: rabbitmq-external
+      host: rabbitmq
     useSES: false
     # Set to false if you want to hand out DynamoDB to store prekeys
     randomPrekeys: true
@@ -141,7 +141,7 @@ galley:
     cassandra:
       host: cassandra-external
     rabbitmq:
-      host: rabbitmq-external
+      host: rabbitmq
     settings:
       # prefix URI used when inviting users to a conversation by link
       conversationCodeURI: https://account.example.com/conversation-join/ # change this