From 7e657ae9fa6c9c0efa471461af200aca15f676b7 Mon Sep 17 00:00:00 2001 From: Arthur Wolf Date: Wed, 11 Oct 2023 18:15:16 +0200 Subject: [PATCH 1/5] Update 99-static --- ansible/inventory/offline/99-static | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/inventory/offline/99-static b/ansible/inventory/offline/99-static index 1e8593548..a7b927751 100644 --- a/ansible/inventory/offline/99-static +++ b/ansible/inventory/offline/99-static @@ -100,7 +100,7 @@ # If you install restund together with other services on the same machine # you need to restund_allowed_private_network_cidrs to allow these services # to communicate on the private network. E.g. If your private network is 172.16.0.1/24 -# restund_allowed_private_network_cidrs = 172.16.0/24 +# restund_allowed_private_network_cidrs = '["172.16.0.0/24"]' # Explicitely specify the restund user id to be "root" to override the default of "997" restund_uid = root From 95b2e181707c4c6cb5e2e8c8884efaccf19bb2f1 Mon Sep 17 00:00:00 2001 From: Arthur Wolf Date: Wed, 11 Oct 2023 18:15:56 +0200 Subject: [PATCH 2/5] Update seed-offline-containerd.yml --- ansible/seed-offline-containerd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/seed-offline-containerd.yml b/ansible/seed-offline-containerd.yml index d0f0a2d3a..1af9a3a2a 100644 --- a/ansible/seed-offline-containerd.yml +++ b/ansible/seed-offline-containerd.yml @@ -1,5 +1,5 @@ - name: Seed system containers - hosts: k8s-cluster:etcd + hosts: k8s-cluster tags: system-containers tasks: - name: load containers From b54c15ec417e6135803a00c85111e9e5547427ed Mon Sep 17 00:00:00 2001 From: Arthur Wolf Date: Wed, 11 Oct 2023 18:16:57 +0200 Subject: [PATCH 3/5] Update offline-cluster.sh --- bin/offline-cluster.sh | 49 +++++++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/bin/offline-cluster.sh b/bin/offline-cluster.sh index c25d240d7..264558b90 100755 --- a/bin/offline-cluster.sh +++ b/bin/offline-cluster.sh @@ -9,36 +9,61 @@ set -x ls $ANSIBLE_DIR/inventory/offline +if [ -f "$ANSIBLE_DIR/inventory/offline/hosts.ini" ] +then + INVENTORY_FILE="$ANSIBLE_DIR/inventory/offline/hosts.ini" +else + if [ -f "$ANSIBLE_DIR/inventory/offline/inventory.yml" ] + then + INVENTORY_FILE="$ANSIBLE_DIR/inventory/offline/inventory.yml" + else + { + echo "no inventory file in ansible/inventory/offline/. please supply an inventory.yml or hosts.ini." + exit -1 + } + fi +fi + +if [ -f "$ANSIBLE_DIR/inventory/offline/hosts.ini" ] && [ -f "$ANSIBLE_DIR/inventory/offline/inventory.ymp" ] +then + { + echo "both hosts.ini and inventory.yml provided in ansible/inventory/offline! pick only one." + exit -1 + } +fi + +echo "using ansible inventory: $INVENTORY_FILE" + # Populate the assethost, and prepare to install images from it. # # Copy over binaries and debs, serves assets from the asset host, and configure # other hosts to fetch debs from it. # # If this step fails partway, and you know that parts of it completed, the `--skip-tags debs,binaries,containers,containers-helm,containers-other` tags may come in handy. -ansible-playbook -i $ANSIBLE_DIR/inventory/offline $ANSIBLE_DIR/setup-offline-sources.yml +ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/setup-offline-sources.yml # Run kubespray until docker is installed and runs. This allows us to preseed the docker containers that # are part of the offline bundle -ansible-playbook -i $ANSIBLE_DIR/inventory/offline $ANSIBLE_DIR/kubernetes.yml --tags bastion,bootstrap-os,preinstall,container-engine +ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/kubernetes.yml --tags bastion,bootstrap-os,preinstall,container-engine # Install docker on the restund nodes -ansible-playbook -i $ANSIBLE_DIR/inventory/offline $ANSIBLE_DIR/restund.yml --tags docker +ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/restund.yml --tags docker # With ctr being installed on all nodes that need it, seed all container images: -ansible-playbook -i $ANSIBLE_DIR/inventory/offline $ANSIBLE_DIR/seed-offline-containerd.yml +ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/seed-offline-containerd.yml # Install NTP -ansible-playbook -i $ANSIBLE_DIR/inventory/offline $ANSIBLE_DIR/sync_time.yml -v +ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/sync_time.yml -v # Run the rest of kubespray. This should bootstrap a kubernetes cluster successfully: -ansible-playbook -i $ANSIBLE_DIR/inventory/offline $ANSIBLE_DIR/kubernetes.yml --skip-tags bootstrap-os,preinstall,container-engine +ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/kubernetes.yml --skip-tags bootstrap-os,preinstall,container-engine ./bin/fix_default_router.sh # Deploy all other services which don't run in kubernetes. -ansible-playbook -i $ANSIBLE_DIR/inventory/offline $ANSIBLE_DIR/cassandra.yml -ansible-playbook -i $ANSIBLE_DIR/inventory/offline $ANSIBLE_DIR/elasticsearch.yml -ansible-playbook -i $ANSIBLE_DIR/inventory/offline $ANSIBLE_DIR/restund.yml -ansible-playbook -i $ANSIBLE_DIR/inventory/offline $ANSIBLE_DIR/minio.yml -ansible-playbook -i $ANSIBLE_DIR/inventory/offline $ANSIBLE_DIR/rabbitmq.yml -ansible-playbook -i $ANSIBLE_DIR/inventory/offline $ANSIBLE_DIR/helm_external.yml +ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/cassandra.yml +ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/elasticsearch.yml +ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/restund.yml +ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/minio.yml +ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/rabbitmq.yml +ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/helm_external.yml From d1eb457310e3abbcb02b088b8f72a74b14cd4493 Mon Sep 17 00:00:00 2001 From: Arthur Wolf Date: Wed, 11 Oct 2023 18:20:05 +0200 Subject: [PATCH 4/5] Update docs_ubuntu_22.04.md --- offline/docs_ubuntu_22.04.md | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/offline/docs_ubuntu_22.04.md b/offline/docs_ubuntu_22.04.md index 3451b27a9..21b0ad660 100644 --- a/offline/docs_ubuntu_22.04.md +++ b/offline/docs_ubuntu_22.04.md @@ -327,7 +327,10 @@ deeplink_title = "wire demo environment, example.com" [restund:vars] restund_uid = root -restund_allowed_private_network_cidrs=172.16.0.1/24 +restund_allowed_private_network_cidrs='["172.16.0.1/24"]' + +[rmq-cluster:vars] +rabbitmq_network_interface = enp1s0 [kube-master] kubenode1 @@ -396,6 +399,8 @@ This should generate two files. `./ansible/inventory/group_vars/all/secrets.yaml ## Deploying Kubernetes, Restund and stateful services +NOTE: Before running `d ./bin/offline-cluster.sh`, comment out the call to `./bin/fix_default_router.sh` (this call/script dsiables DNS resolution in the cluster). If you do not comment out this line, notifications will not occur. + In order to deploy all the services run: ``` d ./bin/offline-cluster.sh @@ -513,21 +518,11 @@ what the IP addresses of cassandra, elasticsearch, minio and rabbitmq are. d ansible-playbook -i ./ansible/inventory/offline/hosts.ini ansible/helm_external.yml ``` -#### Installing Rabbitmq +### Preparation for Federation -To install the rabbitmq, -First copy the value and secret file: -``` -cp ./values/rabbitmq/prod-values.example.yaml ./values/rabbitmq/values.yaml -cp ./values/rabbitmq/prod-secrets.example.yaml ./values/rabbitmq/secrets.yaml -``` +For enabling Federation, we need to have RabbitMQ in place. Please follow the instructions in [offline/federation_preparation.md](./federation_preparation.md) for setting up RabbitMQ. -Now, update the `./values/rabbitmq/values.yaml` and `./values/rabbitmq/secrets.yaml` with correct values as per needed. - -Deploy the rabbitmq helm chart - -``` -d helm upgrade --install rabbitmq ./charts/rabbitmq --values ./values/rabbitmq/values.yaml --values ./values/rabbitmq/secrets.yaml -``` +After that continue to the next steps below. ### Deploying Wire From 7037769ae8b6d3ad1518ebd5e2e42a420bc41dfb Mon Sep 17 00:00:00 2001 From: Arthur Wolf Date: Wed, 11 Oct 2023 18:20:55 +0200 Subject: [PATCH 5/5] Create federation_preparation.md --- offline/federation_preparation.md | 85 +++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 offline/federation_preparation.md diff --git a/offline/federation_preparation.md b/offline/federation_preparation.md new file mode 100644 index 000000000..9e31c686d --- /dev/null +++ b/offline/federation_preparation.md @@ -0,0 +1,85 @@ +## RabbitMQ + +There are two methods to deploy the RabbitMQ cluster: + +### Method 1: Install RabbitMQ inside kubernetes cluster with the help of helm chart + +To install the RabbitMQ service, first copy the value and secret files: +``` +cp ./values/rabbitmq/prod-values.example.yaml ./values/rabbitmq/values.yaml +cp ./values/rabbitmq/prod-secrets.example.yaml ./values/rabbitmq/secrets.yaml +``` +By default this will create a RabbitMQ deployment with ephemeral storage. To use the local persistence storage of Kubernetes nodes, please refer to the related documentation in [offline/local_persistent_storage_k8s.md](./local_persistent_storage_k8s.md). + +Now, update the `./values/rabbitmq/values.yaml` and `./values/rabbitmq/secrets.yaml` with correct values as needed. + +Deploy the `rabbitmq` helm chart: +``` +d helm upgrade --install rabbitmq ./charts/rabbitmq --values ./values/rabbitmq/values.yaml --values ./values/rabbitmq/secrets.yaml +``` + +### Method 2: Install RabbitMQ outside of the Kubernetes cluster with an Ansible playbook + +Add the nodes on which you want to run rabbitmq to the `[rmq-cluster]` group in the `ansible/inventory/offline/hosts.ini` file. Also, update the `ansible/roles/rabbitmq-cluster/defaults/main.yml` file with the correct configurations for your environment. + +If you need RabbitMQ to listen on a different interface than the default gateway, set `rabbitmq_network_interface` + +You should have following entries in the `/ansible/inventory/offline/hosts.ini` file. For example: +``` +[rmq-cluster:vars] +rabbitmq_network_interface = enp1s0 + +[rmq-cluster] +ansnode1 +ansnode2 +ansnode3 +``` + +**Important:** RabbitMQ nodes address each other using a node name, for e.g rabbitmq@ansnode1 +Please refer to the official documentation and configure your DNS based on the setup - https://www.rabbitmq.com/clustering.html#cluster-formation-requirements + + +For adding entries to local host file(`/etc/hosts`), run +``` +d ansible-playbook -i ansible/inventory/offline/hosts.ini ansible/roles/rabbitmq-cluster/tasks/configure_dns.yml +``` + +Create the rabbitmq cluster: + +``` +d ansible-playbook -i ansible/inventory/offline/hosts.ini ansible/rabbitmq.yml +``` + +Uncomment the following section, in the `ansible/helm_external.yml` file: +``` +# - hosts: "rmq-cluster" +# become: false +# tasks: +# - name: Generate rabbitmq IPs for helm +# include_tasks: tasks/helm_external.yml +# vars: +# external_dir_name: rabbitmq-external +# server_type: rmq-cluster +# network_interface: "{{ rabbitmq_network_interface }}" + +``` + +and run the following playbook to create values file for helm charts to look for RabbitMQ IP addresses - + +``` +d ansible-playbook -i ./ansible/inventory/offline/hosts.ini ansible/helm_external.yml +``` + +Make Kubernetes aware of where RabbitMQ external stateful service is running: +``` +d helm install rabbitmq-external ./charts/rabbitmq-external --values ./values/rabbitmq-external/values.yaml +``` + +Configure wire-server to use the external RabbitMQ service: + +Edit the `/values/wire-server/prod-values.yaml` file to update the RabbitMQ host +Under `brig` and `galley` section, you will find the `rabbitmq` config, update the host to `rabbitmq-external`, it should look like this: +``` +rabbitmq: + host: rabbitmq-external +```