From 5144e5339ee25f19bfbb3ed5287d0c83809c992e Mon Sep 17 00:00:00 2001 From: Amit Sagtani Date: Thu, 10 Oct 2024 12:25:50 +0200 Subject: [PATCH 1/7] remove ansible-restund submodules --- .gitmodules | 3 --- ansible/roles-external/ansible-restund | 1 - 2 files changed, 4 deletions(-) delete mode 160000 ansible/roles-external/ansible-restund diff --git a/.gitmodules b/.gitmodules index f747a0e2a..6a2fe1249 100644 --- a/.gitmodules +++ b/.gitmodules @@ -22,9 +22,6 @@ [submodule "ansible/roles-external/ansible-minio"] path = ansible/roles-external/ansible-minio url = https://github.com/wireapp/ansible-minio.git -[submodule "ansible/roles-external/ansible-restund"] - path = ansible/roles-external/ansible-restund - url = https://github.com/wireapp/ansible-restund.git [submodule "ansible/roles-external/ansible-tinc"] path = ansible/roles-external/ansible-tinc url = https://github.com/wireapp/ansible-tinc.git diff --git a/ansible/roles-external/ansible-restund b/ansible/roles-external/ansible-restund deleted file mode 160000 index 8feeb7e9c..000000000 --- a/ansible/roles-external/ansible-restund +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8feeb7e9cfd5ac98020c9bc5397662c75960ff53 From 7828462a2c16929dcffaf01b925aec82ecbb18a2 Mon Sep 17 00:00:00 2001 From: Amit Sagtani Date: Thu, 10 Oct 2024 12:28:38 +0200 Subject: [PATCH 2/7] remove ansible.yml playbook --- ansible/restund.yml | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100644 ansible/restund.yml diff --git a/ansible/restund.yml b/ansible/restund.yml deleted file mode 100644 index 829bd57a2..000000000 --- a/ansible/restund.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Reminder that the pem file should look like: -# -----BEGIN CERTIFICATE----- -# --- ... CERT CONTENT ... -- -# -----END CERTIFICATE----- -# -----BEGIN CERTIFICATE----- -# --- ... INTERMEDIATE ..---- -# -----END CERTIFICATE---- -# -----BEGIN PRIVATE KEY----- -# --- .... PRIV KEY ----- -# -----END PRIVATE KEY----- -- name: provision - hosts: restund - gather_facts: yes - become: yes - any_errors_fatal: True - environment: "{{ proxy_env | default({}) }}" - vars: - # This config will make restund run as root and listen on ports 80 and 443 - - restund_user: root - # - restund_tls_certificate: "{{ lookup('file', '/tmp/tls_cert_and_priv_key.pem') }}" - # - restund_udp_listen_port: 80 # deploy to port 3478 by default - # - restund_tcp_listen_port: 80 # deploy to port 3478 by default - # - restund_tls_listen_port: 443 - roles: - - role: ansible-restund - tags: - - restund From 6d40c7c7a0e9ec626d8d480f8037644cb8b4e1ba Mon Sep 17 00:00:00 2001 From: Amit Sagtani Date: Thu, 10 Oct 2024 14:35:43 +0200 Subject: [PATCH 3/7] remove restund references and code --- ansible/inventory/offline/99-static | 30 ------------------- ansible/inventory/prod/hosts.example.ini | 17 ----------- ansible/seed-offline-containerd.yml | 11 ------- ansible/seed-offline-docker.yml | 10 ------- ansible/setup-offline-sources.yml | 2 +- bin/autodeploy.sh | 4 +-- bin/offline-cluster.sh | 4 --- bin/offline-secrets.sh | 1 - offline/ci.sh | 6 ---- offline/coturn.md | 24 +++------------ terraform/examples/create-infrastructure.tf | 13 -------- terraform/examples/inventory.tpl | 5 ---- .../main.tf | 23 -------------- .../outputs.tf | 12 -------- values/wire-server/prod-values.example.yaml | 10 +++---- 15 files changed, 11 insertions(+), 161 deletions(-) diff --git a/ansible/inventory/offline/99-static b/ansible/inventory/offline/99-static index 30cbeb44b..c219bab68 100644 --- a/ansible/inventory/offline/99-static +++ b/ansible/inventory/offline/99-static @@ -15,9 +15,6 @@ # You could add more if capacity is needed # kubenode4 .... -# restund1 ansible_host=XXXX -# restund2 ansible_host=XXXX - # cassandra1 ansible_host=XXXX # cassandra2 ansible_host=XXXX # cassandra3 ansible_host=XXXX @@ -96,28 +93,6 @@ [rmq-cluster:vars] # rabbitmq_network_interface = enp1s0 -[restund:vars] -# Uncomment if your public IP is not on the default gateway -# restund_network_interface = enp1s0 -# Uncomment and set to the true public IP if you are behind 1:1 NAT -# restund_peer_udp_advertise_addr = a.b.c.d -# -# Uncomment to create firewall exception for private networks -# restund_allowed_private_network_cidrs = a.b.c.d/24 -# If you install restund together with other services on the same machine -# you need to restund_allowed_private_network_cidrs to allow these services -# to communicate on the private network. E.g. If your private network is 172.16.0.0/24 -# restund_allowed_private_network_cidrs = '["172.16.0.0/24"]' - -# Explicitely specify the restund user id to be "root" to override the default of "997" -restund_uid = root - -# For the following groups, add all nodes defined above to the sections below. -# Define any additional variables that should be set for these nodes. - -# Uncomment this is you use the bastion host -# [bastion] -# bastion # Add all nodes that should be the master [kube-master] @@ -156,11 +131,6 @@ restund_uid = root kube-master kube-node -[restund] - -# restund1 -# restund2 - # Add all cassandra nodes here [cassandra] # cassandra1 diff --git a/ansible/inventory/prod/hosts.example.ini b/ansible/inventory/prod/hosts.example.ini index b0af9e887..b095547f7 100644 --- a/ansible/inventory/prod/hosts.example.ini +++ b/ansible/inventory/prod/hosts.example.ini @@ -9,13 +9,6 @@ minio01 ansible_host=X.X.X.X minio02 ansible_host=X.X.X.X minio03 ansible_host=X.X.X.X -# * 'ansible_host' is the IP to ssh into -# * set restund_network_interface to the interface that you want the process to bind to in the [all:vars] section -# * Optional: 'restund_peer_udp_advertise_addr' is the public IP to advertise for other turn servers if different than the ip on the 'restund_network_interface' -# If using 'restund_peer_udp_advertise_addr', make sure that UDP (!) traffic from any restund server (including itself) -# can reach that IP (for restund->restund communication) -restund01 ansible_host=X.X.X.X -restund02 ansible_host=X.X.X.X # * 'ansible_host' is the IP to ssh into # * 'ip' is the IP to bind to (if multiple network interfaces are in use) @@ -65,14 +58,6 @@ prefix = "example-" domain = "example.com" deeplink_title = "example.com environment" -[restund] -restund01 -restund02 - -[restund:vars] -## Set the network interface name for restund to bind to if you have more than one network interface -## If unset, defaults to the ansible_default_ipv4 (if defined) otherwise to eth0 -# restund_network_interface = eth0 ### KUBERNETES ### @@ -141,8 +126,6 @@ is_aws_environment = False ## Set this to a name of a network interface (e.g. 'eth0'), on which you wish minio processes to talk to each other. # minio_network_interface = "ens123" -### RESTUND section ### -# restund_network_interface = "..." ### KUBERNETES section (see kubespray documentation for details) ### diff --git a/ansible/seed-offline-containerd.yml b/ansible/seed-offline-containerd.yml index 85bc50b6f..8cfc7a9b0 100644 --- a/ansible/seed-offline-containerd.yml +++ b/ansible/seed-offline-containerd.yml @@ -30,14 +30,3 @@ # sudo ctr -n=k8s.io images tag registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20220916-gd32f8c343 registry.k8s.io/ingress-nginx/kube-webhook-certgen@sha256:39c5b2e3310dc4264d638ad28d9d1d96c4cbb2b2dcfb52368fe4e3c63f61e10f # sudo ctr -n=k8s.io images tag registry.k8s.io/ingress-nginx/controller:v1.6.4 registry.k8s.io/ingress-nginx/controller:v1.6.4@sha256:15be4666c53052484dd2992efacf2f50ea77a78ae8aa21ccd91af6baaa7ea22f #################################################################################################### - - -- name: Download restund container - hosts: restund - tags: restund-containers - tasks: - - name: load restund container - shell: | - for container in $(curl -q {{ assethost_host }}/containers-other/index.txt);do - curl -q "{{ assethost_host }}/containers-other/$container" | ctr -n=k8s.io images import - - done diff --git a/ansible/seed-offline-docker.yml b/ansible/seed-offline-docker.yml index 2f98016b9..7109b059a 100644 --- a/ansible/seed-offline-docker.yml +++ b/ansible/seed-offline-docker.yml @@ -17,13 +17,3 @@ for container in $(curl -q {{ assethost_host }}/containers-helm/index.txt);do curl -q "{{ assethost_host }}/containers-helm/$container" | docker load done - -- name: Download restund container - hosts: restund - tags: restund-containers - tasks: - - name: load containers - shell: | - for container in $(curl -q {{ assethost_host }}/containers-other/index.txt);do - curl -q "{{ assethost_host }}/containers-other/$container" | docker load - done diff --git a/ansible/setup-offline-sources.yml b/ansible/setup-offline-sources.yml index 986dad22a..6a0eaf468 100644 --- a/ansible/setup-offline-sources.yml +++ b/ansible/setup-offline-sources.yml @@ -47,7 +47,7 @@ daemon-reload: yes - name: Set up offline repositories and remove online ones - hosts: k8s-cluster:etcd:restund:cassandra:elasticsearch:minio:rmq-cluster + hosts: k8s-cluster:etcd:cassandra:elasticsearch:minio:rmq-cluster tasks: - name: Bail if GPG is not installed or installable. apt: diff --git a/bin/autodeploy.sh b/bin/autodeploy.sh index 7c6bfd18b..c602e91ce 100755 --- a/bin/autodeploy.sh +++ b/bin/autodeploy.sh @@ -306,8 +306,8 @@ ufw allow 25672/tcp; cp values/wire-server/prod-values.example.yaml values/wire-server/values.yaml sed -i "s/example.com/$TARGET_SYSTEM/g" values/wire-server/values.yaml - sed -i "s/# - \"turn::80\"/- \"turn:$HOST_IP:3478\"/g" values/wire-server/values.yaml - sed -i "s/# - \"turn::80?transport=tcp\"/- \"turn:$HOST_IP:3478?transport=tcp\"/g" values/wire-server/values.yaml + sed -i "s/# - \"turn::3478\"/- \"turn:$HOST_IP:3478\"/g" values/wire-server/values.yaml + sed -i "s/# - \"turn::3478?transport=tcp\"/- \"turn:$HOST_IP:3478?transport=tcp\"/g" values/wire-server/values.yaml d helm install wire-server ./charts/wire-server --timeout=15m0s --values ./values/wire-server/values.yaml --values ./values/wire-server/secrets.yaml diff --git a/bin/offline-cluster.sh b/bin/offline-cluster.sh index fb263f19d..882224018 100755 --- a/bin/offline-cluster.sh +++ b/bin/offline-cluster.sh @@ -37,9 +37,6 @@ ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/setup-offline-sources.yml # are part of the offline bundle ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/kubernetes.yml --tags bastion,bootstrap-os,preinstall,container-engine -# Install docker on the restund nodes -ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/restund.yml --tags docker - # With ctr being installed on all nodes that need it, seed all container images: ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/seed-offline-containerd.yml @@ -53,7 +50,6 @@ ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/kubernetes.yml --skip-tags boot ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/cassandra.yml ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/elasticsearch.yml ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/minio.yml -ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/restund.yml # create helm values that tell our helm charts what the IP addresses of cassandra, elasticsearch and minio are: ansible-playbook -i $INVENTORY_FILE $ANSIBLE_DIR/helm_external.yml --skip-tags=rabbitmq-external diff --git a/bin/offline-secrets.sh b/bin/offline-secrets.sh index d58d48072..255cbeece 100755 --- a/bin/offline-secrets.sh +++ b/bin/offline-secrets.sh @@ -75,7 +75,6 @@ fi if [[ ! -f $ANSIBLE_DIR/inventory/offline/group_vars/all/secrets.yaml ]]; then echo "Writing $ANSIBLE_DIR/inventory/offline/group_vars/all/secrets.yaml" cat << EOT > $ANSIBLE_DIR/inventory/offline/group_vars/all/secrets.yaml -restund_zrest_secret: "$zrest" minio_access_key: "$minio_access_key" minio_secret_key: "$minio_secret_key" EOT diff --git a/offline/ci.sh b/offline/ci.sh index 3160607c8..0d69e7793 100755 --- a/offline/ci.sh +++ b/offline/ci.sh @@ -105,11 +105,6 @@ list-system-containers | create-container-dump containers-system tar cf containers-system.tar containers-system [[ "$INCREMENTAL" -eq 0 ]] && rm -r containers-system -# Used for ansible-restund role -echo "quay.io/wire/restund:v0.6.0-rc.2" | create-container-dump containers-other -tar cf containers-other.tar containers-other -[[ "$INCREMENTAL" -eq 0 ]] && rm -r containers-other - legacy_chart_release() { # Note: if you want to ship from the develop branch, replace 'repo' url below # repo=https://s3-eu-west-1.amazonaws.com/public.wire.com/charts-develop @@ -143,7 +138,6 @@ legacy_chart_release() { calling_charts=( sftd - restund coturn ) for chartName in "${calling_charts[@]}"; do diff --git a/offline/coturn.md b/offline/coturn.md index 353d6c583..a43ccb35e 100644 --- a/offline/coturn.md +++ b/offline/coturn.md @@ -368,26 +368,10 @@ You will find a section that looks like this (default): turnStatic: v1: [] v2: - # - "turn::80" - # - "turn::80?transport=tcp" - # - "turn::80?transport=tcp" - # - "turns::443?transport=tcp" - # - "turns::443?transport=tcp" - -``` - -Or if you have already configured Restund, something like this: - -```yaml - - turnStatic: - v1: [] - v2: - - "turn::80" - - "turn::80" - - "turn::80?transport=tcp" - - "turn::80?transport=tcp" + # - "turn::3478" + # - "turn::3478" + # - "turn::3478?transport=tcp" + # - "turn::3478?transport=tcp" ``` diff --git a/terraform/examples/create-infrastructure.tf b/terraform/examples/create-infrastructure.tf index 2d74ac9ec..94147ddcf 100644 --- a/terraform/examples/create-infrastructure.tf +++ b/terraform/examples/create-infrastructure.tf @@ -49,17 +49,6 @@ resource "hcloud_server" "redis" { location = "nbg1" } -resource "hcloud_server" "restund" { - count = 2 - name = "restund${count.index}" - image = "ubuntu-22.04" - server_type = "cx22" - ssh_keys = ["hetznerssh-key"] - - # Nuremberg (for choices see `hcloud datacenter list`) - location = "nbg1" -} - resource "hcloud_server" "minio" { count = 3 name = "minio${count.index}" @@ -152,7 +141,6 @@ data "template_file" "inventory" { connection_strings_elasticsearch = "${join("\n", formatlist("%s ansible_host=%s vpn_ip=%s", hcloud_server.elasticsearch.*.name, hcloud_server.elasticsearch.*.ipv4_address, null_resource.vpnes.*.triggers.ip))}" connection_strings_minio = "${join("\n", formatlist("%s ansible_host=%s vpn_ip=%s", hcloud_server.minio.*.name, hcloud_server.minio.*.ipv4_address, null_resource.vpnminio.*.triggers.ip))}" connection_strings_redis = "${join("\n", formatlist("%s ansible_host=%s vpn_ip=%s", hcloud_server.redis.*.name, hcloud_server.redis.*.ipv4_address, null_resource.vpnredis.*.triggers.ip))}" - connection_strings_restund = "${join("\n", formatlist("%s ansible_host=%s", hcloud_server.restund.*.name, hcloud_server.restund.*.ipv4_address))}" list_master = "${join("\n",hcloud_server.node.*.name)}" list_etcd = "${join("\n",hcloud_server.etcd.*.name)}" list_node = "${join("\n",hcloud_server.node.*.name)}" @@ -160,7 +148,6 @@ data "template_file" "inventory" { list_elasticsearch = "${join("\n",hcloud_server.elasticsearch.*.name)}" list_minio = "${join("\n",hcloud_server.minio.*.name)}" list_redis = "${join("\n",hcloud_server.redis.*.name)}" - list_restund = "${join("\n",hcloud_server.restund.*.name)}" } } diff --git a/terraform/examples/inventory.tpl b/terraform/examples/inventory.tpl index 26c5d965a..932e2fb5f 100644 --- a/terraform/examples/inventory.tpl +++ b/terraform/examples/inventory.tpl @@ -5,7 +5,6 @@ ${connection_strings_minio} ${connection_strings_elasticsearch} ${connection_strings_cassandra} ${connection_strings_redis} -${connection_strings_restund} [vpn:children] k8s-cluster @@ -56,9 +55,6 @@ ${list_minio} [redis] ${list_redis} -[restund] -${list_restund} - [all:vars] ## path to the ssh private key # ansible_ssh_private_key_file = @@ -78,7 +74,6 @@ ansible_python_interpreter = /usr/bin/python3 # cassandra_network_interface = vpn0 # redis_network_interface = vpn0 # registry_network_interface = vpn0 -# restund_network_interface = vpn0 ## configure a proxy if one is needed to access the Internet # http_proxy = "" diff --git a/terraform/examples/wire-server-deploy-offline-hetzner/main.tf b/terraform/examples/wire-server-deploy-offline-hetzner/main.tf index dfd9b7b61..854a24309 100644 --- a/terraform/examples/wire-server-deploy-offline-hetzner/main.tf +++ b/terraform/examples/wire-server-deploy-offline-hetzner/main.tf @@ -4,7 +4,6 @@ locals { minio_count = 2 elasticsearch_count = 2 cassandra_count = 3 - restund_count = 2 ssh_keys = [hcloud_ssh_key.adminhost.name] # TODO: IPv6 @@ -107,28 +106,6 @@ resource "hcloud_server_network" "assethost" { subnet_id = hcloud_network_subnet.main.id } - -resource "random_pet" "restund" { - count = local.restund_count -} - -resource "hcloud_server" "restund" { - count = local.restund_count - location = "nbg1" - name = "restund-${random_pet.restund[count.index].id}" - image = "ubuntu-22.04" - ssh_keys = local.ssh_keys - server_type = "cx22" - user_data = local.disable_network_cfg -} - -resource "hcloud_server_network" "restund" { - count = local.restund_count - server_id = hcloud_server.restund[count.index].id - subnet_id = hcloud_network_subnet.main.id -} - - resource "random_pet" "kubenode" { count = local.kubenode_count } diff --git a/terraform/examples/wire-server-deploy-offline-hetzner/outputs.tf b/terraform/examples/wire-server-deploy-offline-hetzner/outputs.tf index c1535555f..f951b28e4 100644 --- a/terraform/examples/wire-server-deploy-offline-hetzner/outputs.tf +++ b/terraform/examples/wire-server-deploy-offline-hetzner/outputs.tf @@ -96,17 +96,5 @@ output "static-inventory" { minio_network_interface = "eth0" } } - restund = { - hosts = { - for index, server in hcloud_server.restund : server.name => { - ansible_host = hcloud_server_network.restund[index].ip - ansible_user = "root" - } - } - vars = { - restund_network_interface = "eth0" - } - } - } } diff --git a/values/wire-server/prod-values.example.yaml b/values/wire-server/prod-values.example.yaml index 369680c84..a4d21876d 100644 --- a/values/wire-server/prod-values.example.yaml +++ b/values/wire-server/prod-values.example.yaml @@ -93,12 +93,10 @@ brig: turnStatic: v1: [] v2: - # - "turn::80" - # - "turn::80?transport=tcp" - # - "turn::80?transport=tcp" - # - "turns::443?transport=tcp" - # - "turns::443?transport=tcp" + # - "turn::3478" + # - "turn::3478" + # - "turn::3478?transport=tcp" + # - "turn::3478?transport=tcp" proxy: From 47e8368f77b5cfdd6819f2fe86b600d3712f6f55 Mon Sep 17 00:00:00 2001 From: Amit Sagtani Date: Thu, 10 Oct 2024 14:45:14 +0200 Subject: [PATCH 4/7] update docs.md --- offline/docs_ubuntu_22.04.md | 101 +++-------------------------------- 1 file changed, 8 insertions(+), 93 deletions(-) diff --git a/offline/docs_ubuntu_22.04.md b/offline/docs_ubuntu_22.04.md index eba8459e9..d684da62d 100644 --- a/offline/docs_ubuntu_22.04.md +++ b/offline/docs_ubuntu_22.04.md @@ -133,9 +133,6 @@ The following artifacts are provided: - `containers-helm.tar` These are the container images our charts (and charts we depend on) refer to. Also come as tarballs, and are seeded like the system containers. - - `containers-other.tar` - These are other container images, not deployed inside k8s. Currently, only - contains `restund`. - `debs-jammy.tar` This acts as a self-contained dump of all packages required to install kubespray, as well as all other packages that are installed by ansible @@ -175,7 +172,7 @@ It's recommended to update the lists of what nodes belong to which group, so ans For our Wire internal offline deployments using seven VMs, we edit the inventory to run all services outside of K8s on three `ansnode` VMs. For productive on-prem deployments, these sections can be divided into individual host groups, reflecting the architecture of the target infrastructure. -Examples with individual nodes for Elastic, MinIO, Cassandra and Restund are commented out below. +Examples with individual nodes for Elastic, MinIO, and Cassandra are commented out below. ``` [elasticsearch] # elasticsearch1 @@ -205,11 +202,6 @@ ansnode3 # cassandraseed1 ansnode1 -[restund] -# restund1 -# restund2 -ansnode1 -ansnode2 ``` ### Configuring kubernetes and etcd @@ -258,26 +250,7 @@ Do this for all of the instances. * Make sure that `cassandra_network_interface` is set to the name of the network interface on which the kubenodes should talk to cassandra and on which the cassandra nodes should communicate among each other. Run `ip addr` on one of the cassandra nodes to determine the network interface names, and which networks they correspond to. In Ubuntu 22.04 for example, interface names are predictable and individualized, eg. `enp41s0`. * Similarly `elasticsearch_network_interface` and `minio_network_interface` should be set to the network interface names you want elasticsearch and minio to communicate with kubernetes with, as well. - - -### Configuring Restund - -Restund is deployed for NAT-hole punching and relaying. So that 1-to-1 calls -can be established between Wire users. Restund needs to be directly publicly -reachable on a public IP. -If you need Restund to listen on a different interface than the default gateway, set `restund_network_interface` - -If the interface on which Restund is listening does not know its own public IP -(e.g. because it is behind NAT itself) extra configuration is necessary. Please provide the public IP on which -Restund is available as `restund_peer_udp_advertise_addr`. - -Due to this *NAT-hole punching* relay purpose and depending on where the Restund instance resides within your network -topology, it could be used to access private services. We consider this to be unintended and thus set a couple -of network rules on a Restund instance. If egress traffic to certain private network ranges should still -be allowed, you may adjust `restund_allowed_private_network_cidrs` according to your setup. -If you install restund together with other services on the same machine you need to set `restund_allowed_private_network_cidrs` -for these services to communicate over the private network. ### Marking kubenode for calling server (SFT) @@ -333,10 +306,6 @@ prefix = "" domain = "example.com" deeplink_title = "wire demo environment, example.com" -[restund:vars] -restund_uid = root -restund_allowed_private_network_cidrs='["192.168.122.0/24"]' - [rmq-cluster:vars] rabbitmq_network_interface = enp1s0 @@ -359,12 +328,6 @@ kubenode3 kube-master kube-node -# Note: If you install restund on the same nodes as other services -# you need to set `restund_allowed_private_network_cidrs` -[restund] -ansnode1 -ansnode2 - [cassandra] ansnode1 ansnode2 @@ -395,7 +358,7 @@ ansnode3 ## Generating secrets -Minio and restund services have shared secrets with the `wire-server` helm chart. Run the folllowing script that generates a fresh set of secrets for these components: +Minio and coturn services have shared secrets with the `wire-server` helm chart. Run the folllowing script that generates a fresh set of secrets for these components: ``` ./bin/offline-secrets.sh @@ -452,7 +415,7 @@ Hash: SHA1, RIPEMD160, SHA256, SHA384, SHA512, SHA224 Compression: Uncompressed, ZIP, ZLIB, BZIP2 ``` -## Deploying Kubernetes, Restund and stateful services +## Deploying Kubernetes and stateful services In order to deploy all mentioned services, run: ``` @@ -470,54 +433,6 @@ d kubectl get nodes -owide ``` They should all report ready. - -#### Troubleshooting restund - -In case the restund firewall fails to start. Fix - -On each ansnode you set in the `[restund]` section of the `hosts.ini` file - -Delete the outbound rule to 172.16.0.0/12 - -``` -sudo ufw status numbered; -``` - -Then find the right number and delete it - -``` -ufw delete ; -``` - - -and enable the ports for colocated services running on these nodes: - -``` -sudo bash -c ' -set -eo pipefail; - -# cassandra -ufw allow 9042/tcp; -ufw allow 9160/tcp; -ufw allow 7000/tcp; -ufw allow 7199/tcp; - -# elasticsearch -ufw allow 9300/tcp; -ufw allow 9200/tcp; - -# minio -ufw allow 9000/tcp; -ufw allow 9092/tcp; - -#rabbitmq -ufw allow 5671/tcp; -ufw allow 5672/tcp; -ufw allow 4369/tcp; -ufw allow 25672/tcp; -' -``` - ### Deploy RabbitMQ cluster Follow the steps mentioned here to create a RabbitMQ cluster based on your setup - [offline/rabbitmq_setup.md](./rabbitmq_setup.md) @@ -566,15 +481,15 @@ cp ./values/wire-server/prod-values.example.yaml ./values/wire-server/values.yam Inspect all the values and adjust domains to your domains where needed. -Add the IPs of your `restund` servers to the `turnStatic.v2` list: +Add the IPs of your `coturn` servers to the `turnStatic.v2` list: ```yaml turnStatic: v1: [] v2: - - "turn::3478" - - "turn::3478" - - "turn::3478?transport=tcp" - - "turn::3478?transport=tcp" + - "turn::3478" + - "turn::3478" + - "turn::3478?transport=tcp" + - "turn::3478?transport=tcp" ``` Open up `./values/wire-server/secrets.yaml` and inspect the values. In theory From 94af95a64680ee4cdeb47ef6ae5a3b9353e5881e Mon Sep 17 00:00:00 2001 From: Amit Sagtani Date: Thu, 10 Oct 2024 15:26:05 +0200 Subject: [PATCH 5/7] update docs --- ansible/setup-offline-sources.yml | 7 ---- offline/ci.sh | 2 +- offline/coturn.md | 66 +++++++++++++++---------------- offline/upgrading.md | 3 -- 4 files changed, 34 insertions(+), 44 deletions(-) diff --git a/ansible/setup-offline-sources.yml b/ansible/setup-offline-sources.yml index 6a0eaf468..9fafa7844 100644 --- a/ansible/setup-offline-sources.yml +++ b/ansible/setup-offline-sources.yml @@ -30,13 +30,6 @@ tags: - containers-helm - containers - - name: Copy other containers - unarchive: - src: ../containers-other.tar - dest: /opt/assets - tags: - - containers-other - - containers - copy: src: files/serve-assets.service dest: /etc/systemd/system/serve-assets.service diff --git a/offline/ci.sh b/offline/ci.sh index 0d69e7793..05d5bf6d0 100755 --- a/offline/ci.sh +++ b/offline/ci.sh @@ -239,6 +239,6 @@ tar cf containers-helm.tar containers-helm echo "docker_ubuntu_repo_repokey: '${fingerprint}'" > ansible/inventory/offline/group_vars/all/key.yml -tar czf assets.tgz debs-jammy.tar binaries.tar containers-adminhost containers-helm.tar containers-other.tar containers-system.tar ansible charts values bin +tar czf assets.tgz debs-jammy.tar binaries.tar containers-adminhost containers-helm.tar containers-system.tar ansible charts values bin echo "Done" diff --git a/offline/coturn.md b/offline/coturn.md index a43ccb35e..8cf0ede65 100644 --- a/offline/coturn.md +++ b/offline/coturn.md @@ -11,7 +11,7 @@ This document explains how to install Coturn on a newly deployed Wire-Server ins This presumes you already have: * Followed the [single Hetzner machine installation](single_hetzner_machine_installation.md) guide or otherwise have a machine ready to accept a Wire-Server deployment. -* Have followed the [Wire-Server installation](docs_ubuntu_22.04.md) guide and have Wire-Server deployed and working (with Restund as the TURN server, which is currently the default, and will be replaced by Coturn as part of this process). +* Have followed the [Wire-Server installation](docs_ubuntu_22.04.md) guide and have Wire-Server deployed and working. ## Plan. @@ -22,8 +22,7 @@ To setup Coturn, we will: * Configure the Coturn labels to select on which machine(s) it will run. * Configure the SFT labels for Coturn and SFT to share a port range. * Configure the port redirection in Nftables. -* Change the Wire-Server configuration to use Coturn instead of Restund. -* Disable Restund. +* Change the Wire-Server configuration to use Coturn. * Install Coturn using Helm. * Verify that Coturn is working. @@ -239,7 +238,7 @@ Note: This section is only relevant if you are running Wire-Server/Coturn/SFT be We must configure the port redirection in Nftables to allow traffic to reach Coturn and SFT. -Calling and TURN services (Coturn, Restund, SFT) require being reachable on a range of ports used to transmit the calling data. +Calling and TURN services (Coturn, SFT) require being reachable on a range of ports used to transmit the calling data. Both SFT and Coturn both want to use the same port range, therefore predicting which node is using which port range ahead of time requires dividing/configuring port ranges in advance. @@ -335,9 +334,9 @@ sudo systemctl restart nftables ``` -## Change the Wire-Server configuration to use Coturn instead of Restund. +## Change the Wire-Server configuration to use Coturn. -We must change the Wire-Server configuration to use Coturn instead of Restund. +We must change the Wire-Server configuration to use Coturn. First, we must locate what the "external" IP address of the machine is. @@ -393,35 +392,10 @@ d helm upgrade --install wire-server ./charts/wire-server --timeout=15m0s --valu ``` -## Disable Restund. - -As we are no longer using Restund, we should now disable it entirely. - -We do this by editing the `hosts.ini` file: - -Edit `ansible/inventory/offline/hosts.ini`, and comment out the restund section by adding `#` at the beginning of each line : - -``` -[restund] -# ansnode1 -# ansnode2 -``` - -Then connect to each ansnode and do: - -```bash -sudo service restund stop -``` - -And check it is stopped with: - -```bash -sudo service restund status -``` ## Install Coturn with Helm. -We have now configured our Coturn `value` and `secret` files, configured `wire-server` to use Coturn, and disabled Restund. +We have now configured our Coturn `value` and `secret` files, configured `wire-server` to use Coturn. It is time to actually deploy Coturn. @@ -499,4 +473,30 @@ These are the additional steps to ensure a smooth transition: 2. Change the `turnStatic` call configuration in the `values/wire-server/values.yaml` file to use the Coturn IPs instead of the Restund IPs. 3. Re-deploy the Wire-Server chart to apply the new configuration. 4. Wait at least 24 hours for all clients to retrieve the new configuration. -5. Once you are sure all clients have migrated to Coturn, you can disable Restund as described in this guide. +5. Once you are sure all clients have migrated to Coturn, you can disable Restund as described in this guide below. + +## Disable Restund. + +As we are no longer using Restund, we should now disable it entirely. + +We do this by editing the `hosts.ini` file: + +Edit `ansible/inventory/offline/hosts.ini`, and comment out the restund section by adding `#` at the beginning of each line : + +``` +[restund] +# ansnode1 +# ansnode2 +``` + +Then connect to each ansnode and do: + +```bash +sudo service restund stop +``` + +And check it is stopped with: + +```bash +sudo service restund status +``` \ No newline at end of file diff --git a/offline/upgrading.md b/offline/upgrading.md index 5e192fcaa..be403e734 100644 --- a/offline/upgrading.md +++ b/offline/upgrading.md @@ -114,9 +114,6 @@ The following is a list of important artifacts which are provided: - `containers-helm.tar` These are the container images our charts (and charts we depend on) refer to. Also come as tarballs, and are seeded like the system containers. - - `containers-other.tar` - These are other container images, not deployed inside k8s. Currently, only - contains Restund. - `debs-*.tar` This acts as a self-contained dump of all packages required to install kubespray, as well as all other packages that are installed by ansible From 3535c896738259bee34229f6b9f009d058a9908c Mon Sep 17 00:00:00 2001 From: Amit Sagtani Date: Wed, 30 Oct 2024 11:50:45 +0100 Subject: [PATCH 6/7] remove restund calling network config from docs --- offline/docs_ubuntu_22.04.md | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/offline/docs_ubuntu_22.04.md b/offline/docs_ubuntu_22.04.md index d684da62d..a0dab0a40 100644 --- a/offline/docs_ubuntu_22.04.md +++ b/offline/docs_ubuntu_22.04.md @@ -711,37 +711,6 @@ iptables -t nat -A PREROUTING -i $INTERNALINTERFACE -d $PUBLICIPADDRESS -p tcp - or add the corresponding rules to a config file (for UFW, /etc/ufw/before.rules) so they persist after rebooting. -### Incoming Calling Traffic - -Make sure `OUTBOUNDINTERFACE` and `PUBLICIPADDRESS` are exported (see above). - -Select one of your kubernetes nodes that hosts restund: - -``` -export RESTUND01IP= -``` - -then run the following: -``` -sudo bash -c " -set -eo pipefail; - -iptables -t nat -A PREROUTING -d $PUBLICIPADDRESS -i $OUTBOUNDINTERFACE -p tcp --dport 80 -j DNAT --to-destination $RESTUND01IP:80; -iptables -t nat -A PREROUTING -d $PUBLICIPADDRESS -i $OUTBOUNDINTERFACE -p udp --dport 80 -j DNAT --to-destination $RESTUND01IP:80; -iptables -t nat -A PREROUTING -d $PUBLICIPADDRESS -i $OUTBOUNDINTERFACE -p udp -m udp --dport 32768:60999 -j DNAT --to-destination $RESTUND01IP; -" -``` - -or add the corresponding rules to a config file (for UFW, /etc/ufw/before.rules) so they persist after rebooting. - -Using nftables, the firewall deployed via single_hetzner_machine_installation.md should already DNAT restund traffic to the correct node (ansnode1, 192.168.122.31). -To verify, check the NAT table status: - -``` -sudo nft list table nat -``` - - ### Changing the TURN port FIXME: ansibleize this! From ba8fc9b5f503a33bb40aed380bf0c57e2adae91b Mon Sep 17 00:00:00 2001 From: Amit Sagtani Date: Wed, 11 Dec 2024 23:08:01 +0100 Subject: [PATCH 7/7] update docs --- ansible/inventory/offline/99-static | 6 ++++++ offline/docs_ubuntu_22.04.md | 30 +++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/ansible/inventory/offline/99-static b/ansible/inventory/offline/99-static index c219bab68..c661ba4da 100644 --- a/ansible/inventory/offline/99-static +++ b/ansible/inventory/offline/99-static @@ -93,6 +93,12 @@ [rmq-cluster:vars] # rabbitmq_network_interface = enp1s0 +# For the following groups, add all nodes defined above to the sections below. +# Define any additional variables that should be set for these nodes. + +# Uncomment this is you use the bastion host +# [bastion] +# bastion # Add all nodes that should be the master [kube-master] diff --git a/offline/docs_ubuntu_22.04.md b/offline/docs_ubuntu_22.04.md index a0dab0a40..20582f16a 100644 --- a/offline/docs_ubuntu_22.04.md +++ b/offline/docs_ubuntu_22.04.md @@ -433,6 +433,36 @@ d kubectl get nodes -owide ``` They should all report ready. +### Troubleshooting external services +Cassandra, Minio and Elasticsearch are running outside Kubernets cluster, make sure those machines have necessary ports open - + +On each of the machines running Cassandra, Minio and Elasticsearch, run the following commands to open the necessary ports, if needed: +``` +sudo bash -c ' +set -eo pipefail; + +# cassandra +ufw allow 9042/tcp; +ufw allow 9160/tcp; +ufw allow 7000/tcp; +ufw allow 7199/tcp; + +# elasticsearch +ufw allow 9300/tcp; +ufw allow 9200/tcp; + +# minio +ufw allow 9000/tcp; +ufw allow 9092/tcp; + +#rabbitmq +ufw allow 5671/tcp; +ufw allow 5672/tcp; +ufw allow 4369/tcp; +ufw allow 25672/tcp; +' +``` + ### Deploy RabbitMQ cluster Follow the steps mentioned here to create a RabbitMQ cluster based on your setup - [offline/rabbitmq_setup.md](./rabbitmq_setup.md)