Test internet connectivity of the machines

Makefile

@@ -19,14 +19,17 @@ MINI_LAB_VM_IMAGE := $(or $(MINI_LAB_VM_IMAGE),ghcr.io/metal-stack/mini-lab-vms:
 MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic:latest)
 
 MACHINE_OS=ubuntu-22.04
+MAX_RETRIES := 10
 
 # Machine flavors
 ifeq ($(MINI_LAB_FLAVOR),cumulus)
 LAB_MACHINES=machine01,machine02
 LAB_TOPOLOGY=mini-lab.cumulus.yaml
+VRF=vrf20
 else ifeq ($(MINI_LAB_FLAVOR),sonic)
 LAB_MACHINES=machine01,machine02
 LAB_TOPOLOGY=mini-lab.sonic.yaml
+VRF=Vrf20
 else
 $(error Unknown flavor $(MINI_LAB_FLAVOR))
 endif
@@ -133,11 +136,11 @@ _privatenet: env
 
 .PHONY: machine
 machine: _privatenet
-	docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl machine create --description test --name test --hostname test --project 00000000-0000-0000-0000-000000000000 --partition mini-lab --image $(MACHINE_OS) --size v1-small-x86 --networks $(shell docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network -o template --template '{{ .id }}')
+	docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl machine create --description test --name test --hostname test --project 00000000-0000-0000-0000-000000000000 --partition mini-lab --image $(MACHINE_OS) --size v1-small-x86 --userdata "@/tmp/ignition.json" --networks $(shell docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network -o template --template '{{ .id }}')
 
 .PHONY: firewall
 firewall: _ips _privatenet
-	docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl firewall create --description fw --name fw --hostname fw --project 00000000-0000-0000-0000-000000000000 --partition mini-lab --image firewall-ubuntu-3.0 --size v1-small-x86 --networks internet-mini-lab,$(shell docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network -o template --template '{{ .id }}')
+	docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl firewall create --description fw --name fw --hostname fw --project 00000000-0000-0000-0000-000000000000 --partition mini-lab --image firewall-ubuntu-3.0 --size v1-small-x86 --userdata "@/tmp/ignition.json" --networks internet-mini-lab,$(shell docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network -o template --template '{{ .id }}')
 
 .PHONY: ls
 ls: env
@@ -223,6 +226,41 @@ console-machine02:
 console-machine03:
 	@$(MAKE) --no-print-directory _console-machine	CONSOLE_PORT=4002
 
+## SSH TARGETS FOR MACHINES ##
+# Python code could be replaced by jq, but it is not preinstalled on Cumulus
+.PHONY: ssh-fw
+ssh-fw:
+	$(eval fw = $(shell ssh -F files/ssh/config leaf01 "vtysh -c 'show bgp neighbors fw json' | \
+		python3 -c 'import sys, json; data = json.load(sys.stdin); key = next(iter(data)); print(data[key][\"bgpNeighborAddr\"] + \"%\" + key)'" \
+	))
+	ssh -F files/ssh/config $(fw) $(COMMAND)
+
+.PHONY: ssh-machine
+ssh-machine:
+	$(eval machine = $(shell ssh -F files/ssh/config leaf01 "vtysh -c 'show bgp vrf $(VRF) neighbors test json' | \
+		python3 -c 'import sys, json; data = json.load(sys.stdin); key = next(iter(data)); print(data[key][\"bgpNeighborAddr\"] + \"%\" + key)'" \
+	))
+	ssh -F files/ssh/config $(machine) $(COMMAND)
+
+.PHONY: ping-cloudflare
+ping-cloudflare:
+	@echo "Attempting to ping 1.1.1.1..."
+	@for i in $$(seq 1 $(MAX_RETRIES)); do \
+		if $(MAKE) ssh-machine COMMAND="ping -c 1 1.1.1.1" > /dev/null 2>&1; then \
+			echo "Ping successful"; \
+			exit 0; \
+		else \
+			echo "Ping failed"; \
+			if [ $$i -lt $(MAX_RETRIES) ]; then \
+				echo "Retrying in 3 seconds..."; \
+				sleep 3; \
+			else \
+				echo "Max retries reached"; \
+				exit 1; \
+			fi; \
+		fi; \
+	done
+
 ## DEV TARGETS ##
 
 .PHONY: dev-env

compose.yaml

@@ -79,6 +79,7 @@ services:
       - METALCTL_API_URL=http://api.172.17.0.1.nip.io:8080/metal
     volumes:
       - ./files/ssh:/root/.ssh:ro
+      - ./files/ignition.json:/tmp/ignition.json
     network_mode: host
     dns:
       - 172.17.0.1

files/ignition.json

@@ -0,0 +1,25 @@
+{
+  "ignition": {
+    "config": {},
+    "security": {},
+    "timeouts": {},
+    "version": "2.3.0"
+  },
+  "networkd": {},
+  "passwd": {},
+  "storage": {
+    "files": [
+      {
+        "filesystem": "root",
+        "path": "/etc/hosts.allow",
+        "append": true,
+        "contents": {
+          "source": "data:,ALL%3A%20%5Bfe80%3A%3A%5D%2F10%0D%0A",
+          "verification": {}
+        },
+        "mode": 644
+      }
+    ]
+  },
+  "systemd": {}
+}

files/ssh/config

@@ -0,0 +1,17 @@
+Host leaf01
+    HostName leaf01
+    User root
+    IdentityFile files/ssh/id_rsa
+    PasswordAuthentication no
+    StrictHostKeyChecking no
+    UserKnownHostsFile /dev/null
+    PubkeyAcceptedKeyTypes +ssh-rsa
+
+Host * !leaf01
+    User metal
+    IdentityFile files/ssh/id_rsa
+    PasswordAuthentication no
+    StrictHostKeyChecking no
+    UserKnownHostsFile /dev/null
+    # bash could be replaced with ncat, but it is not preinstalled on Cumulus
+    ProxyCommand ssh -q -F files/ssh/config leaf01 'sudo ip vrf exec default bash -c "exec 3<>/dev/tcp/%h/%p; cat<&0 >&3 & cat<&3 >&1"'

inventories/group_vars/cumulus/main.yaml

@@ -3,7 +3,8 @@ ports:
   1: 100G
 interfaces:
 - name: swp1
-uplinks: []
+uplinks:
+- name: swp31
 
 # The best practice recommendation is to set an MTU of 9,216 for the inter-switch links,
 # and an MTU of 9,000 for the server-facing ports, which don’t carry the VXLAN header.

mini-lab.cumulus.yaml

@@ -14,8 +14,6 @@ topology:
         ansible-group: cumulus
       binds:
         - files/ssh/id_rsa.pub:/root/.ssh/authorized_keys
-    linux:
-      image: ${MINI_LAB_VM_IMAGE}
 
   nodes:
     leaf01:
@@ -26,8 +24,19 @@ topology:
       kind: cvx
       binds:
         - apt-transport-https.tar.gz:/root/jessie-apt-transport-fix.tar.gz
+    inet:
+      kind: linux
+      image: quay.io/frrouting/frr:9.1.0
+      binds:
+        - files/inet/daemons:/etc/frr/daemons
+        - files/inet/frr.conf:/etc/frr/frr.conf
+        - files/inet/vtysh.conf:/etc/frr/vtysh.conf
+        - files/inet/network.sh:/root/network.sh
+      exec:
+        - sh /root/network.sh
     vms:
       kind: linux
+      image: ${MINI_LAB_VM_IMAGE}
       binds:
         - /dev:/dev
         - scripts:/mini-lab
@@ -39,3 +48,5 @@ topology:
     - endpoints: ["leaf02:swp2", "vms:lan3"]
     - endpoints: ["leaf01:swp3", "vms:lan4"]
     - endpoints: ["leaf02:swp3", "vms:lan5"]
+    - endpoints: ["leaf01:swp31", "inet:eth1"]
+    - endpoints: ["leaf02:swp31", "inet:eth2"]

test/integration.sh

@@ -44,6 +44,9 @@ echo "$phoned/$minPhoned machines have phoned home"
 
 sleep 10
 
+echo "Test connectivity to outside"
+make ping-cloudflare
+
 echo "Adding route to leaf01 and leaf02"
 make route