Add mini-lab flavor for cluster-api (#61)

Makefile

@@ -1,10 +1,26 @@
 .DEFAULT_GOAL := up
+.EXPORT_ALL_VARIABLES:
 
 KUBECONFIG := $(shell pwd)/.kubeconfig
+MINI_LAB_FLAVOR := $(or $(MINI_LAB_FLAVOR),default)
+
+# Default values
+VAGRANT_VAGRANTFILE=Vagrantfile
+DOCKER_COMPOSE_OVERRIDE=
+
+ifeq ($(MINI_LAB_FLAVOR),default)
+VAGRANT_MACHINES=machine01 machine02
+MACHINE_OS=ubuntu-20.04
+else ifeq ($(MINI_LAB_FLAVOR),cluster-api)
+VAGRANT_MACHINES=machine01 machine02 machine03
+MACHINE_OS=ubuntu-cloud-init-20.04
+else
+$(error Unknown flavor $(MINI_LAB_FLAVOR))
+endif
 
 .PHONY: up
 up: bake env
-	docker-compose up --remove-orphans --force-recreate control-plane partition && vagrant up machine01 machine02
+	docker-compose up --remove-orphans --force-recreate control-plane partition && vagrant up $(VAGRANT_MACHINES)
 
 .PHONY: restart
 restart: down up
@@ -35,17 +51,20 @@ partition-bake:
 
 .PHONY: partition
 partition: partition-bake
-	docker-compose up --remove-orphans --force-recreate partition && vagrant up machine01 machine02
+	docker-compose -f docker-compose.yml $(DOCKER_COMPOSE_OVERRIDE) up --remove-orphans --force-recreate partition && vagrant up $(VAGRANT_MACHINES)
 
 .PHONY: route
 route: _ips
-	@echo "sudo ip r a $(staticR)"
+	eval "sudo ip r a ${staticR}"
 
 .PHONY: fwrules
 fwrules: _ips
-	@echo "sudo -- iptables -I LIBVIRT_FWO -s 100.255.254.0/24 -i $(dev) -j ACCEPT;"
-	@echo "sudo -- iptables -I LIBVIRT_FWI -d 100.255.254.0/24 -o $(dev) -j ACCEPT;"
-	@echo "sudo -- iptables -t nat -I LIBVIRT_PRT -s 100.255.254.0/24 ! -d 100.255.254.0/24 -j MASQUERADE"
+	eval "sudo -- iptables -I LIBVIRT_FWO -s 100.255.254.0/24 -i $(dev) -j ACCEPT;"
+	eval "sudo -- iptables -I LIBVIRT_FWO -s 10.0.1.0/24 -i $(dev) -j ACCEPT;"
+	eval "sudo -- iptables -I LIBVIRT_FWI -d 100.255.254.0/24 -o $(dev) -j ACCEPT;"
+	eval "sudo -- iptables -I LIBVIRT_FWI -d 10.0.1.0/24 -o $(dev) -j ACCEPT;"
+	eval "sudo -- iptables -t nat -I LIBVIRT_PRT -s 100.255.254.0/24 ! -d 100.255.254.0/24 -j MASQUERADE"
+	eval "sudo -- iptables -t nat -I LIBVIRT_PRT -s 10.0.1.0/24 ! -d 10.0.1.0/24 -j MASQUERADE"
 
 .PHONY: cleanup
 cleanup: caddy-down registry-down
@@ -72,6 +91,11 @@ reboot-machine02:
 	vagrant destroy -f machine02
 	vagrant up machine02
 
+.PHONY: reboot-machine03
+reboot-machine03:
+	vagrant destroy -f machine03
+	vagrant up machine03
+
 .PHONY: password01
 password01: env
 	docker-compose run metalctl machine ls --id e0ab02d2-27cd-5a5e-8efc-080ba80cf258 -o template --template "{{ .allocation.console_password }}"
@@ -80,17 +104,21 @@ password01: env
 password02: env
 	docker-compose run metalctl machine ls --id 2294c949-88f6-5390-8154-fa53d93a3313 -o template --template "{{ .allocation.console_password }}"
 
+.PHONY: password03
+password03: env
+	docker-compose run metalctl machine ls --id 2294c949-88f6-5390-8154-fa53d93a3314 -o template --template "{{ .allocation.console_password }}"
+
 .PHONY: _privatenet
 _privatenet: env
 	docker-compose run metalctl network list --name user-private-network | grep user-private-network || docker-compose run metalctl network allocate --partition vagrant --project 00000000-0000-0000-0000-000000000000 --name user-private-network
 
 .PHONY: machine
 machine: _privatenet
-	docker-compose run metalctl machine create --description test --name test --hostname test --project 00000000-0000-0000-0000-000000000000 --partition vagrant --image ubuntu-20.04 --size v1-small-x86 --networks $(shell docker-compose run metalctl network list --name user-private-network -o template --template '{{ .id }}')
+	docker-compose run metalctl machine create --description test --name test --hostname test --project 00000000-0000-0000-0000-000000000000 --partition vagrant --image $(MACHINE_OS) --size v1-small-x86 --networks $(shell docker-compose run metalctl network list --name user-private-network -o template --template '{{ .id }}')
 
 .PHONY: firewall
 firewall: _ips _privatenet
-	docker-compose run metalctl firewall create --description fw --name fw --hostname fw --project 00000000-0000-0000-0000-000000000000 --partition vagrant --image firewall-ubuntu-2.0 --size v1-small-x86 --networks internet-vagrant-lab,$(shell docker-compose run metalctl network list --name user-private-network -o template --template '{{ .id }}')
+	docker-compose run metalctl firewall create --description fw --name fw --hostname fw --project 00000000-0000-0000-0000-000000000000 --partition vagrant --image firewall-ubuntu-2.0 --size v1-small-x86 --networks internet-vagrant-lab,$(shell docker-compose run metalctl network list --name user-private-network -o template --template '{{ .id }}') --id=e0ab02d2-27cd-5a5e-8efc-080ba80cf258
 
 .PHONY: reinstall-machine01
 reinstall-machine01: env
@@ -112,6 +140,11 @@ delete-machine02: env
 	docker-compose run metalctl machine rm 2294c949-88f6-5390-8154-fa53d93a3313
 	@$(MAKE) --no-print-directory reboot-machine02
 
+.PHONY: delete-machine03
+delete-machine03: env
+	docker-compose run metalctl machine rm 2294c949-88f6-5390-8154-fa53d93a3314
+	@$(MAKE) --no-print-directory reboot-machine03
+
 .PHONY: console-machine01
 console-machine01:
 	@echo "exit console with CTRL+5"
@@ -135,7 +168,7 @@ env:
 .PHONY: dev
 dev: caddy registry build-hammer-initrd build-api-image build-core-image push-core-image control-plane-bake load-api-image partition-bake
 	docker-compose -f docker-compose.yml -f docker-compose.dev.yml up
-	vagrant up machine01 machine02
+	vagrant up $(VAGRANT_MACHINES)
 
 .PHONY: load-api-image
 load-api-image:

README.md

@@ -200,6 +200,20 @@ Remove a machine with
 docker-compose run metalctl machine rm e0ab02d2-27cd-5a5e-8efc-080ba80cf258
 ```
 
+## Flavors
+
+There's few versions of mini-lab environment that you can run. We call them flavors. There's 2 flavors at the moment:
+
+- `default` -- runs 2 machines.
+- `cluster-api` -- runs 3 machines. Usefull for testing Control plane and worker node deployment with [Cluster API provider](https://github.com/metal-stack/cluster-api-provider-metalstack).
+
+In order to start specific flavor, you can define the flavor as follows:
+
+```bash
+export MINI_LAB_FLAVOR=cluster-api
+make
+```
+
 ## Development of metal-api, metal-hammer and metal-core
 
 To simplify developing changes for the `metal-api`, `metal-hammer` and `metal-core`, it is possible to use development artifacts from within the mini-lab.

Vagrantfile

@@ -27,6 +27,7 @@ Vagrant.configure("2") do |config|
     box device: device, hostname: "leaf02", box: "CumulusCommunity/cumulus-vx", box_version: "3.7.13", memory: 512
     cable device: device, iface: "swp1", mac: "44:38:39:00:00:04", port: "9003", remote_port: "8003" # -> lan1@machine01
     cable device: device, iface: "swp2", mac: "44:38:39:00:00:19", port: "9017", remote_port: "8017" # -> lan1@machine02
+    cable device: device, iface: "swp3", mac: "44:38:39:00:00:21", port: "9019", remote_port: "8019" # -> lan1@machine03
     device.vm.provision :shell , path: "./vagrant/provision/config_switch.sh"
     device.vm.provision :shell , path: "./vagrant/provision/udev_leaf02.sh"
     device.vm.provision :shell , path: "./vagrant/provision/common.sh"
@@ -35,18 +36,24 @@ Vagrant.configure("2") do |config|
     box device: device, hostname: "leaf01", box: "CumulusCommunity/cumulus-vx", box_version: "3.7.13", memory: 512
     cable device: device, iface: "swp1", mac: "44:38:39:00:00:1a", port: "9018", remote_port: "8018" # -> lan0@machine01
     cable device: device, iface: "swp2", mac: "44:38:39:00:00:18", port: "9016", remote_port: "8016" # -> lan0@machine02
+    cable device: device, iface: "swp3", mac: "44:38:39:00:00:20", port: "9015", remote_port: "8015" # -> lan0@machine03
     device.vm.provision :shell , path: "./vagrant/provision/config_switch.sh"
     device.vm.provision :shell , path: "./vagrant/provision/udev_leaf01.sh"
     device.vm.provision :shell , path: "./vagrant/provision/common.sh"
   end
   config.vm.define "machine01", autostart: false do |device|
-    pxe device: device, hostname: "machine01", memory: 1536, uuid: "e0ab02d2-27cd-5a5e-8efc-080ba80cf258"
+    pxe device: device, hostname: "machine01", memory: 1300, uuid: "e0ab02d2-27cd-5a5e-8efc-080ba80cf258"
     cable device: device, iface: "lan0", mac: "00:04:00:11:11:01", port: "8018", remote_port: "9018" # -> swp1@leaf01
     cable device: device, iface: "lan1", mac: "00:04:00:11:12:01", port: "8003", remote_port: "9003" # -> swp1@leaf02
   end
   config.vm.define "machine02", autostart: false do |device|
-    pxe device: device, hostname: "machine02", memory: 1536, uuid: "2294c949-88f6-5390-8154-fa53d93a3313"
+    pxe device: device, hostname: "machine02", memory: 1300, uuid: "2294c949-88f6-5390-8154-fa53d93a3313"
     cable device: device, iface: "lan0", mac: "00:04:00:22:21:02", port: "8016", remote_port: "9016" # -> swp2@leaf01
     cable device: device, iface: "lan1", mac: "00:04:00:22:22:02", port: "8017", remote_port: "9017" # -> swp2@leaf02
   end
-end
+  config.vm.define "machine03", autostart: false do |device|
+    pxe device: device, hostname: "machine03", memory: 1300, uuid: "2294c949-88f6-5390-8154-fa53d93a3314"
+    cable device: device, iface: "lan0", mac: "00:04:00:23:21:02", port: "8015", remote_port: "9015" # -> swp3@leaf01
+    cable device: device, iface: "lan1", mac: "00:04:00:23:22:02", port: "8019", remote_port: "9019" # -> swp3@leaf02
+  end
+end

docker-compose.yml

@@ -92,4 +92,4 @@ services:
       - 192.168.121.1
       - 1.1.1.1
       - 1.0.0.1
-    command: --version
+    command: --version

inventories/group_vars/control-plane/metal.yml

@@ -24,6 +24,12 @@ metal_api_images:
   url: https://images.metal-stack.io/metal-os/master/ubuntu/20.04/20201126/img.tar.lz4
   features:
     - machine
+- id: ubuntu-cloud-init-20.04
+  name: Ubuntu 20.04 cloud-init
+  description: Ubuntu 20.04 with cloud-init support
+  url: http://images.metal-stack.io/metal-os/pull_requests/62-cloud-init-support/ubuntu/20.04/20201107/img.tar.lz4
+  features:
+    - machine
 
 metal_api_sizes:
 - id: v1-small-x86

inventories/partition-static.yaml

@@ -27,9 +27,11 @@ leaves:
     ports:
       1: 100G
       2: 100G
+      3: 100G
     interfaces:
     - name: swp1
     - name: swp2
+    - name: swp3
     uplinks: []
     dhcp_server_ip: 10.0.1.1
 

obtain_role_requirements.yaml

@@ -20,4 +20,4 @@
           - src: {{ role_params.get('repository') }}
             name: {{ role_name }}
             version: {{ hostvars[inventory_hostname][role_name | lower | replace('-', '_') + '_version'] | default(role_params.get('version'), true) }}
-          {% endfor %}
+          {% endfor %}

test/ci-cleanup.sh

@@ -6,7 +6,7 @@ echo "Cleanup artifacts of previous runs"
 make cleanup
 
 # cleanup does not work 100% on the CI-runner - use virsh commands directly
-for i in metalleaf01 metalleaf02 metalmachine01 metalmachine02; do \
+for i in metalleaf01 metalleaf02 metalmachine01 metalmachine02 metalmachine03; do \
     virsh destroy $i || true; \
     virsh undefine $i || true; \
     virsh vol-delete --pool default "$i-sda.qcow2" || true; \

test/integration.sh

@@ -45,10 +45,10 @@ echo "$phoned/$minPhoned machines have phoned home"
 sleep 10
 
 echo "Adding route to leaf01 and leaf02"
-eval $(make route)
+make route
 
 echo "Adding iptables forwarding rules for libvirt networking"
-eval $(make fwrules)
+make fwrules
 
 echo "Check if SSH login to firewall works"
 # FIXME: Again this is unstable in CI integration tests 

vagrant/provision/udev_leaf01.sh

@@ -12,5 +12,7 @@ echo "  INFO: Adding UDEV Rule: 44:38:39:00:00:1a --> swp1"
 echo 'ACTION=="add", SUBSYSTEM=="net", ATTR{address}=="44:38:39:00:00:1a", NAME="swp1", SUBSYSTEMS=="pci"' >> /etc/udev/rules.d/70-persistent-net.rules
 echo "  INFO: Adding UDEV Rule: 44:38:39:00:00:18 --> swp2"
 echo 'ACTION=="add", SUBSYSTEM=="net", ATTR{address}=="44:38:39:00:00:18", NAME="swp2", SUBSYSTEMS=="pci"' >> /etc/udev/rules.d/70-persistent-net.rules
+echo "  INFO: Adding UDEV Rule: 44:38:39:00:00:20 --> swp3"
+echo 'ACTION=="add", SUBSYSTEM=="net", ATTR{address}=="44:38:39:00:00:20", NAME="swp3", SUBSYSTEMS=="pci"' >> /etc/udev/rules.d/70-persistent-net.rules
 
 cat /etc/udev/rules.d/70-persistent-net.rules

vagrant/provision/udev_leaf02.sh

@@ -12,5 +12,7 @@ echo "  INFO: Adding UDEV Rule: 44:38:39:00:00:04 --> swp1"
 echo 'ACTION=="add", SUBSYSTEM=="net", ATTR{address}=="44:38:39:00:00:04", NAME="swp1", SUBSYSTEMS=="pci"' >> /etc/udev/rules.d/70-persistent-net.rules
 echo "  INFO: Adding UDEV Rule: 44:38:39:00:00:19 --> swp2"
 echo 'ACTION=="add", SUBSYSTEM=="net", ATTR{address}=="44:38:39:00:00:19", NAME="swp2", SUBSYSTEMS=="pci"' >> /etc/udev/rules.d/70-persistent-net.rules
+echo "  INFO: Adding UDEV Rule: 44:38:39:00:00:21 --> swp3"
+echo 'ACTION=="add", SUBSYSTEM=="net", ATTR{address}=="44:38:39:00:00:21", NAME="swp3", SUBSYSTEMS=="pci"' >> /etc/udev/rules.d/70-persistent-net.rules
 
 cat /etc/udev/rules.d/70-persistent-net.rules