diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 2f77350c..c6eedba5 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -21,6 +21,7 @@ jobs: flavors: - name: cumulus - name: sonic + - name: mixed steps: - name: Gain back workspace permissions # https://github.com/actions/checkout/issues/211 diff --git a/Makefile b/Makefile index a9af06be..00b7b2a2 100644 --- a/Makefile +++ b/Makefile @@ -25,11 +25,18 @@ MAX_RETRIES := 30 ifeq ($(MINI_LAB_FLAVOR),cumulus) LAB_MACHINES=machine01,machine02 LAB_TOPOLOGY=mini-lab.cumulus.yaml +LAB_SWITCHES=leaf01 leaf02 VRF=vrf20 else ifeq ($(MINI_LAB_FLAVOR),sonic) LAB_MACHINES=machine01,machine02 LAB_TOPOLOGY=mini-lab.sonic.yaml +LAB_SWITCHES=leaf01 leaf02 VRF=Vrf20 +else ifeq ($(MINI_LAB_FLAVOR),mixed) +LAB_MACHINES=machine01,machine02 +LAB_TOPOLOGY=mini-lab.mixed.yaml +LAB_SWITCHES=leaf01 leaf02 leaf02-sonic +VRF=vrf20 else $(error Unknown flavor $(MINI_LAB_FLAVOR)) endif @@ -54,8 +61,7 @@ up: env control-plane-bake partition-bake # without restarting the metal-core # TODO: should be investigated and fixed if possible sleep 10 - ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o "PubkeyAcceptedKeyTypes +ssh-rsa" root@leaf01 -i files/ssh/id_rsa 'systemctl restart metal-core' - ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o "PubkeyAcceptedKeyTypes +ssh-rsa" root@leaf02 -i files/ssh/id_rsa 'systemctl restart metal-core' + @$(MAKE) --no-print-directory restart-metal-core .PHONY: restart restart: down up @@ -83,9 +89,7 @@ partition: partition-bake .PHONY: partition-bake partition-bake: external_network docker pull $(MINI_LAB_VM_IMAGE) -ifeq ($(MINI_LAB_FLAVOR),sonic) docker pull $(MINI_LAB_SONIC_IMAGE) -endif @if ! sudo $(CONTAINERLAB) --topo $(LAB_TOPOLOGY) inspect | grep -i leaf01 > /dev/null; then \ sudo --preserve-env $(CONTAINERLAB) deploy --topo $(LAB_TOPOLOGY) --reconfigure && \ ./scripts/deactivate_offloading.sh; fi @@ -120,6 +124,7 @@ cleanup-partition: mkdir -p clab-mini-lab sudo --preserve-env $(CONTAINERLAB) destroy --topo mini-lab.cumulus.yaml sudo --preserve-env $(CONTAINERLAB) destroy --topo mini-lab.sonic.yaml + sudo --preserve-env $(CONTAINERLAB) destroy --topo mini-lab.mixed.yaml docker network rm --force mini_lab_ext .PHONY: _privatenet @@ -166,6 +171,20 @@ ssh-leaf01: ssh-leaf02: ssh -o StrictHostKeyChecking=no -o "PubkeyAcceptedKeyTypes +ssh-rsa" -i files/ssh/id_rsa root@leaf02 +.PHONY: restart-metal-core +restart-metal-core: + $(foreach host,$(LAB_SWITCHES),ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o "PubkeyAcceptedKeyTypes +ssh-rsa" root@$(host) -i files/ssh/id_rsa 'systemctl restart metal-core';) + +.PHONY: migrate-cumulus-to-sonic +migrate-cumulus-to-sonic: + docker exec vms /mini-lab/migrate.sh tap1 lan1 lan4 + docker exec vms /mini-lab/migrate.sh tap3 lan3 lan5 + +.PHONY: migrate-sonic-to-cumulus +migrate-sonic-to-cumulus: + docker exec vms /mini-lab/migrate.sh tap1 lan4 lan1 + docker exec vms /mini-lab/migrate.sh tap3 lan5 lan3 + ## MACHINE MANAGEMENT ## .PHONY: start-machines diff --git a/README.md b/README.md index 24e76bd3..51150d5b 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,8 @@ The mini-lab is a small, virtual setup to locally run the metal-stack. It deploy - [Requirements](#requirements) - [Known Limitations](#known-limitations) - [Try it out](#try-it-out) - - [Reinstall machine](#reinstall-machine) - - [Free machine](#free-machine) + - [Reinstall machine](#reinstall-machine) + - [Free machine](#free-machine) - [Flavors](#flavors) @@ -29,7 +29,7 @@ The mini-lab is a small, virtual setup to locally run the metal-stack. It deploy Here is some code that should help you to set up most of the requirements: - ```bash +```bash # If UFW enabled. # Disable the firewall or allow traffic through Docker network IP range. sudo ufw status @@ -58,11 +58,11 @@ sudo chmod +x /usr/local/bin/kind The following ports are used statically on your host machine: | Port | Bind Address | Description | -|:----:|:------------ |:---------------------------------- | -| 6443 | 0.0.0.0 | kube-apiserver of the kind cluster | -| 4443 | 0.0.0.0 | HTTPS ingress | -| 4150 | 0.0.0.0 | nsqd | -| 8080 | 0.0.0.0 | HTTP ingress | +| :--: | :----------- | :--------------------------------- | +| 6443 | 0.0.0.0 | kube-apiserver of the kind cluster | +| 4443 | 0.0.0.0 | HTTPS ingress | +| 4150 | 0.0.0.0 | nsqd | +| 8080 | 0.0.0.0 | HTTP ingress | ## Known Limitations @@ -111,7 +111,7 @@ make firewall make machine ``` -__Alternatively__, you may want to issue the `metalctl` commands on your own: +**Alternatively**, you may want to issue the `metalctl` commands on your own: ```bash docker compose run --rm metalctl network allocate \ @@ -197,6 +197,7 @@ There are two versions, or flavors, of the mini-lab environment which differ in - `cumulus` -- runs 2 Cumulus switches. - `sonic` -- runs 2 SONiC switches +- `mixed` -- starts 2 Cumulus switches and one SONiC switch, where the SONiC switch is not connected to any machines yet. Read the `Switch migration` section to see how this setup can be used to simulate switch migration from Cumulus to SONiC or vice versa. In order to start specific flavor, you can define the flavor as follows: @@ -204,3 +205,18 @@ In order to start specific flavor, you can define the flavor as follows: export MINI_LAB_FLAVOR=sonic make ``` + +# Switch migration + +The strategy for migrating a Cumulus rack to SONiC without downtime involves migrating one switch first creating a mixed setup with Cumulus and SONiC running alongside each other and then migrating the second one. +To simulate the first half of this procedure in the mini-lab, the following steps are performed: + +- start the mini-lab with the `mixed` flavor. +- run `eval $(make dev-env)` to get access to the lab via `metalctl`. +- run `metalctl switch migrate mini-lab-leaf02 leaf02-sonic` to adjust machine connections in the database. +- run `make migrate-cumulus-to-sonic` to adjust the redirect rules in the `vms` container. This simulates changing the cables of the machines from one switch to another. +- run `make firewall machine` to allocate a firewall and a machine + +Wait for a few minutes and run `metalctl machine ls`. If everything works as expected you should see that both machines have `Phoned Home`. + +Although migrating from Cumulus to SONiC is the more important use case, the other direction is supported as well. Just run `metalctl switch migrate leaf02-sonic mini-lab-leaf02` followed by `make migrate-sonic-to-cumulus`. diff --git a/deploy_partition.yaml b/deploy_partition.yaml index 7a59227e..fa86448c 100644 --- a/deploy_partition.yaml +++ b/deploy_partition.yaml @@ -85,6 +85,6 @@ retries: 60 delay: 3 until: - - lookup('metal', 'search', 'switch', api_url=metal_partition_metal_api_protocol+'://'+metal_partition_metal_api_addr+':'+metal_partition_metal_api_port|string+metal_partition_metal_api_basepath, api_hmac=metal_partition_metal_api_hmac_edit_key) | length == 2 + - lookup('metal', 'search', 'switch', api_url=metal_partition_metal_api_protocol+'://'+metal_partition_metal_api_addr+':'+metal_partition_metal_api_port|string+metal_partition_metal_api_basepath, api_hmac=metal_partition_metal_api_hmac_edit_key) | length >= 2 - lookup('metal', 'search', 'switch', api_url=metal_partition_metal_api_protocol+'://'+metal_partition_metal_api_addr+':'+metal_partition_metal_api_port|string+metal_partition_metal_api_basepath, api_hmac=metal_partition_metal_api_hmac_edit_key)[0]["last_sync"] != None - lookup('metal', 'search', 'switch', api_url=metal_partition_metal_api_protocol+'://'+metal_partition_metal_api_addr+':'+metal_partition_metal_api_port|string+metal_partition_metal_api_basepath, api_hmac=metal_partition_metal_api_hmac_edit_key)[1]["last_sync"] != None diff --git a/images/sonic/config_db.json b/images/sonic/config_db.json index b34fca11..f185025e 100644 --- a/images/sonic/config_db.json +++ b/images/sonic/config_db.json @@ -41,25 +41,25 @@ "PORT": { "Ethernet0": { "lanes": "25,26,27,28", - "alias": "fortyGigE0/0", - "index": "0", - "speed": "40000", + "alias": "Eth1/1(Port1)", + "index": "1", + "speed": "25000", "admin_status": "up", "mtu": "9100" }, "Ethernet4": { "lanes": "29,30,31,32", - "alias": "fortyGigE0/4", - "index": "1", - "speed": "40000", + "alias": "Eth2/1(Port2)", + "index": "2", + "speed": "25000", "admin_status": "up", "mtu": "9100" }, "Ethernet120": { "lanes": "33,34,35,36", - "alias": "fortyGigE0/8", - "index": "2", - "speed": "40000", + "alias": "Eth31(Port31)", + "index": "31", + "speed": "100000", "admin_status": "up", "mtu": "9100" } diff --git a/inventories/group_vars/all/images.yaml b/inventories/group_vars/all/images.yaml index 86b83341..34a7772f 100644 --- a/inventories/group_vars/all/images.yaml +++ b/inventories/group_vars/all/images.yaml @@ -7,7 +7,7 @@ metal_stack_release_version: develop # metal_hammer_image_url: https://images.metal-stack.io/metal-hammer/pull-requests//metal-hammer-initrd.img.lz4 # metal_api_image_name: -# metal_api_image_tag: +metal_api_image_tag: switch-migrate # metal_metalctl_image_name: # metal_metalctl_image_tag: # metal_masterdata_api_image_name: diff --git a/inventories/host_vars/leaf01.yaml b/inventories/host_vars/leaf01.yaml new file mode 100644 index 00000000..a166b256 --- /dev/null +++ b/inventories/host_vars/leaf01.yaml @@ -0,0 +1,15 @@ +lo: 10.0.0.11 +asn: 4200000011 +metal_core_cidr: 10.0.1.1/{{ metal_core_cidr_mask }} + +dhcp_default_lease_time: 600 +dhcp_max_lease_time: 600 +dhcp_subnets: + - network: 10.0.1.0 + netmask: 255.255.255.0 + range: + begin: 10.0.1.2 + end: 10.0.1.255 + options: + - routers {{ dhcp_server_ip }} + - domain-name-servers 1.1.1.1, 8.8.8.8 diff --git a/inventories/host_vars/leaf02-sonic.yaml b/inventories/host_vars/leaf02-sonic.yaml new file mode 100644 index 00000000..255acc2f --- /dev/null +++ b/inventories/host_vars/leaf02-sonic.yaml @@ -0,0 +1,3 @@ +lo: 10.0.0.13 +asn: 4200000013 +metal_core_cidr: 10.0.1.255/{{ metal_core_cidr_mask }} diff --git a/inventories/host_vars/leaf02.yaml b/inventories/host_vars/leaf02.yaml new file mode 100644 index 00000000..7ab3f9e8 --- /dev/null +++ b/inventories/host_vars/leaf02.yaml @@ -0,0 +1,3 @@ +lo: 10.0.0.12 +asn: 4200000012 +metal_core_cidr: 10.0.1.128/{{ metal_core_cidr_mask }} diff --git a/inventories/partition.yaml b/inventories/partition.yaml index 23c0490e..018ce9f7 100644 --- a/inventories/partition.yaml +++ b/inventories/partition.yaml @@ -10,27 +10,9 @@ partition: leaves: leaves: - hosts: - leaf01: - lo: 10.0.0.11 - asn: 4200000011 - metal_core_cidr: 10.0.1.1/{{ metal_core_cidr_mask }} - - dhcp_default_lease_time: 600 - dhcp_max_lease_time: 600 - dhcp_subnets: - - network: 10.0.1.0 - netmask: 255.255.255.0 - range: - begin: 10.0.1.2 - end: 10.0.1.255 - options: - - routers {{ dhcp_server_ip }} - - domain-name-servers 1.1.1.1, 8.8.8.8 - leaf02: - lo: 10.0.0.12 - asn: 4200000012 - metal_core_cidr: 10.0.1.128/{{ metal_core_cidr_mask }} + children: + cumulus: + sonic: vars: pixie_server_ip: 10.0.1.1 dhcp_server_ip: 10.0.1.1 diff --git a/mini-lab.mixed.yaml b/mini-lab.mixed.yaml new file mode 100644 index 00000000..52b72bc6 --- /dev/null +++ b/mini-lab.mixed.yaml @@ -0,0 +1,74 @@ +name: mini-lab +prefix: "" + +mgmt: + network: bridge + +topology: + kinds: + cvx: + image: networkop/cx:3.7.0 + kernel: docker.io/grigoriymikh/kernel:4.1.0 + sandbox: grigoriymikh/sandbox:latest + labels: + ansible-group: cumulus + binds: + - apt-transport-https.tar.gz:/root/jessie-apt-transport-fix.tar.gz + - files/ssh/id_rsa.pub:/root/.ssh/authorized_keys + linux: + image: ${MINI_LAB_VM_IMAGE} + + nodes: + mini_lab_ext: + kind: bridge + leaf01: + kind: cvx + leaf02: + kind: cvx + leaf02-sonic: + kind: linux + image: ${MINI_LAB_SONIC_IMAGE} + labels: + ansible-group: sonic + binds: + - /dev:/dev + - files/ssh/id_rsa.pub:/authorized_keys + inet: + kind: linux + image: quay.io/frrouting/frr:10.0.1 + binds: + - files/inet/daemons:/etc/frr/daemons + - files/inet/frr.conf:/etc/frr/frr.conf + - files/inet/vtysh.conf:/etc/frr/vtysh.conf + - files/inet/network.sh:/root/network.sh + exec: + - sh /root/network.sh + vms: + kind: linux + binds: + - /dev:/dev + - scripts:/mini-lab + www: + kind: linux + image: docker.io/library/nginx:alpine-slim + network-mode: none + exec: + - ip addr add 203.0.113.3/24 dev ext + - ip route add 203.0.113.128/25 via 203.0.113.2 dev ext + + links: + - endpoints: ["inet:ext", "mini_lab_ext:inet"] + mtu: 9000 + - endpoints: ["www:ext", "mini_lab_ext:www"] + + - endpoints: ["leaf01:swp1", "vms:lan0"] + - endpoints: ["leaf01:swp2", "vms:lan2"] + - endpoints: ["leaf01:swp31", "inet:eth1"] + + - endpoints: ["leaf02:swp1", "vms:lan1"] + - endpoints: ["leaf02:swp2", "vms:lan3"] + - endpoints: ["leaf02:swp31", "inet:eth2"] + + - endpoints: ["leaf02-sonic:eth1", "vms:lan4"] + - endpoints: ["leaf02-sonic:eth2", "vms:lan5"] + - endpoints: ["leaf02-sonic:eth3", "inet:eth3"] diff --git a/scripts/migrate.sh b/scripts/migrate.sh new file mode 100755 index 00000000..63fac018 --- /dev/null +++ b/scripts/migrate.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -o errexit +TAP_IF=$1 +FROM_IF=$2 +TO_IF=$3 + +tc qdisc del dev $FROM_IF ingress +tc qdisc del dev $TAP_IF ingress + +tc qdisc add dev $TO_IF ingress +tc filter add dev $TO_IF parent ffff: protocol all u32 match u8 0 0 action mirred egress redirect dev $TAP_IF + +tc qdisc add dev $TAP_IF ingress +tc filter add dev $TAP_IF parent ffff: protocol all u32 match u8 0 0 action mirred egress redirect dev $TO_IF diff --git a/scripts/mirror_tap_to_lan.sh b/scripts/mirror_tap_to_lan.sh index f62b6d5e..55e1ba3a 100755 --- a/scripts/mirror_tap_to_lan.sh +++ b/scripts/mirror_tap_to_lan.sh @@ -1,6 +1,5 @@ #!/bin/bash -# Script is taken from https://netdevops.me/2021/transparently-redirecting-packets/frames-between-interfaces/ # Script is taken from https://netdevops.me/2021/transparently-redirecting-packetsframes-between-interfaces/ # Read it for better understanding