diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 80805ad5..b53a5201 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -33,7 +33,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # we can remove desired version again after this fix was released: https://github.com/srl-labs/containerlab/pull/2000 - DESIRED_VERSION: v0.52.0 + DESIRED_VERSION: v0.56.0 - name: Log in to the container registry uses: docker/login-action@v3 diff --git a/Makefile b/Makefile index ef8b0dee..8432bfa4 100644 --- a/Makefile +++ b/Makefile @@ -19,14 +19,17 @@ MINI_LAB_VM_IMAGE := $(or $(MINI_LAB_VM_IMAGE),ghcr.io/metal-stack/mini-lab-vms: MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic:latest) MACHINE_OS=ubuntu-24.04 +MAX_RETRIES := 30 # Machine flavors ifeq ($(MINI_LAB_FLAVOR),cumulus) LAB_MACHINES=machine01,machine02 LAB_TOPOLOGY=mini-lab.cumulus.yaml +VRF=vrf20 else ifeq ($(MINI_LAB_FLAVOR),sonic) LAB_MACHINES=machine01,machine02 LAB_TOPOLOGY=mini-lab.sonic.yaml +VRF=Vrf20 else $(error Unknown flavor $(MINI_LAB_FLAVOR)) endif @@ -78,7 +81,7 @@ partition: partition-bake docker compose up --remove-orphans --force-recreate partition .PHONY: partition-bake -partition-bake: +partition-bake: external_network docker pull $(MINI_LAB_VM_IMAGE) ifeq ($(MINI_LAB_FLAVOR),sonic) docker pull $(MINI_LAB_SONIC_IMAGE) @@ -87,20 +90,22 @@ endif sudo --preserve-env $(CONTAINERLAB) deploy --topo $(LAB_TOPOLOGY) --reconfigure && \ ./scripts/deactivate_offloading.sh; fi +.PHONY: external_network +external_network: + @if ! docker network ls | grep -q mini_lab_ext; then \ + docker network create mini_lab_ext \ + --driver=bridge \ + --gateway=203.0.113.1 \ + --subnet=203.0.113.0/24 \ + --opt "com.docker.network.driver.mtu=9000" \ + --opt "com.docker.network.bridge.name=mini_lab_ext" \ + --opt "com.docker.network.bridge.enable_ip_masquerade=true" && \ + sudo ip route add 203.0.113.128/25 via 203.0.113.2 dev mini_lab_ext; fi + .PHONY: env env: @./env.sh -.PHONY: _ips -_ips: - $(eval ipL1 = $(shell ${YQ} --unwrapScalar=true '.nodes.leaf01."mgmt-ipv4-address"' clab-mini-lab/topology-data.json)) - $(eval ipL2 = $(shell ${YQ} --unwrapScalar=true '.nodes.leaf02."mgmt-ipv4-address"' clab-mini-lab/topology-data.json)) - $(eval staticR = "100.255.254.0/24 nexthop via $(ipL1) dev docker0 nexthop via $(ipL2) dev docker0") - -.PHONY: route -route: _ips - eval "sudo ip r a ${staticR}" - .PHONY: cleanup cleanup: cleanup-control-plane cleanup-partition @@ -113,20 +118,26 @@ cleanup-control-plane: .PHONY: cleanup-partition cleanup-partition: mkdir -p clab-mini-lab - sudo $(CONTAINERLAB) destroy --topo mini-lab.cumulus.yaml - sudo $(CONTAINERLAB) destroy --topo mini-lab.sonic.yaml + sudo --preserve-env $(CONTAINERLAB) destroy --topo mini-lab.cumulus.yaml + sudo --preserve-env $(CONTAINERLAB) destroy --topo mini-lab.sonic.yaml + docker network rm --force mini_lab_ext .PHONY: _privatenet _privatenet: env - docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network | grep user-private-network || docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network allocate --partition mini-lab --project 00000000-0000-0000-0000-000000000000 --name user-private-network + docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network | grep user-private-network || docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network allocate --partition mini-lab --project 00000000-0000-0000-0000-000000000001 --name user-private-network + +.PHONY: _public_ips +_public_ips: env + docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network ip list --name firewall | grep firewall || docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network ip create --network internet-mini-lab --project 00000000-0000-0000-0000-000000000001 --ipaddress 203.0.113.129 --name firewall + docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network ip list --name machine | grep machine || docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network ip create --network internet-mini-lab --project 00000000-0000-0000-0000-000000000001 --ipaddress 203.0.113.130 --name machine .PHONY: machine -machine: _privatenet - docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl machine create --description test --name test --hostname test --project 00000000-0000-0000-0000-000000000000 --partition mini-lab --image $(MACHINE_OS) --size v1-small-x86 --networks $(shell docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network -o template --template '{{ .id }}') +machine: _privatenet _public_ips + docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl machine create --description test --name test --hostname test --project 00000000-0000-0000-0000-000000000001 --partition mini-lab --image $(MACHINE_OS) --size v1-small-x86 --userdata "@/tmp/ignition.json" --ips 203.0.113.130 --networks internet-mini-lab,$(shell docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network -o template --template '{{ .id }}') .PHONY: firewall -firewall: _ips _privatenet - docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl firewall create --description fw --name fw --hostname fw --project 00000000-0000-0000-0000-000000000000 --partition mini-lab --image firewall-ubuntu-3.0 --size v1-small-x86 --networks internet-mini-lab,$(shell docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network -o template --template '{{ .id }}') +firewall: _privatenet _public_ips + docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl firewall create --description fw --name fw --hostname fw --project 00000000-0000-0000-0000-000000000001 --partition mini-lab --image firewall-ubuntu-3.0 --size v1-small-x86 --userdata "@/tmp/ignition.json" --ips 203.0.113.129 --firewall-rules-file=/tmp/rules.yaml --networks internet-mini-lab,$(shell docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network -o template --template '{{ .id }}') .PHONY: ls ls: env @@ -212,6 +223,41 @@ console-machine02: console-machine03: @$(MAKE) --no-print-directory _console-machine CONSOLE_PORT=4002 +## SSH TARGETS FOR MACHINES ## +# Python code could be replaced by jq, but it is not preinstalled on Cumulus +.PHONY: ssh-firewall +ssh-firewall: + $(eval fw = $(shell ssh -F files/ssh/config leaf01 "vtysh -c 'show bgp neighbors fw json' | \ + python3 -c 'import sys, json; data = json.load(sys.stdin); key = next(iter(data)); print(data[key][\"bgpNeighborAddr\"] + \"%\" + key)'" \ + )) + ssh -F files/ssh/config $(fw) $(COMMAND) + +.PHONY: ssh-machine +ssh-machine: + $(eval machine = $(shell ssh -F files/ssh/config leaf01 "vtysh -c 'show bgp vrf $(VRF) neighbors test json' | \ + python3 -c 'import sys, json; data = json.load(sys.stdin); key = next(iter(data)); print(data[key][\"bgpNeighborAddr\"] + \"%\" + key)'" \ + )) + ssh -F files/ssh/config $(machine) $(COMMAND) + +.PHONY: connect-to-cloudflare +connect-to-cloudflare: + @echo "Attempting to connect to Cloudflare..." + @for i in $$(seq 1 $(MAX_RETRIES)); do \ + if $(MAKE) ssh-machine COMMAND="sudo curl --connect-timeout 1 --fail --silent https://1.1.1.1" > /dev/null 2>&1; then \ + echo "Connected successfully"; \ + exit 0; \ + else \ + echo "Connection failed"; \ + if [ $$i -lt $(MAX_RETRIES) ]; then \ + echo "Retrying in 2 seconds..."; \ + sleep 2; \ + else \ + echo "Max retries reached"; \ + exit 1; \ + fi; \ + fi; \ + done + ## DEV TARGETS ## .PHONY: dev-env diff --git a/README.md b/README.md index cb0a08c1..c18c448e 100644 --- a/README.md +++ b/README.md @@ -23,8 +23,8 @@ The mini-lab is a small, virtual setup to locally run the metal-stack. It deploy - kvm as hypervisor for the VMs (you can check through the `kvm-ok` command) - [docker](https://www.docker.com/) >= 24.x.y (for using kind and our deployment base image) - [kind](https://github.com/kubernetes-sigs/kind/releases) == v0.23.0 (for hosting the metal control plane) -- [containerlab](https://containerlab.dev/install/) >= v0.55.0 -- the lab creates a docker network on your host machine (`172.17.0.1`), this hopefully does not overlap with other networks you have +- [containerlab](https://containerlab.dev/install/) >= v0.56.0 +- the lab creates a docker network on your host machine with the address block `203.0.113.0/24`, designated as TEST-NET-3 for documentation and examples. - (recommended) haveged to have enough random entropy (only needed if the PXE process does not work) Here is some code that should help you to set up most of the requirements: @@ -167,16 +167,6 @@ Login with user name metal and the console password from docker compose run --rm metalctl machine consolepassword e0ab02d2-27cd-5a5e-8efc-080ba80cf258 ``` -If you want to access the firewall with SSH or have internet connectivity from the firewall and machine, you'll need to have a static route configured that points to the leaf switches: - -```bash -# Add the route to the network internet-mini-lab 100.255.254.0/24 via leaf01 and leaf02, whose IPs are dynamically allocated. Make sure there's no old route before execution. -make route - -# Connect to the firewall -ssh metal@100.255.254.1 -``` - To remove the kind cluster, the switches and machines, run: ```bash diff --git a/compose.yaml b/compose.yaml index 02ca5dc5..7389d839 100644 --- a/compose.yaml +++ b/compose.yaml @@ -79,6 +79,8 @@ services: - METALCTL_API_URL=http://api.172.17.0.1.nip.io:8080/metal volumes: - ./files/ssh:/root/.ssh:ro + - ./files/ignition.json:/tmp/ignition.json + - ./files/rules.yaml:/tmp/rules.yaml network_mode: host dns: - 172.17.0.1 diff --git a/files/ignition.json b/files/ignition.json new file mode 100644 index 00000000..7c2eefe3 --- /dev/null +++ b/files/ignition.json @@ -0,0 +1,25 @@ +{ + "ignition": { + "config": {}, + "security": {}, + "timeouts": {}, + "version": "2.3.0" + }, + "networkd": {}, + "passwd": {}, + "storage": { + "files": [ + { + "filesystem": "root", + "path": "/etc/hosts.allow", + "append": true, + "contents": { + "source": "data:,ALL%3A%20%5Bfe80%3A%3A%5D%2F10%0D%0AALL%3A%20203.0.113.1%0D%0A", + "verification": {} + }, + "mode": 644 + } + ] + }, + "systemd": {} +} diff --git a/files/inet/frr.conf b/files/inet/frr.conf index 0e32b873..813941d2 100644 --- a/files/inet/frr.conf +++ b/files/inet/frr.conf @@ -5,17 +5,20 @@ log syslog informational ! vrf vrfInternet vni 104009 - ip route 0.0.0.0/0 172.17.0.1 + ip route 0.0.0.0/0 203.0.113.1 exit-vrf ! -interface eth2 +interface eth1 ipv6 nd ra-interval 6 no ipv6 nd suppress-ra ! -interface eth3 +interface eth2 ipv6 nd ra-interval 6 no ipv6 nd suppress-ra ! +interface ext + ip address 203.0.113.2/24 +! interface lo ip address 10.0.0.21/32 ! diff --git a/files/inet/network.sh b/files/inet/network.sh index 5b0313c2..418d74db 100644 --- a/files/inet/network.sh +++ b/files/inet/network.sh @@ -3,7 +3,7 @@ set -o errexit -o xtrace ip link add vrfInternet type vrf table 1000 ip link set dev vrfInternet up -ip link set dev eth0 master vrfInternet +ip link set dev ext master vrfInternet ip link add name bridge type bridge stp_state 0 ip link set dev bridge type bridge vlan_filtering 1 @@ -18,12 +18,10 @@ bridge vlan add vid 1000 dev bridge self ip link set dev vlanInternet up ip link add vniInternet type vxlan id 104009 dstport 4789 local 10.0.0.21 nolearning -ip link set dev vlanInternet mtu 9000 +ip link set dev vniInternet mtu 9000 ip link set dev vniInternet master bridge bridge vlan del vid 1 dev vniInternet bridge vlan del vid 1 untagged pvid dev vniInternet bridge vlan add vid 1000 dev vniInternet bridge vlan add vid 1000 untagged pvid dev vniInternet ip link set up dev vniInternet - -iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE diff --git a/files/rules.yaml b/files/rules.yaml new file mode 100644 index 00000000..2b20c744 --- /dev/null +++ b/files/rules.yaml @@ -0,0 +1,30 @@ +--- +egress: + - comment: allow outgoing https + ports: + - 443 + protocol: TCP + to: + - 0.0.0.0/0 + - comment: allow outgoing dns via tcp + ports: + - 53 + protocol: TCP + to: + - 0.0.0.0/0 + - comment: allow outgoing dns and ntp via udp + ports: + - 53 + - 123 + protocol: UDP + to: + - 0.0.0.0/0 +ingress: + - comment: allow incoming ssh + ports: + - 22 + protocol: TCP + from: + - 203.0.113.1/25 + to: + - 203.0.113.128/25 diff --git a/files/ssh/config b/files/ssh/config new file mode 100644 index 00000000..3bfcfc11 --- /dev/null +++ b/files/ssh/config @@ -0,0 +1,17 @@ +Host leaf01 + HostName leaf01 + User root + IdentityFile files/ssh/id_rsa + PasswordAuthentication no + StrictHostKeyChecking no + UserKnownHostsFile /dev/null + PubkeyAcceptedKeyTypes +ssh-rsa + +Host * !leaf01 + User metal + IdentityFile files/ssh/id_rsa + PasswordAuthentication no + StrictHostKeyChecking no + UserKnownHostsFile /dev/null + # bash could be replaced with ncat, but it is not preinstalled on Cumulus + ProxyCommand ssh -q -F files/ssh/config leaf01 'sudo ip vrf exec default bash -c "exec 3<>/dev/tcp/%h/%p; cat<&0 >&3 & cat<&3 >&1"' diff --git a/inventories/group_vars/control-plane/metal.yml b/inventories/group_vars/control-plane/metal.yml index 6cf4cb36..8a36552b 100644 --- a/inventories/group_vars/control-plane/metal.yml +++ b/inventories/group_vars/control-plane/metal.yml @@ -74,7 +74,7 @@ metal_api_networks: partitionid: "mini-lab" vrf: 104009 prefixes: - - 100.255.254.0/24 + - 203.0.113.128/25 labels: network.metal-stack.io/default: "" network.metal-stack.io/default-external: "" diff --git a/inventories/group_vars/cumulus/main.yaml b/inventories/group_vars/cumulus/main.yaml index 15d599f0..bd9f059f 100644 --- a/inventories/group_vars/cumulus/main.yaml +++ b/inventories/group_vars/cumulus/main.yaml @@ -3,7 +3,8 @@ ports: 1: 100G interfaces: - name: swp1 -uplinks: [] +uplinks: +- name: swp31 # The best practice recommendation is to set an MTU of 9,216 for the inter-switch links, # and an MTU of 9,000 for the server-facing ports, which don’t carry the VXLAN header. diff --git a/mini-lab.cumulus.yaml b/mini-lab.cumulus.yaml index a686ea3d..407fb2cb 100644 --- a/mini-lab.cumulus.yaml +++ b/mini-lab.cumulus.yaml @@ -14,10 +14,10 @@ topology: ansible-group: cumulus binds: - files/ssh/id_rsa.pub:/root/.ssh/authorized_keys - linux: - image: ${MINI_LAB_VM_IMAGE} nodes: + mini_lab_ext: + kind: bridge leaf01: kind: cvx binds: @@ -26,16 +26,31 @@ topology: kind: cvx binds: - apt-transport-https.tar.gz:/root/jessie-apt-transport-fix.tar.gz + inet: + kind: linux + image: quay.io/frrouting/frr:10.0.1 + binds: + - files/inet/daemons:/etc/frr/daemons + - files/inet/frr.conf:/etc/frr/frr.conf + - files/inet/vtysh.conf:/etc/frr/vtysh.conf + - files/inet/network.sh:/root/network.sh + exec: + - sh /root/network.sh vms: kind: linux + image: ${MINI_LAB_VM_IMAGE} binds: - /dev:/dev - scripts:/mini-lab links: + - endpoints: ["inet:ext", "mini_lab_ext:inet"] + mtu: 9000 - endpoints: ["leaf01:swp1", "vms:lan0"] - endpoints: ["leaf02:swp1", "vms:lan1"] - endpoints: ["leaf01:swp2", "vms:lan2"] - endpoints: ["leaf02:swp2", "vms:lan3"] - endpoints: ["leaf01:swp3", "vms:lan4"] - endpoints: ["leaf02:swp3", "vms:lan5"] + - endpoints: ["leaf01:swp31", "inet:eth1"] + - endpoints: ["leaf02:swp31", "inet:eth2"] diff --git a/mini-lab.sonic.yaml b/mini-lab.sonic.yaml index 1e10b974..337a13da 100644 --- a/mini-lab.sonic.yaml +++ b/mini-lab.sonic.yaml @@ -6,6 +6,8 @@ mgmt: topology: nodes: + mini_lab_ext: + kind: bridge leaf01: kind: linux image: ${MINI_LAB_SONIC_IMAGE} @@ -39,6 +41,8 @@ topology: - /dev:/dev - scripts:/mini-lab links: + - endpoints: ["inet:ext", "mini_lab_ext:inet"] + mtu: 9000 - endpoints: ["leaf01:eth1", "inet:eth1"] - endpoints: ["leaf02:eth1", "inet:eth2"] - endpoints: ["leaf01:eth2", "vms:lan0"] diff --git a/roles/cumulus/files/frr.tpl b/roles/cumulus/files/frr.tpl deleted file mode 100644 index 1dfef936..00000000 --- a/roles/cumulus/files/frr.tpl +++ /dev/null @@ -1,151 +0,0 @@ -{{- $ASN := .ASN -}}{{- $RouterId := .Loopback -}}! The frr version is not rendered since it seems to be optional. -frr defaults datacenter -hostname {{ .Name }} -username cumulus nopassword -service integrated-vtysh-config -! -log syslog {{ .LogLevel }} -debug bgp updates -debug bgp nht -debug bgp update-groups -debug bgp zebra -! -vrf vrfInternet - vni 104009 - ip route 0.0.0.0/0 172.17.0.1 nexthop-vrf mgmt -exit-vrf -{{- range $vrf, $t := .Ports.Vrfs }} -! -vrf vrf{{ $t.VNI }} - vni {{ $t.VNI }} - exit-vrf -{{- end }} -{{- range .Ports.Underlay }} -! -interface {{ . }} - ipv6 nd ra-interval 6 - no ipv6 nd suppress-ra -{{- end }} -{{- range .Ports.Firewalls }} -! -interface {{ .Port }} - ipv6 nd ra-interval 6 - no ipv6 nd suppress-ra -{{- end }} -{{- range $vrf, $t := .Ports.Vrfs }} -{{- range $t.Neighbors }} -! -interface {{ . }} vrf {{ $vrf }} - ipv6 nd ra-interval 6 - no ipv6 nd suppress-ra -{{- end }} -{{- end }} -! -router bgp {{ $ASN }} - bgp router-id {{ $RouterId }} - bgp bestpath as-path multipath-relax - neighbor FABRIC peer-group - neighbor FABRIC remote-as external - neighbor FABRIC timers 1 3 - {{- range .Ports.Underlay }} - neighbor {{ . }} interface peer-group FABRIC - {{- end }} - neighbor FIREWALL peer-group - neighbor FIREWALL remote-as external - neighbor FIREWALL timers 1 3 - {{- range .Ports.Firewalls }} - neighbor {{ .Port }} interface peer-group FIREWALL - {{- end }} - ! - address-family ipv4 unicast - redistribute connected route-map LOOPBACKS - neighbor FIREWALL allowas-in 1 - {{- range $k, $f := .Ports.Firewalls }} - neighbor {{ $f.Port }} route-map fw-{{ $k }}-in in - {{- end }} - exit-address-family - ! - address-family l2vpn evpn - advertise-all-vni - neighbor FABRIC activate - neighbor FIREWALL activate - neighbor FIREWALL allowas-in 1 - {{- range $k, $f := .Ports.Firewalls }} - neighbor {{ $f.Port }} route-map fw-{{ $k }}-vni out - {{- end }} - exit-address-family -! -route-map LOOPBACKS permit 10 - match interface lo -! -{{- range $k, $f := .Ports.Firewalls }} -# route-maps for firewall@{{ $k }} - {{- range $f.IPPrefixLists }} -ip prefix-list {{ .Name }} {{ .Spec }} - {{- end}} - {{- range $f.RouteMaps }} -route-map {{ .Name }} {{ .Policy }} {{ .Order }} - {{- range .Entries }} - {{ . }} - {{- end }} - {{- end }} -! -{{- end }} -ip route 0.0.0.0/0 {{ .Ports.Eth0.Gateway }} nexthop-vrf mgmt -! -{{- range $vrf, $t := .Ports.Vrfs }} -router bgp {{ $ASN }} vrf {{ $vrf }} - bgp router-id {{ $RouterId }} - bgp bestpath as-path multipath-relax - neighbor MACHINE peer-group - neighbor MACHINE remote-as external - neighbor MACHINE timers 1 3 - {{- range $t.Neighbors }} - neighbor {{ . }} interface peer-group MACHINE - {{- end }} - ! - address-family ipv4 unicast - redistribute connected - neighbor MACHINE maximum-prefix 24000 - {{- if gt (len $t.IPPrefixLists) 0 }} - neighbor MACHINE route-map {{ $vrf }}-in in - {{- end }} - exit-address-family - ! - address-family l2vpn evpn - advertise ipv4 unicast - exit-address-family -! -{{- if gt (len $t.IPPrefixLists) 0 }} -# route-maps for {{ $vrf }} - {{- range $t.IPPrefixLists }} -ip prefix-list {{ .Name }} {{ .Spec }} - {{- end}} - {{- range $t.RouteMaps }} -route-map {{ .Name }} {{ .Policy }} {{ .Order }} - {{- range .Entries }} - {{ . }} - {{- end }} - {{- end }} -!{{- end }}{{- end }} -! -router bgp {{ $ASN }} vrf vrfInternet - bgp router-id {{ $RouterId }} - bgp bestpath as-path multipath-relax - ! - address-family ipv4 unicast - import vrf mgmt - network 0.0.0.0/0 - exit-address-family - ! - address-family l2vpn evpn - advertise ipv4 unicast - exit-address-family -! -vrf mgmt - ip route 10.0.1.0/24 {{ .Loopback }} nexthop-vrf default - ip route 100.255.254.0/24 vrfInternet nexthop-vrf vrfInternet -exit-vrf -! -line vty -! \ No newline at end of file diff --git a/roles/cumulus/tasks/internet.yaml b/roles/cumulus/tasks/internet.yaml deleted file mode 100644 index 0cab4702..00000000 --- a/roles/cumulus/tasks/internet.yaml +++ /dev/null @@ -1,35 +0,0 @@ ---- -- name: place customized frr template file for mini-lab - copy: - src: frr.tpl - dest: /etc/frr/frr.tpl - -# add additional interfaces vniInternet, vlanInternet and vrfInternet -# this is done with a dedicated file that is included by /etc/network/interfaces -# note: metal-core has no knowledge of this interfaces and would otherwise simply overwrite them if they are present in /etc/network/interfaces -- name: add internet interfaces configuration to /etc/network/interfaces.d - template: - src: internet.intf.j2 - dest: /etc/network/interfaces.d/internet.intf - -# add the vniInternet interface to the bridge and configure the new vid at the bridge -# otherwise ifreload will not work -- name: add internet vni to bridge - command: net add bridge bridge ports vniInternet - -- name: add internet vlan id to bridge - command: net add bridge bridge vids 104009 - -- name: commit - command: net commit - -- name: set envs for metal-core - set_fact: - metal_core_frr_tpl_file: "/etc/frr/frr.tpl" - metal_core_additional_bridge_ports: - - vniInternet - metal_core_additional_bridge_vids: - - 4009 - -- name: add masquerade rule that skips virtual internet network - command: iptables -t nat -A POSTROUTING ! -s 100.255.254.0/24 -o eth0 -j MASQUERADE diff --git a/roles/cumulus/tasks/main.yaml b/roles/cumulus/tasks/main.yaml index c75a9fd1..b7042195 100644 --- a/roles/cumulus/tasks/main.yaml +++ b/roles/cumulus/tasks/main.yaml @@ -27,5 +27,3 @@ src: bridgemac.json dest: /etc/network/ifupdown2/policy.d/bridgemac.json notify: reload interfaces - -- import_tasks: internet.yaml diff --git a/roles/cumulus/templates/internet.intf.j2 b/roles/cumulus/templates/internet.intf.j2 deleted file mode 100644 index fd99ce96..00000000 --- a/roles/cumulus/templates/internet.intf.j2 +++ /dev/null @@ -1,20 +0,0 @@ -auto vrfInternet -iface vrfInternet - vrf-table auto - -auto vlanInternet -iface vlanInternet - mtu 9000 - vlan-id 4009 - vlan-raw-device bridge - vrf vrfInternet - -auto vniInternet -iface vniInternet - mtu 9000 - bridge-access 4009 - bridge-learning off - mstpctl-bpduguard yes - mstpctl-portbpdufilter yes - vxlan-id 104009 - vxlan-local-tunnelip {{ lo }} diff --git a/test/ci-cleanup.sh b/test/ci-cleanup.sh index 7eb15589..a2160898 100755 --- a/test/ci-cleanup.sh +++ b/test/ci-cleanup.sh @@ -15,5 +15,3 @@ if [ ! -z "$running_containers" ]; then fi make cleanup - -sudo ip r d 100.255.254.0/24 || true diff --git a/test/integration.sh b/test/integration.sh index ab08f030..bb6fae07 100755 --- a/test/integration.sh +++ b/test/integration.sh @@ -42,13 +42,10 @@ do done echo "$phoned/$minPhoned machines have phoned home" -sleep 10 +echo "Test connectivity to outside" +make connect-to-cloudflare -echo "Adding route to leaf01 and leaf02" -make route - -echo "Check if SSH login to firewall works" -# FIXME: Again this is unstable in CI integration tests -# ssh -o StrictHostKeyChecking=no metal@100.255.254.1 -C exit +echo "Test connectivity from outside" +ssh -o StrictHostKeyChecking=no -o "PubkeyAcceptedKeyTypes +ssh-rsa" -i files/ssh/id_rsa metal@203.0.113.130 -C exit echo "Successfully started mini-lab"