diff --git a/README.md b/README.md index 2789a1cb..fe753945 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ project itself, consider reading the [docs of the upstream repo](https://github. ## What is this fork about? At [containerlab](https://containerlab.srlinux.dev) we needed to have -[a way to run virtual routers](https://containerlab.srlinux.dev/manual/vrnetlab/) +[a way to run virtual routers](https://containerlab.dev/manual/vrnetlab/) alongside the containerized Network Operating Systems. Vrnetlab provides perfect machinery to package most-common routing VMs in @@ -80,6 +80,24 @@ Full list of connection mode values: | ovs-bridge | :white_check_mark: | Same as a regular bridge, but uses OvS (Open vSwitch). | macvtap | :x: | Requires mounting entire `/dev` to a container namespace. Needs file descriptor manipulation due to no native qemu support. +## Management interface + +There are two types of management connectivity for NOS VMs: _pass-through_ and _host-forwarded_ (legacy) management interfaces. + +_Pass-through management_ interfaces allows the use of the assigned management IP within the NOS VM, management traffic is transparently passed through to the VM, and the NOS configuration can accurately reflect the management IP. However, it is no longer possible to send or receive traffic directly in the vrnetlab container (e.g. for installing additional packages within the container), other than to pre-defined exceptions, such as the QEMU serial port on TCP port 5000. + +NOSes defaulting to _pass-through_ management interfaces are: + +* None so far, we are gathering feedback on this, and will update this list as feedback is received. Please contact us in [Discord](https://discord.gg/vAyddtaEV9) or open up an issue here if you have found any issues when trying the passthrough mode. + +In case of _host-forwarded_ management interfaces, certain ports are forwarded to the NOS VM IP, which is always 10.0.0.15/24. The management gateway in this case is 10.0.0.2/24, and outgoing traffic is NATed to the container management IP. This management interface connection mode does not allow for traffic such as LLDP to pass through the management interface. + +NOSes defaulting to _host-forwarded_ management interfaces are: + +* all current systems + +It is possible to change from the default management interface mode by setting the `CLAB_MGMT_PASSTHROUGH` environment variable to 'true' or 'false', however, it is left up to the user to provide a startup configuration compatible with the requested mode. + ## Which vrnetlab routers are supported? Since the changes we made in this fork are VM specific, we added a few popular diff --git a/c8000v/docker/launch.py b/c8000v/docker/launch.py index 9c129b5c..71454769 100755 --- a/c8000v/docker/launch.py +++ b/c8000v/docker/launch.py @@ -151,6 +151,8 @@ def bootstrap_config(self): """Do the actual bootstrap config""" self.logger.info("applying bootstrap configuration") + v4_mgmt_address = vrnetlab.cidr_to_ddn(self.mgmt_address_ipv4) + self.wait_write("", None) self.wait_write("enable", wait=">") self.wait_write("configure terminal", wait=">") @@ -164,18 +166,24 @@ def bootstrap_config(self): else: self.wait_write("ip domain-name example.com") self.wait_write("crypto key generate rsa modulus 2048") - + + self.wait_write("ipv6 unicast-routing") + self.wait_write("vrf definition clab-mgmt") + self.wait_write("description Containerlab management VRF (DO NOT DELETE)") self.wait_write("address-family ipv4") self.wait_write("exit") - self.wait_write("description Containerlab management VRF (DO NOT DELETE)") + self.wait_write("address-family ipv6") + self.wait_write("exit") self.wait_write("exit") - self.wait_write("ip route vrf clab-mgmt 0.0.0.0 0.0.0.0 10.0.0.2") + self.wait_write(f"ip route vrf clab-mgmt 0.0.0.0 0.0.0.0 {self.mgmt_gw_ipv4}") + self.wait_write(f"ipv6 route vrf clab-mgmt ::/0 {self.mgmt_gw_ipv6}") self.wait_write("interface GigabitEthernet1") self.wait_write("vrf forwarding clab-mgmt") - self.wait_write("ip address 10.0.0.15 255.255.255.0") + self.wait_write(f"ip address {v4_mgmt_address[0]} {v4_mgmt_address[1]}") + self.wait_write(f"ipv6 address {self.mgmt_address_ipv6}") self.wait_write("no shut") self.wait_write("exit") self.wait_write("restconf") diff --git a/cat9kv/docker/launch.py b/cat9kv/docker/launch.py index 56e00fcc..acccb2c2 100755 --- a/cat9kv/docker/launch.py +++ b/cat9kv/docker/launch.py @@ -157,6 +157,8 @@ def bootstrap_spin(self): def bootstrap_config(self): """Do the actual bootstrap config""" self.logger.info("applying bootstrap configuration") + + v4_mgmt_address = vrnetlab.cidr_to_ddn(self.mgmt_address_ipv4) self.wait_write("", None) self.wait_write("enable", wait=">") @@ -173,12 +175,16 @@ def bootstrap_config(self): self.wait_write("crypto key generate rsa modulus 2048") self.wait_write("no ip domain lookup") + + self.wait_write("ipv6 unicast-routing") # add mgmt vrf static route - self.wait_write("ip route vrf Mgmt-vrf 0.0.0.0 0.0.0.0 10.0.0.2") + self.wait_write(f"ip route vrf clab-mgmt 0.0.0.0 0.0.0.0 {self.mgmt_gw_ipv4}") + self.wait_write(f"ipv6 route vrf clab-mgmt ::/0 {self.mgmt_gw_ipv6}") self.wait_write("interface GigabitEthernet0/0") - self.wait_write("ip address 10.0.0.15 255.255.255.0") + self.wait_write(f"ip address {v4_mgmt_address[0]} {v4_mgmt_address[1]}") + self.wait_write(f"ipv6 address {self.mgmt_address_ipv6}") self.wait_write("no shut") self.wait_write("exit") diff --git a/common/vrnetlab.py b/common/vrnetlab.py index 58258262..b683e413 100644 --- a/common/vrnetlab.py +++ b/common/vrnetlab.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import datetime +import ipaddress import json import logging import math @@ -10,7 +11,6 @@ import subprocess import telnetlib import time -import ipaddress from pathlib import Path MAX_RETRIES = 60 @@ -78,6 +78,7 @@ def __init__( provision_pci_bus=True, cpu="host", smp="1", + mgmt_passthrough=False, min_dp_nics=0, ): self.logger = logging.getLogger() @@ -98,7 +99,7 @@ def __init__( self._cpu = cpu self._smp = smp - # various settings + # various settings self.uuid = None self.fake_start_date = None self.nic_type = "e1000" @@ -109,6 +110,26 @@ def __init__( # to have them allocated sequential from eth1 self.highest_provisioned_nic_num = 0 + # Whether the management interface is pass-through or host-forwarded. + # Host-forwarded is the original vrnetlab mode where a VM gets a static IP for its management address, + # which **does not** match the eth0 interface of a container. + # In pass-through mode the VM container uses the same IP as the container's eth0 interface and transparently forwards traffic between the two interfaces. + # See https://github.com/hellt/vrnetlab/issues/286 + self.mgmt_passthrough = mgmt_passthrough + mgmt_passthrough_override = os.environ.get("CLAB_MGMT_PASSTHROUGH", "") + if mgmt_passthrough_override: + self.mgmt_passthrough = mgmt_passthrough_override.lower() == "true" + + # Populate management IP and gateway + if self.mgmt_passthrough: + self.mgmt_address_ipv4, self.mgmt_address_ipv6 = self.get_mgmt_address() + self.mgmt_gw_ipv4, self.mgmt_gw_ipv6 = self.get_mgmt_gw() + else: + self.mgmt_address_ipv4 = "10.0.0.15/24" + self.mgmt_address_ipv6 = "2001:db8::2/64" + self.mgmt_gw_ipv4 = "10.0.0.2" + self.mgmt_gw_ipv6 = "2001:db8::1" + self.insuffucient_nics = False self.min_nics = 0 # if an image needs minimum amount of dataplane nics to bootup, specify @@ -116,9 +137,9 @@ def __init__( self.min_nics = min_dp_nics # management subnet properties, defaults - self.mgmt_subnet = "10.0.0.0/24" - self.mgmt_host_ip = 2 - self.mgmt_guest_ip = 15 + self.mgmt_subnet = "10.0.0.0/24" + self.mgmt_host_ip = 2 + self.mgmt_guest_ip = 15 # Default TCP ports forwarded (TODO tune per platform): # 80 - http @@ -129,7 +150,7 @@ def __init__( # 9339 - iana gnmi/gnoi # 32767 - gnmi/gnoi juniper # 57400 - nokia gnmi/gnoi - self.mgmt_tcp_ports = [80,443,830,6030,8080,9339,32767,57400] + self.mgmt_tcp_ports = [80, 443, 830, 6030, 8080, 9339, 32767, 57400] # we setup pci bus by default self.provision_pci_bus = provision_pci_bus @@ -299,20 +320,51 @@ def create_tc_tap_ifup(self): ip link set $TAP_IF mtu 65000 # create tc eth<->tap redirect rules - tc qdisc add dev eth$INDEX ingress - tc filter add dev eth$INDEX parent ffff: protocol all u32 match u8 0 0 action mirred egress redirect dev tap$INDEX + tc qdisc add dev eth$INDEX clsact + tc filter add dev eth$INDEX ingress flower action mirred egress redirect dev tap$INDEX - tc qdisc add dev $TAP_IF ingress - tc filter add dev $TAP_IF parent ffff: protocol all u32 match u8 0 0 action mirred egress redirect dev eth$INDEX + tc qdisc add dev $TAP_IF clsact + tc filter add dev $TAP_IF ingress flower action mirred egress redirect dev eth$INDEX """ with open("/etc/tc-tap-ifup", "w") as f: f.write(ifup_script) os.chmod("/etc/tc-tap-ifup", 0o777) + def create_tc_tap_mgmt_ifup(self): + """Create tap ifup script that is used in tc datapath mode, specifically for the management interface""" + ifup_script = """#!/bin/bash + + ip link set tap0 up + ip link set tap0 mtu 65000 + + # create tc eth<->tap redirect rules + + tc qdisc add dev eth0 clsact + # exception for TCP ports 5000-5007 + tc filter add dev eth0 ingress prio 1 protocol ip flower ip_proto tcp dst_port 5000-5007 action pass + # mirror ARP traffic to container + tc filter add dev eth0 ingress prio 2 protocol arp flower action mirred egress mirror dev tap0 + # redirect rest of ingress traffic of eth0 to egress of tap0 + tc filter add dev eth0 ingress prio 3 flower action mirred egress redirect dev tap0 + + tc qdisc add dev tap0 clsact + # redirect all ingress traffic of tap0 to egress of eth0 + tc filter add dev tap0 ingress flower action mirred egress redirect dev eth0 + + # clone management MAC of the VM + ip link set dev eth0 address {MGMT_MAC} + """ + + ifup_script = ifup_script.replace("{MGMT_MAC}", self.mgmt_mac) + + with open("/etc/tc-tap-mgmt-ifup", "w") as f: + f.write(ifup_script) + os.chmod("/etc/tc-tap-mgmt-ifup", 0o777) + def gen_mgmt(self): """Generate qemu args for the mgmt interface(s) - + Default TCP ports forwarded: 80 - http 443 - https @@ -323,34 +375,97 @@ def gen_mgmt(self): 32767 - gnmi/gnoi juniper 57400 - nokia gnmi/gnoi """ - if self.mgmt_host_ip+1>=self.mgmt_guest_ip: - self.logger.error("Guest IP (%s) must be at least 2 higher than host IP(%s)", - self.mgmt_guest_ip, self.mgmt_host_ip) + if self.mgmt_host_ip + 1 >= self.mgmt_guest_ip: + self.logger.error( + "Guest IP (%s) must be at least 2 higher than host IP(%s)", + self.mgmt_guest_ip, + self.mgmt_host_ip, + ) network = ipaddress.ip_network(self.mgmt_subnet) host = str(network[self.mgmt_host_ip]) - dns = str(network[self.mgmt_host_ip+1]) + dns = str(network[self.mgmt_host_ip + 1]) guest = str(network[self.mgmt_guest_ip]) res = [] - # mgmt interface is special - we use qemu user mode network with DHCP res.append("-device") - mac = ( + self.mgmt_mac = ( "c0:00:01:00:ca:fe" if getattr(self, "_static_mgmt_mac", False) else gen_mac(0) ) - res.append(self.nic_type + f",netdev=p00,mac={mac}") + + res.append(self.nic_type + f",netdev=p00,mac={self.mgmt_mac}") res.append("-netdev") - res.append( - f"user,id=p00,net={self.mgmt_subnet},host={host},dns={dns},dhcpstart={guest}," + - f"hostfwd=tcp:0.0.0.0:22-{guest}:22," + # ssh - f"hostfwd=udp:0.0.0.0:161-{guest}:161," + # snmp - (",".join([ f"hostfwd=tcp:0.0.0.0:{p}-{guest}:{p}" for p in self.mgmt_tcp_ports ])) + - ",tftp=/tftpboot" - ) + + if self.mgmt_passthrough: + # mgmt interface is passthrough - we just create a normal mirred tap interface + res.append( + "tap,id=p00,ifname=tap0,script=/etc/tc-tap-mgmt-ifup,downscript=no" + ) + self.create_tc_tap_mgmt_ifup() + else: + # mgmt interface is host-forwarded - we use qemu user mode network + # with hostfwd rules to forward ports from the host to the guest + res.append( + f"user,id=p00,net={self.mgmt_subnet},host={host},dns={dns},dhcpstart={guest}," + + f"hostfwd=tcp:0.0.0.0:22-{guest}:22," # ssh + + f"hostfwd=udp:0.0.0.0:161-{guest}:161," # snmp + + ( + ",".join( + [ + f"hostfwd=tcp:0.0.0.0:{p}-{guest}:{p}" + for p in self.mgmt_tcp_ports + ] + ) + ) + + ",tftp=/tftpboot" + ) + return res + def get_mgmt_address(self): + """Returns the IPv4 and IPv6 address of the eth0 interface of the container""" + stdout, _ = run_command(["ip", "--json", "address", "show", "dev", "eth0"]) + command_json = json.loads(stdout.decode("utf-8")) + intf_addrinfos = command_json[0]["addr_info"] + mgmt_cidr_v4 = None + mgmt_cidr_v6 = None + for addrinfo in intf_addrinfos: + if addrinfo["family"] == "inet" and addrinfo["scope"] == "global": + mgmt_address_v4 = addrinfo["local"] + mgmt_prefixlen_v4 = addrinfo["prefixlen"] + mgmt_cidr_v4 = mgmt_address_v4 + "/" + str(mgmt_prefixlen_v4) + if addrinfo["family"] == "inet6" and addrinfo["scope"] == "global": + mgmt_address_v6 = addrinfo["local"] + mgmt_prefixlen_v6 = addrinfo["prefixlen"] + mgmt_cidr_v6 = mgmt_address_v6 + "/" + str(mgmt_prefixlen_v6) + + if not mgmt_cidr_v4: + raise ValueError("No IPv4 address set on management interface eth0!") + + return mgmt_cidr_v4, mgmt_cidr_v6 + + def get_mgmt_gw(self): + """Returns the IPv4 and IPv6 default gateways of the container, used for generating the management default route""" + stdout_v4, _ = run_command(["ip", "--json", "-4", "route", "show", "default"]) + command_json_v4 = json.loads(stdout_v4.decode("utf-8")) + try: + mgmt_gw_v4 = command_json_v4[0]["gateway"] + except IndexError as e: + raise IndexError( + "No default gateway route on management interface eth0!" + ) from e + + stdout_v6, _ = run_command(["ip", "--json", "-6", "route", "show", "default"]) + command_json_v6 = json.loads(stdout_v6.decode("utf-8")) + try: + mgmt_gw_v6 = command_json_v6[0]["gateway"] + except IndexError: + mgmt_gw_v6 = None + + return mgmt_gw_v4, mgmt_gw_v6 + def nic_provision_delay(self) -> None: self.logger.debug( f"number of provisioned data plane interfaces is {self.num_provisioned_nics}" @@ -526,7 +641,9 @@ def restart(self): self.stop() self.start() - def wait_write(self, cmd, wait="__defaultpattern__", con=None, clean_buffer=False, hold=""): + def wait_write( + self, cmd, wait="__defaultpattern__", con=None, clean_buffer=False, hold="" + ): """Wait for something on the serial port and then send command Defaults to using self.tn as connection but this can be overridden @@ -548,8 +665,10 @@ def wait_write(self, cmd, wait="__defaultpattern__", con=None, clean_buffer=Fals self.logger.trace(f"waiting for '{wait}' on {con_name}") res = con.read_until(wait.encode()) - while (hold and (hold in res.decode())): - self.logger.trace(f"Holding pattern '{hold}' detected: {res.decode()}, retrying in 10s...") + while hold and (hold in res.decode()): + self.logger.trace( + f"Holding pattern '{hold}' detected: {res.decode()}, retrying in 10s..." + ) con.write("\r".encode()) time.sleep(10) res = con.read_until(wait.encode()) @@ -661,9 +780,19 @@ def qemu_additional_args(self): class VR: - def __init__(self, username, password): + def __init__(self, username, password, mgmt_passthrough: bool = False): self.logger = logging.getLogger() + # Whether the management interface is pass-through or host-forwarded. + # Host-forwarded is the original vrnetlab mode where a VM gets a static IP for its management address, + # which **does not** match the eth0 interface of a container. + # In pass-through mode the VM container uses the same IP as the container's eth0 interface and transparently forwards traffic between the two interfaces. + # See https://github.com/hellt/vrnetlab/issues/286 + self.mgmt_passthrough = mgmt_passthrough + mgmt_passthrough_override = os.environ.get("CLAB_MGMT_PASSTHROUGH", "") + if mgmt_passthrough_override: + self.mgmt_passthrough = mgmt_passthrough_override.lower() == "true" + try: os.mkdir("/tftpboot") except: @@ -708,3 +837,18 @@ def get_digits(input_str: str) -> int: non_string_chars = re.findall(r"\d", input_str) return int("".join(non_string_chars)) + + +def cidr_to_ddn(prefix: str) -> list[str]: + """ + Convert a IPv4 CIDR notation prefix to address + mask in DDN notation + + Returns a list of IP address (str) and mask (str) in dotted decimal + + Example: + get_ddn_mask('192.168.0.1/24') + returns ['192.168.0.1' ,'255.255.255.0'] + """ + + network = ipaddress.IPv4Interface(prefix) + return [str(network.ip), str(network.netmask)] diff --git a/csr/docker/Dockerfile b/csr/docker/Dockerfile index 7606384d..9cf2e68b 100644 --- a/csr/docker/Dockerfile +++ b/csr/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 +FROM public.ecr.aws/docker/library/debian:bookworm-slim MAINTAINER Kristian Larsson MAINTAINER Denis Pointer diff --git a/csr/docker/launch.py b/csr/docker/launch.py index a8598f3f..c4b3283d 100755 --- a/csr/docker/launch.py +++ b/csr/docker/launch.py @@ -144,6 +144,8 @@ def bootstrap_spin(self): def bootstrap_config(self): """Do the actual bootstrap config""" self.logger.info("applying bootstrap configuration") + + v4_mgmt_address = vrnetlab.cidr_to_ddn(self.mgmt_address_ipv4) self.wait_write("", None) self.wait_write("enable", wait=">") @@ -159,17 +161,23 @@ def bootstrap_config(self): self.wait_write("ip domain-name example.com") self.wait_write("crypto key generate rsa modulus 2048") + self.wait_write("ipv6 unicast-routing") + self.wait_write("vrf definition clab-mgmt") + self.wait_write("description Containerlab management VRF (DO NOT DELETE)") self.wait_write("address-family ipv4") self.wait_write("exit") - self.wait_write("description Containerlab management VRF (DO NOT DELETE)") + self.wait_write("address-family ipv6") + self.wait_write("exit") self.wait_write("exit") - self.wait_write("ip route vrf clab-mgmt 0.0.0.0 0.0.0.0 10.0.0.2") + self.wait_write(f"ip route vrf clab-mgmt 0.0.0.0 0.0.0.0 {self.mgmt_gw_ipv4}") + self.wait_write(f"ipv6 route vrf clab-mgmt ::/0 {self.mgmt_gw_ipv6}") self.wait_write("interface GigabitEthernet1") self.wait_write("vrf forwarding clab-mgmt") - self.wait_write("ip address 10.0.0.15 255.255.255.0") + self.wait_write(f"ip address {v4_mgmt_address[0]} {v4_mgmt_address[1]}") + self.wait_write(f"ipv6 address {self.mgmt_address_ipv6}") self.wait_write("no shut") self.wait_write("exit") self.wait_write("restconf") diff --git a/dell_sonic/Makefile b/dell_sonic/Makefile index 7627496d..11b8133e 100644 --- a/dell_sonic/Makefile +++ b/dell_sonic/Makefile @@ -1,10 +1,11 @@ -VENDOR=Sonic -NAME=dell_sonic +VENDOR=Dell +NAME=sonic IMAGE_FORMAT=qcow IMAGE_GLOB=*.qcow2 -# match versions like: -# 4.2.1 +# rename the disk image file as dell-sonic-.qcow2 +# examples: +# for a file named "dell-sonic-4.2.1.qcow2" the image will be named "vrnetlab/dell_sonic:4.2.1" VERSION=$(shell echo $(IMAGE) | sed -e 's/dell-sonic-//' | sed -e 's/.qcow2//') -include ../makefile-sanity.include diff --git a/dell_sonic/docker/launch.py b/dell_sonic/docker/launch.py index a5151e48..209e96d7 100755 --- a/dell_sonic/docker/launch.py +++ b/dell_sonic/docker/launch.py @@ -103,7 +103,14 @@ def bootstrap_config(self): """Do the actual bootstrap config""" self.logger.info("applying bootstrap configuration") self.wait_write("sudo -i", "$") - self.wait_write("/usr/sbin/ip address add 10.0.0.15/24 dev eth0", "#") + # set ipv4/6 address to the management interface + self.wait_write( + f"sudo /usr/sbin/ip address add {self.mgmt_address_ipv4} dev eth0", "#" + ) + # note, v6 address is not being applied for whatever reason + self.wait_write( + f"sudo /usr/sbin/ip -6 address add {self.mgmt_address_ipv6} dev eth0", "#" + ) self.wait_write("passwd -q %s" % (self.username)) self.wait_write(self.password, "New password:") self.wait_write(self.password, "password:") diff --git a/n9kv/Makefile b/n9kv/Makefile index b673c695..d68927bb 100644 --- a/n9kv/Makefile +++ b/n9kv/Makefile @@ -1,17 +1,12 @@ VENDOR=Cisco -NAME=NXOS 9000v +NAME=n9kv IMAGE_FORMAT=qcow2 IMAGE_GLOB=*.qcow2 -# Match versions similar to the following: -# - nxosv-final.7.0.3.I7.5a.qcow2 -# - nxosv-final.7.0.3.I7.9.qcow2 -# - nxosv.9.2.1.qcow2 -# - nxosv.9.2.4.qcow2 -# - nexus9300v.9.3.9.qcow2 -# - nexus9300v.9.3.10.qcow2 -# - nexus9300v64.10.2.2.F.qcow -VERSION=$(shell echo $(IMAGE) | sed -e 's/.\+\?\.\(\(7\.0\.3\.I[0-9]\.[0-9a-z]\+\)\|\([0-9]\+\.[0-9]\+\.[0-9]\+\)\)\(\..*\|$$\)/\1/') +# rename the disk image file as n9kv-.qcow2 +# examples: +# for a file named "n9kv-9300-10.5.2.qcow2" the image will be named "vrnetlab/cisco_n9kv:9300-10.5.2" +VERSION=$(shell echo $(IMAGE) | sed -e 's/n9kv-\(.*\)\.qcow2/\1/') -include ../makefile-sanity.include -include ../makefile.include diff --git a/n9kv/docker/Dockerfile b/n9kv/docker/Dockerfile index 53c78a09..f0814d73 100644 --- a/n9kv/docker/Dockerfile +++ b/n9kv/docker/Dockerfile @@ -1,11 +1,9 @@ -FROM ubuntu:20.04 -LABEL maintainer="Kristian Larsson " -LABEL maintainer="Roman Dodin " +FROM public.ecr.aws/docker/library/debian:bookworm-slim +LABEL maintainer="Roman Dodin , Kaelem Chandra " ARG DEBIAN_FRONTEND=noninteractive RUN apt-get update -qy \ - && apt-get upgrade -qy \ - && apt-get install -y \ + && apt-get install -y --no-install-recommends \ bridge-utils \ iproute2 \ python3-ipy \ diff --git a/n9kv/docker/launch.py b/n9kv/docker/launch.py index 1115f595..ab36a422 100755 --- a/n9kv/docker/launch.py +++ b/n9kv/docker/launch.py @@ -142,13 +142,19 @@ def bootstrap_config(self): # configure management vrf self.wait_write("vrf context management") - self.wait_write("ip route 0.0.0.0/0 10.0.0.2") + self.wait_write(f"ip route 0.0.0.0/0 {self.mgmt_gw_ipv4}") + self.wait_write(f"ipv6 route ::/0 {self.mgmt_gw_ipv6}") self.wait_write("exit") # configure mgmt interface self.wait_write("interface mgmt0") - self.wait_write("ip address 10.0.0.15/24") + self.wait_write(f"ip address {self.mgmt_address_ipv4}") + self.wait_write(f"ipv6 address {self.mgmt_address_ipv6}") self.wait_write("exit") + + # configure longer ssh keys + self.wait_write("ssh key rsa 2048 force") + self.wait_write("feature ssh") # setup nxapi/scp server self.wait_write("feature scp-server") diff --git a/nxos/docker/Dockerfile b/nxos/docker/Dockerfile index 705006df..6ec3ca93 100644 --- a/nxos/docker/Dockerfile +++ b/nxos/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 +FROM public.ecr.aws/docker/library/debian:bookworm-slim LABEL org.opencontainers.image.authors="roman@dodin.dev" ENV DEBIAN_FRONTEND=noninteractive diff --git a/nxos/docker/launch.py b/nxos/docker/launch.py index a06eefe9..26aa2a72 100755 --- a/nxos/docker/launch.py +++ b/nxos/docker/launch.py @@ -108,10 +108,23 @@ def bootstrap_config(self): ) self.wait_write("hostname %s" % (self.hostname)) + # configure management vrf + self.wait_write("vrf context management") + self.wait_write(f"ip route 0.0.0.0/0 {self.mgmt_gw_ipv4}") + self.wait_write(f"ipv6 route ::/0 {self.mgmt_gw_ipv6}") + self.wait_write("exit") + # configure mgmt interface self.wait_write("interface mgmt0") - self.wait_write("ip address 10.0.0.15/24") + self.wait_write(f"ip address {self.mgmt_address_ipv4}") + self.wait_write(f"ipv6 address {self.mgmt_address_ipv6}") self.wait_write("exit") + + # configure longer ssh keys + self.wait_write("no feature ssh") + self.wait_write("ssh key rsa 2048 force") + self.wait_write("feature ssh") + self.wait_write("exit") self.wait_write("copy running-config startup-config") diff --git a/sros/docker/healthcheck.py b/sros/docker/healthcheck.py deleted file mode 100755 index 49053b63..00000000 --- a/sros/docker/healthcheck.py +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python3 - -import sys - -try: - health_file = open("/health", "r") - health = health_file.read() - health_file.close() -except FileNotFoundError: - print("health status file not found") - sys.exit(2) - -exit_status, message = health.strip().split(" ", 1) - -if message != "": - print(message) - -sys.exit(int(exit_status)) diff --git a/sros/docker/launch.py b/sros/docker/launch.py index cfa92712..13a5d702 100755 --- a/sros/docker/launch.py +++ b/sros/docker/launch.py @@ -764,8 +764,8 @@ def get_version_specific_config(major_version: int): """ -# to allow writing config to tftp location we needed to spin up a normal -# tftp server in container host system. To access the host from qemu VM +# In host-forwarded mgmt mode, to allow writing config to tftp location we needed to spin up a normal +# tftp server in a container host system. To access the host from qemu VM # we needed to put SR OS management interface in the container host network namespace # this is done by putting SR OS management interface with into a br-mgmt bridge # the bridge and SR OS mgmt interfaces will be addressed as follows @@ -776,6 +776,11 @@ def get_version_specific_config(major_version: int): SROS_MGMT_V6_ADDR = "200::1" V6_PREFIX_LENGTH = "127" +# In pass-through mode, we also spin up a tftp server, but in this case we create a new namespace +# inside the container that simulates the IP addressing of the host. +# we redirect traffic to this ns by using tc flower filters +FAKEHOST_VETH_MAC_ADDR = "3a:3a:3a:3a:3a:3a" + def parse_variant_line(cfg, obj, skip_nics=False): if not obj: @@ -893,32 +898,6 @@ def uuid_rev_part(part): return res -def gen_bof_config(): - """generate bof configuration commands based on env vars and SR OS version""" - cmds = [] - if "DOCKER_NET_V4_ADDR" in os.environ and os.getenv("DOCKER_NET_V4_ADDR") != "": - if SROS_VERSION.major >= 23 and not SROS_VERSION.magc: - cmds.append( - f'/bof router static-routes route {os.getenv("DOCKER_NET_V4_ADDR")} next-hop {BRIDGE_V4_ADDR}' - ) - else: - cmds.append( - f'/bof static-route {os.getenv("DOCKER_NET_V4_ADDR")} next-hop {BRIDGE_V4_ADDR}' - ) - if "DOCKER_NET_V6_ADDR" in os.environ and os.getenv("DOCKER_NET_V6_ADDR") != "": - if SROS_VERSION.major >= 23 and not SROS_VERSION.magc: - cmds.append( - f'/bof router static-routes route {os.getenv("DOCKER_NET_V6_ADDR")} next-hop {BRIDGE_V6_ADDR}' - ) - else: - cmds.append( - f'/bof static-route {os.getenv("DOCKER_NET_V6_ADDR")} next-hop {BRIDGE_V6_ADDR}' - ) - # if "docker-net-v6-addr" in m: - # cmds.append(f"/bof static-route {m[docker-net-v6-addr]} next-hop {BRIDGE_ADDR}") - return cmds - - class SROS_vm(vrnetlab.VM): def __init__(self, username, password, ram, conn_mode, cpu=2, num=0): if not cpu or cpu == 0 or cpu == "0": @@ -943,6 +922,72 @@ def __init__(self, username, password, ram, conn_mode, cpu=2, num=0): # override default wait pattern with hash followed by the space self.wait_pattern = "# " + def create_tc_tap_mgmt_ifup(self): + # override the parent's function with sros requirements + # this is used when using pass-through mode for mgmt connectivity + """Create tap ifup script that is used in tc datapath mode, specifically for the management interface""" + ifup_script = """#!/bin/bash + + ip link set tap0 up + ip link set tap0 mtu 65000 + + # create tc eth<->tap redirect rules + + tc qdisc add dev eth0 clsact + # exception for TCP ports 5000-5007 + tc filter add dev eth0 ingress prio 1 protocol ip flower ip_proto tcp dst_port 5000-5007 action pass + # mirror ARP traffic to container + tc filter add dev eth0 ingress prio 2 protocol arp flower action mirred egress mirror dev tap0 + # redirect rest of ingress traffic of eth0 to egress of tap0 + tc filter add dev eth0 ingress prio 3 flower action mirred egress redirect dev tap0 + + tc qdisc add dev tap0 clsact + # redirect tftp traffic to fakehost ns + tc filter add dev tap0 ingress protocol ip prio 1 \ + flower ip_proto udp dst_port 69 dst_ip {MGMT_CONTAINER_GW} \ + action pedit ex munge eth dst set {FAKEHOST_VETH_MAC_ADDR} pipe \ + action mirred egress redirect dev RA + + tc filter add dev tap0 ingress protocol ip prio 2 \ + flower ip_proto udp dst_port 52400-52500 dst_ip {MGMT_CONTAINER_GW} \ + action pedit ex munge eth dst set {FAKEHOST_VETH_MAC_ADDR} pipe \ + action mirred egress redirect dev RA + + # redirect all ingress traffic of tap0 to egress of eth0 + tc filter add dev tap0 ingress flower action mirred egress redirect dev eth0 + + # redirect tftp traffic coming from ns to the mgmt address of the sros VM + tc qdisc add dev RA clsact + tc filter add dev RA ingress protocol ip prio 1 \ + flower ip_proto udp src_port 69 dst_ip {MGMT_IP_ADDRESS} \ + action mirred egress redirect dev tap0 + + tc filter add dev RA ingress protocol ip prio 2 \ + flower ip_proto udp src_port 52400-52500 dst_ip {MGMT_IP_ADDRESS} \ + action mirred egress redirect dev tap0 + + # clone management MAC of the VM + ip link set dev eth0 address {MGMT_MAC} + + # configure the ip address of the namespace as it was the host and remove the temporary one + ip netns exec fakehost ip addr add {MGMT_CONTAINER_GW}/{MGMT_IP_PREFIXLEN} dev FA + ip netns exec fakehost ip addr del 169.254.254.254/16 dev FA + """ + + mgmt_ip_v4_address, mgmt_ip_v4_prefixlen = self.mgmt_address_ipv4.split("/") + + ifup_script = ifup_script.replace("{MGMT_MAC}", self.mgmt_mac) + ifup_script = ifup_script.replace( + "{FAKEHOST_VETH_MAC_ADDR}", FAKEHOST_VETH_MAC_ADDR + ) + ifup_script = ifup_script.replace("{MGMT_CONTAINER_GW}", self.mgmt_gw_ipv4) + ifup_script = ifup_script.replace("{MGMT_IP_PREFIXLEN}", mgmt_ip_v4_prefixlen) + ifup_script = ifup_script.replace("{MGMT_IP_ADDRESS}", mgmt_ip_v4_address) + + with open("/etc/tc-tap-mgmt-ifup", "w") as f: + f.write(ifup_script) + os.chmod("/etc/tc-tap-mgmt-ifup", 0o777) + def attach_cf(self, slot, cfname, size): """Attach extra CF. Create if needed.""" cfname = cfname.lower() @@ -1118,6 +1163,61 @@ def switchConfigEngine(self): f"/configure system management-interface configuration-mode {self.mode}" ) + def gen_bof_config(self): + """generate bof configuration commands based on env vars and SR OS version""" + cmds = [] + if "DOCKER_NET_V4_ADDR" in os.environ and os.getenv("DOCKER_NET_V4_ADDR") != "": + if self.mgmt_passthrough: + # in pass-trough mode we configure static routes for the IPv4 private space + if SROS_VERSION.major >= 23 and not SROS_VERSION.magc: + cmds.append( + f"/bof router static-routes route 100.64.0.0/10 next-hop {self.mgmt_gw_ipv4}" + ) + cmds.append( + f"/bof router static-routes route 10.0.0.0/8 next-hop {self.mgmt_gw_ipv4}" + ) + cmds.append( + f"/bof router static-routes route 172.16.0.0/12 next-hop {self.mgmt_gw_ipv4}" + ) + cmds.append( + f"/bof router static-routes route 192.168.0.0/16 next-hop {self.mgmt_gw_ipv4}" + ) + else: + cmds.append( + f"/bof static-route 100.64.0.0/10 next-hop {self.mgmt_gw_ipv4}" + ) + cmds.append( + f"/bof static-route 10.0.0.0/8 next-hop {self.mgmt_gw_ipv4}" + ) + cmds.append( + f"/bof static-route 172.16.0.0/12 next-hop {self.mgmt_gw_ipv4}" + ) + cmds.append( + f"/bof static-route 192.168.0.0/16 next-hop {self.mgmt_gw_ipv4}" + ) + else: + if SROS_VERSION.major >= 23 and not SROS_VERSION.magc: + cmds.append( + f'/bof router static-routes route {os.getenv("DOCKER_NET_V4_ADDR")} next-hop {BRIDGE_V4_ADDR}' + ) + else: + cmds.append( + f'/bof static-route {os.getenv("DOCKER_NET_V4_ADDR")} next-hop {BRIDGE_V4_ADDR}' + ) + if "DOCKER_NET_V6_ADDR" in os.environ and os.getenv("DOCKER_NET_V6_ADDR") != "": + if not self.mgmt_passthrough: + if SROS_VERSION.major >= 23 and not SROS_VERSION.magc: + cmds.append( + f'/bof router static-routes route {os.getenv("DOCKER_NET_V6_ADDR")} next-hop {BRIDGE_V6_ADDR}' + ) + else: + cmds.append( + f'/bof static-route {os.getenv("DOCKER_NET_V6_ADDR")} next-hop {BRIDGE_V6_ADDR}' + ) + # if "docker-net-v6-addr" in m: + # cmds.append(f"/bof static-route {m[docker-net-v6-addr]} next-hop {BRIDGE_ADDR}") + return cmds + def bootstrap_config(self): """Common function used to push initial configuration for bof and config to both integrated and distributed nodes.""" @@ -1126,7 +1226,7 @@ def bootstrap_config(self): # since bof statements are not part of the config file # thus it must be applied unconditionally self.enterBofConfig() - for line in iter(gen_bof_config()): + for line in iter(self.gen_bof_config()): self.wait_write(line) self.commitBofConfig() # save bof config on disk @@ -1196,12 +1296,22 @@ def __init__( self.mode = mode self.role = "integrated" self.num_nics = num_nics - self.smbios = [ - f"type=1,product=TIMOS:address={SROS_MGMT_V4_ADDR}/{V4_PREFIX_LENGTH}@active " - f"address={SROS_MGMT_V6_ADDR}/{V6_PREFIX_LENGTH}@active license-file=tftp://{BRIDGE_V4_ADDR}/" - f"license.txt primary-config=tftp://{BRIDGE_V4_ADDR}/config.txt system-base-mac={vrnetlab.gen_mac(0)} " - f"{variant['timos_line']}" - ] + + if self.mgmt_passthrough: + self.smbios = [ + f"type=1,product=TIMOS:address={self.mgmt_address_ipv4}@active " + f"address={self.mgmt_address_ipv6}@active " + f"license-file=tftp://{self.mgmt_gw_ipv4}/" + f"license.txt primary-config=tftp://{self.mgmt_gw_ipv4}/config.txt system-base-mac={vrnetlab.gen_mac(0)} " + f"{variant['timos_line']}" + ] + else: + self.smbios = [ + f"type=1,product=TIMOS:address={SROS_MGMT_V4_ADDR}/{V4_PREFIX_LENGTH}@active " + f"address={SROS_MGMT_V6_ADDR}/{V6_PREFIX_LENGTH}@active license-file=tftp://{BRIDGE_V4_ADDR}/" + f"license.txt primary-config=tftp://{BRIDGE_V4_ADDR}/config.txt system-base-mac={vrnetlab.gen_mac(0)} " + f"{variant['timos_line']}" + ] self.logger.info("Acting timos line: {}".format(self.smbios)) self.variant = variant self.hostname = hostname @@ -1219,13 +1329,26 @@ def gen_mgmt(self): res = [] - res.append("-device") + mac = vrnetlab.gen_mac(0) + self.mgmt_mac = mac - res.append( - self.nic_type + ",netdev=br-mgmt,mac=%(mac)s" % {"mac": vrnetlab.gen_mac(0)} - ) - res.append("-netdev") - res.append("bridge,br=br-mgmt,id=br-mgmt" % {"i": 0}) + if self.mgmt_passthrough: + res.append("-device") + res.append("virtio-net-pci,netdev=p00,mac=%s" % self.mgmt_mac) + res.append("-netdev") + res.append( + "tap,ifname=tap0,id=p00,script=/etc/tc-tap-mgmt-ifup,downscript=no" + ) + self.create_tc_tap_mgmt_ifup() + + else: + res.append("-device") + + res.append( + self.nic_type + ",netdev=br-mgmt,mac=%(mac)s" % {"mac": self.mgmt_mac} + ) + res.append("-netdev") + res.append("bridge,br=br-mgmt,id=br-mgmt" % {"i": 0}) if any( chassis in self.variant["timos_line"] @@ -1267,14 +1390,22 @@ def __init__(self, hostname, username, password, mode, variant, conn_mode): self.num_nics = 0 self.hostname = hostname self.variant = variant - - self.smbios = [ - f"type=1,product=TIMOS:address={SROS_MGMT_V4_ADDR}/{V4_PREFIX_LENGTH}@active " - f"address={SROS_MGMT_V6_ADDR}/{V6_PREFIX_LENGTH}@active " - f"license-file=tftp://{BRIDGE_V4_ADDR}/license.txt " - f"primary-config=tftp://{BRIDGE_V4_ADDR}/config.txt " - f"system-base-mac={vrnetlab.gen_mac(0)} {variant['cp']['timos_line']}" - ] + if self.mgmt_passthrough: + self.smbios = [ + f"type=1,product=TIMOS:address={self.mgmt_address_ipv4}@active " + f"address={self.mgmt_address_ipv6}@active " + f"license-file=tftp://{self.mgmt_gw_ipv4}/" + f"license.txt primary-config=tftp://{self.mgmt_gw_ipv4}/config.txt system-base-mac={vrnetlab.gen_mac(0)} " + f"{variant['cp']['timos_line']}" + ] + else: + self.smbios = [ + f"type=1,product=TIMOS:address={SROS_MGMT_V4_ADDR}/{V4_PREFIX_LENGTH}@active " + f"address={SROS_MGMT_V6_ADDR}/{V6_PREFIX_LENGTH}@active " + f"license-file=tftp://{BRIDGE_V4_ADDR}/license.txt " + f"primary-config=tftp://{BRIDGE_V4_ADDR}/config.txt " + f"system-base-mac={vrnetlab.gen_mac(0)} {variant['cp']['timos_line']}" + ] self.logger.info("Acting timos line: {}".format(self.smbios)) # Optional CFs indicated by environment variable. The value indicate the SIZE to be passed directly to qemu-img create (eg: CF1=1G) @@ -1304,13 +1435,25 @@ def gen_mgmt(self): """ res = [] - res.append("-device") + mac = vrnetlab.gen_mac(0) + self.mgmt_mac = mac - res.append( - self.nic_type + ",netdev=br-mgmt,mac=%(mac)s" % {"mac": vrnetlab.gen_mac(0)} - ) - res.append("-netdev") - res.append("bridge,br=br-mgmt,id=br-mgmt") + if self.mgmt_passthrough: + res.append("-device") + res.append("virtio-net-pci,netdev=p00,mac=%s" % self.mgmt_mac) + res.append("-netdev") + res.append( + "tap,ifname=tap0,id=p00,script=/etc/tc-tap-mgmt-ifup,downscript=no" + ) + self.create_tc_tap_mgmt_ifup() + else: + res.append("-device") + + res.append( + self.nic_type + ",netdev=br-mgmt,mac=%(mac)s" % {"mac": self.mgmt_mac} + ) + res.append("-netdev") + res.append("bridge,br=br-mgmt,id=br-mgmt") # add virtio NIC for internal control plane interface to vFPC res.append("-device") @@ -1389,8 +1532,17 @@ def bootstrap_spin(self): # SROS is main class for VSR-SIM class SROS(vrnetlab.VR): - def __init__(self, hostname, username, password, mode, variant_name, conn_mode): - super().__init__(username, password) + def __init__( + self, + hostname, + username, + password, + mode, + variant_name, + conn_mode, + mgmt_passthrough, + ): + super().__init__(username, password, mgmt_passthrough) if variant_name.lower() in SROS_VARIANTS: variant = SROS_VARIANTS[variant_name.lower()] @@ -1426,7 +1578,9 @@ def __init__(self, hostname, username, password, mode, variant_name, conn_mode): self.logger.info(f"Number of NICs: {variant['max_nics']}") self.logger.info("Configuration mode: " + str(mode)) - self.setupMgmtBridge() + # if we are in host-forwarded then we need to create Mgmt bridge + if not self.mgmt_passthrough: + self.setupMgmtBridge() if variant["deployment_model"] == "distributed": # CP VM instantiation @@ -1545,7 +1699,9 @@ def setupMgmtBridge(self): def extractVersion(self): """extractVersion extracts the SR OS version from the qcow2 image name""" # https://regex101.com/r/V9jNHc/1 - pattern = r"(magc-)?\S+-((\d{1,3})\.(\d{1,2})\.\w(\d{1,2}(?:-\d{1,2})?))\.qcow2$" + pattern = ( + r"(magc-)?\S+-((\d{1,3})\.(\d{1,2})\.\w(\d{1,2}(?:-\d{1,2})?))\.qcow2$" + ) match_found = False for e in os.listdir("/"): @@ -1622,59 +1778,122 @@ def getDefaultConfig() -> str: if args.trace: logger.setLevel(1) - vrnetlab.run_command( - [ - "in.tftpd", - "--listen", - "--user", - "root", - "-a", - "0.0.0.0:69", - "-s", - "-c", - "-v", - "-p", - "/tftpboot", - ] - ) + # set management interface mode to pass-through or host-forwarded + # host-forwarded is the original vrnetlab mode where SR OS gets a static IP for its bof address, + # which does not match the eth0 interface of a container. + # In pass-through mode the SR OS container uses the same IP as the container's eth0 interface and transparently forwards traffic between the two interfaces. + mgmt_passthrough = False + if os.getenv("CLAB_MGMT_PASSTHROUGH", "").lower() == "true": + mgmt_passthrough = True + logger.debug("Management passthrough mode is ON") + + # In host-forwarded mode the container runs a tftp server in the root namespace of the container. + if not mgmt_passthrough: + vrnetlab.run_command( + [ + "in.tftpd", + "--listen", + "--user", + "root", + "-a", + "0.0.0.0:69", + "-s", + "-c", + "-v", + "-p", + "/tftpboot", + ] + ) - # make tftpboot writable for saving SR OS config - vrnetlab.run_command(["chmod", "-R", "777", "/tftpboot"]) + # make tftpboot writable for saving SR OS config + vrnetlab.run_command(["chmod", "-R", "777", "/tftpboot"]) - # kill origin socats since we use bridge interface - # for SR OS management interface - # thus we need to forward connections to a different address - vrnetlab.run_command(["pkill", "socat"]) + # kill origin socats since we use bridge interface + # for SR OS management interface + # thus we need to forward connections to a different address + vrnetlab.run_command(["pkill", "socat"]) - # redirecting incoming tcp traffic (except serial port 5000) from eth0 to SR management interface - vrnetlab.run_command( - f"iptables-nft -t nat -A PREROUTING -i eth0 -p tcp ! --dport 5000 -j DNAT --to-destination {SROS_MGMT_V4_ADDR}".split() - ) - vrnetlab.run_command( - f"ip6tables-nft -t nat -A PREROUTING -i eth0 -p tcp ! --dport 5000 -j DNAT --to-destination {SROS_MGMT_V6_ADDR}".split() - ) - # same redirection but for UDP - vrnetlab.run_command( - f"iptables-nft -t nat -A PREROUTING -i eth0 -p udp -j DNAT --to-destination {SROS_MGMT_V4_ADDR}".split() - ) - vrnetlab.run_command( - f"ip6tables-nft -t nat -A PREROUTING -i eth0 -p udp -j DNAT --to-destination {SROS_MGMT_V6_ADDR}".split() - ) - # masquerading the incoming traffic so SR OS is able to reply back - vrnetlab.run_command( - "iptables-nft -t nat -A POSTROUTING -o br-mgmt -j MASQUERADE".split() - ) - vrnetlab.run_command( - "ip6tables-nft -t nat -A POSTROUTING -o br-mgmt -j MASQUERADE".split() - ) - # allow sros breakout to management network by NATing via eth0 - vrnetlab.run_command( - "iptables-nft -t nat -A POSTROUTING -o eth0 -j MASQUERADE".split() - ) - vrnetlab.run_command( - "ip6tables-nft -t nat -A POSTROUTING -o eth0 -j MASQUERADE".split() - ) + # redirecting incoming tcp traffic (except serial port 5000) from eth0 to SR management interface + vrnetlab.run_command( + f"iptables-nft -t nat -A PREROUTING -i eth0 -p tcp ! --dport 5000 -j DNAT --to-destination {SROS_MGMT_V4_ADDR}".split() + ) + vrnetlab.run_command( + f"ip6tables-nft -t nat -A PREROUTING -i eth0 -p tcp ! --dport 5000 -j DNAT --to-destination {SROS_MGMT_V6_ADDR}".split() + ) + # same redirection but for UDP + vrnetlab.run_command( + f"iptables-nft -t nat -A PREROUTING -i eth0 -p udp -j DNAT --to-destination {SROS_MGMT_V4_ADDR}".split() + ) + vrnetlab.run_command( + f"ip6tables-nft -t nat -A PREROUTING -i eth0 -p udp -j DNAT --to-destination {SROS_MGMT_V6_ADDR}".split() + ) + # masquerading the incoming traffic so SR OS is able to reply back + vrnetlab.run_command( + "iptables-nft -t nat -A POSTROUTING -o br-mgmt -j MASQUERADE".split() + ) + vrnetlab.run_command( + "ip6tables-nft -t nat -A POSTROUTING -o br-mgmt -j MASQUERADE".split() + ) + # allow sros breakout to management network by NATing via eth0 + vrnetlab.run_command( + "iptables-nft -t nat -A POSTROUTING -o eth0 -j MASQUERADE".split() + ) + vrnetlab.run_command( + "ip6tables-nft -t nat -A POSTROUTING -o eth0 -j MASQUERADE".split() + ) + + # In management pass-through mode the container runs a tftp server in a dedicated namepace. + # This namespace will use the IPv4 default gateway of the container as interface + # tc flower rules will intercept tftp traffic and redirect it to this namespace + else: + # create namespace + vrnetlab.run_command("ip netns add fakehost".split()) + # create vethts: FA in fakehost ns, RA in "root" ns + vrnetlab.run_command("ip link add FA type veth peer name RA".split()) + # assign FA veth to ns + vrnetlab.run_command("ip link set FA netns fakehost".split()) + # enable veth root ns + vrnetlab.run_command("ip link set RA up".split()) + # enable loop in ns + vrnetlab.run_command("ip netns exec fakehost ip link set dev lo up".split()) + # enable veth in fakehost ns + vrnetlab.run_command("ip netns exec fakehost ip link set FA up".split()) + # assign a dummy mac that will not collide with the real docker bridge mac address + vrnetlab.run_command( + f"ip netns exec fakehost ip link set dev FA address {FAKEHOST_VETH_MAC_ADDR}".split() + ) + # configure a temporary ip address so the tftp server can start. + # modified later in the startup process in the create_tc_tap_mgmt_ifup function + vrnetlab.run_command( + f"ip netns exec fakehost ip addr add 169.254.254.254/16 dev FA".split() + ) + # block arp responses in fakehost namespace so it doesn't interfere with root namespace + vrnetlab.run_command( + "ip netns exec fakehost sysctl -w net.ipv4.conf.all.arp_ignore=8".split() + ) + # start tftp in ns, assign ports to server so it's easier to track it with flower filters + vrnetlab.run_command( + [ + "ip", + "netns", + "exec", + "fakehost", + "in.tftpd", + "--listen", + "--user", + "root", + "-a", + "0.0.0.0:69", + "-R", + "52400:52500", + "-s", + "-c", + "-v", + "-p", + "/tftpboot", + ] + ) logger.debug( f"acting flags: username '{args.username}', password '{args.password}', connection-mode '{args.connection_mode}', variant '{args.variant}'" ) @@ -1690,5 +1909,6 @@ def getDefaultConfig() -> str: mode=args.mode, variant_name=args.variant, conn_mode=args.connection_mode, + mgmt_passthrough=mgmt_passthrough, ) ia.start() diff --git a/vios/README.md b/vios/README.md index ba1787f7..febf8e08 100644 --- a/vios/README.md +++ b/vios/README.md @@ -53,8 +53,8 @@ The following protocols are enabled on the management interface: | ID | Description | Default | |-----------------|---------------------------|------------| -| USERNAME | SSH username | vrnetlab | -| PASSWORD | SSH password | VR-netlab9 | +| USERNAME | SSH username | admin | +| PASSWORD | SSH password | admin | | HOSTNAME | device hostname | vios | | TRACE | enable trace logging | false | | CONNECTION_MODE | interface connection mode | tc | @@ -75,10 +75,7 @@ name: vios-lab topology: kinds: linux: - image: vrnetlab/vr-vios:15.9.3M6 - env: - USERNAME: admin - PASSWORD: admin + image: vrnetlab/cisco_vios:15.9.3M6 nodes: vios1: kind: linux diff --git a/vios/docker/launch.py b/vios/docker/launch.py index a7b70339..be55ced2 100755 --- a/vios/docker/launch.py +++ b/vios/docker/launch.py @@ -121,6 +121,9 @@ def _bootstrap_config(self): self.wait_write(f"hostname {self.hostname}") self.wait_write(f"ip domain-name {self.hostname}.clab") self.wait_write("no ip domain-lookup") + + # Explicitly enable IPv6 + self.wait_write("ipv6 unicast-routing") self.wait_write(f"username {self.username} privilege 15 secret {self.password}") @@ -138,17 +141,24 @@ def _bootstrap_config(self): self.wait_write("exit") self.wait_write("vrf definition clab-mgmt") + self.wait_write("description Containerlab management VRF (DO NOT DELETE)") self.wait_write("address-family ipv4") self.wait_write("exit") - self.wait_write("description Management network") + self.wait_write("address-family ipv6") self.wait_write("exit") + self.wait_write("exit") + + v4_mgmt_address = vrnetlab.cidr_to_ddn(self.mgmt_address_ipv4) self.wait_write("interface GigabitEthernet0/0") self.wait_write("vrf forwarding clab-mgmt") - self.wait_write("ip address 10.0.0.15 255.255.255.0") + self.wait_write(f"ip address {v4_mgmt_address[0]} {v4_mgmt_address[1]}") + self.wait_write(f"ipv6 address {self.mgmt_address_ipv6}") self.wait_write("no shutdown") self.wait_write("exit") - self.wait_write("ip route vrf clab-mgmt 0.0.0.0 0.0.0.0 10.0.0.2") + + self.wait_write(f"ip route vrf clab-mgmt 0.0.0.0 0.0.0.0 {self.mgmt_gw_ipv4}") + self.wait_write(f"ipv6 route vrf clab-mgmt ::/0 {self.mgmt_gw_ipv6}") self.wait_write("crypto key generate rsa modulus 2048") self.wait_write("ip ssh version 2") @@ -195,10 +205,10 @@ def __init__(self, hostname: str, username: str, password: str, conn_mode: str): default=os.getenv("TRACE", "false").lower() == "true", ) parser.add_argument( - "--username", help="Username", default=os.getenv("USERNAME", "vrnetlab") + "--username", help="Username", default=os.getenv("USERNAME", "admin") ) parser.add_argument( - "--password", help="Password", default=os.getenv("PASSWORD", "VR-netlab9") + "--password", help="Password", default=os.getenv("PASSWORD", "admin") ) parser.add_argument( "--hostname", help="Router hostname", default=os.getenv("HOSTNAME", "vios") diff --git a/vjunosevolved/docker/init.conf b/vjunosevolved/docker/init.conf index 8537f7ab..07ea9742 100644 --- a/vjunosevolved/docker/init.conf +++ b/vjunosevolved/docker/init.conf @@ -25,7 +25,10 @@ interfaces { re0:mgmt-0 { unit 0 { family inet { - address 10.0.0.15/24; + address {MGMT_IP_IPV4}; + } + family inet { + address {MGMT_IP_IPV6}; } } } @@ -34,7 +37,12 @@ routing-instances { mgmt_junos { routing-options { static { - route 0.0.0.0/0 next-hop 10.0.0.2; + route 0.0.0.0/0 next-hop {MGMT_GW_IPV4}; + } + rib mgmt_junos.inet6.0 { + static { + route ::/0 next-hop {MGMT_GW_IPV6}; + } } } } diff --git a/vjunosevolved/docker/launch.py b/vjunosevolved/docker/launch.py index a2fbfd30..172c75ec 100755 --- a/vjunosevolved/docker/launch.py +++ b/vjunosevolved/docker/launch.py @@ -9,9 +9,8 @@ import sys import uuid -from passlib.hash import sha512_crypt - import vrnetlab +from passlib.hash import sha512_crypt # loadable startup config STARTUP_CONFIG_FILE = "/config/startup-config.cfg" @@ -55,6 +54,7 @@ def __init__(self, hostname, username, password, conn_mode): driveif="virtio", cpu="IvyBridge,vme=on,ss=on,vmx=on,f16c=on,rdrand=on,hypervisor=on,arat=on,tsc-adjust=on,umip=on,arch-capabilities=on,pdpe1gb=on,skip-l1dfl-vmentry=on,pschange-mc-no=on,bmi1=off,avx2=off,bmi2=off,erms=off,invpcid=off,rdseed=off,adx=off,smap=off,xsaveopt=off,abm=off,svm=off", smp="4,sockets=1,cores=4,threads=1", + mgmt_passthrough=False, ) # device hostname @@ -67,16 +67,18 @@ def __init__(self, hostname, username, password, conn_mode): with open("init.conf", "r") as file: cfg = file.read() - # replace HOSTNAME file var with nodes given hostname + cfg = cfg.replace("{MGMT_IP_IPV4}", self.mgmt_address_ipv4) + cfg = cfg.replace("{MGMT_GW_IPV4}", self.mgmt_gw_ipv4) + cfg = cfg.replace("{MGMT_IP_IPV6}", self.mgmt_address_ipv6) + cfg = cfg.replace("{MGMT_GW_IPV6}", self.mgmt_gw_ipv6) + cfg = cfg.replace("{HOSTNAME}", self.hostname) # replace CRYPT_PSWD file var with nodes given password # (Evo does not accept plaintext passwords in config) - new_cfg = cfg.replace("{HOSTNAME}", hostname).replace( - "{CRYPT_PSWD}", password_hash - ) + cfg = cfg.replace("{CRYPT_PSWD}", password_hash) # write changes to init.conf file with open("init.conf", "w") as file: - file.write(new_cfg) + file.write(cfg) # pass in user startup config self.startup_config() diff --git a/vjunosrouter/docker/init.conf b/vjunosrouter/docker/init.conf index b8f3ef56..d4361b75 100644 --- a/vjunosrouter/docker/init.conf +++ b/vjunosrouter/docker/init.conf @@ -25,7 +25,10 @@ interfaces { fxp0 { unit 0 { family inet { - address 10.0.0.15/24; + address {MGMT_IP_IPV4}; + } + family inet6 { + address {MGMT_IP_IPV6}; } } } @@ -34,7 +37,12 @@ routing-instances { mgmt_junos { routing-options { static { - route 0.0.0.0/0 next-hop 10.0.0.2; + route 0.0.0.0/0 next-hop {MGMT_GW_IPV4}; + } + rib mgmt_junos.inet6.0 { + static { + route ::/0 next-hop {MGMT_GW_IPV6}; + } } } } diff --git a/vjunosrouter/docker/launch.py b/vjunosrouter/docker/launch.py index 6c07034e..7125b51a 100755 --- a/vjunosrouter/docker/launch.py +++ b/vjunosrouter/docker/launch.py @@ -52,6 +52,7 @@ def __init__(self, hostname, username, password, conn_mode): driveif="virtio", cpu="IvyBridge,vme=on,ss=on,vmx=on,f16c=on,rdrand=on,hypervisor=on,arat=on,tsc-adjust=on,umip=on,arch-capabilities=on,pdpe1gb=on,skip-l1dfl-vmentry=on,pschange-mc-no=on,bmi1=off,avx2=off,bmi2=off,erms=off,invpcid=off,rdseed=off,adx=off,smap=off,xsaveopt=off,abm=off,svm=on", smp="4,sockets=1,cores=4,threads=1", + mgmt_passthrough=False, ) # device hostname self.hostname = hostname @@ -61,12 +62,15 @@ def __init__(self, hostname, username, password, conn_mode): with open("init.conf", "r") as file: cfg = file.read() - # replace HOSTNAME file var with nodes given hostname - new_cfg = cfg.replace("{HOSTNAME}", hostname) + cfg = cfg.replace("{MGMT_IP_IPV4}", self.mgmt_address_ipv4) + cfg = cfg.replace("{MGMT_GW_IPV4}", self.mgmt_gw_ipv4) + cfg = cfg.replace("{MGMT_IP_IPV6}", self.mgmt_address_ipv6) + cfg = cfg.replace("{MGMT_GW_IPV6}", self.mgmt_gw_ipv6) + cfg = cfg.replace("{HOSTNAME}", self.hostname) # write changes to init.conf file with open("init.conf", "w") as file: - file.write(new_cfg) + file.write(cfg) # pass in user startup config self.startup_config() diff --git a/vjunosswitch/docker/init.conf b/vjunosswitch/docker/init.conf index b8f3ef56..d4361b75 100644 --- a/vjunosswitch/docker/init.conf +++ b/vjunosswitch/docker/init.conf @@ -25,7 +25,10 @@ interfaces { fxp0 { unit 0 { family inet { - address 10.0.0.15/24; + address {MGMT_IP_IPV4}; + } + family inet6 { + address {MGMT_IP_IPV6}; } } } @@ -34,7 +37,12 @@ routing-instances { mgmt_junos { routing-options { static { - route 0.0.0.0/0 next-hop 10.0.0.2; + route 0.0.0.0/0 next-hop {MGMT_GW_IPV4}; + } + rib mgmt_junos.inet6.0 { + static { + route ::/0 next-hop {MGMT_GW_IPV6}; + } } } } diff --git a/vjunosswitch/docker/launch.py b/vjunosswitch/docker/launch.py index 7f23d13d..605878b3 100755 --- a/vjunosswitch/docker/launch.py +++ b/vjunosswitch/docker/launch.py @@ -52,6 +52,7 @@ def __init__(self, hostname, username, password, conn_mode): driveif="virtio", cpu="IvyBridge,vme=on,ss=on,vmx=on,f16c=on,rdrand=on,hypervisor=on,arat=on,tsc-adjust=on,umip=on,arch-capabilities=on,pdpe1gb=on,skip-l1dfl-vmentry=on,pschange-mc-no=on,bmi1=off,avx2=off,bmi2=off,erms=off,invpcid=off,rdseed=off,adx=off,smap=off,xsaveopt=off,abm=off,svm=on", smp="4,sockets=1,cores=4,threads=1", + mgmt_passthrough=False, ) # device hostname self.hostname = hostname @@ -61,12 +62,15 @@ def __init__(self, hostname, username, password, conn_mode): with open("init.conf", "r") as file: cfg = file.read() - # replace HOSTNAME file var with nodes given hostname - new_cfg = cfg.replace("{HOSTNAME}", hostname) + cfg = cfg.replace("{MGMT_IP_IPV4}", self.mgmt_address_ipv4) + cfg = cfg.replace("{MGMT_GW_IPV4}", self.mgmt_gw_ipv4) + cfg = cfg.replace("{MGMT_IP_IPV6}", self.mgmt_address_ipv6) + cfg = cfg.replace("{MGMT_GW_IPV6}", self.mgmt_gw_ipv6) + cfg = cfg.replace("{HOSTNAME}", self.hostname) # write changes to init.conf file with open("init.conf", "w") as file: - file.write(new_cfg) + file.write(cfg) # pass in user startup config self.startup_config() diff --git a/vsrx/docker/init.conf b/vsrx/docker/init.conf index ca0c5bf7..30339247 100644 --- a/vsrx/docker/init.conf +++ b/vsrx/docker/init.conf @@ -25,7 +25,10 @@ interfaces { fxp0 { unit 0 { family inet { - address 10.0.0.15/24; + address {MGMT_IP_IPV4}; + } + family inet6 { + address {MGMT_IP_IPV6}; } } } @@ -34,7 +37,12 @@ routing-instances { mgmt_junos { routing-options { static { - route 0.0.0.0/0 next-hop 10.0.0.2; + route 0.0.0.0/0 next-hop {MGMT_GW_IPV4}; + } + rib mgmt_junos.inet6.0 { + static { + route ::/0 next-hop {MGMT_GW_IPV6}; + } } } } diff --git a/vsrx/docker/launch.py b/vsrx/docker/launch.py index e6f886dc..6df420cf 100755 --- a/vsrx/docker/launch.py +++ b/vsrx/docker/launch.py @@ -52,6 +52,7 @@ def __init__(self, hostname, username, password, conn_mode): driveif="virtio", cpu="SandyBridge,vme=on,ss=on,vmx=on,f16c=on,rdrand=on,hypervisor=on,arat=on,tsc-adjust=on,umip=on,arch-capabilities=on,pdpe1gb=on,skip-l1dfl-vmentry=on,pschange-mc-no=on,bmi1=off,avx2=off,bmi2=off,erms=off,invpcid=off,rdseed=off,adx=off,smap=off,xsaveopt=off,abm=off,svm=on,aes=on", smp="2,sockets=1,cores=2,threads=1", + mgmt_passthrough=False, ) self.nic_type = "virtio-net-pci" self.conn_mode = conn_mode @@ -61,10 +62,14 @@ def __init__(self, hostname, username, password, conn_mode): with open("init.conf", "r") as file: cfg = file.read() - new_cfg = cfg.replace("{HOSTNAME}", hostname) + cfg = cfg.replace("{MGMT_IP_IPV4}", self.mgmt_address_ipv4) + cfg = cfg.replace("{MGMT_GW_IPV4}", self.mgmt_gw_ipv4) + cfg = cfg.replace("{MGMT_IP_IPV6}", self.mgmt_address_ipv6) + cfg = cfg.replace("{MGMT_GW_IPV6}", self.mgmt_gw_ipv6) + cfg = cfg.replace("{HOSTNAME}", self.hostname) with open("init.conf", "w") as file: - cfg = file.write(new_cfg) + cfg = file.write(cfg) self.startup_config() diff --git a/xrv/docker/Dockerfile b/xrv/docker/Dockerfile index fb4a41aa..a5cbc2ac 100644 --- a/xrv/docker/Dockerfile +++ b/xrv/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 +FROM public.ecr.aws/docker/library/debian:bookworm-slim LABEL org.opencontainers.image.authors="roman@dodin.dev" ARG DEBIAN_FRONTEND=noninteractive diff --git a/xrv/docker/launch.py b/xrv/docker/launch.py index 7c0557cd..662214d3 100755 --- a/xrv/docker/launch.py +++ b/xrv/docker/launch.py @@ -172,13 +172,18 @@ def bootstrap_config(self): self.wait_write("description Containerlab management VRF (DO NOT DELETE)") self.wait_write("address-family ipv4 unicast") self.wait_write("exit") + self.wait_write("address-family ipv6 unicast") + self.wait_write("exit") self.wait_write("exit") # add static route for management self.wait_write("router static") self.wait_write("vrf clab-mgmt") self.wait_write("address-family ipv4 unicast") - self.wait_write("0.0.0.0/0 10.0.0.2") + self.wait_write(f"0.0.0.0/0 {self.mgmt_gw_ipv4}") + self.wait_write("exit") + self.wait_write("address-family ipv6 unicast") + self.wait_write(f"::/0 {self.mgmt_gw_ipv6}") self.wait_write("exit") self.wait_write("exit") self.wait_write("exit") @@ -203,7 +208,8 @@ def bootstrap_config(self): self.wait_write("interface MgmtEth 0/0/CPU0/0") self.wait_write("vrf clab-mgmt") self.wait_write("no shutdown") - self.wait_write("ipv4 address 10.0.0.15/24") + self.wait_write(f"ipv4 address {self.mgmt_address_ipv4}") + self.wait_write(f"ipv6 address {self.mgmt_address_ipv6}") self.wait_write("exit") self.wait_write("commit") self.wait_write("exit") diff --git a/xrv9k/docker/Dockerfile b/xrv9k/docker/Dockerfile index 4658bdf3..73e3a8f5 100644 --- a/xrv9k/docker/Dockerfile +++ b/xrv9k/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 +FROM public.ecr.aws/docker/library/debian:bookworm-slim LABEL org.opencontainers.image.authors="roman@dodin.dev" ARG DEBIAN_FRONTEND=noninteractive diff --git a/xrv9k/docker/launch.py b/xrv9k/docker/launch.py index eeacca70..50ba30c5 100755 --- a/xrv9k/docker/launch.py +++ b/xrv9k/docker/launch.py @@ -72,22 +72,9 @@ def __init__(self, hostname, username, password, nics, conn_mode, vcpu, ram, ins def gen_mgmt(self): """Generate qemu args for the mgmt interface(s)""" - res = [] - # mgmt interface - res.extend( - ["-device", "virtio-net-pci,netdev=mgmt,mac=%s" % vrnetlab.gen_mac(0)] - ) - res.extend( - [ - "-netdev", - "user,id=mgmt,net=10.0.0.0/24," - "tftp=/tftpboot," - "hostfwd=tcp:0.0.0.0:22-10.0.0.15:22," - "hostfwd=udp:0.0.0.0:161-10.0.0.15:161," - "hostfwd=tcp:0.0.0.0:830-10.0.0.15:830," - "hostfwd=tcp:0.0.0.0:57400-10.0.0.15:57400" - ] - ) + + res = super().gen_mgmt() + # dummy interface for xrv9k ctrl interface res.extend( [ @@ -244,13 +231,18 @@ def bootstrap_config(self): self.wait_write("description Containerlab management VRF (DO NOT DELETE)") self.wait_write("address-family ipv4 unicast") self.wait_write("exit") + self.wait_write("address-family ipv6 unicast") + self.wait_write("exit") self.wait_write("exit") # add static route for management self.wait_write("router static") self.wait_write("vrf clab-mgmt") self.wait_write("address-family ipv4 unicast") - self.wait_write("0.0.0.0/0 10.0.0.2") + self.wait_write(f"0.0.0.0/0 {self.mgmt_gw_ipv4}") + self.wait_write("exit") + self.wait_write("address-family ipv6 unicast") + self.wait_write(f"::/0 {self.mgmt_gw_ipv6}") self.wait_write("exit") self.wait_write("exit") self.wait_write("exit") @@ -274,10 +266,10 @@ def bootstrap_config(self): self.wait_write("interface MgmtEth0/RP0/CPU0/0") self.wait_write("vrf clab-mgmt") self.wait_write("no shutdown") - self.wait_write("ipv4 address 10.0.0.15/24") - self.wait_write("exit") + self.wait_write(f"ipv4 address {self.mgmt_address_ipv4}") + self.wait_write(f"ipv6 address {self.mgmt_address_ipv6}") self.wait_write("commit") - self.wait_write("exit") + self.wait_write("end") return True @@ -365,7 +357,7 @@ def install(self): parser.add_argument("--nics", type=int, default=128, help="Number of NICS") parser.add_argument('--install', action="store_true", help="Pre-install image") parser.add_argument( - "--vcpu", type=int, default=2, help="Number of cpu cores to use" + "--vcpu", type=int, default=4, help="Number of cpu cores to use" ) parser.add_argument( "--ram", type=int, default=16384, help="Number RAM to use in MB"