diff --git a/README.md b/README.md index e2485366..cb21eadd 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,22 @@ Other connection mode values are: * ovs-bridge - same as a regular bridge, but uses OvS. Can pass LACP traffic. * macvtap +## Management interface + +There are two types of management connectivity for NOS VMs: _pass-through_ and _host-forwarded_ (legacy) management interfaces. + +_Pass-through management_ interfaces allows the use of the assigned management IP within the NOS VM, management traffic is transparently passed through to the VM, and the NOS configuration can accurately reflect the management IP. However, it is no longer possible to send or receive traffic directly in the vrnetlab container (e.g. for installing additional packages within the container), other than to pre-defined exceptions, such as the QEMU serial port on TCP port 5000. + +NOSes defaulting to _pass-through_ management interfaces are: +- All vJunos routers + +In case of _host-forwarded_ management interfaces, certain ports are forwarded to the NOS VM IP, which is always 10.0.0.15/24. The management gateway in this case is 10.0.0.2/24, and outgoing traffic is NATed to the container management IP. This management interface connection mode does not allow for traffic such as LLDP to pass through the management interface. + +NOSes defaulting to _host-forwarded_ management interfaces are: +- Every NOS not listed as pass-through management + +It is possible to change from the default management interface mode by setting the `CLAB_MGMT_PASSTHROUGH` environment variable to 'true' or 'false', however, it is left up to the user to provide a startup configuration compatible with the requested mode. + ## Which vrnetlab routers are supported? Since the changes we made in this fork are VM specific, we added a few popular routing products: diff --git a/common/vrnetlab.py b/common/vrnetlab.py index 58258262..9e2252e6 100644 --- a/common/vrnetlab.py +++ b/common/vrnetlab.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import datetime +import ipaddress import json import logging import math @@ -10,7 +11,6 @@ import subprocess import telnetlib import time -import ipaddress from pathlib import Path MAX_RETRIES = 60 @@ -78,6 +78,7 @@ def __init__( provision_pci_bus=True, cpu="host", smp="1", + mgmt_passthrough=False, min_dp_nics=0, ): self.logger = logging.getLogger() @@ -109,6 +110,22 @@ def __init__( # to have them allocated sequential from eth1 self.highest_provisioned_nic_num = 0 + # Whether the management interface is pass-through or host-forwarded + self.mgmt_nic_passthrough = mgmt_passthrough + mgmt_passthrough_override = os.environ.get("CLAB_MGMT_PASSTHROUGH", "") + if mgmt_passthrough_override: + self.mgmt_nic_passthrough = mgmt_passthrough_override.lower() == "true" + + # Populate management IP and gateway + if self.mgmt_nic_passthrough: + self.mgmt_address_ipv4, self.mgmt_address_ipv6 = self.get_mgmt_address() + self.mgmt_gw_ipv4, self.mgmt_gw_ipv6 = self.get_mgmt_gw() + else: + self.mgmt_address_ipv4 = "10.0.0.15/24" + self.mgmt_address_ipv6 = "2001:db8::2/64" + self.mgmt_gw_ipv4 = "10.0.0.2" + self.mgmt_gw_ipv6 = "2001:db8::1" + self.insuffucient_nics = False self.min_nics = 0 # if an image needs minimum amount of dataplane nics to bootup, specify @@ -116,9 +133,9 @@ def __init__( self.min_nics = min_dp_nics # management subnet properties, defaults - self.mgmt_subnet = "10.0.0.0/24" - self.mgmt_host_ip = 2 - self.mgmt_guest_ip = 15 + self.mgmt_subnet = "10.0.0.0/24" + self.mgmt_host_ip = 2 + self.mgmt_guest_ip = 15 # Default TCP ports forwarded (TODO tune per platform): # 80 - http @@ -129,7 +146,7 @@ def __init__( # 9339 - iana gnmi/gnoi # 32767 - gnmi/gnoi juniper # 57400 - nokia gnmi/gnoi - self.mgmt_tcp_ports = [80,443,830,6030,8080,9339,32767,57400] + self.mgmt_tcp_ports = [80, 443, 830, 6030, 8080, 9339, 32767, 57400] # we setup pci bus by default self.provision_pci_bus = provision_pci_bus @@ -299,20 +316,51 @@ def create_tc_tap_ifup(self): ip link set $TAP_IF mtu 65000 # create tc eth<->tap redirect rules - tc qdisc add dev eth$INDEX ingress - tc filter add dev eth$INDEX parent ffff: protocol all u32 match u8 0 0 action mirred egress redirect dev tap$INDEX + tc qdisc add dev eth$INDEX clsact + tc filter add dev eth$INDEX ingress flower action mirred egress redirect dev tap$INDEX - tc qdisc add dev $TAP_IF ingress - tc filter add dev $TAP_IF parent ffff: protocol all u32 match u8 0 0 action mirred egress redirect dev eth$INDEX + tc qdisc add dev $TAP_IF clsact + tc filter add dev $TAP_IF ingress flower action mirred egress redirect dev eth$INDEX """ with open("/etc/tc-tap-ifup", "w") as f: f.write(ifup_script) os.chmod("/etc/tc-tap-ifup", 0o777) + def create_tc_tap_mgmt_ifup(self): + """Create tap ifup script that is used in tc datapath mode, specifically for the management interface""" + ifup_script = """#!/bin/bash + + ip link set tap0 up + ip link set tap0 mtu 65000 + + # create tc eth<->tap redirect rules + + tc qdisc add dev eth0 clsact + # exception for TCP ports 5000-5007 + tc filter add dev eth0 ingress prio 1 protocol ip flower ip_proto tcp dst_port 5000-5007 action pass + # mirror ARP traffic to container + tc filter add dev eth0 ingress prio 2 protocol arp flower action mirred egress mirror dev tap0 + # redirect rest of ingress traffic of eth0 to egress of tap0 + tc filter add dev eth0 ingress prio 3 flower action mirred egress redirect dev tap0 + + tc qdisc add dev tap0 clsact + # redirect all ingress traffic of tap0 to egress of eth0 + tc filter add dev tap0 ingress flower action mirred egress redirect dev eth0 + + # clone management MAC of the VM + ip link set dev eth0 address {MGMT_MAC} + """ + + ifup_script = ifup_script.replace("{MGMT_MAC}", self.mgmt_mac) + + with open("/etc/tc-tap-mgmt-ifup", "w") as f: + f.write(ifup_script) + os.chmod("/etc/tc-tap-mgmt-ifup", 0o777) + def gen_mgmt(self): """Generate qemu args for the mgmt interface(s) - + Default TCP ports forwarded: 80 - http 443 - https @@ -323,34 +371,98 @@ def gen_mgmt(self): 32767 - gnmi/gnoi juniper 57400 - nokia gnmi/gnoi """ - if self.mgmt_host_ip+1>=self.mgmt_guest_ip: - self.logger.error("Guest IP (%s) must be at least 2 higher than host IP(%s)", - self.mgmt_guest_ip, self.mgmt_host_ip) + if self.mgmt_host_ip + 1 >= self.mgmt_guest_ip: + self.logger.error( + "Guest IP (%s) must be at least 2 higher than host IP(%s)", + self.mgmt_guest_ip, + self.mgmt_host_ip, + ) network = ipaddress.ip_network(self.mgmt_subnet) host = str(network[self.mgmt_host_ip]) - dns = str(network[self.mgmt_host_ip+1]) + dns = str(network[self.mgmt_host_ip + 1]) guest = str(network[self.mgmt_guest_ip]) res = [] - # mgmt interface is special - we use qemu user mode network with DHCP res.append("-device") - mac = ( + self.mgmt_mac = ( "c0:00:01:00:ca:fe" if getattr(self, "_static_mgmt_mac", False) else gen_mac(0) ) - res.append(self.nic_type + f",netdev=p00,mac={mac}") + + res.append(self.nic_type + f",netdev=p00,mac={self.mgmt_mac}") res.append("-netdev") - res.append( - f"user,id=p00,net={self.mgmt_subnet},host={host},dns={dns},dhcpstart={guest}," + - f"hostfwd=tcp:0.0.0.0:22-{guest}:22," + # ssh - f"hostfwd=udp:0.0.0.0:161-{guest}:161," + # snmp - (",".join([ f"hostfwd=tcp:0.0.0.0:{p}-{guest}:{p}" for p in self.mgmt_tcp_ports ])) + - ",tftp=/tftpboot" - ) + + if self.mgmt_nic_passthrough: + # mgmt interface is passthrough - we just create a normal mirred tap interface + res.append( + "tap,id=p00,ifname=tap0,script=/etc/tc-tap-mgmt-ifup,downscript=no" + ) + self.create_tc_tap_mgmt_ifup() + else: + # mgmt interface is host-forwarded - we use qemu user mode network + # with hostfwd rules to forward ports from the host to the guest + res.append( + f"user,id=p00,net={self.mgmt_subnet},host={host},dns={dns},dhcpstart={guest}," + + f"hostfwd=tcp:0.0.0.0:22-{guest}:22," # ssh + + f"hostfwd=udp:0.0.0.0:161-{guest}:161," # snmp + + ( + ",".join( + [ + f"hostfwd=tcp:0.0.0.0:{p}-{guest}:{p}" + for p in self.mgmt_tcp_ports + ] + ) + ) + + ",tftp=/tftpboot" + ) + return res + def get_mgmt_address(self): + """Returns the IPv4 and IPv6 address of the eth0 interface of the container""" + stdout, _ = run_command(["ip", "--json", "address", "show", "dev", "eth0"]) + command_json = json.loads(stdout.decode("utf-8")) + intf_addrinfos = command_json[0]["addr_info"] + + mgmt_cidr_v4 = None + mgmt_cidr_v6 = None + for addrinfo in intf_addrinfos: + if addrinfo["family"] == "inet" and addrinfo["scope"] == "global": + mgmt_address_v4 = addrinfo["local"] + mgmt_prefixlen_v4 = addrinfo["prefixlen"] + mgmt_cidr_v4 = mgmt_address_v4 + "/" + str(mgmt_prefixlen_v4) + if addrinfo["family"] == "inet6" and addrinfo["scope"] == "global": + mgmt_address_v6 = addrinfo["local"] + mgmt_prefixlen_v6 = addrinfo["prefixlen"] + mgmt_cidr_v6 = mgmt_address_v6 + "/" + str(mgmt_prefixlen_v6) + + if not mgmt_cidr_v4: + raise ValueError("No IPv4 address set on management interface eth0!") + + return mgmt_cidr_v4, mgmt_cidr_v6 + + def get_mgmt_gw(self): + """Returns the IPv4 and IPv6 default gateways of the container, used for generating the management default route""" + stdout_v4, _ = run_command(["ip", "--json", "-4", "route", "show", "default"]) + command_json_v4 = json.loads(stdout_v4.decode("utf-8")) + try: + mgmt_gw_v4 = command_json_v4[0]["gateway"] + except IndexError as e: + raise IndexError( + "No default gateway route on management interface eth0!" + ) from e + + stdout_v6, _ = run_command(["ip", "--json", "-6", "route", "show", "default"]) + command_json_v6 = json.loads(stdout_v6.decode("utf-8")) + try: + mgmt_gw_v6 = command_json_v6[0]["gateway"] + except IndexError: + mgmt_gw_v6 = None + + return mgmt_gw_v4, mgmt_gw_v6 + def nic_provision_delay(self) -> None: self.logger.debug( f"number of provisioned data plane interfaces is {self.num_provisioned_nics}" @@ -526,7 +638,9 @@ def restart(self): self.stop() self.start() - def wait_write(self, cmd, wait="__defaultpattern__", con=None, clean_buffer=False, hold=""): + def wait_write( + self, cmd, wait="__defaultpattern__", con=None, clean_buffer=False, hold="" + ): """Wait for something on the serial port and then send command Defaults to using self.tn as connection but this can be overridden @@ -548,8 +662,10 @@ def wait_write(self, cmd, wait="__defaultpattern__", con=None, clean_buffer=Fals self.logger.trace(f"waiting for '{wait}' on {con_name}") res = con.read_until(wait.encode()) - while (hold and (hold in res.decode())): - self.logger.trace(f"Holding pattern '{hold}' detected: {res.decode()}, retrying in 10s...") + while hold and (hold in res.decode()): + self.logger.trace( + f"Holding pattern '{hold}' detected: {res.decode()}, retrying in 10s..." + ) con.write("\r".encode()) time.sleep(10) res = con.read_until(wait.encode()) diff --git a/vjunosevolved/docker/init.conf b/vjunosevolved/docker/init.conf index 8537f7ab..07ea9742 100644 --- a/vjunosevolved/docker/init.conf +++ b/vjunosevolved/docker/init.conf @@ -25,7 +25,10 @@ interfaces { re0:mgmt-0 { unit 0 { family inet { - address 10.0.0.15/24; + address {MGMT_IP_IPV4}; + } + family inet { + address {MGMT_IP_IPV6}; } } } @@ -34,7 +37,12 @@ routing-instances { mgmt_junos { routing-options { static { - route 0.0.0.0/0 next-hop 10.0.0.2; + route 0.0.0.0/0 next-hop {MGMT_GW_IPV4}; + } + rib mgmt_junos.inet6.0 { + static { + route ::/0 next-hop {MGMT_GW_IPV6}; + } } } } diff --git a/vjunosevolved/docker/launch.py b/vjunosevolved/docker/launch.py index a2fbfd30..d3bb46c3 100755 --- a/vjunosevolved/docker/launch.py +++ b/vjunosevolved/docker/launch.py @@ -55,6 +55,7 @@ def __init__(self, hostname, username, password, conn_mode): driveif="virtio", cpu="IvyBridge,vme=on,ss=on,vmx=on,f16c=on,rdrand=on,hypervisor=on,arat=on,tsc-adjust=on,umip=on,arch-capabilities=on,pdpe1gb=on,skip-l1dfl-vmentry=on,pschange-mc-no=on,bmi1=off,avx2=off,bmi2=off,erms=off,invpcid=off,rdseed=off,adx=off,smap=off,xsaveopt=off,abm=off,svm=off", smp="4,sockets=1,cores=4,threads=1", + mgmt_passthrough=True ) # device hostname @@ -67,16 +68,18 @@ def __init__(self, hostname, username, password, conn_mode): with open("init.conf", "r") as file: cfg = file.read() - # replace HOSTNAME file var with nodes given hostname + cfg = cfg.replace("{MGMT_IP_IPV4}", self.mgmt_address_ipv4) + cfg = cfg.replace("{MGMT_GW_IPV4}", self.mgmt_gw_ipv4) + cfg = cfg.replace("{MGMT_IP_IPV6}", self.mgmt_address_ipv6) + cfg = cfg.replace("{MGMT_GW_IPV6}", self.mgmt_gw_ipv6) + cfg = cfg.replace("{HOSTNAME}", self.hostname) # replace CRYPT_PSWD file var with nodes given password # (Evo does not accept plaintext passwords in config) - new_cfg = cfg.replace("{HOSTNAME}", hostname).replace( - "{CRYPT_PSWD}", password_hash - ) + cfg = cfg.replace("{CRYPT_PSWD}", password_hash) # write changes to init.conf file with open("init.conf", "w") as file: - file.write(new_cfg) + file.write(cfg) # pass in user startup config self.startup_config() diff --git a/vjunosrouter/docker/init.conf b/vjunosrouter/docker/init.conf index b8f3ef56..d4361b75 100644 --- a/vjunosrouter/docker/init.conf +++ b/vjunosrouter/docker/init.conf @@ -25,7 +25,10 @@ interfaces { fxp0 { unit 0 { family inet { - address 10.0.0.15/24; + address {MGMT_IP_IPV4}; + } + family inet6 { + address {MGMT_IP_IPV6}; } } } @@ -34,7 +37,12 @@ routing-instances { mgmt_junos { routing-options { static { - route 0.0.0.0/0 next-hop 10.0.0.2; + route 0.0.0.0/0 next-hop {MGMT_GW_IPV4}; + } + rib mgmt_junos.inet6.0 { + static { + route ::/0 next-hop {MGMT_GW_IPV6}; + } } } } diff --git a/vjunosrouter/docker/launch.py b/vjunosrouter/docker/launch.py index 6c07034e..53aa9238 100755 --- a/vjunosrouter/docker/launch.py +++ b/vjunosrouter/docker/launch.py @@ -52,6 +52,7 @@ def __init__(self, hostname, username, password, conn_mode): driveif="virtio", cpu="IvyBridge,vme=on,ss=on,vmx=on,f16c=on,rdrand=on,hypervisor=on,arat=on,tsc-adjust=on,umip=on,arch-capabilities=on,pdpe1gb=on,skip-l1dfl-vmentry=on,pschange-mc-no=on,bmi1=off,avx2=off,bmi2=off,erms=off,invpcid=off,rdseed=off,adx=off,smap=off,xsaveopt=off,abm=off,svm=on", smp="4,sockets=1,cores=4,threads=1", + mgmt_passthrough=True ) # device hostname self.hostname = hostname @@ -61,12 +62,15 @@ def __init__(self, hostname, username, password, conn_mode): with open("init.conf", "r") as file: cfg = file.read() - # replace HOSTNAME file var with nodes given hostname - new_cfg = cfg.replace("{HOSTNAME}", hostname) + cfg = cfg.replace("{MGMT_IP_IPV4}", self.mgmt_address_ipv4) + cfg = cfg.replace("{MGMT_GW_IPV4}", self.mgmt_gw_ipv4) + cfg = cfg.replace("{MGMT_IP_IPV6}", self.mgmt_address_ipv6) + cfg = cfg.replace("{MGMT_GW_IPV6}", self.mgmt_gw_ipv6) + cfg = cfg.replace("{HOSTNAME}", self.hostname) # write changes to init.conf file with open("init.conf", "w") as file: - file.write(new_cfg) + file.write(cfg) # pass in user startup config self.startup_config() diff --git a/vjunosswitch/docker/init.conf b/vjunosswitch/docker/init.conf index b8f3ef56..d4361b75 100644 --- a/vjunosswitch/docker/init.conf +++ b/vjunosswitch/docker/init.conf @@ -25,7 +25,10 @@ interfaces { fxp0 { unit 0 { family inet { - address 10.0.0.15/24; + address {MGMT_IP_IPV4}; + } + family inet6 { + address {MGMT_IP_IPV6}; } } } @@ -34,7 +37,12 @@ routing-instances { mgmt_junos { routing-options { static { - route 0.0.0.0/0 next-hop 10.0.0.2; + route 0.0.0.0/0 next-hop {MGMT_GW_IPV4}; + } + rib mgmt_junos.inet6.0 { + static { + route ::/0 next-hop {MGMT_GW_IPV6}; + } } } } diff --git a/vjunosswitch/docker/launch.py b/vjunosswitch/docker/launch.py index 7f23d13d..40c0d3d3 100755 --- a/vjunosswitch/docker/launch.py +++ b/vjunosswitch/docker/launch.py @@ -52,6 +52,7 @@ def __init__(self, hostname, username, password, conn_mode): driveif="virtio", cpu="IvyBridge,vme=on,ss=on,vmx=on,f16c=on,rdrand=on,hypervisor=on,arat=on,tsc-adjust=on,umip=on,arch-capabilities=on,pdpe1gb=on,skip-l1dfl-vmentry=on,pschange-mc-no=on,bmi1=off,avx2=off,bmi2=off,erms=off,invpcid=off,rdseed=off,adx=off,smap=off,xsaveopt=off,abm=off,svm=on", smp="4,sockets=1,cores=4,threads=1", + mgmt_passthrough=True ) # device hostname self.hostname = hostname @@ -61,12 +62,15 @@ def __init__(self, hostname, username, password, conn_mode): with open("init.conf", "r") as file: cfg = file.read() - # replace HOSTNAME file var with nodes given hostname - new_cfg = cfg.replace("{HOSTNAME}", hostname) + cfg = cfg.replace("{MGMT_IP_IPV4}", self.mgmt_address_ipv4) + cfg = cfg.replace("{MGMT_GW_IPV4}", self.mgmt_gw_ipv4) + cfg = cfg.replace("{MGMT_IP_IPV6}", self.mgmt_address_ipv6) + cfg = cfg.replace("{MGMT_GW_IPV6}", self.mgmt_gw_ipv6) + cfg = cfg.replace("{HOSTNAME}", self.hostname) # write changes to init.conf file with open("init.conf", "w") as file: - file.write(new_cfg) + file.write(cfg) # pass in user startup config self.startup_config()